I am trying to use aiohttp to send requests one after another like this
import aiohttp
import asyncio
from datetime import datetime
async def main():
request_url = "https://..."
async with aiohttp.ClientSession() as session:
while True:
print(datetime.now())
async with session.get(request_url) as response:
json_data = await response.json()
print(json_data)
await asyncio.sleep(0.2)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
So I would expect each datetime print to be 0.2s apart. However, they seem to be about 0.35s apart as I think it takes 0.15s to get the data from the response. Why is this happening? I want it to be asynchronous so it should just go onto the next one?
How can I fix this?
When you use await all next code will wait for end of this code.
If you want to run asyncio code asynchronously, you should use functions like asyncio.gather
import asyncio
import aiohttp
import datetime
async def fetch(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
print('#', response.status)
async def worker(queue):
print('START WORKER')
while True:
url = await queue.get()
await fetch(url)
queue.task_done()
async def control(queue):
while True:
print(datetime.datetime.now())
queue.put_nowait('https://docs.python.org/')
await asyncio.sleep(0.2)
async def main():
queue = asyncio.Queue()
await asyncio.gather(
control(queue),
asyncio.gather(*[worker(queue) for _ in range(10)])
)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Sending http request and fetching response back takes some time. You need excluding this time from asyncio.sleep() call:
import aiohttp
import asyncio
import time
from datetime import datetime
async def main():
request_url = "https://..."
async with aiohttp.ClientSession() as session:
while True:
print(datetime.now())
t0 = time.monotonic()
async with session.get(request_url) as response:
json_data = await response.json()
print(json_data)
t1 = time.monotonic()
await asyncio.sleep(0.2 - (t1 - t0))
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Related
I am using aiohttp and asyncio to run multiple requests asynchronously, the problem is when i try to print the data i receive i end up getting the data of another request in the task queue. I have tried to debug this and look at the docs for any answers but i am unable to solve this problem.
here's my code:
from time import sleep
import aiohttp
import asyncio
async def search(query, session):
search_params = {
"query":query
}
async with session.get(
url,
params=search_params,
) as response:
json_response = await response.json()
data = json_response["data"]
print(data)
"""the above line always prints the data from the response of the first task to get executed
and not the current data from this request with a different query"""
async def main():
async with aiohttp.ClientSession() as session:
await init_session(session)
await enable_search(session)
while True:
tasks = [asyncio.create_task(search(session=session, query)) for query in inputs]
await asyncio.gather(*tasks)
sleep(5)
if __name__ == "__main__":
asyncio.run(main())
I'm new to Async and, I wanna create a script that will execute a request every half second to check if a website is available. So even if the website response time is like '4s' it will execute another request every 0.5 seconds. Once one of the requests receives the "200" status code the event loop will break.
URL = "https://stackoverflow.com"
async def load(session, url):
async with session.get(url) as response:
return await response.status == 200
async def create_session():
complete = False
while not complete:
async with aiohttp.ClientSession() as session:
task = await asyncio.create_task(load(session, URL))
if task:
break
await asyncio.sleep(0.5)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(create_session())
Right now I got something like this which obviously won't work.
I was able to create the desired program using the asyncio.Event().
import asyncio
import aiohttp
url = "https://www.somesite.com"
async def load(session, url, flag):
async with session.get(url) as response:
if await response.status == 200: #Check if the site is available.
flag.set() # Flag is set
async def create_session():
flag = asyncio.Event()
async with aiohttp.ClientSession() as session: # Create aiohttp Session
while 1:
asyncio.create_task(load(session, url, flag))
await asyncio.sleep(0.5) # Wait 0.5 s between the requests
if flag.is_set():# If flag is set then break the loop
break
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(create_session())
I found this package aiomultiprocess that seems like it can do both multiprocessing and asyncio.
from aiohttp import request
from aiomultiprocess import Pool
async def get(url):
async with request("GET", url) as response:
return await response.text("utf-8")
async def main():
urls = ["https://jreese.sh", "https://www.google.com", ]
async with Pool() as pool:
async for result in pool.map(get, urls):
print(result)
Trying to run the sample code, though, does absolutely nothing.
Trying to call the main() gives me an error RuntimeWarning: coroutine 'main' was never awaited. I can't find an actual example of how to trigger the code.
The only other question about this isn't answered.
The aiomultiprocess documentation example does not cover how to call the loop. The function needs to be called via asyncio.
import asyncio
from aiohttp import request
from aiomultiprocess import Pool
async def get(url):
async with request("GET", url) as response:
return await response.read()
async def main():
urls = ["https://jreese.sh", "https://www.google.com", ]
async with Pool() as pool:
async for result in pool.map(get, urls):
print(result)
if __name__ == '__main__':
# for Python 3.7
asyncio.run(main())
# for Python 3.6
# loop = asyncio.get_event_loop()
# loop.run_until_complete(main())
I need to know how much Mbytes per second there are in the request with is sent with asyncio in python. I tried with resp.read() or with resp.content.read() but it isn't working.
import aiohttp
import asyncio
async def get_requests(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
result = resp
return result
big_urls = ['http://google.com', 'http://yahoo.com']
loop = asyncio.get_event_loop()
coroutines = [get_requests(url) for url in big_urls]
results = loop.run_until_complete(asyncio.gather(*coroutines))
print(results)
I have the following code
import asyncio
import aiohttp
urls = [
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
]
async def query(urls):
out = []
with aiohttp.ClientSession() as session:
for url in urls:
try:
async with session.get(url, timeout=5) as resp:
text = await resp.text()
out.append(resp.status)
except:
print('timeout')
return out
loop = asyncio.get_event_loop()
out = loop.run_until_complete(query(urls))
loop.close()
print(str(out))
The code is much slower than the one that uses a threadpool and keep increasing if you increase the number of urls (lets say 20, 50 etc.)
I have a feeling that when the initial connection establishment is not done in an async way.
(Note that I am connecting here to an non-existing server to deliberately produce a connection timeout).
Can someone point out what is wrong here?
Warning: I don't promise this code works, as I can't install aiohttp atm, but looking at the example in the docs
async def fetch(session, url):
async with async_timeout.timeout(10):
async with session.get(url) as response:
return await response.text()
async def main():
async with aiohttp.ClientSession() as session:
html = await fetch(session, 'http://python.org')
print(html)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Notice how they're calling the aiohttp.ClientSession() with the async keyword. Additionally, I was getting some error in your line data = await async with session.get(url) as resp:
async def fetch(session, url):
async with session.get(url) as response:
return await response.text()
async def main():
out = []
async with aiohttp.ClientSession() as session:
for url in urls:
data = await fetch(session, url)
out.append(data)
return out
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())