I have the following code
import asyncio
import aiohttp
urls = [
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
'http://54.224.27.241',
]
async def query(urls):
out = []
with aiohttp.ClientSession() as session:
for url in urls:
try:
async with session.get(url, timeout=5) as resp:
text = await resp.text()
out.append(resp.status)
except:
print('timeout')
return out
loop = asyncio.get_event_loop()
out = loop.run_until_complete(query(urls))
loop.close()
print(str(out))
The code is much slower than the one that uses a threadpool and keep increasing if you increase the number of urls (lets say 20, 50 etc.)
I have a feeling that when the initial connection establishment is not done in an async way.
(Note that I am connecting here to an non-existing server to deliberately produce a connection timeout).
Can someone point out what is wrong here?
Warning: I don't promise this code works, as I can't install aiohttp atm, but looking at the example in the docs
async def fetch(session, url):
async with async_timeout.timeout(10):
async with session.get(url) as response:
return await response.text()
async def main():
async with aiohttp.ClientSession() as session:
html = await fetch(session, 'http://python.org')
print(html)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Notice how they're calling the aiohttp.ClientSession() with the async keyword. Additionally, I was getting some error in your line data = await async with session.get(url) as resp:
async def fetch(session, url):
async with session.get(url) as response:
return await response.text()
async def main():
out = []
async with aiohttp.ClientSession() as session:
for url in urls:
data = await fetch(session, url)
out.append(data)
return out
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Related
I'm new to Async and, I wanna create a script that will execute a request every half second to check if a website is available. So even if the website response time is like '4s' it will execute another request every 0.5 seconds. Once one of the requests receives the "200" status code the event loop will break.
URL = "https://stackoverflow.com"
async def load(session, url):
async with session.get(url) as response:
return await response.status == 200
async def create_session():
complete = False
while not complete:
async with aiohttp.ClientSession() as session:
task = await asyncio.create_task(load(session, URL))
if task:
break
await asyncio.sleep(0.5)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(create_session())
Right now I got something like this which obviously won't work.
I was able to create the desired program using the asyncio.Event().
import asyncio
import aiohttp
url = "https://www.somesite.com"
async def load(session, url, flag):
async with session.get(url) as response:
if await response.status == 200: #Check if the site is available.
flag.set() # Flag is set
async def create_session():
flag = asyncio.Event()
async with aiohttp.ClientSession() as session: # Create aiohttp Session
while 1:
asyncio.create_task(load(session, url, flag))
await asyncio.sleep(0.5) # Wait 0.5 s between the requests
if flag.is_set():# If flag is set then break the loop
break
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(create_session())
I am new to asynchronous programming in python, have been working on a script using aiohttp that fetches data from a get request and passes a specific variable from the response onto another post request. A sample of what I have tried is below:
async def fetch1(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp: # First hit to the url
data = resp.json() # Grab response
return await fetch2(data['uuid']) # Pass uuid to the second function for post request
async def fetch2(id):
url2 = "http://httpbin.org/post"
params = {'id': id}
async with aiohttp.ClientSession() as session:
async with session.post(url2,data=params) as resp:
return await resp.json()
async def main():
url = 'http://httpbin.org/uuid'
data = await fetch1(url)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
When I execute the script, I get the following error:
Traceback (most recent call last):
File ".\benchmark.py", line 27, in <module>
loop.run_until_complete(main())
File "C:\ProgramFiles\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2288.0_x64__qbz5n2kfra8p0\lib\asyncio\base_events.py", line 616, in run_until_complete
return future.result()
File ".\benchmark.py", line 22, in main
data = await fetch1(url)
File ".\benchmark.py", line 10, in fetch1
return fetch2(data['uuid'])
TypeError: 'coroutine' object is not subscriptable
sys:1: RuntimeWarning: coroutine 'ClientResponse.json' was never awaited
I know that the coroutine is a generator, but how do I go ahead, any help will be appreciated.
The error says coroutine 'ClientResponse.json' was never awaited which means it must have an await before the json part. This is because you are using an async function.
async def fetch1(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp: # First hit to the url
data = await resp.json() # Grab response
return await fetch2(data['uuid']) # Pass uuid to the second function for post request
async def fetch2(id):
url2 = "http://httpbin.org/post"
params = {'id': id}
async with aiohttp.ClientSession() as session:
async with session.post(url2,data=params) as resp:
return await resp.json()
async def main():
url = 'http://httpbin.org/uuid'
data = await fetch1(url)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I am trying to achieve aiohttp async processing of requests that have been defined in my class as follows:
class Async():
async def get_service_1(self, zip_code, session):
url = SERVICE1_ENDPOINT.format(zip_code)
response = await session.request('GET', url)
return await response
async def get_service_2(self, zip_code, session):
url = SERVICE2_ENDPOINT.format(zip_code)
response = await session.request('GET', url)
return await response
async def gather(self, zip_code):
async with aiohttp.ClientSession() as session:
return await asyncio.gather(
self.get_service_1(zip_code, session),
self.get_service_2(zip_code, session)
)
def get_async_requests(self, zip_code):
asyncio.set_event_loop(asyncio.SelectorEventLoop())
loop = asyncio.get_event_loop()
results = loop.run_until_complete(self.gather(zip_code))
loop.close()
return results
When running to get the results from the get_async_requests function, i am getting the following error:
TypeError: object ClientResponse can't be used in 'await' expression
Where am i going wrong in the code? Thank you in advance
When you await something like session.response, the I/O starts, but aiohttp returns when it receives the headers; it doesn't want for the response to finish. (This would let you react to a status code without waiting for the entire body of the response.)
You need to await something that does that. If you're expecting a response that contains text, that would be response.text. If you're expecting JSON, that's response.json. This would look something like
response = await session.get(url)
return await response.text()
I need to know how much Mbytes per second there are in the request with is sent with asyncio in python. I tried with resp.read() or with resp.content.read() but it isn't working.
import aiohttp
import asyncio
async def get_requests(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp:
result = resp
return result
big_urls = ['http://google.com', 'http://yahoo.com']
loop = asyncio.get_event_loop()
coroutines = [get_requests(url) for url in big_urls]
results = loop.run_until_complete(asyncio.gather(*coroutines))
print(results)
I am trying to use aiohttp to send requests one after another like this
import aiohttp
import asyncio
from datetime import datetime
async def main():
request_url = "https://..."
async with aiohttp.ClientSession() as session:
while True:
print(datetime.now())
async with session.get(request_url) as response:
json_data = await response.json()
print(json_data)
await asyncio.sleep(0.2)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
So I would expect each datetime print to be 0.2s apart. However, they seem to be about 0.35s apart as I think it takes 0.15s to get the data from the response. Why is this happening? I want it to be asynchronous so it should just go onto the next one?
How can I fix this?
When you use await all next code will wait for end of this code.
If you want to run asyncio code asynchronously, you should use functions like asyncio.gather
import asyncio
import aiohttp
import datetime
async def fetch(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
print('#', response.status)
async def worker(queue):
print('START WORKER')
while True:
url = await queue.get()
await fetch(url)
queue.task_done()
async def control(queue):
while True:
print(datetime.datetime.now())
queue.put_nowait('https://docs.python.org/')
await asyncio.sleep(0.2)
async def main():
queue = asyncio.Queue()
await asyncio.gather(
control(queue),
asyncio.gather(*[worker(queue) for _ in range(10)])
)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
Sending http request and fetching response back takes some time. You need excluding this time from asyncio.sleep() call:
import aiohttp
import asyncio
import time
from datetime import datetime
async def main():
request_url = "https://..."
async with aiohttp.ClientSession() as session:
while True:
print(datetime.now())
t0 = time.monotonic()
async with session.get(request_url) as response:
json_data = await response.json()
print(json_data)
t1 = time.monotonic()
await asyncio.sleep(0.2 - (t1 - t0))
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())