I am trying to achieve aiohttp async processing of requests that have been defined in my class as follows:
class Async():
async def get_service_1(self, zip_code, session):
url = SERVICE1_ENDPOINT.format(zip_code)
response = await session.request('GET', url)
return await response
async def get_service_2(self, zip_code, session):
url = SERVICE2_ENDPOINT.format(zip_code)
response = await session.request('GET', url)
return await response
async def gather(self, zip_code):
async with aiohttp.ClientSession() as session:
return await asyncio.gather(
self.get_service_1(zip_code, session),
self.get_service_2(zip_code, session)
)
def get_async_requests(self, zip_code):
asyncio.set_event_loop(asyncio.SelectorEventLoop())
loop = asyncio.get_event_loop()
results = loop.run_until_complete(self.gather(zip_code))
loop.close()
return results
When running to get the results from the get_async_requests function, i am getting the following error:
TypeError: object ClientResponse can't be used in 'await' expression
Where am i going wrong in the code? Thank you in advance
When you await something like session.response, the I/O starts, but aiohttp returns when it receives the headers; it doesn't want for the response to finish. (This would let you react to a status code without waiting for the entire body of the response.)
You need to await something that does that. If you're expecting a response that contains text, that would be response.text. If you're expecting JSON, that's response.json. This would look something like
response = await session.get(url)
return await response.text()
Related
I am using aiohttp and asyncio to run multiple requests asynchronously, the problem is when i try to print the data i receive i end up getting the data of another request in the task queue. I have tried to debug this and look at the docs for any answers but i am unable to solve this problem.
here's my code:
from time import sleep
import aiohttp
import asyncio
async def search(query, session):
search_params = {
"query":query
}
async with session.get(
url,
params=search_params,
) as response:
json_response = await response.json()
data = json_response["data"]
print(data)
"""the above line always prints the data from the response of the first task to get executed
and not the current data from this request with a different query"""
async def main():
async with aiohttp.ClientSession() as session:
await init_session(session)
await enable_search(session)
while True:
tasks = [asyncio.create_task(search(session=session, query)) for query in inputs]
await asyncio.gather(*tasks)
sleep(5)
if __name__ == "__main__":
asyncio.run(main())
I am new to asynchronous programming in python, have been working on a script using aiohttp that fetches data from a get request and passes a specific variable from the response onto another post request. A sample of what I have tried is below:
async def fetch1(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp: # First hit to the url
data = resp.json() # Grab response
return await fetch2(data['uuid']) # Pass uuid to the second function for post request
async def fetch2(id):
url2 = "http://httpbin.org/post"
params = {'id': id}
async with aiohttp.ClientSession() as session:
async with session.post(url2,data=params) as resp:
return await resp.json()
async def main():
url = 'http://httpbin.org/uuid'
data = await fetch1(url)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
When I execute the script, I get the following error:
Traceback (most recent call last):
File ".\benchmark.py", line 27, in <module>
loop.run_until_complete(main())
File "C:\ProgramFiles\WindowsApps\PythonSoftwareFoundation.Python.3.8_3.8.2288.0_x64__qbz5n2kfra8p0\lib\asyncio\base_events.py", line 616, in run_until_complete
return future.result()
File ".\benchmark.py", line 22, in main
data = await fetch1(url)
File ".\benchmark.py", line 10, in fetch1
return fetch2(data['uuid'])
TypeError: 'coroutine' object is not subscriptable
sys:1: RuntimeWarning: coroutine 'ClientResponse.json' was never awaited
I know that the coroutine is a generator, but how do I go ahead, any help will be appreciated.
The error says coroutine 'ClientResponse.json' was never awaited which means it must have an await before the json part. This is because you are using an async function.
async def fetch1(url):
async with aiohttp.ClientSession() as session:
async with session.get(url) as resp: # First hit to the url
data = await resp.json() # Grab response
return await fetch2(data['uuid']) # Pass uuid to the second function for post request
async def fetch2(id):
url2 = "http://httpbin.org/post"
params = {'id': id}
async with aiohttp.ClientSession() as session:
async with session.post(url2,data=params) as resp:
return await resp.json()
async def main():
url = 'http://httpbin.org/uuid'
data = await fetch1(url)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I am trying to add the user's response into the URL. My code is
async def dungeondata():
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
async with session.get('https://sky.shiiyu.moe/api/v2/dungeons/{}/{}'.format(name, cutename)) as resp:
return await resp.json()
#bot.command(name='dungeon', aliases=['dungeons'])
async def dungeon(ctx, name, cutename):
JSONData = await dungeondata(name, cutename)
When the user does ?dungeons , I am trying to add the and to the URL so the url becomes https://sky.shiiyu.moe/api/v2/dungeons/name/cutename. How do I do that?
Add name and cutename as arguments in dungeon function.
async def dungeon(name, cutename):
async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session:
async with session.get('https://sky.shiiyu.moe/api/v2/dungeons/{}/{}'.format(name, cutename)) as resp:
return await resp.json()
Question is regarding aiohttp libriary usage.
My goal here is to check list of urls by sending bunch of HEAD requests, potentially asynchronously ,and eventually create dict of
url: status pairs.
I am new in asyncio and stuff and I found a lot of examples where people use GET requests to fetch html ,for example ,and they use await resp.read() or await resp.text() and it works fine but with HEAD request I don’t have body, I just have header, that's it. If I try to await resp.status or resp itself as an object – it does not work as they are not awaitable.
Code below works only synchronously step by step and I can’t figure out how to make it run asynchronously. Seems like whatever i do with status turns code to sync mode somehow...
I would be glad to see your ideas.
Thanks.
import asyncio
import aiohttp
urls_list = [url1, url2, url3, etc, etc, etc, ]
status_dict = {}
async def main():
async with aiohttp.ClientSession() as session:
for individual_url in urls_list:
async with session.head(individual_url) as resp:
status_dict.update({url: resp.status})
asyncio.run(main())
You can you asyncio.gather:
import asyncio
import aiohttp
urls_list = ["https://google.com", "https://yahoo.com", "http://hello123456789.com"]
status_dict = {}
async def head_status(session, url) -> dict:
async with session.head(url) as resp:
return {url: resp.status}
async def main():
async with aiohttp.ClientSession() as session:
statuses = await asyncio.gather(*[head_status(session, url) for url in urls_list], return_exceptions=True)
for a in statuses:
if not isinstance(a, Exception):
status_dict.update(a)
asyncio.run(main())
I have a script that checks the status code for a couple hundred thousand supplied websites, and I was trying to integrate a Semaphore to the flow to speed up processing. The problem is that whenever I integrate a Semaphore, I just get a list populated with None objects, and I'm not entirely sure why.
I have been mostly copying code from other sources as I don't fully grok asynchronous programming fully yet, but it seems like when I debug I should be getting results out of the function, but something is going wrong when I gather the results. I've tried juggling around my looping, my gathering, ensuring futures, etc, but nothing seems to return a list of things that work.
async def fetch(session, url):
try:
async with session.head(url, allow_redirects=True) as resp:
return url, resp.real_url, resp.status, resp.reason
except Exception as e:
return url, None, e, 'Error'
async def bound_fetch(sem, session, url):
async with sem:
await fetch(session, url)
async def run(urls):
timeout = 15
tasks = []
sem = asyncio.Semaphore(100)
conn = aiohttp.TCPConnector(limit=64, ssl=False)
async with aiohttp.ClientSession(connector=conn) as session:
for url in urls:
task = asyncio.wait_for(bound_fetch(sem, session, url), timeout)
tasks.append(task)
responses = await asyncio.gather(*tasks)
# responses = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(tasks))]
return responses
urls = ['https://google.com', 'https://yahoo.com']
loop = asyncio.ProactorEventLoop()
data = loop.run_until_complete(run(urls))
I've commented out the progress bar component, but that implementation returns the desired results when there is no semaphore.
Any help would be greatly appreciated. I am furiously reading up on asynchronous programming, but I can't wrap my mind around it yet.
You should explicitly return results of awaiting coroutines.
Replace this code...
async def bound_fetch(sem, session, url):
async with sem:
await fetch(session, url)
... with this:
async def bound_fetch(sem, session, url):
async with sem:
return await fetch(session, url)