I am new with async functions and i'm trying to make multiple calls from an external API.
concurrent.Futures is not quite enough to retrieve the responses so i tried with asyncio and httpx but the process is throwing an error unknown and difficult to debug for me.
It seems that the coroutine is having an empty value or is never being called.
This is my client
async def get_product_by_id_async(self, product_id: str) -> Response:
if product_id is None:
return None
response_await = await self.async_client.get(
url=f"{self.base_url}/api/catalog/stock{product_id}",
headers=self.headers,
)
if response_await.status_code == 200:
response = json.loads(response_await.text)
return response
And this is my coordinating function
async def async_get_products(data_parser):
path = data_parser.options["path"]
sku_list = client.run()
products_list = []
tasks = [await client.get_product_by_id(sku) for sku in sku_list]
breakpoint()
completed = await asyncio.gather(*tasks)
for product in completed:
products_list = build_product_df(products_list, product)
products_df = pd.DataFrame(products_list)
products_df.to_csv(path, index=False)
return products_df
def products_processor_concurrent(data_parser):
return async_get_products(data_parser)
After the solution given there is a new error in the code:
This is the trace:
Traceback (most recent call last):
File "manage.py", line 32, in <module>
module.run()
File "/opt/project/providers/api/data_collector.py", line 90, in run
integrator.run()
File "/opt/project/common/integrator.py", line 448, in run
self.start_processing()
File "/opt/project/providers/api/data_collector.py", line 81, in start_processing
products.run()
File "/opt/project/common/data_parser.py", line 94, in run
self.build_dataframe()
File "/opt/project/common/data_parser.py", line 54, in build_dataframe
df = func(self)
File "/opt/project/providers/data_collector.py", line 68, in products_processor_concurrent
return run(async_get_products(data_parser))
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/opt/project/providers/api/data_collector.py", line 57, in async_get_products
tasks = [await client.get_product_by_id(sku) for sku in sku_list]
File "/opt/project/providers/api/data_collector.py", line 57, in <listcomp>
tasks = [await client.get_product_by_id(sku) for sku in sku_list]
TypeError: object dict can't be used in 'await' expression
When you want to run asynchronous functions from synchronous functions, you have to use the asyncio library to run them. Your last function should look like this.
def products_processor_concurrent(data_parser):
from asyncio import run
return run(async_get_products(data_parser))
Related
I made a simple asynchronous python program to get the list of all Virtual Machines in a subscription. But I'm getting an error when iterating through the list.
The complete program:
from azure.identity.aio import AzureCliCredential
from azure.mgmt.compute.aio import ComputeManagementClient
from azure.mgmt.network.aio import NetworkManagementClient
from azure.mgmt.web.aio import WebSiteManagementClient
from azure.mgmt.sql.aio import SqlManagementClient
import asyncio
credential = AzureCliCredential()
compute_client = ComputeManagementClient(credential, "XXXXX")
web_client = WebSiteManagementClient(credential, "XXXX")
async def get_as_list():
as_list = web_client.web_apps.list()
async for _as in as_list:
pass
return as_list
async def get_vm_list():
vm_list = compute_client.virtual_machines.list_all()
async for vm in vm_list:
pass
return vm_list
async def main():
await get_as_list()
await get_vm_list()
if __name__ == '__main__':
asyncio.run(main())
The Error:
Traceback (most recent call last):
File "c:\Users\Kristian-Laptop\Desktop\Work\Azure Inventory\Inventory with UI\test.py", line 32, in <module>
asyncio.run(main())
File "C:\Users\Kristian-Laptop\AppData\Local\Programs\Python\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Users\Kristian-Laptop\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 647, in run_until_complete
return future.result()
File "c:\Users\Kristian-Laptop\Desktop\Work\Azure Inventory\Inventory with UI\test.py", line 28, in main
await get_vm_list()
File "c:\Users\Kristian-Laptop\Desktop\Work\Azure Inventory\Inventory with UI\test.py", line 22, in get_vm_list
async for vm in vm_list:
File "C:\Users\Kristian-Laptop\.virtualenvs\Inventory_with_UI-ioU1jGkq\lib\site-packages\azure\core\async_paging.py", line 163, in __anext__
return await self.__anext__()
File "C:\Users\Kristian-Laptop\.virtualenvs\Inventory_with_UI-ioU1jGkq\lib\site-packages\azure\core\async_paging.py", line 157, in __anext__
self._page = await self._page_iterator.__anext__()
File "C:\Users\Kristian-Laptop\.virtualenvs\Inventory_with_UI-ioU1jGkq\lib\site-packages\azure\core\async_paging.py", line 99, in __anext__
self._response = await self._get_next(self.continuation_token)
File "C:\Users\Kristian-Laptop\.virtualenvs\Inventory_with_UI-ioU1jGkq\lib\site-packages\azure\mgmt\compute\v2022_08_01\aio\operations\_virtual_machines_operations.py", line 1486, in get_next
request = prepare_request(next_link)
File "C:\Users\Kristian-Laptop\.virtualenvs\Inventory_with_UI-ioU1jGkq\lib\site-packages\azure\mgmt\compute\v2022_08_01\aio\operations\_virtual_machines_operations.py", line 1471, in prepare_request
_next_request_params["api-version"] = self._config.api_version
AttributeError: 'ComputeManagementClientConfiguration' object has no attribute 'api_version'
No problem iterating through the Web app list. I'm only getting the error with the VM list.
I am trying to make some API calls to firebase using the google.cloud.firestore.AsyncClient module. Fire2 is just a wrapper for this object.
The traceback suggests its a problem with asyncio, but if I make a dummy await it actually runs just fine. What is the problem and why does it occur?
Sample code:
import Fire2
class Test:
def __init__(self):
doc = asyncio.run(self.wait())
async def wait(self):
doc = await Fire2("test1").get(g1) # gives the error
# doc = await asyncio.sleep(1) # runs without error
return doc
def test(self):
x = Test2(p1)
class Test2:
def __init__(self, p):
doc = asyncio.run(self.run(p))
print(doc.to_dict())
async def run(self, p):
doc = await Fire2('test2').get(p)
return doc
p1 = 'foo'
g1 = 'bar'
h = Test()
h.test()
Traceback:
Traceback (most recent call last):
File "<project path>\scratch.py", line 137, in <module>
h.test()
File "<project path>\scratch.py", line 123, in test
x = Test2(p1)
File "<project path>\scratch.py", line 127, in __init__
doc = asyncio.run(self.run(p))
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "<project path>\scratch.py", line 131, in run
doc = await Fire2('test2').get(p)
File "<project path>\<Fire2 file>", line 422, in get
res = await self._collection.document(doc_id).get()
File "<project path>\venv\lib\site-packages\google\cloud\firestore_v1\async_document.py", line 364, in get
response_iter = await self._client._firestore_api.batch_get_documents(
File "<project path>\venv\lib\site-packages\google\api_core\grpc_helpers_async.py", line 171, in error_remapped_callable
call = callable_(*args, **kwargs)
File "<project path>\venv\lib\site-packages\grpc\aio\_channel.py", line 165, in __call__
call = UnaryStreamCall(request, deadline, metadata, credentials,
File "<project path>\venv\lib\site-packages\grpc\aio\_call.py", line 553, in __init__
self._send_unary_request_task = loop.create_task(
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 431, in create_task
self._check_closed()
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 510, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
sys:1: RuntimeWarning: coroutine 'UnaryStreamCall._send_unary_request' was never awaited
Process finished with exit code 1
You are making two separate calls to asyncio.run. We can see from the stack trace that the error is coming from inside the grpc library. I suspect that the same grpc connection is being used both times, and that during its initialization it integrates itself with the async event loop in order to maintain the connection in the background between calls.
The problem is that every time you call asyncio.run, you will create a new event loop, and every time it returns that event loop will be closed. So by the time you call this function again, and the grpc layer tries to do its work, it realizes that the original event loop has been closed and it returns this error.
To fix this, You can have a single call to asyncio.run, typically calling a "main" function, or similar. This would then make calls to async functions which perform the actual work.
If the problem persists then there might be some issues in the “Fire2”class.
For more reference you can check this link which outlines high-level asyncio APIs to work with coroutines and Tasks with the help of various examples.
Use pytest with using fixture with scope="session" in conftest.py
#pytest.fixture(scope="session")
def event_loop():
loop = get_event_loop()
yield loop
loop.close()
I am trying to add a web ui to a discord bot using Quart. From what I've seen the appropriate way to do this is to from a instance of discord.Bot create a task and to run it. I'm currently doing it this way
def start():
bot.loop.create_task(app.run_task("0.0.0.0"))
load_commands()
print(f"loaded commands: {loaded_commands}")
bot.run(TOKEN)
however when this is run I get the following error
Task exception was never retrieved
future: <Task finished name='Task-1' coro=<serve() done, defined at /var/home/nullrequest/.local/share/virtualenvs/lunbot-ldM1Y48e/lib/python3.10/site-packages/hypercorn/asyncio/__init__.py:9> exception=TypeError("BaseEventLoop.create_server() got an unexpected keyword argument 'loop'")>
Traceback (most recent call last):
File "/var/home/nullrequest/.local/share/virtualenvs/lunbot-ldM1Y48e/lib/python3.10/site-packages/hypercorn/asyncio/__init__.py", line 39, in serve
await worker_serve(app, config, shutdown_trigger=shutdown_trigger)
File "/var/home/nullrequest/.local/share/virtualenvs/lunbot-ldM1Y48e/lib/python3.10/site-packages/hypercorn/asyncio/run.py", line 128, in worker_serve
await asyncio.start_server(
File "/usr/lib64/python3.10/asyncio/streams.py", line 84, in start_server
return await loop.create_server(factory, host, port, **kwds)
TypeError: BaseEventLoop.create_server() got an unexpected keyword argument 'loop'
Task was destroyed but it is pending!
task: <Task pending name='Task-3' coro=<Lifespan.handle_lifespan() running at /var/home/nullrequest/.local/share/virtualenvs/lunbot-ldM1Y48e/lib/python3.10/site-packages/hypercorn/asyncio/lifespan.py:30> wait_for=<Future pending cb=[Task.task_wakeup()]>>
I am using python 3.10rc1 if that makes any difference.
I have same error. Python 3.10 released on 04.10.2021. May be however it depends on the version of interpreter.
for message in client.iter_history(chat.id):
if message.from_user.id == user_id:
# My code...
My traceback:
Traceback (most recent call last):
File "D:\freelance\messagesParser\main.py", line 74, in <module>
main(sys.argv)
File "D:\freelance\messagesParser\main.py", line 64, in main
messages[chat] = get_messages(client, chat, **user_data)
File "D:\freelance\messagesParser\main.py", line 22, in get_messages
for message in client.iter_history(chat.id):
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\sync.py", line 59, in async_to_sync_wra
p
return loop.run_until_complete(consume_generator(coroutine))
File "C:\Program Files\Python310\lib\asyncio\base_events.py", line 641, in run_until_complete
return future.result()
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\sync.py", line 34, in consume_generator
return types.List([i async for i in coroutine])
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\sync.py", line 34, in <listcomp>
return types.List([i async for i in coroutine])
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\methods\messages\iter_history.py", line
79, in iter_history
messages = await self.get_history(
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\methods\messages\get_history.py", line
86, in get_history
messages = await utils.parse_messages(
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\utils.py", line 91, in parse_messages
parsed_messages.append(await types.Message._parse(client, message, users, chats, replies=0))
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\types\messages_and_media\message.py", l
ine 619, in _parse
sticker = await types.Sticker._parse(
File "D:\freelance\messagesParser\venv\lib\site-packages\pyrogram\types\messages_and_media\sticker.py", l
ine 135, in _parse
set_name = await Sticker._get_sticker_set_name(client.send, input_sticker_set_id)
File "D:\freelance\messagesParser\venv\lib\site-packages\async_lru.py", line 237, in wrapped
return (yield from asyncio.shield(fut, loop=_loop))
TypeError: shield() got an unexpected keyword argument 'loop'
I am using a third party API Wrapper that supports async. I am having a difficult time figuring out how to use it as I am new to asyncio, and I'm not sure if the errors I'm getting are my usage (most likely) or the wrapper itself.
The API can be found here: https://clashroyale.readthedocs.io/en/latest/api.html#royaleapi
The goal of my code is to make three API requests at once (the .get_clan(*lClanGroup)), asynchronously instead of synchronously.
My Code:
import clashroyale
import asyncio
import aiohttp
# Define Tokens
unofficialAPIToken = "secretToken"
# Get Client Objects
session1 = aiohttp.ClientSession()
unofficialClient = clashroyale.royaleapi.Client(unofficialAPIToken, session=session1, is_async=True)
lClanGroups = ['PPCLCJG9', '2LRU2J', 'PGLQ0VQ', 'YU2RQG9', '2LVRQ29'],['PYP8UPJV', 'P9L0CYY0', 'Y2RGQPJ', '8P2GYJ8', '9VQJPL2L'],['RYPRGCJ', '809R8PG8', 'PJY9PP98', '2GCQLC', '2GL2QPPL']
async def x(lClanGroup):
print(*lClanGroup)
a = await unofficialClient.get_clan(*lClanGroup) # Iterates five tags for the API to request at once
return a
async def y(lClanGroups):
result = await asyncio.gather(x(lClanGroups[0]),x(lClanGroups[1]),x(lClanGroups[2]))
return result
async def close_sessions():
await session1.close()
asyncio.run(y(lClanGroups))
asyncio.run(close_sessions())
The error I'm getting is long and difficult to make sense of:
C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\Scripts\python.exe C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py
PPCLCJG9 2LRU2J PGLQ0VQ YU2RQG9 2LVRQ29
PYP8UPJV P9L0CYY0 Y2RGQPJ 8P2GYJ8 9VQJPL2L
RYPRGCJ 809R8PG8 PJY9PP98 2GCQLC 2GL2QPPL
C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py:16: DeprecationWarning: The object should be created from async function
session1 = aiohttp.ClientSession()
Traceback (most recent call last):
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 33, in <module>
asyncio.run(y(lClanGroups))
File "C:\Program Files\Python38\lib\asyncio\runners.py", line 43, in run
return loop.run_until_complete(main)
File "C:\Program Files\Python38\lib\asyncio\base_events.py", line 612, in run_until_complete
return future.result()
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 27, in y
result = await asyncio.gather(x(lClanGroups[0]),x(lClanGroups[1]),x(lClanGroups[2]))
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 23, in x
a = await unofficialClient.get_clan(*lClanGroup)
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 203, in _aget_model
raise e
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 196, in _aget_model
data, cached, ts, resp = await self._request(url, **params)
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 150, in _arequest
async with self.session.get(url, timeout=timeout, headers=self.headers, params=params) as resp:
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\client.py", line 1012, in __aenter__
self._resp = await self._coro
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\client.py", line 426, in _request
with timer:
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\helpers.py", line 579, in __enter__
raise RuntimeError('Timeout context manager should be used '
RuntimeError: Timeout context manager should be used inside a task
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x0000016C5AD50EB0>
Process finished with exit code 1
Thanks for your help!
Each asyncio.run() creates new event loop. Moreover, if you create objects like aiohttp.ClientSession() globally, they may be bind to default event loop.
Using multiple event loops in your program is dangerous and may lead to non-obvious problems. To avoid the situation:
Run asyncio.run() only once
Create all asyncio-related stuff inside main coroutine
I didn't work with royaleapi, but it seems API developers do things differently than you in their tests. Here's how they innit/close stuff and here's how they get clan.
Let's give it a try?
import clashroyale
import asyncio
TOKEN = "secretToken"
clan_groups = [
['PPCLCJG9', '2LRU2J', 'PGLQ0VQ', 'YU2RQG9', '2LVRQ29'],
['PYP8UPJV', 'P9L0CYY0', 'Y2RGQPJ', '8P2GYJ8', '9VQJPL2L'],
['RYPRGCJ', '809R8PG8', 'PJY9PP98', '2GCQLC', '2GL2QPPL']
]
async def get_clans(cr, clan_groups):
return await asyncio.gather(*[
cr.get_clan(tag)
for group in clan_groups
for tag in group
])
async def main():
cr = clashroyale.RoyaleAPI(
TOKEN,
is_async=True,
timeout=30
)
try:
results = await get_clans(cr, clan_groups)
print(results)
finally:
await cr.close()
await asyncio.sleep(2)
asyncio.run(main())
Didn't test it.
I have the following Sanic route:
md = Markdown()
#app.route('/md_file')
async def md_file(request):
async with aiofiles.open('./file.md')) as f:
content = await f.read()
content = md.convert(content)
return html(content)
This works just fine but the conversion takes a very long time and blocks the endpoint. When benchmarking, the endpoint can only handle 4 requests per second.
Since there is no asyncio markdown library, I figured I would offload the conversion into a separate thread to free up the blocking code:
loop = asyncio.get_event_loop()
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert(content))
However, this throws a traceback:
2017-07-22 12:02:24 - (sanic)[ERROR]: Traceback (most recent call last):
File "/home/user/app/venv/lib64/python3.5/site-packages/sanic/app.py", line 471, in handle_request
response = await response
File "app.py", line 127, in blog_posts
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert(content))
File "uvloop/future.pyx", line 241, in __await__ (uvloop/loop.c:110786)
File "uvloop/future.pyx", line 432, in uvloop.loop.BaseTask._fast_wakeup (uvloop/loop.c:113980)
File "uvloop/future.pyx", line 101, in uvloop.loop.BaseFuture._result_impl (uvloop/loop.c:108900)
File "/opt/rh/rh-python35/root/usr/lib64/python3.5/concurrent/futures/thread.py", line 55, in run
result = self.fn(*self.args, **self.kwargs)
TypeError: 'str' object is not callable
Is it not possible to use the event loop from within Sanic? Are there any other options to make the conversion non-blocking?
md.convert(content) actually runs the function. This:
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert(content))
is the same as this:
content = await loop.run_in_executor(ThreadPoolExecutor(), "some HTML")
And that is quite obviously wrong. You don't want to run the function. You want to pass the function; the executor will handle running it. The signature of run_in_executor is
coroutine AbstractEventLoop.run_in_executor(executor, func, *args)
So use this instead
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert, content)