I am trying to make some API calls to firebase using the google.cloud.firestore.AsyncClient module. Fire2 is just a wrapper for this object.
The traceback suggests its a problem with asyncio, but if I make a dummy await it actually runs just fine. What is the problem and why does it occur?
Sample code:
import Fire2
class Test:
def __init__(self):
doc = asyncio.run(self.wait())
async def wait(self):
doc = await Fire2("test1").get(g1) # gives the error
# doc = await asyncio.sleep(1) # runs without error
return doc
def test(self):
x = Test2(p1)
class Test2:
def __init__(self, p):
doc = asyncio.run(self.run(p))
print(doc.to_dict())
async def run(self, p):
doc = await Fire2('test2').get(p)
return doc
p1 = 'foo'
g1 = 'bar'
h = Test()
h.test()
Traceback:
Traceback (most recent call last):
File "<project path>\scratch.py", line 137, in <module>
h.test()
File "<project path>\scratch.py", line 123, in test
x = Test2(p1)
File "<project path>\scratch.py", line 127, in __init__
doc = asyncio.run(self.run(p))
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "<project path>\scratch.py", line 131, in run
doc = await Fire2('test2').get(p)
File "<project path>\<Fire2 file>", line 422, in get
res = await self._collection.document(doc_id).get()
File "<project path>\venv\lib\site-packages\google\cloud\firestore_v1\async_document.py", line 364, in get
response_iter = await self._client._firestore_api.batch_get_documents(
File "<project path>\venv\lib\site-packages\google\api_core\grpc_helpers_async.py", line 171, in error_remapped_callable
call = callable_(*args, **kwargs)
File "<project path>\venv\lib\site-packages\grpc\aio\_channel.py", line 165, in __call__
call = UnaryStreamCall(request, deadline, metadata, credentials,
File "<project path>\venv\lib\site-packages\grpc\aio\_call.py", line 553, in __init__
self._send_unary_request_task = loop.create_task(
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 431, in create_task
self._check_closed()
File "<user AppData>\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 510, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
sys:1: RuntimeWarning: coroutine 'UnaryStreamCall._send_unary_request' was never awaited
Process finished with exit code 1
You are making two separate calls to asyncio.run. We can see from the stack trace that the error is coming from inside the grpc library. I suspect that the same grpc connection is being used both times, and that during its initialization it integrates itself with the async event loop in order to maintain the connection in the background between calls.
The problem is that every time you call asyncio.run, you will create a new event loop, and every time it returns that event loop will be closed. So by the time you call this function again, and the grpc layer tries to do its work, it realizes that the original event loop has been closed and it returns this error.
To fix this, You can have a single call to asyncio.run, typically calling a "main" function, or similar. This would then make calls to async functions which perform the actual work.
If the problem persists then there might be some issues in the “Fire2”class.
For more reference you can check this link which outlines high-level asyncio APIs to work with coroutines and Tasks with the help of various examples.
Use pytest with using fixture with scope="session" in conftest.py
#pytest.fixture(scope="session")
def event_loop():
loop = get_event_loop()
yield loop
loop.close()
Related
I am new with async functions and i'm trying to make multiple calls from an external API.
concurrent.Futures is not quite enough to retrieve the responses so i tried with asyncio and httpx but the process is throwing an error unknown and difficult to debug for me.
It seems that the coroutine is having an empty value or is never being called.
This is my client
async def get_product_by_id_async(self, product_id: str) -> Response:
if product_id is None:
return None
response_await = await self.async_client.get(
url=f"{self.base_url}/api/catalog/stock{product_id}",
headers=self.headers,
)
if response_await.status_code == 200:
response = json.loads(response_await.text)
return response
And this is my coordinating function
async def async_get_products(data_parser):
path = data_parser.options["path"]
sku_list = client.run()
products_list = []
tasks = [await client.get_product_by_id(sku) for sku in sku_list]
breakpoint()
completed = await asyncio.gather(*tasks)
for product in completed:
products_list = build_product_df(products_list, product)
products_df = pd.DataFrame(products_list)
products_df.to_csv(path, index=False)
return products_df
def products_processor_concurrent(data_parser):
return async_get_products(data_parser)
After the solution given there is a new error in the code:
This is the trace:
Traceback (most recent call last):
File "manage.py", line 32, in <module>
module.run()
File "/opt/project/providers/api/data_collector.py", line 90, in run
integrator.run()
File "/opt/project/common/integrator.py", line 448, in run
self.start_processing()
File "/opt/project/providers/api/data_collector.py", line 81, in start_processing
products.run()
File "/opt/project/common/data_parser.py", line 94, in run
self.build_dataframe()
File "/opt/project/common/data_parser.py", line 54, in build_dataframe
df = func(self)
File "/opt/project/providers/data_collector.py", line 68, in products_processor_concurrent
return run(async_get_products(data_parser))
File "/usr/local/lib/python3.8/asyncio/runners.py", line 44, in run
return loop.run_until_complete(main)
File "/usr/local/lib/python3.8/asyncio/base_events.py", line 616, in run_until_complete
return future.result()
File "/opt/project/providers/api/data_collector.py", line 57, in async_get_products
tasks = [await client.get_product_by_id(sku) for sku in sku_list]
File "/opt/project/providers/api/data_collector.py", line 57, in <listcomp>
tasks = [await client.get_product_by_id(sku) for sku in sku_list]
TypeError: object dict can't be used in 'await' expression
When you want to run asynchronous functions from synchronous functions, you have to use the asyncio library to run them. Your last function should look like this.
def products_processor_concurrent(data_parser):
from asyncio import run
return run(async_get_products(data_parser))
I have the following function.
async def f(url):
async with httpx.AsyncClient() as client:
async with client.stream('GET', url) as resp:
async for c in resp.aiter_bytes():
yield c
And I'm writing a test code to test if the function yields.
#pytest.mark.asyncio
async def test():
c = None
async for c in f(URL):
c = chunk
break # there will be no error if removing this line. However, I don't need to loop all the items
assert c
However, the code runs but with the following warnings/errors. What's the error?
lib\site-packages\httpx_client.py:2012: UserWarning: Unclosed <httpx.AsyncClient object at 0x000001AC1E8CF3A0>. See https://www.python-httpx.org/async/#opening-and-closing-clients for details.
Is the "Unclosed" warning caused by break?
Exception ignored in: <function _ProactorBasePipeTransport.__del__ at 0x000001AC1D884DC0>
Traceback (most recent call last):
File "C:\anaconda3\envs\s\lib\asyncio\proactor_events.py", line 116, in __del__
self.close()
File "C:\anaconda3\envs\s\lib\asyncio\proactor_events.py", line 108, in close
self._loop.call_soon(self._call_connection_lost, None)
File "C:\anaconda3\envs\s\lib\asyncio\base_events.py", line 719, in call_soon
self._check_closed()
File "C:\anaconda3\envs\s\lib\asyncio\base_events.py", line 508, in _check_closed
raise RuntimeError('Event loop is closed')
RuntimeError: Event loop is closed
The error disappears if removing the break statement.
In the answer of Async generator is not an iterator?, the following code will get rid of the error. However, it needs to call a -private- "special" function __anext()__().
#pytest.mark.asyncio
async def test():
it = f(URL):
c = it.__anext__()
assert c
I am using a third party API Wrapper that supports async. I am having a difficult time figuring out how to use it as I am new to asyncio, and I'm not sure if the errors I'm getting are my usage (most likely) or the wrapper itself.
The API can be found here: https://clashroyale.readthedocs.io/en/latest/api.html#royaleapi
The goal of my code is to make three API requests at once (the .get_clan(*lClanGroup)), asynchronously instead of synchronously.
My Code:
import clashroyale
import asyncio
import aiohttp
# Define Tokens
unofficialAPIToken = "secretToken"
# Get Client Objects
session1 = aiohttp.ClientSession()
unofficialClient = clashroyale.royaleapi.Client(unofficialAPIToken, session=session1, is_async=True)
lClanGroups = ['PPCLCJG9', '2LRU2J', 'PGLQ0VQ', 'YU2RQG9', '2LVRQ29'],['PYP8UPJV', 'P9L0CYY0', 'Y2RGQPJ', '8P2GYJ8', '9VQJPL2L'],['RYPRGCJ', '809R8PG8', 'PJY9PP98', '2GCQLC', '2GL2QPPL']
async def x(lClanGroup):
print(*lClanGroup)
a = await unofficialClient.get_clan(*lClanGroup) # Iterates five tags for the API to request at once
return a
async def y(lClanGroups):
result = await asyncio.gather(x(lClanGroups[0]),x(lClanGroups[1]),x(lClanGroups[2]))
return result
async def close_sessions():
await session1.close()
asyncio.run(y(lClanGroups))
asyncio.run(close_sessions())
The error I'm getting is long and difficult to make sense of:
C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\Scripts\python.exe C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py
PPCLCJG9 2LRU2J PGLQ0VQ YU2RQG9 2LVRQ29
PYP8UPJV P9L0CYY0 Y2RGQPJ 8P2GYJ8 9VQJPL2L
RYPRGCJ 809R8PG8 PJY9PP98 2GCQLC 2GL2QPPL
C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py:16: DeprecationWarning: The object should be created from async function
session1 = aiohttp.ClientSession()
Traceback (most recent call last):
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 33, in <module>
asyncio.run(y(lClanGroups))
File "C:\Program Files\Python38\lib\asyncio\runners.py", line 43, in run
return loop.run_until_complete(main)
File "C:\Program Files\Python38\lib\asyncio\base_events.py", line 612, in run_until_complete
return future.result()
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 27, in y
result = await asyncio.gather(x(lClanGroups[0]),x(lClanGroups[1]),x(lClanGroups[2]))
File "C:/Users/Adam/PycharmProjects/ClashRoyaleMeta/testme2.py", line 23, in x
a = await unofficialClient.get_clan(*lClanGroup)
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 203, in _aget_model
raise e
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 196, in _aget_model
data, cached, ts, resp = await self._request(url, **params)
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\clashroyale\royaleapi\client.py", line 150, in _arequest
async with self.session.get(url, timeout=timeout, headers=self.headers, params=params) as resp:
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\client.py", line 1012, in __aenter__
self._resp = await self._coro
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\client.py", line 426, in _request
with timer:
File "C:\Users\Adam\PycharmProjects\ClashRoyaleMeta\venv2\lib\site-packages\aiohttp\helpers.py", line 579, in __enter__
raise RuntimeError('Timeout context manager should be used '
RuntimeError: Timeout context manager should be used inside a task
Unclosed client session
client_session: <aiohttp.client.ClientSession object at 0x0000016C5AD50EB0>
Process finished with exit code 1
Thanks for your help!
Each asyncio.run() creates new event loop. Moreover, if you create objects like aiohttp.ClientSession() globally, they may be bind to default event loop.
Using multiple event loops in your program is dangerous and may lead to non-obvious problems. To avoid the situation:
Run asyncio.run() only once
Create all asyncio-related stuff inside main coroutine
I didn't work with royaleapi, but it seems API developers do things differently than you in their tests. Here's how they innit/close stuff and here's how they get clan.
Let's give it a try?
import clashroyale
import asyncio
TOKEN = "secretToken"
clan_groups = [
['PPCLCJG9', '2LRU2J', 'PGLQ0VQ', 'YU2RQG9', '2LVRQ29'],
['PYP8UPJV', 'P9L0CYY0', 'Y2RGQPJ', '8P2GYJ8', '9VQJPL2L'],
['RYPRGCJ', '809R8PG8', 'PJY9PP98', '2GCQLC', '2GL2QPPL']
]
async def get_clans(cr, clan_groups):
return await asyncio.gather(*[
cr.get_clan(tag)
for group in clan_groups
for tag in group
])
async def main():
cr = clashroyale.RoyaleAPI(
TOKEN,
is_async=True,
timeout=30
)
try:
results = await get_clans(cr, clan_groups)
print(results)
finally:
await cr.close()
await asyncio.sleep(2)
asyncio.run(main())
Didn't test it.
I have the following Sanic route:
md = Markdown()
#app.route('/md_file')
async def md_file(request):
async with aiofiles.open('./file.md')) as f:
content = await f.read()
content = md.convert(content)
return html(content)
This works just fine but the conversion takes a very long time and blocks the endpoint. When benchmarking, the endpoint can only handle 4 requests per second.
Since there is no asyncio markdown library, I figured I would offload the conversion into a separate thread to free up the blocking code:
loop = asyncio.get_event_loop()
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert(content))
However, this throws a traceback:
2017-07-22 12:02:24 - (sanic)[ERROR]: Traceback (most recent call last):
File "/home/user/app/venv/lib64/python3.5/site-packages/sanic/app.py", line 471, in handle_request
response = await response
File "app.py", line 127, in blog_posts
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert(content))
File "uvloop/future.pyx", line 241, in __await__ (uvloop/loop.c:110786)
File "uvloop/future.pyx", line 432, in uvloop.loop.BaseTask._fast_wakeup (uvloop/loop.c:113980)
File "uvloop/future.pyx", line 101, in uvloop.loop.BaseFuture._result_impl (uvloop/loop.c:108900)
File "/opt/rh/rh-python35/root/usr/lib64/python3.5/concurrent/futures/thread.py", line 55, in run
result = self.fn(*self.args, **self.kwargs)
TypeError: 'str' object is not callable
Is it not possible to use the event loop from within Sanic? Are there any other options to make the conversion non-blocking?
md.convert(content) actually runs the function. This:
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert(content))
is the same as this:
content = await loop.run_in_executor(ThreadPoolExecutor(), "some HTML")
And that is quite obviously wrong. You don't want to run the function. You want to pass the function; the executor will handle running it. The signature of run_in_executor is
coroutine AbstractEventLoop.run_in_executor(executor, func, *args)
So use this instead
content = await loop.run_in_executor(ThreadPoolExecutor(), md.convert, content)
I am trying to make an API call from within my service and I am running into event loop issues. Can someone help me understand what I am doing wrong?
Basically I want to make a service that does some calculations based on data pulled from a different service.
I can call this code below from a cli, but not when I start up a web app (i.e.) hitting http://127.0.0.1:8080/add
loop = asyncio.get_event_loop()
data = loop.run_until_complete(run_fetch(loop, 'http://google.com'))
Sample code:
from aiohttp import web
import aiohttp
import asyncio
async def add(request):
loop = asyncio.get_event_loop()
data = loop.run_until_complete(run_fetch(loop, 'http://google.com'))
return web.json_response(data)
async def fetch(client, url):
async with client.get(url) as resp:
assert resp.status == 200
return await resp.text()
async def run_fetch(loop, url):
async with aiohttp.ClientSession(loop=loop) as client:
html = await fetch(client, url)
return html
app = web.Application()
app.router.add_get('/add', add)
web.run_app(app, host='127.0.0.1', port=8080)
Exception:
Error handling request
Traceback (most recent call last):
File ".../aiohttp/web_protocol.py", line 417, in start
resp = yield from self._request_handler(request)
File ".../aiohttp/web.py", line 289, in _handle
resp = yield from handler(request)
File ".../sample.py", line 11, in add data = loop.run_until_complete(run_fetch(loop, 'http://google.com'))
File ".../python3.6/asyncio/base_events.py", line 454, in run_until_complete
self.run_forever()
File ".../python3.6/asyncio/base_events.py", line 408, in run_forever
raise RuntimeError('This event loop is already running')
RuntimeError: This event loop is already running
The run_until_complete is the way to run some async code from the sync context. Under the hood it adds provided future to the given ioloop and calls run_forever then returns result or throws exception (resolved future).
Actually you need to await run_fetch(loop, 'http://google.com'), since the caller function is asynchronous.