Let's say I have a class which uses asyncio loop internally and doesn't have async interface:
class Fetcher:
_loop = None
def get_result(...):
"""
After 3 nested sync calls async tasks are finally called with *run_until_complete*
"""
...
I use all advantages of asyncio internally and don't have to care about it in the outer code.
But then I want to call 3 Fetcher instances in one event loop. If I had async def interface there would be no problem: asyncio.gather could help me. Is there really no other way to do it without supporting both interfaces? Come on! It makes you change all your project because of one asyncio usage. Tell me this is not true.
Come on! It makes you change all your project because of one asyncio
usage. Tell me this is not true.
It's true
Whole idea of using await keyword is to execute concurrent jobs in one event loop from different places of the code (which you can't do with regular function code).
asyncio - is not some utility, but whole style of writing asynchronous programs.
On the other hand Python is very flexible, so you can still try to hide using of asyncio. If you really want to get sync result of 3 Fetcher instances, you can for example do something like this:
import asyncio
def sync_exec(coro):
loop = asyncio.get_event_loop()
return loop.run_until_complete(coro)
class Fetcher:
async def async_get_result(self):
# async interface:
async def async_job():
await asyncio.sleep(1)
return id(self)
return (await async_job())
def get_result(self):
# sync interface:
return sync_exec(self.async_get_result())
#classmethod
def get_results(cls, *fetchers):
# sync interface multiple:
return sync_exec(
asyncio.gather(*[fetcher.async_get_result() for fetcher in fetchers])
)
# single sync get_result:
f1 = Fetcher()
print('Result: ', f1.get_result())
# multiple sync get_result:
f2 = Fetcher()
f3 = Fetcher()
print('Results: ', Fetcher.get_results(f1, f2, f3))
Output:
Result: 2504097887120
Results: [2504097887120, 2504104854416, 2504104854136]
But, again, you'll really regret someday if you continue to write code this way, believe me. If you want to get full advantage of asynchronous programming - use coroutines and await explicitly.
Related
In short, the problem is that the future returned by asyncio.run_coroutine_threadsafe is blocking when I call future.result()
The problem is also documented in the following question with (currently) no satisfactory answer: Future from asyncio.run_coroutine_threadsafe hangs forever?
What I'm trying to achieve is to call async code from sync code, where the sync code is actually itself wrapped in async code with an existing running event loop (to make things more concrete: it's a Jupyter notebook).
I would want to send async tasks from nested sync code to the existing 'outer' event loop and 'await' its results within the nested sync code. Implied constraint: I do not want to run those tasks on a new event loop (multiple reasons).
Since it's not possible to just 'await' an async result from sync code without blocking and without using asyncio.run which creates a new event loop, I thought using a separate thread would somehow help.
From the documentation description, asyncio.run_coroutine_threadsafe sounds like the perfect candidate.
But it's still blocking...
Bellow full snippet, with a timeout when calling the future's result.
How can I get this code to work correctly?
import asyncio
from concurrent.futures import ThreadPoolExecutor
async def gather_coroutines(*coroutines):
return await asyncio.gather(*coroutines)
def run_th_safe(loop, coroutines):
future = asyncio.run_coroutine_threadsafe(gather_coroutines(*coroutines), loop)
res = future.result(timeout=3) # **** BLOCKING *****
return res
def async2sync(*coroutines):
try:
loop = asyncio.get_running_loop()
except RuntimeError:
return asyncio.run(gather_coroutines(*coroutines))
# BLOW DOESN'T WORK BECAUSE run_th_safe IS BLOCKING
with ThreadPoolExecutor(max_workers=1) as ex:
thread_future = ex.submit(run_th_safe, loop, coroutines)
return thread_future.result()
# Testing
async def some_async_task(n):
"""Some async function to test"""
print('Task running with n =', n)
await asyncio.sleep(n/10)
print('Inside coro', n)
return list(range(n))
async def main_async():
coro3 = some_async_task(30)
coro1 = some_async_task(10)
coro2 = some_async_task(20)
results = async2sync(coro3, coro1, coro2)
return results
def main_sync():
coro3 = some_async_task(30)
coro1 = some_async_task(10)
coro2 = some_async_task(20)
results = async2sync(coro3, coro1, coro2)
return results
if __name__ == '__main__':
# Testing functionnality with asyncio.run()
# This works
print(main_sync())
# Testing functionnality with outer-loop (asyncio.run) and nested asyncio.run_coroutine_threadsafe
# **DOESN'T WORK**
print(asyncio.run(main_async()))
I'm working on making a previously sync webserver as sync one.
Most of my functions are sync and I'd like to simply make async calls from existing code to avoid async creep. nest_asyncio appears to allow this by making run_until_complete re-entrant.
However, while this works for 1 reentrant call, I get a deadlock with two:
import asyncio
import functools
import time
from random import random
import nest_asyncio
nest_asyncio.apply()
def sync(corot, loop=None):
"""
Make a synchronous function from an asynchronous one.
:param corot:
:return:
"""
if loop is None:
loop = asyncio.get_event_loop()
result, = loop.run_until_complete(asyncio.gather(corot))
return result
async def sync_to_corountine(func, *args, **kw):
"""
Make a coroutine from a synchronous function.
"""
try:
return func(*args, *kw)
finally:
# every async needs an await.
await asyncio.sleep(0)
def main():
async def background(timeout):
await asyncio.sleep(timeout)
print(f"Background: {timeout}")
loop = asyncio.get_event_loop()
# Run some background work to check we are never blocked
bg_tasks = [
loop.create_task(background(i))
for i in range(10)
]
async def long_running_async_task(result):
# Simulate slow IO
print(f"...START long_running_async_task [{result}]")
await asyncio.sleep(1)
print(f"...END long_running_async_task [{result}]")
return result
def sync_function_with_async_dependency(result):
print(f"...START sync_function_with_async_dependency [{result}]")
result = sync(long_running_async_task(result), loop=loop)
print(f"...END sync_function_with_async_dependency [{result}]")
return result
# Call sync_function_with_async_dependency
# One reentrant task is OK
# Multiple reentrant tasks=>fails to exit
n = 2
for i in range(n):
bg_tasks.append(sync_to_corountine(sync_function_with_async_dependency, i))
# for i in range(n):
# bg_tasks.append(long_running_async_task(i))
# OK
# bg_tasks.append(long_running_async_task(123))
# bg_tasks.append(long_running_async_task(456))
task = asyncio.gather(*bg_tasks) # , loop=loop)
loop.run_until_complete(task)
if __name__ == '__main__':
main()
...START sync_function_with_async_dependency [0]
...START sync_function_with_async_dependency [1]
Background: 0
...START long_running_async_task [0]
...START long_running_async_task [1]
Background: 1
...END long_running_async_task [0]
...END long_running_async_task [1]
...END sync_function_with_async_dependency [1]
Background: 2
Background: 3
Background: 4
... we are missing
...END sync_function_with_async_dependency [0]
Am I using nest_asyncio correctly?
You want to use gevent if you're going to nest async calls inside synchronous calls, otherwise it can lead to starvation issues.
The whole design of asyncio is async routines call async routines, but sync routines can not (except via run). They do this as they feel that anything else will lead to overly complex code that is difficult to follow. Probably bugs in asyncio are harder to fix as well.
The monkey patch nest_async tries to break that design principle, but it doesn't do a very good job of it as nested runs will not give time to tasks scheduled outside the nested run. This can potentially leading to starvation if you spend too much time on code inside the nested run.
I tried to explain this to the author, but he's still struggling:
https://github.com/erdewit/nest_asyncio/issues/36
I am in the process of trying to port a flask app to quart to utilise asyncio. I don't think my current approach is working, as my entire function chain is written without async in mind - consider the following:
def long_running_task(task):
result = some_synchronous_function(task)
return result
#app.route('/<task>', methods=['GET'])
async def do_task(task):
ok = await long_running_task(task)
if ok:
return (ok.result)
else:
return ('Something went wrong')
If long_running_task and its whole chain of function calls are not declared as async, am I actually getting any benefit from my route being declared as async?
To run a blocking synchronous function from asyncio, without blocking the main event loop, you can use loop.run_in_executor() to run the blocking function in a ThreadPoolExecutor or ProcessPoolExecutor` (i.e. in its own thread or process).
From within the async function you want to call it from:
loop = asyncio.get_event_loop()
result = await loop.run_in_executor(None, long_running_task, task)
The first argument None is to tell it to use the default executor for the loop. Obviously do_task() will still have to wait for result to complete, but while it is waiting, other async tasks will be able to run in event-loop.
I've read tons of articles and tutorial about Python's 3.5 async/await thing. I have to say I'm pretty confused, because some use get_event_loop() and run_until_complete(), some use ensure_future(), some use asyncio.wait(), and some use call_soon().
It seems like I have a lot choices, but I have no idea if they are completely identical or there are cases where you use loops and there are cases where you use wait().
But the thing is all examples work with asyncio.sleep() as simulation of real slow operation which returns an awaitable object. Once I try to swap this line for some real code the whole thing fails. What the heck are the differences between approaches written above and how should I run a third-party library which is not ready for async/await. I do use the Quandl service to fetch some stock data.
import asyncio
import quandl
async def slow_operation(n):
# await asyncio.sleep(1) # Works because it's await ready.
await quandl.Dataset(n) # Doesn't work because it's not await ready.
async def main():
await asyncio.wait([
slow_operation("SIX/US9884981013EUR4"),
slow_operation("SIX/US88160R1014EUR4"),
])
# You don't have to use any code for 50 requests/day.
quandl.ApiConfig.api_key = "MY_SECRET_CODE"
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I hope you get the point how lost I feel and how simple thing I would like to have running in parallel.
If a third-party library is not compatible with async/await then obviously you can't use it easily. There are two cases:
Let's say that the function in the library is asynchronous and it gives you a callback, e.g.
def fn(..., clb):
...
So you can do:
def on_result(...):
...
fn(..., on_result)
In that case you can wrap such functions into the asyncio protocol like this:
from asyncio import Future
def wrapper(...):
future = Future()
def my_clb(...):
future.set_result(xyz)
fn(..., my_clb)
return future
(use future.set_exception(exc) on exception)
Then you can simply call that wrapper in some async function with await:
value = await wrapper(...)
Note that await works with any Future object. You don't have to declare wrapper as async.
If the function in the library is synchronous then you can run it in a separate thread (probably you would use some thread pool for that). The whole code may look like this:
import asyncio
import time
from concurrent.futures import ThreadPoolExecutor
# Initialize 10 threads
THREAD_POOL = ThreadPoolExecutor(10)
def synchronous_handler(param1, ...):
# Do something synchronous
time.sleep(2)
return "foo"
# Somewhere else
async def main():
loop = asyncio.get_event_loop()
futures = [
loop.run_in_executor(THREAD_POOL, synchronous_handler, param1, ...),
loop.run_in_executor(THREAD_POOL, synchronous_handler, param1, ...),
loop.run_in_executor(THREAD_POOL, synchronous_handler, param1, ...),
]
await asyncio.wait(futures)
for future in futures:
print(future.result())
with THREAD_POOL:
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
If you can't use threads for whatever reason then using such a library simply makes entire asynchronous code pointless.
Note however that using synchronous library with async is probably a bad idea. You won't get much and yet you complicate the code a lot.
You can take a look at the following simple working example from here. By the way it returns a string worth reading :-)
import aiohttp
import asyncio
async def fetch(client):
async with client.get('https://docs.aiohttp.org/en/stable/client_reference.html') as resp:
assert resp.status == 200
return await resp.text()
async def main():
async with aiohttp.ClientSession() as client:
html = await fetch(client)
print(html)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I'm trying to consume multiple queues concurrently using python, asyncio and asynqp.
I don't understand why my asyncio.sleep() function call does not have any effect. The code doesn't pause there. To be fair, I actually don't understand in which context the callback is executed, and whether I can yield control bavck to the event loop at all (so that the asyncio.sleep() call would make sense).
What If I had to use a aiohttp.ClientSession.get() function call in my process_msg callback function? I'm not able to do it since it's not a coroutine. There has to be a way which is beyond my current understanding of asyncio.
#!/usr/bin/env python3
import asyncio
import asynqp
USERS = {'betty', 'bob', 'luis', 'tony'}
def process_msg(msg):
asyncio.sleep(10)
print('>> {}'.format(msg.body))
msg.ack()
async def connect():
connection = await asynqp.connect(host='dev_queue', virtual_host='asynqp_test')
channel = await connection.open_channel()
exchange = await channel.declare_exchange('inboxes', 'direct')
# we have 10 users. Set up a queue for each of them
# use different channels to avoid any interference
# during message consumption, just in case.
for username in USERS:
user_channel = await connection.open_channel()
queue = await user_channel.declare_queue('Inbox_{}'.format(username))
await queue.bind(exchange, routing_key=username)
await queue.consume(process_msg)
# deliver 10 messages to each user
for username in USERS:
for msg_idx in range(10):
msg = asynqp.Message('Msg #{} for {}'.format(msg_idx, username))
exchange.publish(msg, routing_key=username)
loop = asyncio.get_event_loop()
loop.run_until_complete(connect())
loop.run_forever()
I don't understand why my asyncio.sleep() function call does not have
any effect.
Because asyncio.sleep() returns a future object that has to be used in combination with an event loop (or async/await semantics).
You can't use await in simple def declaration because the callback is called outside of async/await context which is attached to some event loop under the hood. In other words mixing callback style with async/await style is quite tricky.
The simple solution though is to schedule the work back to the event loop:
async def process_msg(msg):
await asyncio.sleep(10)
print('>> {}'.format(msg.body))
msg.ack()
def _process_msg(msg):
loop = asyncio.get_event_loop()
loop.create_task(process_msg(msg))
# or if loop is always the same one single line is enough
# asyncio.ensure_future(process_msg(msg))
# some code
await queue.consume(_process_msg)
Note that there is no recursion in _process_msg function, i.e. the body of process_msg is not executed while in _process_msg. The inner process_msg function will be called once the control goes back to the event loop.
This can be generalized with the following code:
def async_to_callback(coro):
def callback(*args, **kwargs):
asyncio.ensure_future(coro(*args, **kwargs))
return callback
async def process_msg(msg):
# the body
# some code
await queue.consume(async_to_callback(process_msg))
See Drizzt1991's response on github for a solution.