There is a global list to store data. Different async functions maybe add or remove value from it.
Example:
a = [] # List[Connection]
async def foo():
for v in a:
await v.send('msg')
async def bar():
await SomeAsyncFunc()
a.pop(0)
Both foo and bar will give up the control to let other coroutines run, so in foo, it is not safe to remove value from the list.
The following example shows how to use the lock for this:
Create a connection manager:
import asyncio
class ConnectionsManager:
def __init__(self, timeout=5):
self.timeout = timeout
self._lock = asyncio.Lock()
self._connections = []
async def __aenter__(self):
await asyncio.wait_for(self._lock.acquire(), timeout=self.timeout)
return self._connections
async def __aexit__(self, *exc):
self._lock.release()
The timeout is a security measure to break bugs with circular waits.
The manager can be used as follows:
async def foo():
for _ in range(10):
async with cm as connections:
# do stuff with connection
await asyncio.sleep(0.25)
connections.append('foo')
async def bar():
for _ in range(5):
async with cm as connections:
# do stuff with connection
await asyncio.sleep(0.5)
if len(connections) > 1:
connections.pop()
else:
connections.append('bar')
cm = ConnectionsManager()
t1 = asyncio.create_task(foo())
t2 = asyncio.create_task(bar())
await t1
await t2
async with cm as connections:
print(connections)
Note, that you could also be more explicit with connections here:
async def foo(cm):
...
async def bar(cm):
...
Just to make a comment why being explicit is so beneficial in contrast to globals. At some point you may need to write unit tests for your code, where you will need to specify all inputs to your functions/methods. Forgetting conditions on implicit inputs to your function (used globals) can easily result in untested states. For example your bar coroutine expects an element in the list a and will break if it is empty. Most of the time it might do the right thing, but one day in production...
Related
In most of the asynchronous coroutines I write, I only need to replace the function definition def func() -> async def func() and the sleep time.sleep(s) -> await asyncio.sleep(s).
Is it possible to convert a standard python function into an async function where all the time.sleep(s) is converted to await asyncio.sleep(s)?
Example
Performance during task
Measure performance during a task
import asyncio
import random
async def performance_during_task(task):
stop_event = asyncio.Event()
target_task = asyncio.create_task(task(stop_event))
perf_task = asyncio.create_task(measure_performance(stop_event))
await target_task
await perf_task
async def measure_performance(event):
while not event.is_set():
print('Performance: ', random.random())
await asyncio.sleep(.2)
if __name__ == "__main__":
asyncio.run(
performance_during_task(task)
)
Task
The task has to be defined with async def and await asyncio.sleep(s)
async def task(event):
for i in range(10):
print('Step: ', i)
await asyncio.sleep(.2)
event.set()
into ->
Easy task definition
To have others not worrying about async etc. I want them to be able to define a task normally (e.g. with a decorator?)
#as_async
def easy_task(event):
for i in range(10):
print('Step: ', i)
time.sleep(.2)
event.set()
So that it can be used as an async function with e.g. performance_during_task()
I think I found a solution similar to the interesting GitHub example mentioned in the comments and a similar post here.
We can write a decorator like
from functools import wraps, partial
def to_async(func):
#wraps(func) # Makes sure that function is returned for e.g. func.__name__ etc.
async def run(*args, loop=None, executor=None, **kwargs):
if loop is None:
loop = asyncio.get_event_loop(). # Make event loop of nothing exists
pfunc = partial(func, *args, **kwargs) # Return function with variables (event) filled in
return await loop.run_in_executor(executor, pfunc).
return run
Such that easy task becomes
#to_async
def easy_task(event):
for i in range(10):
print('Step: ', i)
time.sleep(.2)
event.set()
Where wraps makes sure we can call attributes of the original function (explained here).
And partial fills in the variables as explained here.
I have a complex function Vehicle.set_data, which has many nested functions, API calls, DB calls, etc. For the sake of this example, I will simplify it.
I am trying to use Async IO to run Vehicle.set_data on multiple vehicles at once. Here is my Vehicle model:
class Vehicle:
def __init__(self, token):
self.token = token
# Works async
async def set_data(self):
await asyncio.sleep(random.random() * 10)
# Does not work async
# def set_data(self):
# time.sleep(random.random() * 10)
And here is my Async IO routinue:
async def set_vehicle_data(vehicle):
# sleep for T seconds on average
await vehicle.set_data()
def get_random_string():
return ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(5))
async def producer(queue):
count = 0
while True:
count += 1
# produce a token and send it to a consumer
token = get_random_string()
vehicle = Vehicle(token)
print(f'produced {vehicle.token}')
await queue.put(vehicle)
if count > 3:
break
async def consumer(queue):
while True:
vehicle = await queue.get()
# process the token received from a producer
print(f'Starting consumption for vehicle {vehicle.token}')
await set_vehicle_data(vehicle)
queue.task_done()
print(f'Ending consumption for vehicle {vehicle.token}')
async def main():
queue = asyncio.Queue()
# #todo now, do I need multiple producers
producers = [asyncio.create_task(producer(queue))
for _ in range(3)]
consumers = [asyncio.create_task(consumer(queue))
for _ in range(3)]
# with both producers and consumers running, wait for
# the producers to finish
await asyncio.gather(*producers)
print('---- done producing')
# wait for the remaining tasks to be processed
await queue.join()
# cancel the consumers, which are now idle
for c in consumers:
c.cancel()
asyncio.run(main())
In the example above, this commented section of code does not allow multiple vehicles to process at once:
# Does not work async
# def set_data(self):
# time.sleep(random.random() * 10)
Because this is such a complex query in our actual codebase, it would be a tremendous refactor to go flag every single nested function with async and await. Is there any way I can make this function work async without marking up my whole codebase with async?
You can run the function in a separate thread with asyncio.to_thread
await asyncio.to_thread(self.set_data)
If you're using python <3.9 use loop.run_in_executor
loop = asyncio.get_event_loop()
await loop.run_in_executor(None, self.set_data)
I am connecting to aioredis from __init__ (I do not want to move it out since this means I have to some extra major changes). How can I wait for aioredis connection task in below __init__ example code and have it print self.sub and self.pub object? Currently it gives an error saying
abc.py:42> exception=AttributeError("'S' object has no attribute
'pub'")
I do see redis connections created and coro create_connetion done.
Is there a way to call blocking asyncio calls from __init__. If I replace asyncio.wait with asyncio.run_until_complete I get an error that roughly says
loop is already running.
asyncio.gather is
import sys, json
from addict import Dict
import asyncio
import aioredis
class S():
def __init__(self, opts):
print(asyncio.Task.all_tasks())
task = asyncio.wait(asyncio.create_task(self.create_connection()), return_when="ALL_COMPLETED")
print(asyncio.Task.all_tasks())
print(task)
print(self.pub, self.sub)
async def receive_message(self, channel):
while await channel.wait_message():
message = await channel.get_json()
await asyncio.create_task(self.callback_loop(Dict(json.loads(message))))
async def run_s(self):
asyncio.create_task(self.listen())
async def callback_loop(msg):
print(msg)
self.callback_loop = callback_loop
async def create_connection(self):
self.pub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.sub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.db = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.listener = await self.sub.subscribe(f"abc")
async def listen(self):
self.tsk = asyncio.ensure_future(self.receive_message(self.listener[0]))
await self.tsk
async def periodic(): #test function to show current tasks
number = 5
while True:
await asyncio.sleep(number)
print(asyncio.Task.all_tasks())
async def main(opts):
loop.create_task(periodic())
s = S(opts)
print(s.pub, s.sub)
loop.create_task(s.run_s())
if __name__ == "__main__":
loop = asyncio.get_event_loop()
main_task = loop.create_task(main(sys.argv[1:]))
loop.run_forever() #I DONT WANT TO MOVE THIS UNLESS IT IS NECESSARY
I think what you want to do is to make sure the function create_connections runs to completion BEFORE the S constructor. A way to do that is to rearrange your code a little bit. Move the create_connections function outside the class:
async def create_connection():
pub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
sub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
db = await aioredis.create_redis("redis://c8:7070/0", password="abc")
listener = await self.sub.subscribe(f"abc")
return pub, sub, db, listener
Now await that function before constructing S. So your main function becomes:
async def main(opts):
loop.create_task(periodic())
x = await create_connections()
s = S(opts, x) # pass the result of create_connections to S
print(s.pub, s.sub)
loop.create_task(s.run_s())
Now modify the S constructor to receive the objects created:
def __init__(self, opts, x):
self.pub, self.sub, self.db, self.listener = x
I'm not sure what you're trying to do with the return_when argument and the call to asyncio.wait. The create_connections function doesn't launch a set of parallel tasks, but rather awaits each of the calls before moving on to the next one. Perhaps you could improve performance by running the four calls in parallel but that's a different question.
I need to suspend a coroutine until a condition is met. Currently, I have:
class Awaiter:
def __init__(self):
self.ready = False
def __await__(self):
while not self.ready:
yield
And the caller code:
await awaiter
This works, but it requires boilerplate code. Is it necessary boilerplate or is there a special syntax to await on a predicate, such as:
await condition
which would yield until condition is false?
At the asyncio package there is a builtin Condition object that you can use.
An asyncio condition primitive can be used by a task to wait for some event to happen and then get exclusive access to a shared resource.
How to use the condition (from the same source):
cond = asyncio.Condition()
# The preferred way to use a Condition is an async with statement
async with cond:
await cond.wait()
# It can also be used as follow
await cond.acquire()
try:
await cond.wait()
finally:
cond.release()
A code example:
import asyncio
cond = asyncio.Condition()
async def func1():
async with cond:
print('It\'s look like I will need to wait')
await cond.wait()
print('Now it\'s my turn')
async def func2():
async with cond:
print('Notifying....')
cond.notify()
print('Let me finish first')
# Main function
async def main(loop):
t1 = loop.create_task(func1())
t2 = loop.create_task(func2())
await asyncio.wait([t1, t2])
if __name__ == '__main__':
l = asyncio.get_event_loop()
l.run_until_complete(main(l))
l.close()
This will results with:
It's look like I will need to wait
Notifying....
Let me finish first
Now it's my turn
An alternative way is to use the asyncio.Event.
import asyncio
event = asyncio.Event()
async def func1():
print('It\'s look like I will need to wait')
await event.wait()
print('Now it\'s my turn')
async def func2():
print('Notifying....')
event.set()
print('Let me finish first')
It will have the same results as the Condition code example.
The toy script shows an application using a class that is dependent on an implementation that is not asyncio-aware, and obviously doesn't work.
How would the fetch method of MyFetcher be implemented, using the asyncio-aware client, while still maintaining the contract with the _internal_validator method of FetcherApp? To be very clear, FetcherApp and AbstractFetcher cannot be modified.
To use async fetch_data function inside fetch both fetch and is_fetched_data_valid functions should be async too. You can change them in child classes without modify parent:
import asyncio
class AsyncFetcherApp(FetcherApp):
async def is_fetched_data_valid(self): # async here
data = await self.fetcher_implementation.fetch() # await here
return self._internal_validator(data)
class AsyncMyFetcher(AbstractFetcher):
def __init__(self, client):
super().__init__()
self.client = client
async def fetch(self): # async here
result = await self.client.fetch_data() # await here
return result
class AsyncClient:
async def fetch_data(self):
await asyncio.sleep(1) # Just to sure it works
return 1
async def main():
async_client = AsyncClient()
my_fetcher = AsyncMyFetcher(async_client)
fetcherApp = AsyncFetcherApp(my_fetcher)
# ...
is_valid = await fetcherApp.is_fetched_data_valid() # await here
print(repr(is_valid))
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.run_until_complete(main())