I'm using python 3.5 to asynchronously return data from one method to another as follows:
async def A():
# Need to get data here from B continuously
val = await B()
async def B():
# Need to get data here from C continuously as they get generated inside while loop of method C
data = await C()
# Modify and process the data and return to A
return await D(data)
async def C():
i = 0
while i < 5:
await asyncio.sleep(1)
# Return this data to method B one by one, Not sure how to do this ??
return i
async def D(val):
# Do some processing of val and return it
return val
I want to continuously stream data from method C and return it to method B, process each item as they are received and return it to method A.
One way is use an asyncio queue and pass it to method B from A, from where it further gets passed on to C.
Method C would keep writing the content in the queue.
Method B would read from queue, process the data and update the queue.
Method A reads the queue at the end for finally processed data.
Can we achieve it using coroutines or async method itself in any other way ? Wish to avoid calls for reading and writing to queues continuously for every request.
import asyncio
from async_generator import async_generator, yield_, yield_from_
async def fun(n):
print("Finding %d-1" % n)
await asyncio.sleep(n/2)
result = n - 1
print("%d - 1 = %d" % (n, result))
return result
#async_generator
async def main(l):
futures = [ fun(n) for n in l ]
for i, future in enumerate(asyncio.as_completed(futures)):
result = await future
print("inside the main..")
print(result)
await yield_(result)
#async_generator
async def dealer():
l = [2, 4, 6]
gen = main(l)
async for item in gen:
print("inside the dealer....")
await yield_(item)
async def dealer1():
gen = dealer()
async for item in gen:
print("inside dealer 1")
print(item)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
#loop.run_until_complete(cc.main())
loop.run_until_complete(dealer1())
loop.close()
You have support for async generators in python3.6. If you are working with python 3.5 you may use async_generator library(https://pypi.python.org/pypi/async_generator/1.5)
Related
Current versions of Python (Dec 2022) still allow using #coroutine decorator and a generation can be as:
import asyncio
asyncify = asyncio.coroutine
data_ready = False # Status of a pipe, just to test
def gen():
global data_ready
while not data_ready:
print("not ready")
data_ready = True # Just to test
yield
return "done"
async def main():
result = await asyncify(gen)()
print(result)
loop = asyncio.new_event_loop()
loop.create_task(main())
loop.run_forever()
However, new Python versions 3.8+ will deprecate #coroutine decorator (the asyncify function alias), how to wait for (await) generator to end as above?
I tried to use async def as expected by the warning but not working:
import asyncio
asyncify = asyncio.coroutine
data_ready = False # Just to test
async def gen():
global data_ready
while not data_ready:
print("not ready")
data_ready = True # Just to test
yield
yield "done"
return
async def main():
# this has error: TypeError: object async_generator can't be used in 'await' expression
result = await gen()
print(result)
loop = asyncio.new_event_loop()
loop.create_task(main())
loop.run_forever()
Asynchronous generators inherit asynchronous iterator and are aimed for asynchronous iterations. You can not directly await them as regular coroutines.
With that in mind, returning to your experimental case and your question "how to wait for (await) generator to end?": to get the final yielded value - perform asynchronous iterations:
import asyncio
data_ready = False # Just to test
async def gen():
global data_ready
while not data_ready:
print("not ready")
data_ready = True # Just to test
yield "processing"
yield "done"
return
async def main():
a_gen = gen()
async for result in a_gen: # assign to result on each async iteration
pass
print('result:', result)
asyncio.run(main())
Prints:
not ready
result: done
Naturally, you can also advance the async generator in steps with anext:
a_gen = gen()
val_1 = await anext(a_gen)
Summing up, follow the guidlines on PEP 525 – Asynchronous Generators and try to not mix old-depreceted things with the actual ones.
I am trying to connect and recieve messages from multiple websockets concurrently.
For this purpose I made it with asyncio, and it prints messages correctly. But the problem is that I just can print it, not return it.
The simplified example of pseudo code which I am struggle with is as below:
import websockets
import json
symbols_id = [1,2]
## LOOP RUNNING EXAMPLE OF ASYNCIO
async def get_connect(symbols_id):
tasks = []
for _id in symbols_id:
print('conncetion to', _id)
if _id == 1:
a = 0
elif _id == 2:
a = 200
tasks.append(asyncio.create_task(_loop(a)))
return tasks
async def _loop(a):
while True:
print(a)
a+=1
await asyncio.sleep(2.5)
async def ping_func():
while True:
print('------ ping')
await asyncio.sleep(5)
async def main():
tasks = await get_connect(symbols_id)
asyncio.create_task(ping_func())
await asyncio.gather(*tasks)
asyncio.run(main())
As you can see from the code above I used print(a) to print a in each loop.
I test return a instead of print(a) but it was not helpful.
thanks
yield a? return a will exit the function and the loop, yield is usually what you want in asyncio for looped tasks
Finally I found the way of using yield and async for to read data in each loop.
It will work correctly, by changing the code to the following one.
import websockets
import json
symbols_id = [1,2]
global a
a=0
## LOOP RUNNING EXAMPLE OF ASYNCIO
async def get_connect(symbols_id):
tasks = []
for _id in symbols_id:
print('conncetion to', _id)
if _id == 1:
a = 0
elif _id == 2:
a = 200
tasks.append(asyncio.create_task(_loop(a)))
return tasks
async def _loop(param):
global a
a = param
while True:
print(a)
a+=1
await asyncio.sleep(2.5)
async def ping_func():
while True:
print('------ ping')
await asyncio.sleep(5)
async def get_result():
global a
while True:
yield a
await asyncio.sleep(1)
async def main():
tasks = await get_connect(symbols_id)
asyncio.create_task(ping_func())
async for x in get_result():
print(x)
await asyncio.gather(*tasks)
asyncio.run(main())
I was confused with how to use generated data from this code snippet inside the other code snippet. what I found is:
1- Generated data can be accessible with global variables.
2- By defining a class and a property, it can be accessible from every part of the code.
In asyncio, it there a way to attach a callback function to a Task such that this callback function will run after the Task has been completed?
So far, the only way I can figure out is to use asyncio.completed in a loop, as shown below. But this requires 2 lists (tasks and cb_tasks) to hold all the tasks/futures.
Is there a better way to do this?
import asyncio
import random
class Foo:
async def start(self):
tasks = []
cb_tasks = []
# Start ten `do_work` tasks simultaneously
for i in range(10):
task = asyncio.create_task(self.do_work(i))
tasks.append(task)
# How to run `self.handle_work_done` as soon as this `task` is completed?
for f in asyncio.as_completed(tasks):
res = await f
t = asyncio.create_task(self.work_done_cb(res))
cb_tasks.append(t)
await asyncio.wait(tasks + cb_tasks)
async def do_work(self, i):
""" Simulate doing some work """
x = random.randint(1, 10)
await asyncio.sleep(x)
print(f"Finished work #{i}")
return x
async def work_done_cb(self, x):
""" Callback after `do_work` has been completed """
await asyncio.sleep(random.randint(1, 3))
print(f"Finished additional work {x}")
if __name__ == "__main__":
foo = Foo()
asyncio.run(foo.start())
Given a regular generator, you can get an iterator from it that can only be consumed once and continue where you left off. Like this -
sync_gen = (i in range(10))
def fetch_batch_sync(num_tasks, job_list):
for i, job in enumerate(job_list):
yield job
if i == num_tasks - 1:
break
>>> sync_gen_iter = sync_gen.__iter__()
>>> for i in fetch_batch_sync(2, sync_gen_iter):
... print i
...
0
1
>>> for i in fetch_batch_sync(3, sync_gen_iter):
... print i
...
2
3
4
Is there a way to do the same with an async generator?
async def fetch_batch_async(num_tasks, job_list_iter):
async for i, job in enumerate(job_list_iter):
yield job
if i == num_tasks - 1:
break
The only difference between regular and async generators is that async generators' equivalents of __next__ and __iter__ methods are themselves async. This is why ordinary for and enumerate fail to recognize them as iterables.
As with regular generators, it is possible to extract a subset of values out of an async generator, but you need to use the appropriate tools. fetch_batch_async already uses async for, but it should also use an async version of enemuerate; for example:
async def aenumerate(aiterable, start=0):
i = start
async for obj in aiterable:
yield i, obj
i += 1
fetch_batch_async would use it exactly like enumerate:
async def fetch_batch_async(num_tasks, job_list_iter):
async for i, job in aenumerate(job_list_iter):
yield job
if i == num_tasks - 1:
break
Finally, this code uses fetch_batch_async to extract several items out of an infinite async iterator:
import asyncio, time
async def infinite():
while True:
yield time.time()
await asyncio.sleep(.1)
async def main():
async for received in fetch_batch_async(10, infinite()):
print(received)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
I have a simple aiohttp-server with two handlers.
First one does some computations in the async for loop. Second one just returns text response. not_so_long_operation returns 30-th fibonacci number with the slowest recursive implementation, which takes something about one second.
def not_so_long_operation():
return fib(30)
class arange:
def __init__(self, n):
self.n = n
self.i = 0
async def __aiter__(self):
return self
async def __anext__(self):
i = self.i
self.i += 1
if self.i <= self.n:
return i
else:
raise StopAsyncIteration
# GET /
async def index(request):
print('request!')
l = []
async for i in arange(20):
print(i)
l.append(not_so_long_operation())
return aiohttp.web.Response(text='%d\n' % l[0])
# GET /lol/
async def lol(request):
print('request!')
return aiohttp.web.Response(text='just respond\n')
When I'm trying to fetch / and then /lol/, it gives me response for the second one only when the first one gets finished.
What am I doing wrong and how to make index handler release the ioloop on each iteration?
Your example has no yield points (await statements) for switching between tasks.
Asynchronous iterator allows to use await inside __aiter__/__anext__ but don't insert it automatically into your code.
Say,
class arange:
def __init__(self, n):
self.n = n
self.i = 0
async def __aiter__(self):
return self
async def __anext__(self):
i = self.i
self.i += 1
if self.i <= self.n:
await asyncio.sleep(0) # insert yield point
return i
else:
raise StopAsyncIteration
should work as you expected.
In real application most likely you don't need await asyncio.sleep(0) calls because you will wait on database access and similar activities.
Since, fib(30) is CPU bound and sharing little data, you should probably use a ProcessPoolExecutor (as opposed to a ThreadPoolExecutor):
async def index(request):
loop = request.app.loop
executor = request.app["executor"]
result = await loop.run_in_executor(executor, fib, 30)
return web.Response(text="%d" % result)
Setup executor when you create the app:
app = Application(...)
app["exector"] = ProcessPoolExector()
An asynchronous iterator is not really needed here. Instead you can simply give the control back to the event loop inside your loop. In python 3.4, this is done by using a simple yield:
#asyncio.coroutine
def index(self):
for i in range(20):
not_so_long_operation()
yield
In python 3.5, you can define an Empty object that basically does the same thing:
class Empty:
def __await__(self):
yield
Then use it with the await syntax:
async def index(request):
for i in range(20):
not_so_long_operation()
await Empty()
Or simply use asyncio.sleep(0) that has been recently optimized:
async def index(request):
for i in range(20):
not_so_long_operation()
await asyncio.sleep(0)
You could also run the not_so_long_operation in a thread using the default executor:
async def index(request, loop):
for i in range(20):
await loop.run_in_executor(None, not_so_long_operation)