We Keep Coding

please! how to!!! ASYNCIO in RECURSIVE program in python3 [duplicate]

I had the hypothesis that if I wrote mutually recursive coroutines with asyncio, they would not hit the maximum recursion depth exception, since the event loop was calling them (and act like a trampoline). This, however, is not the case when I write them like this:
import asyncio
#asyncio.coroutine
def a(n):
print("A: {}".format(n))
if n > 1000: return n
else: yield from b(n+1)
#asyncio.coroutine
def b(n):
print("B: {}".format(n))
yield from a(n+1)
loop = asyncio.get_event_loop()
loop.run_until_complete(a(0))
When this runs, I get RuntimeError: maximum recursion depth exceeded while calling a Python object.
Is there a way to keep the stack from growing in recursive coroutines with asyncio?

To keep the stack from growing, you have to allow each coroutine to actually exit after it schedules the next recursive call, which means you have to avoid using yield from. Instead, you use asyncio.async (or asyncio.ensure_future if using Python 3.4.4+) to schedule the next coroutine with the event loop, and use Future.add_done_callback to schedule a callback to run once the recursive call returns. Each coroutine then returns an asyncio.Future object, which has its result set inside the callback that's run when the recursive call it scheduled completes.
It's probably easiest to understand if you actually see the code:
import asyncio
#asyncio.coroutine
def a(n):
fut = asyncio.Future() # We're going to return this right away to our caller
def set_result(out): # This gets called when the next recursive call completes
fut.set_result(out.result()) # Pull the result from the inner call and return it up the stack.
print("A: {}".format(n))
if n > 1000:
return n
else:
in_fut = asyncio.async(b(n+1)) # This returns an asyncio.Task
in_fut.add_done_callback(set_result) # schedule set_result when the Task is done.
return fut
#asyncio.coroutine
def b(n):
fut = asyncio.Future()
def set_result(out):
fut.set_result(out.result())
print("B: {}".format(n))
in_fut = asyncio.async(a(n+1))
in_fut.add_done_callback(set_result)
return fut
loop = asyncio.get_event_loop()
print("Out is {}".format(loop.run_until_complete(a(0))))
Output:
A: 0
B: 1
A: 2
B: 3
A: 4
B: 5
...
A: 994
B: 995
A: 996
B: 997
A: 998
B: 999
A: 1000
B: 1001
A: 1002
Out is 1002
Now, your example code doesn't actually return n all the way back up the stack, so you could make something functionally equivalent that's a bit simpler:
import asyncio
#asyncio.coroutine
def a(n):
print("A: {}".format(n))
if n > 1000: loop.stop(); return n
else: asyncio.async(b(n+1))
#asyncio.coroutine
def b(n):
print("B: {}".format(n))
asyncio.async(a(n+1))
loop = asyncio.get_event_loop()
asyncio.async(a(0))
loop.run_forever()
But I suspect you really meant to return n all the way back up.

In Python 3.7, you can achieve the "trampoline" effect by using asyncio.create_task() instead of awaiting the coroutine directly.
import asyncio
async def a(n):
print(f"A: {n}")
if n > 1000: return n
return await asyncio.create_task(b(n+1))
async def b(n):
print(f"B: {n}")
return await asyncio.create_task(a(n+1))
assert asyncio.run(a(0)) == 1002
However, this has the disadvantage that the event loop still needs to keep track of all the intermediate tasks, since each task is awaiting its successor. We can use a Future object to avoid this problem.
import asyncio
async def _a(n, f):
print(f"A: {n}")
if n > 1000:
f.set_result(n)
return
asyncio.create_task(_b(n+1, f))
async def _b(n, f):
print(f"B: {n}}")
asyncio.create_task(_a(n+1, f))
async def a(n):
f = asyncio.get_running_loop().create_future()
asyncio.create_task(_a(0, f))
return await f
assert asyncio.run(a(0)) == 1002

I changed the code to async, await and measured time. I really like how much more readable it is.
Future:
import asyncio
#asyncio.coroutine
def a(n):
fut = asyncio.Future()
def set_result(out):
fut.set_result(out.result())
if n > 1000:
return n
else:
in_fut = asyncio.async(b(n+1))
in_fut.add_done_callback(set_result)
return fut
#asyncio.coroutine
def b(n):
fut = asyncio.Future()
def set_result(out):
fut.set_result(out.result())
in_fut = asyncio.async(a(n+1))
in_fut.add_done_callback(set_result)
return fut
import timeit
print(min(timeit.repeat("""
loop = asyncio.get_event_loop()
loop.run_until_complete(a(0))
""", "from __main__ import a, b, asyncio", number=10)))
Result:
% time python stack_ori.py
0.6602963969999109
python stack_ori.py 2,06s user 0,01s system 99% cpu 2,071 total
Async, await:
import asyncio
async def a(n):
if n > 1000:
return n
else:
ret = await asyncio.ensure_future(b(n + 1))
return ret
async def b(n):
ret = await asyncio.ensure_future(a(n + 1))
return ret
import timeit
print(min(timeit.repeat("""
loop = asyncio.get_event_loop()
loop.run_until_complete(a(0))
""", "from __main__ import a, b, asyncio", number=10)))
Result:
% time python stack.py
0.45157229300002655
python stack.py 1,42s user 0,02s system 99% cpu 1,451 total

asyncio : asynchronous program is slower than synchronous one

The program I wrote loop through a range and find numbers that are prime and palindrome.
As a part of learning asyncio I tried to re-construct it using async. But the result was not good.Here async code is taking much longer that the synchronous code.
synchronous code
import math
import time
def prime(n):
limit=int(math.sqrt(n))
for j in range(2,limit):
if(n%j==0):
return 0
return 1
def pallindrome(n):
n=str(n)
m=n[::-1]
if(m==n):
return 1
return 0
a, b, c = 999999999, 9999999, 0
start = time.time()
for i in range(a, b, -1):
if(pallindrome(i)):
if(prime(i)):
c+=1
print(i)
if(c==20):
break
print("took --> ", time.time()-start)
RESULT :
999727999
999686999
999676999
999565999
999454999
999434999
999272999
999212999
999070999
998979899
998939899
998898899
998757899
998666899
998565899
998333899
998282899
998202899
998171899
998121899
took --> 0.6525201797485352
asynchronous code
import math , time, asyncio
async def is_prime(n):
limit= int(math.sqrt(n))
for j in range(2,limit):
await asyncio.sleep(0)
if(n%j==0):
return 0
return 1
async def is_pallindrome(n):
await asyncio.sleep(0)
n=str(n)
m=n[::-1]
if(m==n):
return 1
return 0
async def waiting(start):
while True:
print("processing --> time took {:.2f} --> still running".format(time.time()-start))
await asyncio.sleep(2)
async def main():
a, b, c = 999999999, 9999999, 0
start = time.time()
for i in range(a, b , -1):
await asyncio.sleep(0)
if(await is_pallindrome(i)):
if(await is_prime(i)):
c+=1
print(i)
if(c==20):
break
print(f"Found {c} results in {time.time()-start}s exiting now")
if __name__ == "__main__":
loop = asyncio.get_event_loop()
loop.create_task(waiting(time.time()))
future = asyncio.ensure_future(main())
loop.run_until_complete(future)
RESULT:
999727999
999686999
999676999
999565999
999454999
999434999
999272999
999212999
999070999
998979899
998939899
998898899
998757899
998666899
998565899
998333899
998282899
998202899
998171899
998121899
Found 20 results in 18.48567509651184s exiting now
another interesting thing is that passing loop.set_debug(True) and running the code tooks
103 seconds to complete.
can someone explain why this happen?

Your use case seem to be CPU intensive only, and does not require IO work.
Async in python is mainly used to keep using the CPU, while IO operations are running (http request, file writing)
I think that you might be confused with threading. Python can only use one CPU core at a time, and async jobs are queued and executed by the same core. This means that in your example, you will not gain anything by using async, but maybe add some overhead that will slow your execution time.

async / await: Continuously stream data from async method

I'm using python 3.5 to asynchronously return data from one method to another as follows:
async def A():
# Need to get data here from B continuously
val = await B()
async def B():
# Need to get data here from C continuously as they get generated inside while loop of method C
data = await C()
# Modify and process the data and return to A
return await D(data)
async def C():
i = 0
while i < 5:
await asyncio.sleep(1)
# Return this data to method B one by one, Not sure how to do this ??
return i
async def D(val):
# Do some processing of val and return it
return val
I want to continuously stream data from method C and return it to method B, process each item as they are received and return it to method A.
One way is use an asyncio queue and pass it to method B from A, from where it further gets passed on to C.
Method C would keep writing the content in the queue.
Method B would read from queue, process the data and update the queue.
Method A reads the queue at the end for finally processed data.
Can we achieve it using coroutines or async method itself in any other way ? Wish to avoid calls for reading and writing to queues continuously for every request.

import asyncio
from async_generator import async_generator, yield_, yield_from_
async def fun(n):
print("Finding %d-1" % n)
await asyncio.sleep(n/2)
result = n - 1
print("%d - 1 = %d" % (n, result))
return result
#async_generator
async def main(l):
futures = [ fun(n) for n in l ]
for i, future in enumerate(asyncio.as_completed(futures)):
result = await future
print("inside the main..")
print(result)
await yield_(result)
#async_generator
async def dealer():
l = [2, 4, 6]
gen = main(l)
async for item in gen:
print("inside the dealer....")
await yield_(item)
async def dealer1():
gen = dealer()
async for item in gen:
print("inside dealer 1")
print(item)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
#loop.run_until_complete(cc.main())
loop.run_until_complete(dealer1())
loop.close()
You have support for async generators in python3.6. If you are working with python 3.5 you may use async_generator library(https://pypi.python.org/pypi/async_generator/1.5)

Python 3.5 async for blocks the ioloop

I have a simple aiohttp-server with two handlers.
First one does some computations in the async for loop. Second one just returns text response. not_so_long_operation returns 30-th fibonacci number with the slowest recursive implementation, which takes something about one second.
def not_so_long_operation():
return fib(30)
class arange:
def __init__(self, n):
self.n = n
self.i = 0
async def __aiter__(self):
return self
async def __anext__(self):
i = self.i
self.i += 1
if self.i <= self.n:
return i
else:
raise StopAsyncIteration
# GET /
async def index(request):
print('request!')
l = []
async for i in arange(20):
print(i)
l.append(not_so_long_operation())
return aiohttp.web.Response(text='%d\n' % l[0])
# GET /lol/
async def lol(request):
print('request!')
return aiohttp.web.Response(text='just respond\n')
When I'm trying to fetch / and then /lol/, it gives me response for the second one only when the first one gets finished.
What am I doing wrong and how to make index handler release the ioloop on each iteration?

Your example has no yield points (await statements) for switching between tasks.
Asynchronous iterator allows to use await inside __aiter__/__anext__ but don't insert it automatically into your code.
Say,
class arange:
def __init__(self, n):
self.n = n
self.i = 0
async def __aiter__(self):
return self
async def __anext__(self):
i = self.i
self.i += 1
if self.i <= self.n:
await asyncio.sleep(0) # insert yield point
return i
else:
raise StopAsyncIteration
should work as you expected.
In real application most likely you don't need await asyncio.sleep(0) calls because you will wait on database access and similar activities.

Since, fib(30) is CPU bound and sharing little data, you should probably use a ProcessPoolExecutor (as opposed to a ThreadPoolExecutor):
async def index(request):
loop = request.app.loop
executor = request.app["executor"]
result = await loop.run_in_executor(executor, fib, 30)
return web.Response(text="%d" % result)
Setup executor when you create the app:
app = Application(...)
app["exector"] = ProcessPoolExector()

An asynchronous iterator is not really needed here. Instead you can simply give the control back to the event loop inside your loop. In python 3.4, this is done by using a simple yield:
#asyncio.coroutine
def index(self):
for i in range(20):
not_so_long_operation()
yield
In python 3.5, you can define an Empty object that basically does the same thing:
class Empty:
def __await__(self):
yield
Then use it with the await syntax:
async def index(request):
for i in range(20):
not_so_long_operation()
await Empty()
Or simply use asyncio.sleep(0) that has been recently optimized:
async def index(request):
for i in range(20):
not_so_long_operation()
await asyncio.sleep(0)
You could also run the not_so_long_operation in a thread using the default executor:
async def index(request, loop):
for i in range(20):
await loop.run_in_executor(None, not_so_long_operation)

Getting values from functions that run as asyncio tasks

I was trying the following code:
import asyncio
#asyncio.coroutine
def func_normal():
print("A")
yield from asyncio.sleep(5)
print("B")
return 'saad'
#asyncio.coroutine
def func_infinite():
i = 0
while i<10:
print("--"+str(i))
i = i+1
return('saad2')
loop = asyncio.get_event_loop()
tasks = [
asyncio.async(func_normal()),
asyncio.async(func_infinite())]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
I can't figure out how to get values in variables from these functions. I can't do this:
asyncio.async(a = func_infinite())
as this would make this a keyword argument. How do I go about accomplishing this?

The coroutines work as is. Just use the returned value from loop.run_until_complete() and call asyncio.gather() to collect multiple results:
#!/usr/bin/env python3
import asyncio
#asyncio.coroutine
def func_normal():
print('A')
yield from asyncio.sleep(5)
print('B')
return 'saad'
#asyncio.coroutine
def func_infinite():
for i in range(10):
print("--%d" % i)
return 'saad2'
loop = asyncio.get_event_loop()
tasks = func_normal(), func_infinite()
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print("func_normal()={a}, func_infinite()={b}".format(**vars()))
loop.close()
Output
--0
--1
--2
--3
--4
--5
--6
--7
--8
--9
A
B
func_normal()=saad, func_infinite()=saad2

loop.run_until_complete returns the value returned by the function you pass into it. So, it will return the output of asyncio.wait:
import asyncio
#asyncio.coroutine
def func_normal():
print("A")
yield from asyncio.sleep(5)
print("B")
return 'saad'
#asyncio.coroutine
def func_infinite():
i = 0
while i<10:
print("--"+str(i))
i = i+1
return('saad2')
loop = asyncio.get_event_loop()
tasks = [
asyncio.async(func_normal()),
asyncio.async(func_infinite())]
done, _ = loop.run_until_complete(asyncio.wait(tasks))
for fut in done:
print("return value is {}".format(fut.result()))
loop.close()
Output:
A
--0
--1
--2
--3
--4
--5
--6
--7
--8
--9
B
return value is saad2
return value is saad
You can also access the results directly from the tasks array:
print(tasks[0].result())
print(tasks[1].result())

If you want to use any value returned by coroutine as soon as coroutine ends you can pass future object into the coro and update this future by computed value. As soon as future is updated it passes its future.result() to the callback function which is bound with given future. See code below:
import asyncio
async def func_normal(future):
print("A")
await asyncio.sleep(5)
print("B")
# return 'saad'
future.set_result('saad')
async def func_infinite(future):
i = 0
while i<10:
print("--"+str(i))
i = i+1
# return('saad2')
future.set_result('saad2')
def got_result(future):
print(future.result())
loop = asyncio.get_event_loop()
future1 = asyncio.Future()
future2 = asyncio.Future()
future1.add_done_callback(got_result)
future2.add_done_callback(got_result)
# Coros are automatically wrapped in Tasks by asyncio.wait()
coros = [
func_normal(future1),
func_infinite(future2)]
loop.run_until_complete(asyncio.wait(coros))
loop.close()
The callback function is called with a single argument - the future object which it is bound with. If you need to pass more arguments into the callback use partial from functools package:
future1.add_done_callback(functools.partial(print, "future:", argin))
will call
print("future:", argin)

I found a solution that I like better:
loop = asyncio.new_event_loop()
task = loop.create_task(awaitable(*args, **kwargs))
loop.run_until_complete(task)
return task.result()