I would like to do something like the following:
import asyncio
async def g():
print('called g')
return 'somevalue'
async def f():
x = g()
loop = asyncio.get_event_loop()
loop.run_until_complete(f())
loop.close()
Where there is no output. Notice that I did not await the g(). This will generate a g was not awaited exception, but I'm looking for behaviour where g most definitely did not run.
This is useful for me where I have a long running operation with complex setup, but I only need its result in certain situations, so why bother running it when it is not needed. Kind of an 'on demand' situation.
How can I do this?
One option is to use simple flags to signal tasks:
import asyncio
import random
async def g(info):
print('> called g')
if not info['skip']:
print('* running g', info['id'])
await asyncio.sleep(random.uniform(1, 3))
else:
print('- skiping g', info['id'])
print('< done g', info['id'])
return info['id']
async def main():
data = [{
'id': i,
'skip': False
} for i in range(10)]
# schedule 10 tasks to run later
tasks = [asyncio.ensure_future(g(info)) for info in data]
# tell some tasks to skip processing
data[2]['skip'] = True
data[5]['skip'] = True
data[6]['skip'] = True
# wait for all results
results = await asyncio.gather(*tasks)
print(results)
print("Done!")
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
A different option would be using task.cancel:
import asyncio
async def coro(x):
print('coro', x)
return x
async def main():
task1 = asyncio.ensure_future(coro(1))
task2 = asyncio.ensure_future(coro(2))
task3 = asyncio.ensure_future(coro(3))
task2.cancel()
for task in asyncio.as_completed([task1, task2, task3]):
try:
result = await task
print("success", result)
except asyncio.CancelledError as e:
print("cancelled", e)
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
Related
In asyncio, it there a way to attach a callback function to a Task such that this callback function will run after the Task has been completed?
So far, the only way I can figure out is to use asyncio.completed in a loop, as shown below. But this requires 2 lists (tasks and cb_tasks) to hold all the tasks/futures.
Is there a better way to do this?
import asyncio
import random
class Foo:
async def start(self):
tasks = []
cb_tasks = []
# Start ten `do_work` tasks simultaneously
for i in range(10):
task = asyncio.create_task(self.do_work(i))
tasks.append(task)
# How to run `self.handle_work_done` as soon as this `task` is completed?
for f in asyncio.as_completed(tasks):
res = await f
t = asyncio.create_task(self.work_done_cb(res))
cb_tasks.append(t)
await asyncio.wait(tasks + cb_tasks)
async def do_work(self, i):
""" Simulate doing some work """
x = random.randint(1, 10)
await asyncio.sleep(x)
print(f"Finished work #{i}")
return x
async def work_done_cb(self, x):
""" Callback after `do_work` has been completed """
await asyncio.sleep(random.randint(1, 3))
print(f"Finished additional work {x}")
if __name__ == "__main__":
foo = Foo()
asyncio.run(foo.start())
import asyncio
import aiohttp
aut_token = ("token")
tasks = []
iter_flag = True
interval = 0
seq = 0
class WAPI:
async def receiver(WAPI_S):
async for msg in WAPI_S:
global interval
global seq
data = msg.json()
seq = data.get("s")
if data.get("op") == 10:
interval = data.get("d").get("heartbeat_interval") / 1000
if data.get("op") == 11:
pass
raise aiohttp.ClientError
async def heartbeating(WAPI_S):
while iter_flag:
await WAPI_S.send_json({
"op": 1,
"d": seq
})
await asyncio.sleep(interval)
async def event_manager():
loop = asyncio.get_running_loop()
try:
async with aiohttp.ClientSession() as session:
async with session.ws_connect("url") as WAPI_S:
task_receive = loop.create_task(WAPI.receiver(WAPI_S)); task_heartbeating = loop.create_task(WAPI.heartbeating(WAPI_S))
tasks.append(task_receive); tasks.append(task_heartbeating)
await asyncio.gather(*tasks)
except aiohttp.ClientError:
global iter_flag
iter_flag = False
await asyncio.sleep(interval)
for task in tasks:
task.cancel()
try:
loop.close()
except:
loop.stop()
asyncio.run(WAPI.event_manager())
I am trying to implement catching ClientError exception with loop closing, however, loop.close throws "RuntimeError: Event loop stopped before Future completed."
How to implement interception correctly?
You do not need to keep track of your tasks manually, you can simply use
asyncio.all_tasks():
Return a set of not yet finished Task objects run by the loop.
And then a:
pending = asyncio.all_tasks()
c in pending:
wait_for(c, timeout=5)
Also, you are trying to stop the loop while you are in it.
This is the pattern I use most of the times:
async def main():
<do some stuff>
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(main())
except ExceptionYouWantToHandle:
<cleaning up>
finally:
loop.stop()
And in the event_manager you simply return once you get an execption or you pass the exception
I have written code for async pool below. in __aexit__ i'm cancelling the _worker tasks after the tasks get finished. But when i run the code, the worker tasks are not getting cancelled and the code is running forever. This what the task looks like: <Task pending coro=<AsyncPool._worker() running at \async_pool.py:17> wait_for=<Future cancelled>>. The asyncio.wait_for is getting cancelled but not the worker tasks.
class AsyncPool:
def __init__(self,coroutine,no_of_workers,timeout):
self._loop = asyncio.get_event_loop()
self._queue = asyncio.Queue()
self._no_of_workers = no_of_workers
self._coroutine = coroutine
self._timeout = timeout
self._workers = None
async def _worker(self):
while True:
try:
ret = False
queue_item = await self._queue.get()
ret = True
result = await asyncio.wait_for(self._coroutine(queue_item), timeout = self._timeout,loop= self._loop)
except Exception as e:
print(e)
finally:
if ret:
self._queue.task_done()
async def push_to_queue(self,item):
self._queue.put_nowait(item)
async def __aenter__(self):
assert self._workers == None
self._workers = [asyncio.create_task(self._worker()) for _ in range(self._no_of_workers)]
return self
async def __aexit__(self,type,value,traceback):
await self._queue.join()
for worker in self._workers:
worker.cancel()
await asyncio.gather(*self._workers, loop=self._loop, return_exceptions =True)
To use the Asyncpool:
async def something(item):
print("got", item)
await asyncio.sleep(item)
async def main():
async with AsyncPool(something, 5, 2) as pool:
for i in range(10):
await pool.push_to_queue(i)
asyncio.run(main())
The Output in my terminal:
The problem is that your except Exception exception clause also catches cancellation, and ignores it. To add to the confusion, print(e) just prints an empty line in case of a CancelledError, which is where the empty lines in the output come from. (Changing it to print(type(e)) shows what's going on.)
To correct the issue, change except Exception to something more specific, like except asyncio.TimeoutError. This change is not needed in Python 3.8 where asyncio.CancelledError no longer derives from Exception, but from BaseException, so except Exception doesn't catch it.
When you have an asyncio task created and then cancelled, you still have the task alive that need to be "reclaimed". So you want to await worker for it. However, once you await such a cancelled task, as it will never give you back the expected return value, the asyncio.CancelledError will be raised and you need to catch it somewhere.
Because of this behavior, I don't think you should gather them but to await for each of the cancelled tasks, as they are supposed to return right away:
async def __aexit__(self,type,value,traceback):
await self._queue.join()
for worker in self._workers:
worker.cancel()
for worker in self._workers:
try:
await worker
except asyncio.CancelledError:
print("worker cancelled:", worker)
This appears to work. The event is a counting timer and when it expires it cancels the tasks.
import asyncio
from datetime import datetime as dt
from datetime import timedelta as td
import random
import time
class Program:
def __init__(self):
self.duration_in_seconds = 20
self.program_start = dt.now()
self.event_has_expired = False
self.canceled_success = False
async def on_start(self):
print("On Start Event Start! Applying Overrides!!!")
await asyncio.sleep(random.randint(3, 9))
async def on_end(self):
print("On End Releasing All Overrides!")
await asyncio.sleep(random.randint(3, 9))
async def get_sensor_readings(self):
print("getting sensor readings!!!")
await asyncio.sleep(random.randint(3, 9))
async def evauluate_data(self):
print("checking data!!!")
await asyncio.sleep(random.randint(3, 9))
async def check_time(self):
if (dt.now() - self.program_start > td(seconds = self.duration_in_seconds)):
self.event_has_expired = True
print("Event is DONE!!!")
else:
print("Event is not done! ",dt.now() - self.program_start)
async def main(self):
# script starts, do only once self.on_start()
await self.on_start()
print("On Start Done!")
while not self.canceled_success:
readings = asyncio.ensure_future(self.get_sensor_readings())
analysis = asyncio.ensure_future(self.evauluate_data())
checker = asyncio.ensure_future(self.check_time())
if not self.event_has_expired:
await readings
await analysis
await checker
else:
# close other tasks before final shutdown
readings.cancel()
analysis.cancel()
checker.cancel()
self.canceled_success = True
print("cancelled hit!")
# script ends, do only once self.on_end() when even is done
await self.on_end()
print('Done Deal!')
async def main():
program = Program()
await program.main()
While working in asyncio, I'm trying to use a list comprehension to build my task list. The basic form of the function is as follows:
import asyncio
import urllib.request as req
#asyncio.coroutine
def coro(term):
print(term)
google = "https://www.google.com/search?q=" + term.replace(" ", "+") + "&num=100&start=0"
request = req.Request(google, None, headers)
(some beautiful soup stuff)
My goal is to use a list of terms to create my task list:
terms = ["pie", "chicken" ,"things" ,"stuff"]
tasks=[
coro("pie"),
coro("chicken"),
coro("things"),
coro("stuff")]
My initial thought was:
loop = asyncio.get_event_loop()
tasks = [my_coroutine(term) for term in terms]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
This doesn't create the task list it runs the function during the list comprehension. Is there a way to use a shortcut to create the task list wihout writing every task?
Your HTTP client does not support asyncio, and you will not get the expected results. Try this to see .wait() does work as you expected:
import asyncio
import random
#asyncio.coroutine
def my_coroutine(term):
print("start", term)
yield from asyncio.sleep(random.uniform(1, 3))
print("end", term)
terms = ["pie", "chicken", "things", "stuff"]
loop = asyncio.get_event_loop()
tasks = [my_coroutine(term) for term in terms]
print("Here we go!")
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
If you use asyncio.gather() you get one future encapsulating all your tasks, which can be easily canceled with .cancel(), here demonstrated with python 3.5+ async def/await syntax (but works the same with #coroutine and yield from):
import asyncio
import random
async def my_coroutine(term):
print("start", term)
n = random.uniform(0.2, 1.5)
await asyncio.sleep(n)
print("end", term)
return "Term {} slept for {:.2f} seconds".format(term, n)
async def stop_all():
"""Cancels all still running tasks after one second"""
await asyncio.sleep(1)
print("stopping")
fut.cancel()
return ":-)"
loop = asyncio.get_event_loop()
terms = ["pie", "chicken", "things", "stuff"]
tasks = (my_coroutine(term) for term in terms)
fut = asyncio.gather(stop_all(), *tasks, return_exceptions=True)
print("Here we go!")
loop.run_until_complete(fut)
for task_result in fut.result():
if not isinstance(task_result, Exception):
print("OK", task_result)
else:
print("Failed", task_result)
loop.close()
And finally, if you want to use an async HTTP client, try aiohttp. First install it with:
pip install aiohttp
then try this example, which uses asyncio.as_completed:
import asyncio
import aiohttp
async def fetch(session, url):
print("Getting {}...".format(url))
async with session.get(url) as resp:
text = await resp.text()
return "{}: Got {} bytes".format(url, len(text))
async def fetch_all():
async with aiohttp.ClientSession() as session:
tasks = [fetch(session, "http://httpbin.org/delay/{}".format(delay))
for delay in (1, 1, 2, 3, 3)]
for task in asyncio.as_completed(tasks):
print(await task)
return "Done."
loop = asyncio.get_event_loop()
resp = loop.run_until_complete(fetch_all())
print(resp)
loop.close()
this works in python 3.5 (added the new async-await syntax):
import asyncio
async def coro(term):
for i in range(3):
await asyncio.sleep(int(len(term))) # just sleep
print("cor1", i, term)
terms = ["pie", "chicken", "things", "stuff"]
tasks = [coro(term) for term in terms]
loop = asyncio.get_event_loop()
cors = asyncio.wait(tasks)
loop.run_until_complete(cors)
should't your version yield from req.Request(google, None, headers)? and (what library is that?) is this library even made for use with asyncio?
(here is the same code with the python <= 3.4 syntax; the missing parts are the same as above):
#asyncio.coroutine
def coro(term):
for i in range(3):
yield from asyncio.sleep(int(len(term))) # just sleep
print("cor1", i, term)
Create queue and run event loop
def main():
while terms:
tasks.append(asyncio.create_task(terms.pop())
responses = asyncio.gather(*tasks, return_exception=True)
loop = asyncio.get_event_loop()
loop.run_until_complete(responses)
I was trying the following code:
import asyncio
#asyncio.coroutine
def func_normal():
print("A")
yield from asyncio.sleep(5)
print("B")
return 'saad'
#asyncio.coroutine
def func_infinite():
i = 0
while i<10:
print("--"+str(i))
i = i+1
return('saad2')
loop = asyncio.get_event_loop()
tasks = [
asyncio.async(func_normal()),
asyncio.async(func_infinite())]
loop.run_until_complete(asyncio.wait(tasks))
loop.close()
I can't figure out how to get values in variables from these functions. I can't do this:
asyncio.async(a = func_infinite())
as this would make this a keyword argument. How do I go about accomplishing this?
The coroutines work as is. Just use the returned value from loop.run_until_complete() and call asyncio.gather() to collect multiple results:
#!/usr/bin/env python3
import asyncio
#asyncio.coroutine
def func_normal():
print('A')
yield from asyncio.sleep(5)
print('B')
return 'saad'
#asyncio.coroutine
def func_infinite():
for i in range(10):
print("--%d" % i)
return 'saad2'
loop = asyncio.get_event_loop()
tasks = func_normal(), func_infinite()
a, b = loop.run_until_complete(asyncio.gather(*tasks))
print("func_normal()={a}, func_infinite()={b}".format(**vars()))
loop.close()
Output
--0
--1
--2
--3
--4
--5
--6
--7
--8
--9
A
B
func_normal()=saad, func_infinite()=saad2
loop.run_until_complete returns the value returned by the function you pass into it. So, it will return the output of asyncio.wait:
import asyncio
#asyncio.coroutine
def func_normal():
print("A")
yield from asyncio.sleep(5)
print("B")
return 'saad'
#asyncio.coroutine
def func_infinite():
i = 0
while i<10:
print("--"+str(i))
i = i+1
return('saad2')
loop = asyncio.get_event_loop()
tasks = [
asyncio.async(func_normal()),
asyncio.async(func_infinite())]
done, _ = loop.run_until_complete(asyncio.wait(tasks))
for fut in done:
print("return value is {}".format(fut.result()))
loop.close()
Output:
A
--0
--1
--2
--3
--4
--5
--6
--7
--8
--9
B
return value is saad2
return value is saad
You can also access the results directly from the tasks array:
print(tasks[0].result())
print(tasks[1].result())
If you want to use any value returned by coroutine as soon as coroutine ends you can pass future object into the coro and update this future by computed value. As soon as future is updated it passes its future.result() to the callback function which is bound with given future. See code below:
import asyncio
async def func_normal(future):
print("A")
await asyncio.sleep(5)
print("B")
# return 'saad'
future.set_result('saad')
async def func_infinite(future):
i = 0
while i<10:
print("--"+str(i))
i = i+1
# return('saad2')
future.set_result('saad2')
def got_result(future):
print(future.result())
loop = asyncio.get_event_loop()
future1 = asyncio.Future()
future2 = asyncio.Future()
future1.add_done_callback(got_result)
future2.add_done_callback(got_result)
# Coros are automatically wrapped in Tasks by asyncio.wait()
coros = [
func_normal(future1),
func_infinite(future2)]
loop.run_until_complete(asyncio.wait(coros))
loop.close()
The callback function is called with a single argument - the future object which it is bound with. If you need to pass more arguments into the callback use partial from functools package:
future1.add_done_callback(functools.partial(print, "future:", argin))
will call
print("future:", argin)
I found a solution that I like better:
loop = asyncio.new_event_loop()
task = loop.create_task(awaitable(*args, **kwargs))
loop.run_until_complete(task)
return task.result()