I have a program that needs to run multiple independent tasks.
I need a way to start them and then stop them when an event occur
I've defined a base class like this
class InterruptibleTask:
def __init__(self, stop_evt=None):
self.stop_evt = stop_evt or asyncio.Event()
async def init(self):
pass
async def cleanup(self):
pass
async def run(self):
await self.init()
await self.stop_evt.wait()
await self.cleanup()
async def stop(self):
self.stop_evt.set()
class MyFirstTask(InterruptibleTask):
async def do_work(self):
while not self.stop_evt.is_set:
print("Do Work")
asyncio.sleep(1)
async def init(self):
await asyncio.create_task(self.do_work())
class MysecondTask(InterruptibleTask):
async def do_work(self):
while not self.stop_evt.is_set:
print("Do 2nd Work")
asyncio.sleep(3)
async def init(self):
await asyncio.create_task(self.do_work())
STOP = asyncio.Event()
tasks = [MyFirstTask(STOP), MysecondTask(STOP)]
async def run_tasks():
await asyncio.gather(*tasks)
def stop_tasks():
for task in tasks:
task.stop()
try
asyncio.run(run_tasks())
except KeyboardInterrupt:
pass
finally:
stop_tasks()
It this the right approach?
Do you have any example?
How can I stop all taks and subtask on exit?
What I do when I have several coroutines and want to stop my loop:
try:
loop.run_until_complete(main())
except KeyboardInterrupt:
pending = asyncio.Task.all_tasks()
for c in pending:
asyncio.wait_for(c, timeout=5)
finally:
loop.stop()
So I don't have to have any bookkeeping of my coroutines and can still make sure that every coroutine is stopped. And since there can be coroutines that can take quite a long time to finish I add a timeout, so I don't have to wait for years for my programm to shut down.
Related
I hope websockets can running in background.
Once received the message, print it.
Other function can call it's 'send' function to send message
But the code under 'loop.run_until_complete(Iao())' will no running.
I tried use 'threading' then put websockets in other thread, it's still can't running.
Is it possible to achieve it? except create a async function
import asyncio
import websockets
import time
class WebsocketHandler():
def __init__(self):
self.conn = None
async def connect(self, url):
self.conn = await websockets.connect(url)
async def send(self, msg):
await self.conn.send(msg)
async def receive(self):
while True:
print(await self.conn.recv())
async def close(self):
await self.conn.close()
async def Iao():
global handler
handler = WebsocketHandler()
await handler.connect('ws://localhost:8765')
await handler.receive()
await handler.send('{"action":"plus"}')
await handler.close()
loop = asyncio.get_event_loop()
loop.run_until_complete(Iao())
while True:
print("1")
asyncio.run(handler.send('{"action":"plus"}'))
print("2")
time.sleep(2)
I want to create multiple nested asyncio task groups. I don't want to use trio nursery also subtasks may contain async generators. However, I am encountering some problems:
Exiting with keyboard interrupt causes could not close task as event loop in closed
Sometimes tasks get cancelled and program gets stuck but this rarely happens
I want to make sure they everything is properly close as I am working with files and sockets
How can I fix those problems?
import asyncio
from asyncio.tasks import Task
from contextlib import AsyncExitStack
from os import error
async def cancel_tasks(tasks: set):
task: Task
for task in tasks:
await task.cancel()
async def demo_producer(queue: asyncio.Queue):
while True:
await queue.put()
async def demo_consumer(queue: asyncio.Queue):
while True:
await queue.get()
async def demo_task():
while True:
await asyncio.sleep(2) # Do Something
async def demo_task_grp(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
async with AsyncExitStack() as stack:
tasks = set()
stack.push_async_callback(cancel_tasks, tasks)
task = asyncio.create_task(demo_producer(qns_queue))
tasks.add(task)
task = asyncio.create_task(demo_consumer(ans_queue))
tasks.add(task)
task = asyncio.create_task(demo_task())
tasks.add(task)
asyncio.gather(*tasks)
async def question_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
while True:
try:
await demo_task_grp(qns_queue, ans_queue)
except error:
print(error)
finally:
await asyncio.sleep(2) # retry
async def answer_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
while True:
try:
await demo_task_grp(qns_queue, ans_queue)
except error:
print(error)
finally:
await asyncio.sleep(2) # retry
async def main():
# await mqtt_setup()
# asyncio.ensure_future(mqtt_setup())
async with AsyncExitStack() as stack:
question_queue = asyncio.Queue()
answer_queue = asyncio.Queue()
tasks = set()
stack.push_async_callback(cancel_tasks, tasks)
task = asyncio.create_task(question_func(question_queue, answer_queue))
tasks.add(task)
task = asyncio.create_task(answer_func(question_queue, answer_queue))
tasks.add(task)
asyncio.gather(*tasks)
asyncio.run(main())
I want to make a timer which is started in a normal function, but in the timer function, it should be able to call an async function
I want to do something like this:
startTimer()
while True:
print("e")
def startTimer(waitForSeconds: int):
# Wait for `waitForSeconds`
await myAsyncFunc()
async def myAsyncFunc():
print("in my async func")
Where the while True loop should do its stuff and after waitForSeconds the timer the async function should execute an other async function, but waiting shouldn't block any other actions and doesn't need to be awaited
If something isn't understandable, I'm sorry, I'll try to explain it then
Thanks
If you want to run your synchronous and asynchronous code in parallel, you will need to run one of them in a separate thread. For example:
def sync_code():
while True:
print("e")
async def start_timer(secs):
await asyncio.sleep(secs)
await async_func()
async def main():
asyncio.create_task(start_timer(1))
loop = asyncio.get_event_loop()
# use run_in_executor to run sync code in a separate thread
# while this thread runs the event loop
await loop.run_in_executor(None, sync_code)
asyncio.run(main())
If the above is not acceptable for you (e.g. because it turns the whole program into an asyncio program), you can also run the event loop in a background thread, and submit tasks to it using asyncio.run_coroutine_threadsafe. That approach would allow startTimer to have the signature (and interface) like you wanted it:
def startTimer(waitForSeconds):
loop = asyncio.new_event_loop()
threading.Thread(daemon=True, target=loop.run_forever).start()
async def sleep_and_run():
await asyncio.sleep(waitForSeconds)
await myAsyncFunc()
asyncio.run_coroutine_threadsafe(sleep_and_run(), loop)
async def myAsyncFunc():
print("in my async func")
startTimer(1)
while True:
print("e")
I'm pretty sure that you are familiar with concurent processing, but you didn't show exactly what you want. So if I understand you correctly you want to have 2 processes. First is doing only while True, and the second process is the timer(waits e.g. 5s) and it will call async task. I assume that you are using asyncio according to tags:
import asyncio
async def myAsyncFunc():
print("in my async func")
async def call_after(delay):
await asyncio.sleep(delay)
await myAsyncFunc()
async def while_true():
while True:
await asyncio.sleep(1) # sleep here to avoid to large output
print("e")
async def main():
task1 = asyncio.create_task(
while_true())
task2 = asyncio.create_task(
call_after(5))
# Wait until both tasks are completed (should take
# around 2 seconds.)
await task1
await task2
asyncio.run(main())
I am new to study about asyncio.I don't know how to
describe my question.But here is a minimal example:
import asyncio
async def work():
await asyncio.sleep(3)
async def check_it():
task = asyncio.create_task(work())
await task
while True:
if task.done():
print("Done")
break
print("Trying...")
asyncio.run(check_it())
My idea is very simple:
create a async task in check_it().And await it.
Use a while loop to check whether the task is finished.
If task.done() return True,break the while loop.Then exit the script.
If my question is duplicate, please flag my question.Thanks!
Try asyncio.wait or use asyncio.sleep. Otherwise, your program will output a lot without some pauses.
import asyncio
async def work():
await asyncio.sleep(3)
async def check_it():
task = asyncio.create_task(work())
# "await" block until the task finish. Do not do here.
timeout = 0 # Probably the first timeout is 0
while True:
done, pending = await asyncio.wait({task}, timeout=timeout)
if task in done:
print('Done')
# Do an await here is favourable in case any exception is raised.
await task
break
print('Trying...')
timeout = 1
asyncio.run(check_it())
Suppose I have a few coroutines running in a loop. How to make so that if some of them failed with exception the whole program would fail with this exception? Because right now asyncio doesn't even prints the error messages from coroutines unless I use logging level "DEBUG".
from asyncio import get_event_loop, sleep
async def c(sleep_time=2, fail=False):
print('c', sleep_time, fail)
if fail:
raise Exception('fail')
while True:
print('doing stuff')
await sleep(sleep_time)
loop = get_event_loop()
loop.create_task(c(sleep_time=10, fail=False))
loop.create_task(c(fail=True))
loop.run_forever()
A graceful way is using error handling api.
https://docs.python.org/3/library/asyncio-eventloop.html#error-handling-api
Example:
import asyncio
async def run_division(a, b):
await asyncio.sleep(2)
return a / b
def custom_exception_handler(loop, context):
# first, handle with default handler
loop.default_exception_handler(context)
exception = context.get('exception')
if isinstance(exception, ZeroDivisionError):
print(context)
loop.stop()
loop = asyncio.get_event_loop()
# Set custom handler
loop.set_exception_handler(custom_exception_handler)
loop.create_task(run_division(1, 0))
loop.run_forever()
Here are some notes that you might want use to craft your solution:
The easiest way to retrieve a couroutine's exception (or result!) is to await for it. asyncio.gather() will create tasks from coroutines and wrap all of them in one encompassing task that will fail if one of the subtasks fails:
import asyncio
import random
async def coro(n):
print("Start", n)
await asyncio.sleep(random.uniform(0.2, 0.5))
if n % 4 == 0:
raise Exception('fail ({})'.format(n))
return "OK: {}".format(n)
async def main():
tasks = [coro(i) for i in range(10)]
await asyncio.gather(*tasks)
print("done")
loop = asyncio.get_event_loop()
try:
asyncio.ensure_future(main())
loop.run_forever()
finally:
loop.close()
This however does not shutdown the loop. To stop a running loop, use loop.stop(). Use this instead:
async def main():
tasks = [coro(i) for i in range(10)]
try:
await asyncio.gather(*tasks)
except Exception as e:
loop.stop()
raise
print("done")
Stopping the loop while some long-running coroutines are running is probably not what you want. You might want to first signal some your coroutines to shut down using an event:
import asyncio
import random
async def repeat(n):
print("start", n)
while not shutting_down.is_set():
print("repeat", n)
await asyncio.sleep(random.uniform(1, 3))
print("done", n)
async def main():
print("waiting 6 seconds..")
await asyncio.sleep(6)
print("shutting down")
shutting_down.set() # not a coroutine!
print("waiting")
await asyncio.wait(long_running)
print("done")
loop.stop()
loop = asyncio.get_event_loop()
shutting_down = asyncio.Event(loop=loop)
long_running = [loop.create_task(repeat(i + 1)) for i in range(5)]
try:
asyncio.ensure_future(main())
loop.run_forever()
finally:
loop.close()
If you don't want to await for your tasks, you might want to use an asyncio.Event (or asyncio.Queue) to signal a global error handler to stop the loop:
import asyncio
async def fail():
try:
print("doing stuff...")
await asyncio.sleep(0.2)
print("doing stuff...")
await asyncio.sleep(0.2)
print("doing stuff...")
raise Exception('fail')
except Exception as e:
error_event.payload = e
error_event.set()
raise # optional
async def error_handler():
await error_event.wait()
e = error_event.payload
print("Got:", e)
raise e
loop = asyncio.get_event_loop()
error_event = asyncio.Event()
try:
loop.create_task(fail())
loop.run_until_complete(error_handler())
finally:
loop.close()
(Used here with run_until_complete() for simplicity, but can be used with loop.stop() as well)
Okay, I've found the solution that doesn't require rewriting any existing code. It may seem hacky, but I think I like it.
Since I already catch KeyboardInterrupt like so.
def main(task=None):
task = task or start()
loop = get_event_loop()
try:
task = loop.create_task(task)
future = ensure_future(task)
loop.run_until_complete(task)
except KeyboardInterrupt:
print('\nperforming cleanup...')
task.cancel()
loop.run_until_complete(future)
loop.close()
sys.exit()
How about sending KeyboardInterrupt to itself from a coroutine? I thought that this would hang the application, because os.kill would wait for application to close and because the application it would wait is the same application it would make kind of a deadlock, but thankfully I was wrong. And this code actually works and prints clean up before exiting.
async def c(sleep_time=2, fail=False):
print('c', sleep_time, fail)
if fail:
await sleep(sleep_time)
os.kill(os.getpid(), signal.SIGINT)
while True:
print('doing stuff')
await sleep(sleep_time)
loop = get_event_loop()
loop.create_task(c(sleep_time=10, fail=False))
loop.create_task(c(fail=True))
try:
loop.run_forever()
except KeyboardInterrupt:
print('clean up')
loop.close()
sys.exit()