I want to create multiple nested asyncio task groups. I don't want to use trio nursery also subtasks may contain async generators. However, I am encountering some problems:
Exiting with keyboard interrupt causes could not close task as event loop in closed
Sometimes tasks get cancelled and program gets stuck but this rarely happens
I want to make sure they everything is properly close as I am working with files and sockets
How can I fix those problems?
import asyncio
from asyncio.tasks import Task
from contextlib import AsyncExitStack
from os import error
async def cancel_tasks(tasks: set):
task: Task
for task in tasks:
await task.cancel()
async def demo_producer(queue: asyncio.Queue):
while True:
await queue.put()
async def demo_consumer(queue: asyncio.Queue):
while True:
await queue.get()
async def demo_task():
while True:
await asyncio.sleep(2) # Do Something
async def demo_task_grp(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
async with AsyncExitStack() as stack:
tasks = set()
stack.push_async_callback(cancel_tasks, tasks)
task = asyncio.create_task(demo_producer(qns_queue))
tasks.add(task)
task = asyncio.create_task(demo_consumer(ans_queue))
tasks.add(task)
task = asyncio.create_task(demo_task())
tasks.add(task)
asyncio.gather(*tasks)
async def question_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
while True:
try:
await demo_task_grp(qns_queue, ans_queue)
except error:
print(error)
finally:
await asyncio.sleep(2) # retry
async def answer_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
while True:
try:
await demo_task_grp(qns_queue, ans_queue)
except error:
print(error)
finally:
await asyncio.sleep(2) # retry
async def main():
# await mqtt_setup()
# asyncio.ensure_future(mqtt_setup())
async with AsyncExitStack() as stack:
question_queue = asyncio.Queue()
answer_queue = asyncio.Queue()
tasks = set()
stack.push_async_callback(cancel_tasks, tasks)
task = asyncio.create_task(question_func(question_queue, answer_queue))
tasks.add(task)
task = asyncio.create_task(answer_func(question_queue, answer_queue))
tasks.add(task)
asyncio.gather(*tasks)
asyncio.run(main())
Related
I have to process every message from redis asynchronously.
Here is my attempt with aioredis:
import asyncio
import aioredis
async def reader(channel: aioredis.client.PubSub):
while True:
data = None
try:
message = await channel.get_message(ignore_subscribe_messages=True)
if message is not None:
print(f"(Reader) Message Received: {message}")
data = message["data"]
except asyncio.TimeoutError:
pass
if data is not None:
await process_message(data)
async def process_message(message):
print(f"start process {message=}")
await asyncio.sleep(10)
print(f"+processed {message=}")
async def publish(redis, channel, message):
print(f"-->publish {message=} to {channel=}")
result = await redis.publish(channel, message)
print(" +published")
return result
async def main():
redis = aioredis.from_url("redis://localhost")
pubsub = redis.pubsub()
await pubsub.subscribe("channel:1", "channel:2")
future = asyncio.create_task(reader(pubsub))
await publish(redis, "channel:1", "Hello")
await publish(redis, "channel:2", "World")
await future
if __name__ == "__main__":
asyncio.run(main())
The problem is that aioredis does not get_message if the previous message was not processed. The messages are processed one by one.
How to solve that issue?
I've found the solution.
Instead of await process_message(data) one should use asyncio.ensure_future(process_message(data))
The idea came from AIORedis and PUB/SUB aren't asnyc
I'm probably having trouble understanding asyncio.shield. I expect that a coroutine wrapped in a shield will not be canceled under any circumstances, allowing it to finish its work. But when I run the example below, the coroutine is still canceled in the process of work :(
Is there a way to protect a coroutine from being cancelled?
import asyncio
import logging
import os
from aiogram import Bot, Dispatcher, types
#API_TOKEN = os.getenv("BOT_TOKEN")
API_TOKEN = "TOKEN"
# Configure logging
logging.basicConfig(level=logging.INFO)
# Initialize bot and dispatcher
bot = Bot(token=API_TOKEN)
dp = Dispatcher(bot)
def shielded(fn):
async def wrapped(*args, **kwargs):
await asyncio.shield(fn(*args, **kwargs))
return wrapped
#dp.message_handler()
#shielded
async def echo(message: types.Message, *args, **kwargs):
try:
await asyncio.sleep(7)
await message.answer(message.text)
except asyncio.CancelledError:
print("handler cancelled :(")
async def cancel_dp_with_delay(dp, sec):
await asyncio.sleep(sec)
dp.stop_polling()
await dp.wait_closed()
async def main():
asyncio.create_task(cancel_dp_with_delay(dp, 5))
await dp.start_polling()
await asyncio.sleep(3)
if __name__ == '__main__':
asyncio.run(main())
I have this code :
import asyncio
import aioconsole
async def imp():
print("Cool I'm printing")
await asyncio.sleep(2)
await imp()
async def console():
while True:
comm = await aioconsole.ainput(">>")
if 'cool' in comm:
await imp()
async def main():
console_use = asyncio.create_task(console())
await console_use
if __name__ == '__main__':
try:
asyncio.run(main())
except AttributeError:
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()
The problem is that when the code enter to the function imp() the console() function stops working and I would like to continue using the console for other functions. How can I solve that?
Ps. I tried doing threads, create tasks and nothing is working...
I am new to study about asyncio.I don't know how to
describe my question.But here is a minimal example:
import asyncio
async def work():
await asyncio.sleep(3)
async def check_it():
task = asyncio.create_task(work())
await task
while True:
if task.done():
print("Done")
break
print("Trying...")
asyncio.run(check_it())
My idea is very simple:
create a async task in check_it().And await it.
Use a while loop to check whether the task is finished.
If task.done() return True,break the while loop.Then exit the script.
If my question is duplicate, please flag my question.Thanks!
Try asyncio.wait or use asyncio.sleep. Otherwise, your program will output a lot without some pauses.
import asyncio
async def work():
await asyncio.sleep(3)
async def check_it():
task = asyncio.create_task(work())
# "await" block until the task finish. Do not do here.
timeout = 0 # Probably the first timeout is 0
while True:
done, pending = await asyncio.wait({task}, timeout=timeout)
if task in done:
print('Done')
# Do an await here is favourable in case any exception is raised.
await task
break
print('Trying...')
timeout = 1
asyncio.run(check_it())
I have a program that needs to run multiple independent tasks.
I need a way to start them and then stop them when an event occur
I've defined a base class like this
class InterruptibleTask:
def __init__(self, stop_evt=None):
self.stop_evt = stop_evt or asyncio.Event()
async def init(self):
pass
async def cleanup(self):
pass
async def run(self):
await self.init()
await self.stop_evt.wait()
await self.cleanup()
async def stop(self):
self.stop_evt.set()
class MyFirstTask(InterruptibleTask):
async def do_work(self):
while not self.stop_evt.is_set:
print("Do Work")
asyncio.sleep(1)
async def init(self):
await asyncio.create_task(self.do_work())
class MysecondTask(InterruptibleTask):
async def do_work(self):
while not self.stop_evt.is_set:
print("Do 2nd Work")
asyncio.sleep(3)
async def init(self):
await asyncio.create_task(self.do_work())
STOP = asyncio.Event()
tasks = [MyFirstTask(STOP), MysecondTask(STOP)]
async def run_tasks():
await asyncio.gather(*tasks)
def stop_tasks():
for task in tasks:
task.stop()
try
asyncio.run(run_tasks())
except KeyboardInterrupt:
pass
finally:
stop_tasks()
It this the right approach?
Do you have any example?
How can I stop all taks and subtask on exit?
What I do when I have several coroutines and want to stop my loop:
try:
loop.run_until_complete(main())
except KeyboardInterrupt:
pending = asyncio.Task.all_tasks()
for c in pending:
asyncio.wait_for(c, timeout=5)
finally:
loop.stop()
So I don't have to have any bookkeeping of my coroutines and can still make sure that every coroutine is stopped. And since there can be coroutines that can take quite a long time to finish I add a timeout, so I don't have to wait for years for my programm to shut down.