I'm probably having trouble understanding asyncio.shield. I expect that a coroutine wrapped in a shield will not be canceled under any circumstances, allowing it to finish its work. But when I run the example below, the coroutine is still canceled in the process of work :(
Is there a way to protect a coroutine from being cancelled?
import asyncio
import logging
import os
from aiogram import Bot, Dispatcher, types
#API_TOKEN = os.getenv("BOT_TOKEN")
API_TOKEN = "TOKEN"
# Configure logging
logging.basicConfig(level=logging.INFO)
# Initialize bot and dispatcher
bot = Bot(token=API_TOKEN)
dp = Dispatcher(bot)
def shielded(fn):
async def wrapped(*args, **kwargs):
await asyncio.shield(fn(*args, **kwargs))
return wrapped
#dp.message_handler()
#shielded
async def echo(message: types.Message, *args, **kwargs):
try:
await asyncio.sleep(7)
await message.answer(message.text)
except asyncio.CancelledError:
print("handler cancelled :(")
async def cancel_dp_with_delay(dp, sec):
await asyncio.sleep(sec)
dp.stop_polling()
await dp.wait_closed()
async def main():
asyncio.create_task(cancel_dp_with_delay(dp, 5))
await dp.start_polling()
await asyncio.sleep(3)
if __name__ == '__main__':
asyncio.run(main())
Related
I have to process every message from redis asynchronously.
Here is my attempt with aioredis:
import asyncio
import aioredis
async def reader(channel: aioredis.client.PubSub):
while True:
data = None
try:
message = await channel.get_message(ignore_subscribe_messages=True)
if message is not None:
print(f"(Reader) Message Received: {message}")
data = message["data"]
except asyncio.TimeoutError:
pass
if data is not None:
await process_message(data)
async def process_message(message):
print(f"start process {message=}")
await asyncio.sleep(10)
print(f"+processed {message=}")
async def publish(redis, channel, message):
print(f"-->publish {message=} to {channel=}")
result = await redis.publish(channel, message)
print(" +published")
return result
async def main():
redis = aioredis.from_url("redis://localhost")
pubsub = redis.pubsub()
await pubsub.subscribe("channel:1", "channel:2")
future = asyncio.create_task(reader(pubsub))
await publish(redis, "channel:1", "Hello")
await publish(redis, "channel:2", "World")
await future
if __name__ == "__main__":
asyncio.run(main())
The problem is that aioredis does not get_message if the previous message was not processed. The messages are processed one by one.
How to solve that issue?
I've found the solution.
Instead of await process_message(data) one should use asyncio.ensure_future(process_message(data))
The idea came from AIORedis and PUB/SUB aren't asnyc
As a proof of concept im trying to integrate multithreading with discord.py, but when i run this code im getting an error ive never seen before. does anyone know how to make this work?
error: TypeError: create() missing 1 required posistional argument 'ctx'
here is a snapshot of the relevant part of the code.
async def create(ctx):
guild = ctx.guild
amount = 150
for i in range(amount):
await guild.create_text_channel("channel")
init()
#client.event
async def on_ready():
print(f"""
threading test discord.py
prefix: {Fore.GREEN}$
""")
#client.command()
async def start(ctx):
thread1 = threading.Thread(target=create)
thread1.start()
client.run(token)```
create is an asynchronous function, threading does not work with them, a solution might be to use the asyncio.run_coroutine_threadsafe method:
import asyncio
def create(ctx):
async def create(ctx):
guild = ctx.guild
amount = 150
for i in range(amount):
await guild.create_text_channel("channel")
loop = asyncio.get_event_loop()
fut = asyncio.run_coroutine_threadsafe(create(ctx), loop)
return fut.result()
# inside command
thread = threading.Thread(target=create, args=(ctx,))
thread.start()
You can make a decorator for simplicity:
from functools import wraps
import asyncio
loop = asyncio.get_event_loop()
def coro_to_thread(fn): # the decorator
#wraps(fn)
def wrapper(*args, **kwargs):
fut = asyncio.run_coroutine_threadsafe(
fn(*args, **kwargs), loop
)
return fut.result()
return wrapper
#coro_to_thread
async def create(ctx):
guild = ctx.guild
amount = 150
for i in range(amount):
await guild.create_text_channel("channel")
# inside command
thread = threading.Thread(target=create, args=(ctx,))
thread.start()
I hope websockets can running in background.
Once received the message, print it.
Other function can call it's 'send' function to send message
But the code under 'loop.run_until_complete(Iao())' will no running.
I tried use 'threading' then put websockets in other thread, it's still can't running.
Is it possible to achieve it? except create a async function
import asyncio
import websockets
import time
class WebsocketHandler():
def __init__(self):
self.conn = None
async def connect(self, url):
self.conn = await websockets.connect(url)
async def send(self, msg):
await self.conn.send(msg)
async def receive(self):
while True:
print(await self.conn.recv())
async def close(self):
await self.conn.close()
async def Iao():
global handler
handler = WebsocketHandler()
await handler.connect('ws://localhost:8765')
await handler.receive()
await handler.send('{"action":"plus"}')
await handler.close()
loop = asyncio.get_event_loop()
loop.run_until_complete(Iao())
while True:
print("1")
asyncio.run(handler.send('{"action":"plus"}'))
print("2")
time.sleep(2)
I want to create multiple nested asyncio task groups. I don't want to use trio nursery also subtasks may contain async generators. However, I am encountering some problems:
Exiting with keyboard interrupt causes could not close task as event loop in closed
Sometimes tasks get cancelled and program gets stuck but this rarely happens
I want to make sure they everything is properly close as I am working with files and sockets
How can I fix those problems?
import asyncio
from asyncio.tasks import Task
from contextlib import AsyncExitStack
from os import error
async def cancel_tasks(tasks: set):
task: Task
for task in tasks:
await task.cancel()
async def demo_producer(queue: asyncio.Queue):
while True:
await queue.put()
async def demo_consumer(queue: asyncio.Queue):
while True:
await queue.get()
async def demo_task():
while True:
await asyncio.sleep(2) # Do Something
async def demo_task_grp(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
async with AsyncExitStack() as stack:
tasks = set()
stack.push_async_callback(cancel_tasks, tasks)
task = asyncio.create_task(demo_producer(qns_queue))
tasks.add(task)
task = asyncio.create_task(demo_consumer(ans_queue))
tasks.add(task)
task = asyncio.create_task(demo_task())
tasks.add(task)
asyncio.gather(*tasks)
async def question_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
while True:
try:
await demo_task_grp(qns_queue, ans_queue)
except error:
print(error)
finally:
await asyncio.sleep(2) # retry
async def answer_func(qns_queue: asyncio.Queue, ans_queue: asyncio.Queue):
while True:
try:
await demo_task_grp(qns_queue, ans_queue)
except error:
print(error)
finally:
await asyncio.sleep(2) # retry
async def main():
# await mqtt_setup()
# asyncio.ensure_future(mqtt_setup())
async with AsyncExitStack() as stack:
question_queue = asyncio.Queue()
answer_queue = asyncio.Queue()
tasks = set()
stack.push_async_callback(cancel_tasks, tasks)
task = asyncio.create_task(question_func(question_queue, answer_queue))
tasks.add(task)
task = asyncio.create_task(answer_func(question_queue, answer_queue))
tasks.add(task)
asyncio.gather(*tasks)
asyncio.run(main())
I have this basic exchange monitor script. I'm trying to create one thread per symbol, apart from the main thread which is handling other work, and have them listen to public Gemini websocket endpoints. I'm getting the first thread running and printing exchange data to the console, but not the second one. I had expected to see data from both threads being printed at approximately the same time. I've tried using the threading library instead of asyncio and encountered the same situation.
I realize my two public API MarketWebsocket classes could be combined to be cleaner, I'm still trying to work out a way to easily add other symbols to the list. Thanks for any nudges in the right direction!
import asyncio
from websockets import connect
symbols_to_watch = [
"BTCUSD",
"ETHUSD"
]
class BTCMarketWebsocket:
disable = False
async def __aenter__(self):
symbol = symbols_to_watch[0]
self._conn = connect("wss://api.gemini.com/v1/marketdata/{}".format(symbol))
self.websocket = await self._conn.__aenter__()
return self
async def __aexit__(self, *args, **kwargs):
await self._conn.__aexit__(*args, **kwargs)
async def receive(self):
return await self.websocket.recv()
class ETHMarketWebsocket:
disable = False
async def __aenter__(self):
symbol = symbols_to_watch[1]
self._conn = connect("wss://api.gemini.com/v1/marketdata/{}".format(symbol))
self.websocket = await self._conn.__aenter__()
return self
async def __aexit__(self, *args, **kwargs):
await self._conn.__aexit__(*args, **kwargs)
async def receive(self):
return await self.websocket.recv()
async def btcMarketWebsocket():
async with BTCMarketWebsocket() as btcMarketWebsocket:
while not btcMarketWebsocket.disable:
print(await btcMarketWebsocket.receive())
async def ethMarketWebsocket():
async with ETHMarketWebsocket() as ethMarketWebsocket:
while not ethMarketWebsocket.disable:
print(await ethMarketWebsocket.receive())
if __name__ == '__main__':
asyncio.run(btcMarketWebsocket())
asyncio.run(ethMarketWebsocket())
You can do
async def multiple_tasks():
Tasks =[]
Tasks.append(btcMarketWebsocket())
Tasks.append(ethMarketWebsocket())
await asyncio.gather(*Tasks, return_exceptions=True)
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(multiple_tasks())