In Spring, to set the concurrency for rabbitmq-consumer is so easy. Like:
container.setConcurrentConsumers(consumerSize);
container.setMaxConcurrentConsumers(consumerMaxSize);
Is it possible in python?
My python code looks like:
async def handle_message(loop):
connection = await connect(SETTINGS.cloudamqp_url, loop = loop)
channel = await connection.channel()
queue = await channel.declare_queue(SETTINGS.request_queue, durable=True)
await queue.consume(on_message, no_ack = True)
I solved my problem with using Thread:
My code looks like:
import threading
from aio_pika import connect, IncomingMessage, Message
import json
class QueueWorker(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.connection = None
self.channel = None
self.queue = None
async def init(self, loop):
self.connection = await connect(cloudamqp_url, loop=loop)
self.channel = await self.connection.channel()
await self.channel.set_qos(prefetch_count=1)
self.queue = await self.channel.declare_queue(queue, durable=True)
await self.queue.consume(self.callback, no_ack=False)
async def callback(self, message: IncomingMessage):
request = json.loads(message.body.decode("utf-8"))
try:
handle(request)
except Exception as e:
handleException...
finally:
await message.ack()
Consume queue with concurrency:
async def listen_queue(loop):
for _ in range(consumer_count):
td = QueueWorker()
await td.init(loop)
Note: Inspired from Consuming rabbitmq queue from inside python threads
Related
Please help me deal with async and threads in the telethon module. I'm trying to make a bot that, with a certain command placed in the queue, will either send a file or send a message, but I ran into difficulty
I'm trying to run telethon async inside the stream, but the clock function does not start, although the function works with the event
import asyncio
import threading
from queue import Queue
from telethon import TelegramClient, events, sync
async def send_message():
loop = asyncio.new_event_loop()
api_id = 11
api_hash = '11'
try:
client = TelegramClient('session', api_id, api_hash, loop=loop)
await client.connect()
if not await client.is_user_authorized():
client.disconnected()
await client.start()
except:
print('qqqqq')
#client.on(events.NewMessage(chats='me'))
async def handler(event):
await client.forward_messages('me', event.message)
async def clock():
global cl
while True:
while (queue_send_al.qsize() == 0):
print("33")
event_send_all.wait()
async with client:
cl = queue_send_al.get()
print(cl)
if cl[0] == 1:
await client.send_file('me', str(cl[1]),use_cache=False, part_size_kb=512)
event_send_all.clear()
elif cl[0] == 2:
print('1111')
event_send_all.clear()
loop.create_task(clock())
print('22')
await client.run_until_disconnected()
def go():
asyncio.run(send_message())
queue_send_al = Queue()
event_send_all = threading.Event()
queue_send_al.put([2, fr"11111111"])
event_send_all.set()
th3 = threading.Thread(target=go).start()
Below is (working) code for a generic websocket streamer.
It creates a daemon thread from which performs asyncio.run(...).
The asyncio code spawns 2 tasks, which never complete.
How to correctly destroy this object?
One of the tasks is executing a keepalive 'ping', so I can easily exit that loop using a flag. But the other is blocking on a message from the websocket.
import json
import aiohttp
import asyncio
import gzip
import asyncio
from threading import Thread
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
self.url = url
self.on_connect = on_connect
self.on_msg = on_msg
self.streams = {}
self.worker_thread = Thread(name='WebSocket', target=self.thread_func, daemon=True).start()
def thread_func(self):
asyncio.run(self.aio_run())
async def aio_run(self):
async with aiohttp.ClientSession() as session:
self.ws = await session.ws_connect(self.url)
await self.on_connect(self)
async def ping():
while True:
print('KEEPALIVE')
await self.ws.ping()
await asyncio.sleep(WebSocket.KEEPALIVE_INTERVAL_S)
async def main_loop():
async for msg in self.ws:
def extract_data(msg):
if msg.type == aiohttp.WSMsgType.BINARY:
as_bytes = gzip.decompress(msg.data)
as_string = as_bytes.decode('utf8')
as_json = json.loads(as_string)
return as_json
elif msg.type == aiohttp.WSMsgType.TEXT:
return json.loads(msg.data)
elif msg.type == aiohttp.WSMsgType.ERROR:
print('⛔️ aiohttp.WSMsgType.ERROR')
return msg.data
data = extract_data(msg)
self.on_msg(data)
# May want this approach if we want to handle graceful shutdown
# W.task_ping = asyncio.create_task(ping())
# W.task_main_loop = asyncio.create_task(main_loop())
await asyncio.gather(
ping(),
main_loop()
)
async def send_json(self, J):
await self.ws.send_json(J)
I'd suggest the use of asyncio.run_coroutine_threadsafe instead of asyncio.run. It returns a concurrent.futures.Future object which you can cancel:
def thread_func(self):
self.future = asyncio.run_coroutine_threadsafe(
self.aio_run(),
asyncio.get_event_loop()
)
# somewhere else
self.future.cancel()
Another approach would be to make ping and main_loop a task, and cancel them when necessary:
# inside `aio_run`
self.task_ping = asyncio.create_task(ping())
self.main_loop_task = asyncio.create_task(main_loop())
await asyncio.gather(
self.task_ping,
self.main_loop_task
return_exceptions=True
)
# somewhere else
self.task_ping.cancel()
self.main_loop_task.cancel()
This doesn't change the fact that aio_run should also be called with asyncio.run_coroutine_threadsafe. asyncio.run should be used as a main entry point for asyncio programs and should be only called once.
I would like to suggest one more variation of the solution. When finishing coroutines (tasks), I prefer minimizing the use of cancel() (but not excluding), since sometimes it can make it difficult to debug business logic (keep in mind that asyncio.CancelledError does not inherit from an Exception).
In your case, the code might look like this(only changes):
class WebSocket:
KEEPALIVE_INTERVAL_S = 10
def __init__(self, url, on_connect, on_msg):
# ...
self.worker_thread = Thread(name='WebSocket', target=self.thread_func)
self.worker_thread.start()
async def aio_run(self):
self._loop = asyncio.get_event_loop()
# ...
self._ping_task = asyncio.create_task(ping())
self._main_task = asyncio.create_task(main_loop())
await asyncio.gather(
self._ping_task,
self._main_task,
return_exceptions=True
)
# ...
async def stop_ping(self):
self._ping_task.cancel()
try:
await self._ping_task
except asyncio.CancelledError:
pass
async def _stop(self):
# wait ping end before socket closing
await self.stop_ping()
# lead to correct exit from `async for msg in self.ws`
await self.ws.close()
def stop(self):
# wait stopping ping and closing socket
asyncio.run_coroutine_threadsafe(
self._stop(), self._loop
).result()
self.worker_thread.join() # wait thread finish
I have an websocket server up and running fine using FastAPI.
However, when i am using those "await", i am getting some latency issues. At first, i though this had something to do with internet connection, or perhaps my linux server. But it appears to be that asyncio waits for other tasks.
Here is my code:
import asyncio
from pydantic import BaseModel
class UserClientWebSocket(BaseModel):
id: str
ws: WebSocket
class Config:
arbitrary_types_allowed = True
class ConnectionManager:
def __init__(self):
self.active_user_client_connections: List[UserClientWebSocket] = []
self.collect_user_IDs = []
async def connect_the_user_client(self, websocket: WebSocket, THE_USER_ID):
await websocket.accept()
await self.send_message_to_absolutely_everybody(f"User: {THE_USER_ID} connected to server!")
print("User: {} ".format(THE_USER_ID) + " Connected")
if THE_USER_ID not in self.collect_user_IDs:
self.collect_user_IDs.append(THE_USER_ID)
else:
await self.send_message_to_absolutely_everybody(f"Somebody connected with the same ID as client: {THE_USER_ID}")
await self.send_message_to_absolutely_everybody("but Vlori is a nice and kind guy, so he wil not get kicked :)")
self.collect_user_IDs.append(THE_USER_ID)
self.active_user_client_connections.append(UserClientWebSocket(ws=websocket, id=THE_USER_ID))
await self.show_number_of_clients()
async def function_one_send_message_to_absolutely_everybody(self, message: str):
try:
await asyncio.sleep(2)
await asyncio.gather(*(conn.ws.send_text(message) for conn in self.active_webpage_client_connections))
await asyncio.gather(*(conn.ws.send_text(message) for conn in self.active_user_client_connections))
except:
print("waiting")
async def function_two_send_personal_message_to_user(self, message: str, websocket: WebSocket):
try:
await websocket.send_text(message)
except:
print("waiting for task..")
...
...
...
...
...
and further down is the channel in which client connects:
#app.websocket("/ws/testchannel")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
try:
while True:
data = await websocket.receive_text()
print_result = print("Received data: {} ".format(data))
send_data_back_to_user = await websocket.send_text(f"you sent message: {data}")
except WebSocketDisconnect as e:
print("client left chat, error = ", e)
The code as it stands now works perfectly, and the performance is good! However, if i add an async def function under the "send_data_back_to_user" line such as this:
await connection_manager.function_one_send_message_to_absolutely_everybody(data)
Then there is a huge latency! why is that?
I am playing around and tried this:
#app.websocket("/ws/testchannel")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
try:
while True:
data = await websocket.receive_text()
print_result = print("Received data: {} ".format(data))
send_data_back_to_user = await websocket.send_text(f"you sent message: {data}") # if i comment this line and the line underneath, and the speed is extremely fast!
the_asyncio_loop = asyncio.get_event_loop()
print_data_on_terminal = asyncio.gather(print_result)
return_data_back_to_user = asyncio.gather(send_data_back_to_user)
broadcast_msg = asyncio.gather(connection_manager.function_one_send_message_to_absolutely_everybody(data))
run_all_loops_together = asyncio.gather(print_data_on_terminal, return_data_back_to_user, broadcast_msg)
results = the_asyncio_loop.run_until_complete(run_all_loops_together)
print(results)
except WebSocketDisconnect as e:
print("client left chat, error = ", e)
but gives me the error:
TypeError: An asyncio.Future, a coroutine or an awaitable is required
could someone help me with this?
thanks.
I am connecting to aioredis from __init__ (I do not want to move it out since this means I have to some extra major changes). How can I wait for aioredis connection task in below __init__ example code and have it print self.sub and self.pub object? Currently it gives an error saying
abc.py:42> exception=AttributeError("'S' object has no attribute
'pub'")
I do see redis connections created and coro create_connetion done.
Is there a way to call blocking asyncio calls from __init__. If I replace asyncio.wait with asyncio.run_until_complete I get an error that roughly says
loop is already running.
asyncio.gather is
import sys, json
from addict import Dict
import asyncio
import aioredis
class S():
def __init__(self, opts):
print(asyncio.Task.all_tasks())
task = asyncio.wait(asyncio.create_task(self.create_connection()), return_when="ALL_COMPLETED")
print(asyncio.Task.all_tasks())
print(task)
print(self.pub, self.sub)
async def receive_message(self, channel):
while await channel.wait_message():
message = await channel.get_json()
await asyncio.create_task(self.callback_loop(Dict(json.loads(message))))
async def run_s(self):
asyncio.create_task(self.listen())
async def callback_loop(msg):
print(msg)
self.callback_loop = callback_loop
async def create_connection(self):
self.pub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.sub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.db = await aioredis.create_redis("redis://c8:7070/0", password="abc")
self.listener = await self.sub.subscribe(f"abc")
async def listen(self):
self.tsk = asyncio.ensure_future(self.receive_message(self.listener[0]))
await self.tsk
async def periodic(): #test function to show current tasks
number = 5
while True:
await asyncio.sleep(number)
print(asyncio.Task.all_tasks())
async def main(opts):
loop.create_task(periodic())
s = S(opts)
print(s.pub, s.sub)
loop.create_task(s.run_s())
if __name__ == "__main__":
loop = asyncio.get_event_loop()
main_task = loop.create_task(main(sys.argv[1:]))
loop.run_forever() #I DONT WANT TO MOVE THIS UNLESS IT IS NECESSARY
I think what you want to do is to make sure the function create_connections runs to completion BEFORE the S constructor. A way to do that is to rearrange your code a little bit. Move the create_connections function outside the class:
async def create_connection():
pub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
sub = await aioredis.create_redis("redis://c8:7070/0", password="abc")
db = await aioredis.create_redis("redis://c8:7070/0", password="abc")
listener = await self.sub.subscribe(f"abc")
return pub, sub, db, listener
Now await that function before constructing S. So your main function becomes:
async def main(opts):
loop.create_task(periodic())
x = await create_connections()
s = S(opts, x) # pass the result of create_connections to S
print(s.pub, s.sub)
loop.create_task(s.run_s())
Now modify the S constructor to receive the objects created:
def __init__(self, opts, x):
self.pub, self.sub, self.db, self.listener = x
I'm not sure what you're trying to do with the return_when argument and the call to asyncio.wait. The create_connections function doesn't launch a set of parallel tasks, but rather awaits each of the calls before moving on to the next one. Perhaps you could improve performance by running the four calls in parallel but that's a different question.
What's wrong with implementation of passing class instance as a dependency in FastAPI router or is it a bug?
1) I have defined router with dependency
app = FastAPI()
dbconnector_is = AsyncDBPool(conn=is_cnx, loop=None)
app.include_router(test_route.router, dependencies=[Depends(dbconnector_is)])
#app.on_event('startup')
async def startup():
app.logger = await AsyncLogger().getlogger(log)
await app.logger.warning('Webservice is starting up...')
await app.logger.info("Establishing RDBS Integration Pool Connection...")
await dbconnector_is.connect()
#app.on_event('startup')
async def startup():
await app.logger.getlogger(log)
await app.logger.warning('Webservice is starting up...')
await dbconnector_is.connect()
router
#router.get('/test')
async def test():
data = await dbconnector_is.callproc('is_processes_get', rows=-1, values=[None, None])
return Response(json.dumps(data, default=str))
custom class for passing it's instance as callable.
class AsyncDBPool:
def __init__(self, conn: dict, loop: None):
self.conn = conn
self.pool = None
self.connected = False
self.loop = loop
def __call__(self):
return self
async def connect(self):
while self.pool is None:
try:
self.pool = await aiomysql.create_pool(**self.conn, loop=self.loop, autocommit=True)
except aiomysql.OperationalError as e:
await asyncio.sleep(1)
continue
else:
return self.pool
And when I send request I receive this error.
data = await dbconnector_is.callproc('is_processes_get', rows=-1, values=[None, None])
NameError: name 'dbconnector_is' is not defined