I've just started using RabbitMQ using aio-pika, and I have multiple queue names to consume.
So far I use this tutorial in worker.py but with multiple queue declarations so it looks like this:
import asyncio
from aio_pika import connect
async def main() -> None:
# Perform connection
connection = await connect(Settings.RABBIT_URL)
async with connection:
# Creating a channel
channel = await connection.channel()
await channel.set_qos(prefetch_count=0)
# Declaring queue
queue = await channel.declare_queue(
"queue_1",
durable=True,
)
queue2 = await channel.declare_queue(
"queue_2",
durable=True,
)
queue3 = await channel.declare_queue(
"queue_3",
durable=True,
)
# Start listening the queue with name 'task_queue'
await queue.consume(on_message)
await queue2.consume(on_message)
await queue3.consume(on_message)
print(" [*] Waiting for messages. To exit press CTRL+C")
await asyncio.Future()
The thing is I need to make it flexible to declare queues as many as how many queue names I could fetch from database, so first is my method to declare multiple queues correct, second how to declare queues based on list of queue names?
Thank you.
Ok, after some trials and errors, basically the way I declare queues is right, the next problem about declaring queues and consuming dynamically can be done by using looping the query results and use locals() to initiate variables dynamically.
queues = dict()
for q in range(0, len(query)):
queue_name = query[q]['result']
locals()['queues_{0}'.format(q)] = await channel.declare_queue(queue_name, durable=True)
await locals()['queues_{0}'.format(q)].consume(on_message)
Related
I found this library for asynchronously consuming kafka messages: https://github.com/aio-libs/aiokafka
It gives this code example:
from aiokafka import AIOKafkaConsumer
import asyncio
async def consume():
consumer = AIOKafkaConsumer(
'redacted',
bootstrap_servers='redacted',
auto_offset_reset="earliest"
#group_id="my-group"
)
# Get cluster layout and join group `my-group`
await consumer.start()
try:
# Consume messages
async for msg in consumer:
print("consumed: ", msg.topic, msg.partition, msg.offset,
msg.key, msg.value, msg.timestamp)
finally:
# Will leave consumer group; perform autocommit if enabled.
await consumer.stop()
asyncio.run(consume())
I would like to find out the biggest kafka message using this code. So, Inside async for I need to do max_size = max(max_size, len(msg.value)). But I think it won't be thread-safe, and I need to lock access to it?
try:
max_size = -1
# Consume messages
async for msg in consumer:
max_size = max(max_size, len(msg.value)) # do I need to lock this code?
How do I do it in python? I've checked out this page: https://docs.python.org/3/library/asyncio-sync.html and I'm confused because those synchronization primitives are not thread-safe? So I can't use them in a multithreaded context? I'm really confused. I come from a Java background and need to write this script, so, pardon me that I haven't read all the asyncio books out there.
Is my understanding correct that the body of the async for loop is a continuation that may be scheduled on a separate thread when the asynchronous operation is done?
I have two async functions that both need to run constantly and one of them just hogs all of the CPU.
The first function handles receiving websocket messages from a client.
async def handle_message(self, ws):
"""Handles a message from the websocket."""
logger.info('awaiting message')
while True:
msg = await ws.receive()
logger.debug('received message: %s', msg)
jmsg = json.loads(msg['text'])
logger.info('received message: {}'.format(jmsg))
param = jmsg['parameter']
val = jmsg['data']['value']
logger.info('setting parameter {} to {}'.format(param, val))
self.camera.change_parameter(param, val)
The second function grabs images from a camera and sends them to the frontend client. This is the one that one that won't give the other guy any time.
async def send_image(self, ws):
"""Sends an image to the websocket."""
for im in self.camera:
await asyncio.sleep(1000)
h, w = im.shape[:2]
resized = cv2.resize(im, (w // 4, h // 4))
await ws.send_bytes(image_to_bytes(resized))
I'm executing these coroutines using asyncio.gather(). The decorator is from FastAPI and Backend() is my class that contains the two async coroutines.
#app.websocket('/ws')
async def websocket_endpoint(websocket: WebSocket):
"""Handle a WebSocket connection."""
backend = Backend()
logger.info('Started backend.')
await websocket.accept()
try:
aws = [backend.send_image(websocket), backend.handle_message(websocket)]
done, pending = await asyncio.gather(*aws)
except WebSocketDisconnect:
await websocket.close()
Both of these coroutines will operate seperately, but if I try to run them together send_image() never gives any time to handle_message and so none of the messages are ever received (or at least that's what I think is going on).
I thought this is what asyncio was trying to solve, but I'm probably using it wrong. I thought about using multiprocessing, but I'm pretty sure FastAPI expects awaitables here. I also read about using the return variables from gather(), but I didn't really understand. Something about canceling the pending tasks and adding them back to the event loop.
Can anyone show me the correct (and preferably modern pythonic) way to make these async coroutines run concurrently?
async def main():
uuids = await get_uuids_from_text_file()
tasks = []
# create a task for each uuid
# and add it to the list of tasks
for uuid in uuids:
task = asyncio.create_task(make_hypixel_request(uuid))
tasks.append(task)
# wait for all the tasks to finish
responses = await asyncio.gather(*tasks)
# run the functions to process the data
for response in responses:
print(response['success'])
data2 = await anti_sniper_request(response)
await store_data_in_json_file(response, data2)
await compare_stats(response)
# loop the main function
async def main_loop():
for _ in itertools.repeat([]):
await main()
# run the loop
loop = asyncio.get_event_loop()
loop.run_until_complete(main_loop())
loop.close()
basically this is my code the functions have very clear and explanatory name
the make_hypixel_requests part i have no issues there, the requests are executed immediately and in parallel,
the problem is after that when "for response in responses" it goes hella slow? how do i get the responses instantly and loop through them very fast? i will try to attach a gif.
basically this is the issue:
the reason is because after waiting for all the responses to return, u process them in a loop instead of asynchronously, since none of the requests seem to depend on each other, waiting for them all to finish before processing them doesn't make sense, the best way to handle this is to couple the request and the processing e.g.
async def request_and_process(uuid):
response = await make_hypixel_request(uuid)
print(response['success'])
compare_stats_task = asyncio.create_task(compare_stats(response))
data2 = await anti_sniper_request(response)
await asyncio.gather(store_data_in_json_file(response, data2), compare_stats_task)
async def main():
while True:
uuids = await get_uuids_from_text_file()
await asyncio.gather(*map(request_and_process, uuids))
asyncio.run(main())
You can use asyncio.wait and return when at least a task is completed, then continue awaiting for the pending tasks. asyncio.wait return a tuple with two sets, the first with the completed tasks, the second with the still pending tasks. You can call to the result method of the done tasks and get its return value.
async def main():
uuids = await get_uuids_from_text_file()
tasks = []
# create a task for each uuid
# and add it to the list of tasks
for uuid in uuids:
task = asyncio.create_task(make_hypixel_request(uuid))
tasks.append(task)
# wait for all the tasks to finish
while tasks:
done_tasks, tasks = await asyncio.wait(tasks, return_when=asyncio.FIRST_COMPLETED)
for done in done_tasks:
response = done.result()
print(response['success'])
data2 = await anti_sniper_request(response)
await store_data_in_json_file(response, data2)
await compare_stats(response)
I'm programming a server in Python3, which takes screenshot and sends it over websockets. I have coroutine for handling connection and I would like to create another coroutine for taking screenshot at some interval. Screenshot coroutine will probably run in different thread and I will need to propagate the result to some shared variable with read-write lock, to be able to send it. My questions: (result should be multiplatform, if possible)
How is it possible to schedule tasks like this? I created server which runs forever, and I can create periodical coroutine, but somehow I can't put them together in one loop.
What is a good way to propagate the result from one thread (or coroutine, if server is single threaded) to another?
I found this piece of code similar to this and I can't get it to work (second coroutine doesn't execute). Can someone correct this with and without multithreading?
async def print_var():
global number
await asyncio.sleep(2)
print(number)
async def inc_var():
global number
await asyncio.sleep(5)
number += 1
number = 0
asyncio.get_event_loop().run_until_complete(print_var())
asyncio.async(inc_var)
asyncio.get_event_loop().run_forever()
Post-answer edit
In the end after more hours of googling, I actually got it to work on a single thread, so there's no danger of race condition. (But I'm still not sure what ensure_future does, and why it isn't called on event loop.)
users = set()
def register(websocket):
users.add(websocket)
def unregister(websocket):
users.remove(websocket)
async def get_screenshot():
global screenshot
while True:
screenshot = screenshot()
await asyncio.sleep(0.2)
async def server(websocket, path):
global screenshot
register(websocket)
try:
async for message in websocket:
respond(screenshot)
finally:
unregister(websocket)
def main():
asyncio.get_event_loop().run_until_complete(
websockets.serve(server, 'localhost', 6789))
asyncio.ensure_future(get_screenshot())
asyncio.get_event_loop().run_forever()
main()
In Python 3.7:
import asyncio
import websockets
CAPTURE_INTERVAL = 1
running = True
queues = set()
async def handle(ws, path):
queue = asyncio.Queue()
queues.add(queue)
while running:
data = await queue.get()
if not data:
break
await ws.send(data)
def capture_screen():
# Do some work here, preferably in C extension without holding the GIL
return b'screenshot data'
async def main():
global running
loop = asyncio.get_running_loop()
server = await websockets.serve(handle, 'localhost', 8765)
try:
while running:
data = await loop.run_in_executor(None, capture_screen)
for queue in queues:
queue.put_nowait(data)
await asyncio.sleep(CAPTURE_INTERVAL)
finally:
running = False
for queue in queues:
queue.put_nowait(None)
server.close()
await server.wait_closed()
if __name__ == '__main__':
asyncio.run(main())
Please note, this is only for demonstrating the producer-consumer fan-out pattern. The queues are not essential - you can simply send data to all server.sockets in main() directly, while in handle() you should worry about incoming websocket messages. For example, client may control image compression rate like this:
import asyncio
import websockets
CAPTURE_INTERVAL = 1
DEFAULT = b'default'
qualities = {}
async def handle(ws, path):
try:
async for req in ws:
qualities[ws] = req
finally:
qualities.pop(ws, None)
def capture_screen():
# Do some work here, preferably in C extension without holding the GIL
return {
DEFAULT: b'default screenshot data',
b'60': b'data at 60% quality',
b'80': b'data at 80% quality',
}
async def main():
loop = asyncio.get_running_loop()
server = await websockets.serve(handle, 'localhost', 8765)
try:
while True:
data = await loop.run_in_executor(None, capture_screen)
for ws in server.sockets:
quality = qualities.get(ws, DEFAULT)
if quality not in data:
quality = DEFAULT
asyncio.create_task(ws.send(data[quality]))
await asyncio.sleep(CAPTURE_INTERVAL)
finally:
server.close()
await server.wait_closed()
if __name__ == '__main__':
asyncio.run(main())
I am trying to create a script in python that listens to multiple sockets using websockets and asyncio, the problem is that no matter what I do it only listen to the first socket I call.
I think its the infinite loop, what are my option to solve this? using threads for each sockets?
async def start_socket(self, event):
payload = json.dumps(event)
loop = asyncio.get_event_loop()
self.tasks.append(loop.create_task(
self.subscribe(event)))
# this should not block the rest of the code
await asyncio.gather(*tasks)
def test(self):
# I want to be able to add corotines at a different time
self.start_socket(event1)
# some code
self.start_socket(event2)
this is what I did eventually, that way its not blocking the main thread and all subscriptions are working in parallel.
def subscribe(self, payload):
ws = websocket.WebSocket(sslopt={"cert_reqs": ssl.CERT_NONE})
ws.connect(url)
ws.send(payload)
while True:
result = ws.recv()
print("Received '%s'" % result)
def start_thread(self, loop):
asyncio.set_event_loop(loop)
loop.run_forever()
def start_socket(self, **kwargs):
worker_loop = asyncio.new_event_loop()
worker = Thread(target=self.start_thread, args=(worker_loop,))
worker.start()
worker_loop.call_soon_threadsafe(self.subscribe, payload)
def listen(self):
self.start_socket(payload1)
# code
self.start_socket(payload2)
# code
self.start_socket(payload3)
Your code appears incomplete, but what you've shown has two issues. One is that run_until_complete accepts a coroutine object (or other kind of future), not a coroutine function. So it should be:
# note parentheses after your_async_function()
asyncio.get_event_loop().run_until_complete(your_async_function())
the problem is that no matter what I do it only listen to the first socket I call. I think its the infinite loop, what are my option to solve this? using threads for each sockets?
The infinite loop is not the problem, asyncio is designed to support such "infinite loops". The problem is that you are trying to do everything in one coroutine, whereas you should be creating one coroutine per websocket. This is not a problem, as coroutines are very lightweight.
For example (untested):
async def subscribe_all(self, payload):
loop = asyncio.get_event_loop()
# create a task for each URL
for url in url_list:
tasks.append(loop.create_task(self.subscribe_one(url, payload)))
# run all tasks in parallel
await asyncio.gather(*tasks)
async def subsribe_one(self, url, payload):
async with websockets.connect(url) as websocket:
await websocket.send(payload)
while True:
msg = await websocket.recv()
print(msg)
One way to efficiently listen to multiple websocket connections from a websocket server is to keep a list of connected clients and essentially juggle multiple conversations in parallel.
E.g. A simple server that sends random # to each connected client every few secs:
import os
import asyncio
import websockets
import random
websocket_clients = set()
async def handle_socket_connection(websocket, path):
"""Handles the whole lifecycle of each client's websocket connection."""
websocket_clients.add(websocket)
print(f'New connection from: {websocket.remote_address} ({len(websocket_clients)} total)')
try:
# This loop will keep listening on the socket until its closed.
async for raw_message in websocket:
print(f'Got: [{raw_message}] from socket [{id(websocket)}]')
except websockets.exceptions.ConnectionClosedError as cce:
pass
finally:
print(f'Disconnected from socket [{id(websocket)}]...')
websocket_clients.remove(websocket)
async def broadcast_random_number(loop):
"""Keeps sending a random # to each connected websocket client"""
while True:
for c in websocket_clients:
num = str(random.randint(10, 99))
print(f'Sending [{num}] to socket [{id(c)}]')
await c.send(num)
await asyncio.sleep(2)
if __name__ == "__main__":
loop = asyncio.get_event_loop()
try:
socket_server = websockets.serve(handle_socket_connection, 'localhost', 6789)
print(f'Started socket server: {socket_server} ...')
loop.run_until_complete(socket_server)
loop.run_until_complete(broadcast_random_number(loop))
loop.run_forever()
finally:
loop.close()
print(f"Successfully shutdown [{loop}].")
A simple client that connects to the server and listens for the numbers:
import asyncio
import random
import websockets
async def handle_message():
uri = "ws://localhost:6789"
async with websockets.connect(uri) as websocket:
msg = 'Please send me a number...'
print(f'Sending [{msg}] to [{websocket}]')
await websocket.send(msg)
while True:
got_back = await websocket.recv()
print(f"Got: {got_back}")
asyncio.get_event_loop().run_until_complete(handle_message())
Mixing up threads and asyncio is more trouble than its worth and you still have code that will block on the most wasteful steps like network IO (which is the essential benefit of using asyncio).
You need to run each coroutine asynchronously in an event loop, call any blocking calls with await and define each method that interacts with any awaitable interactions with an async
See a working e.g.: https://github.com/adnantium/websocket_client_server