I have to process every message from redis asynchronously.
Here is my attempt with aioredis:
import asyncio
import aioredis
async def reader(channel: aioredis.client.PubSub):
while True:
data = None
try:
message = await channel.get_message(ignore_subscribe_messages=True)
if message is not None:
print(f"(Reader) Message Received: {message}")
data = message["data"]
except asyncio.TimeoutError:
pass
if data is not None:
await process_message(data)
async def process_message(message):
print(f"start process {message=}")
await asyncio.sleep(10)
print(f"+processed {message=}")
async def publish(redis, channel, message):
print(f"-->publish {message=} to {channel=}")
result = await redis.publish(channel, message)
print(" +published")
return result
async def main():
redis = aioredis.from_url("redis://localhost")
pubsub = redis.pubsub()
await pubsub.subscribe("channel:1", "channel:2")
future = asyncio.create_task(reader(pubsub))
await publish(redis, "channel:1", "Hello")
await publish(redis, "channel:2", "World")
await future
if __name__ == "__main__":
asyncio.run(main())
The problem is that aioredis does not get_message if the previous message was not processed. The messages are processed one by one.
How to solve that issue?
I've found the solution.
Instead of await process_message(data) one should use asyncio.ensure_future(process_message(data))
The idea came from AIORedis and PUB/SUB aren't asnyc
I'm trying to create a full duplex client that sends and receives asynchronously at the same time, using python's websockets package.
The server simply receives a message and echoes it back.
when the client sends all the messages, but doesn't receive anything at all, as if either the send is blocking the receive handler, or the handler is stuck and never updates the data.
However, the server ensures that it both received and sent the data, so I doubt that it's the problem.
I'm genuinely new to async, multithreading, and network programming in general, but this code will be reflected on an applicated that buffers audios from an incoming systems, and sends it to another service, also it can receive any messages from that service at any time regarding this session.
python 3.9.15
websockets==10.4
I've followed the tutorial on the official websockets documentation:
https://websockets.readthedocs.io/en/stable/howto/patterns.html#consumer-and-producer
Client Code:
`
import asyncio
import websockets
sent = []
received = []
URL = "ws://localhost:8001"
async def update_sent(message):
with open("sent.txt", "a+") as f:
print(message, file=f)
sent.append(message)
return 0
async def update_received(message):
with open("recv.txt", "a+") as f:
print(message, file=f)
received.append(message)
return 0
async def sending_handler(websocket):
while True:
try:
message = input("send message:>")
await websocket.send(message)
await update_sent(message)
except Exception as e:
print("Sender: connection closed due to Exception", e)
break
async def receive_handler(websocket):
while True:
try:
message = await websocket.recv()
await update_received(message)
except Exception as e:
print("Receiver: connection closed due to Exception", e)
break
async def full_duplex_handler(websocket):
receiving_task = asyncio.create_task(receive_handler(websocket))
sending_task = asyncio.create_task(sending_handler(websocket))
done, pending = await asyncio.wait([receiving_task, sending_task],
return_when=asyncio.FIRST_COMPLETED)
# return_when=asyncio.FIRST_EXCEPTION)
for task in pending:
print(task)
task.cancel()
async def gather_handler(websocket):
await asyncio.gather(
sending_handler(websocket),
receive_handler(websocket),
)
# using asyncio.wait
async def main_1(url=URL):
async with websockets.connect(url) as websocket:
await full_duplex_handler(websocket)
# using asyncio.gather
# async def main_2(url=URL):
# async with websockets.connect(url) as websocket:
# await gather_handler(websocket)
if __name__ == "__main__":
asyncio.run(main_1())
# asyncio.run(main_2())
`
Server code:
`
import asyncio
import websockets
msgs = []
sent = []
async def handle_send(websocket, message):
await websocket.send(message)
msgs.append(message)
async def handle_recv(websocket):
message = await websocket.recv()
sent.append(message)
return f"echo {message}"
async def handler(websocket):
while True:
try:
message = await handle_recv(websocket)
await handle_send(websocket, message)
except Exception as e:
print(e)
print(msgs)
print(sent)
break
async def main():
async with websockets.serve(handler, "localhost", 8001):
await asyncio.Future()
if __name__ == "__main__":
print("starting the server now")
asyncio.run(main())
`
After sending some messages, all sent and received messages should be written to a file,
but only sent messages are received and processed.
TL;DR
I've put a sleep statement:
await asyncio.sleep(0.02)
in the sending_handler while loop, and it resolved the problem,
apparently the issue was that the sender is way faster than the receiver, that it keeps locking the resources for its use, while the receiver is being blocked.
Any shorter sleep durations can't solve this problem.
final while loop:
async def sending_handler(websocket):
while True:
await asyncio.sleep(0.02)
try:
message = input("send message:>")
await websocket.send(message)
await update_sent(message)
except Exception as e:
print("Sender: connection closed due to Exception", e)
break
Hope this answer helps anyone else who faces the same problem
Hi I am using AioKafka consumer to read message published by another process. The other process has just published one message and my consumer code is reading same message infinitely. I tried to use manual commit but in vain.
I am using library: https://pypi.org/project/aiokafka-commit/
I want to read each message only once as and when available.
from aiokafka import AIOKafkaConsumer
import asyncio
async def consume():
consumer = AIOKafkaConsumer('websocket_chat_kafka',
bootstrap_servers = "127.0.0.1:9092",
group_id = 'gid',
client_id = 'cid',
# enable_auto_commit = True,
# auto_commit_interval_ms=1000,
auto_offset_reset="latest")
# await consumer.start()
# while True:
# msg = await consumer.getone()
# print(msg)
# await consumer.commit()
await consumer.start()
# await consumer.seek_to_committed()
async for msg in consumer:
print(msg)
await consumer.commit()
await consumer.stop()
asyncio.run(consume())
Any ideas what am I doing wrong?
Thank you.
The code in the docs has a while True: block and I am curious if something like that would deadlock the process. If I get two requests, would the second one just not go thrpugh? why or why not?
#app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
while True:
data = await websocket.receive_text()
await websocket.send_text(f"Message text was: {data}")
I am trying to setup a listener for socket events while concurrently sending a ping message in the background.
The message received events fire, but only after the next ping completes. There seems to be something blocking.
Here's a mockup of the code I'm running.
import asyncio
import websockets
message = "First Message"
ping = "ping"
url = 'www.socketurl.com'
async def consumer_handler(websocket):
message = await websocket.recv()
if message:
print(message)
async def pingFunc(socket):
await asyncio.sleep(4)
await socket.send(ping)
async def connect(uri):
async with websockets.connect(uri,timeout=5) as websocket:
await websocket.send(message) #send hello message that returns a few pieces of info
while True:
await consumer_handler(websocket) #wait for messages and handle them
await pingFunc(websocket) #keep the connection alive
loop = asyncio.get_event_loop()
loop.run_until_complete(
connect(url)
)