I have to process every message from redis asynchronously.
Here is my attempt with aioredis:
import asyncio
import aioredis
async def reader(channel: aioredis.client.PubSub):
while True:
data = None
try:
message = await channel.get_message(ignore_subscribe_messages=True)
if message is not None:
print(f"(Reader) Message Received: {message}")
data = message["data"]
except asyncio.TimeoutError:
pass
if data is not None:
await process_message(data)
async def process_message(message):
print(f"start process {message=}")
await asyncio.sleep(10)
print(f"+processed {message=}")
async def publish(redis, channel, message):
print(f"-->publish {message=} to {channel=}")
result = await redis.publish(channel, message)
print(" +published")
return result
async def main():
redis = aioredis.from_url("redis://localhost")
pubsub = redis.pubsub()
await pubsub.subscribe("channel:1", "channel:2")
future = asyncio.create_task(reader(pubsub))
await publish(redis, "channel:1", "Hello")
await publish(redis, "channel:2", "World")
await future
if __name__ == "__main__":
asyncio.run(main())
The problem is that aioredis does not get_message if the previous message was not processed. The messages are processed one by one.
How to solve that issue?
I've found the solution.
Instead of await process_message(data) one should use asyncio.ensure_future(process_message(data))
The idea came from AIORedis and PUB/SUB aren't asnyc
Related
I'm trying to create a full duplex client that sends and receives asynchronously at the same time, using python's websockets package.
The server simply receives a message and echoes it back.
when the client sends all the messages, but doesn't receive anything at all, as if either the send is blocking the receive handler, or the handler is stuck and never updates the data.
However, the server ensures that it both received and sent the data, so I doubt that it's the problem.
I'm genuinely new to async, multithreading, and network programming in general, but this code will be reflected on an applicated that buffers audios from an incoming systems, and sends it to another service, also it can receive any messages from that service at any time regarding this session.
python 3.9.15
websockets==10.4
I've followed the tutorial on the official websockets documentation:
https://websockets.readthedocs.io/en/stable/howto/patterns.html#consumer-and-producer
Client Code:
`
import asyncio
import websockets
sent = []
received = []
URL = "ws://localhost:8001"
async def update_sent(message):
with open("sent.txt", "a+") as f:
print(message, file=f)
sent.append(message)
return 0
async def update_received(message):
with open("recv.txt", "a+") as f:
print(message, file=f)
received.append(message)
return 0
async def sending_handler(websocket):
while True:
try:
message = input("send message:>")
await websocket.send(message)
await update_sent(message)
except Exception as e:
print("Sender: connection closed due to Exception", e)
break
async def receive_handler(websocket):
while True:
try:
message = await websocket.recv()
await update_received(message)
except Exception as e:
print("Receiver: connection closed due to Exception", e)
break
async def full_duplex_handler(websocket):
receiving_task = asyncio.create_task(receive_handler(websocket))
sending_task = asyncio.create_task(sending_handler(websocket))
done, pending = await asyncio.wait([receiving_task, sending_task],
return_when=asyncio.FIRST_COMPLETED)
# return_when=asyncio.FIRST_EXCEPTION)
for task in pending:
print(task)
task.cancel()
async def gather_handler(websocket):
await asyncio.gather(
sending_handler(websocket),
receive_handler(websocket),
)
# using asyncio.wait
async def main_1(url=URL):
async with websockets.connect(url) as websocket:
await full_duplex_handler(websocket)
# using asyncio.gather
# async def main_2(url=URL):
# async with websockets.connect(url) as websocket:
# await gather_handler(websocket)
if __name__ == "__main__":
asyncio.run(main_1())
# asyncio.run(main_2())
`
Server code:
`
import asyncio
import websockets
msgs = []
sent = []
async def handle_send(websocket, message):
await websocket.send(message)
msgs.append(message)
async def handle_recv(websocket):
message = await websocket.recv()
sent.append(message)
return f"echo {message}"
async def handler(websocket):
while True:
try:
message = await handle_recv(websocket)
await handle_send(websocket, message)
except Exception as e:
print(e)
print(msgs)
print(sent)
break
async def main():
async with websockets.serve(handler, "localhost", 8001):
await asyncio.Future()
if __name__ == "__main__":
print("starting the server now")
asyncio.run(main())
`
After sending some messages, all sent and received messages should be written to a file,
but only sent messages are received and processed.
TL;DR
I've put a sleep statement:
await asyncio.sleep(0.02)
in the sending_handler while loop, and it resolved the problem,
apparently the issue was that the sender is way faster than the receiver, that it keeps locking the resources for its use, while the receiver is being blocked.
Any shorter sleep durations can't solve this problem.
final while loop:
async def sending_handler(websocket):
while True:
await asyncio.sleep(0.02)
try:
message = input("send message:>")
await websocket.send(message)
await update_sent(message)
except Exception as e:
print("Sender: connection closed due to Exception", e)
break
Hope this answer helps anyone else who faces the same problem
How do you get the last message received from server even if there are unread messages in queue?
Also, how could I ignore (delete) the rest of the unread messages?
CODE example:
while True:
msg = await ws_server.recv()
await do_something_with_latest_message(msg)
I Nead something like:
while True:
msg = await ws_server.recv_last_msg() # On next loop I should "await" until a newer msg comes, not te receive the previous msg in LIFO order
await do_something_with_latest_message(msg)
There's no way to do this natively, just with the websockets library. However, you could use an asyncio.LifoQueue:
queue = asyncio.LifoQueue()
async def producer(ws_server):
async for msg in ws_server:
await queue.put(msg)
async def consumer():
while True:
msg = await queue.get()
# clear queue
while not queue.empty():
await queue.get()
await do_something_with_latest_message(msg)
await asyncio.gather(producer(ws_server), consumer())
The code in the docs has a while True: block and I am curious if something like that would deadlock the process. If I get two requests, would the second one just not go thrpugh? why or why not?
#app.websocket("/ws")
async def websocket_endpoint(websocket: WebSocket):
await websocket.accept()
while True:
data = await websocket.receive_text()
await websocket.send_text(f"Message text was: {data}")
I want to return msg from eye tracker data (real-time) using websockets python, but I've got only 1 msg instead real time data every seconds.
import websockets
import asyncio
async def listen():
link = "ws://localhost:5000"
async with websockets.connect(link) as ws:
await ws.send("Device is connected")
while True:
msg = await ws.recv
return msg
What I expected before, the return always give me string every seconds. Now I trying to run code below:
async def main():
print(await listen())
asyncio.run(main())
But I've got only 1 msg. I already tried another code:
async def listen():
link = "ws://localhost:5000"
async def main():
url = await listen()
async with websockets.connect(url) as ws:
await ws.send("Device is connected now!")
while True:
msg = await ws.recv()
print(msg)
asyncio.run(main())
Now, what I want to know is how to return msg? The function above not return msg, even the output is what I need. I curious to return msg in coroutine function. Thank you in advance.
I am trying to messing up with Websockets module and after checking the main page:
https://websockets.readthedocs.io/en/stable/intro.html
I did following:
SERVER
# SERVER
import asyncio
import websockets
import nest_asyncio
USERS = {}
async def set_online(websocket, user_name):
USERS[user_name] = websocket
await notify()
async def set_offline(websocket, user_name):
USERS.pop(user_name, None)
await notify()
async def notify():
if USERS:
message = "Online users: {}\n".format(len(USERS))
print (message)
#await asyncio.wait([user.send(message) for user in USERS])
else:
message = "Online users: 0\n"
print (message)
async def server(websocket, path):
user_name = await websocket.recv()
await set_online(websocket, user_name)
try:
async for message in websocket:
for user_name, user_ws in USERS.items():
if websocket == user_ws:
print (f"{user_name}: {message}")
finally:
await set_offline(websocket, user_name)
start_server = websockets.serve(server, "localhost", 3000,
ping_interval=None)
nest_asyncio.apply()
loop = asyncio.get_event_loop()
loop.run_until_complete(start_server)
loop.run_forever()
and also:
CLIENT
# CLIENT
import asyncio
import websockets
import nest_asyncio
async def client(localhost, port):
uri = "ws://{0}:{1}".format(localhost, str(port))
async with websockets.connect(uri) as websocket:
user_name = input("set your name: ")
await websocket.send(f"{user_name}")
while True:
message = input("> ")
if message == "/quit":
break
else:
await websocket.send(message)
host = "localhost"
port = 3000
nest_asyncio.apply()
loop = asyncio.get_event_loop()
loop.run_until_complete(client(host, port))
so all works as expected but I would like to achieve that each user can receive the answer as well from other users.
I found there is a conflict when I want to use websocket.send(message) in for loop async for message in websocket: on SERVER side
The link which I paste above, I think has a solution but I am struggling to figure out how to use it properly in my script.
I believe I need to create two tasks (send and recv) which will work in parallel.
Like:
async def handler(websocket, path):
consumer_task = asyncio.ensure_future(consumer_handler(websocket, path))
producer_task = asyncio.ensure_future(producer_handler(websocket, path))
done, pending = await asyncio.wait([consumer_task, producer_task],return_when=asyncio.FIRST_COMPLETED)
for task in pending:
task.cancel()
the following is displayed on the website which I provided above, just one thing needs to be changed from asyncio.ensure_future to asyncio.create_task. I implemented function handler, producer, consumer, producer_handler and consumer_handler to make it works but no luck.
Could someone provide an example or how this should be set up correctly?
I believe asyncio.create_task should be used on both (SERVER and CLIENT) so they both receive and send at one time.
This is pretty long but I hope someone can help me with it and also maybe my part of script will be handy for someone as well!