I tested the following code with Ncat. It only sends a single message, then it does not send anything, and does not except. It is also not reading anything.
I have no idea what could be going on. There is no exception, and no data seems to be sent.
import asyncio
loop = asyncio.get_event_loop()
class Client:
def __init__(self):
self.writer = None
self.reader = None
async def connect(self, address, port):
reader, writer = await asyncio.open_connection(address, port)
self.reader = reader
self.writer = writer
print("connected!")
async def send(self, message):
print("writing " + message)
self.writer.write((message + '\n').encode())
await self.writer.drain()
async def receive(self):
print("receiving")
message = (self.reader.readuntil('\n'.encode())).decode()
return message
async def read_loop(self):
while True:
incoming = await self.receive()
print("remote: " + incoming)
async def main():
client = Client()
await client.connect("127.0.0.1", 31416)
loop.create_task(client.read_loop())
while True:
text = input("message: ")
await client.send(text)
loop.run_until_complete(main())
I haven't tested your code but there are things that immediately jump out.
First, you should avoid to use low-level asyncio APIs calling loop.create_task or loop.run_until_complete. Those are intended to be used by frameworks developers. Use asyncio.create_task and asyncio.run instead.
Then, you are getting no exception (or result) because you are not keeping any reference to the task you are creating. You should always keep one to tasks (i.e. assigning it to a variabile or adding it in a list/set).
Last, you are missing an await in the receive method: StreamReader.readuntil is a coroutine function and so you have to await on it.
class Client:
...
async def receive(self):
print('receiving...')
message = await self.reader.readuntil('\n'.encode())
return message.decode()
...
Related
I'm trying to create a full duplex client that sends and receives asynchronously at the same time, using python's websockets package.
The server simply receives a message and echoes it back.
when the client sends all the messages, but doesn't receive anything at all, as if either the send is blocking the receive handler, or the handler is stuck and never updates the data.
However, the server ensures that it both received and sent the data, so I doubt that it's the problem.
I'm genuinely new to async, multithreading, and network programming in general, but this code will be reflected on an applicated that buffers audios from an incoming systems, and sends it to another service, also it can receive any messages from that service at any time regarding this session.
python 3.9.15
websockets==10.4
I've followed the tutorial on the official websockets documentation:
https://websockets.readthedocs.io/en/stable/howto/patterns.html#consumer-and-producer
Client Code:
`
import asyncio
import websockets
sent = []
received = []
URL = "ws://localhost:8001"
async def update_sent(message):
with open("sent.txt", "a+") as f:
print(message, file=f)
sent.append(message)
return 0
async def update_received(message):
with open("recv.txt", "a+") as f:
print(message, file=f)
received.append(message)
return 0
async def sending_handler(websocket):
while True:
try:
message = input("send message:>")
await websocket.send(message)
await update_sent(message)
except Exception as e:
print("Sender: connection closed due to Exception", e)
break
async def receive_handler(websocket):
while True:
try:
message = await websocket.recv()
await update_received(message)
except Exception as e:
print("Receiver: connection closed due to Exception", e)
break
async def full_duplex_handler(websocket):
receiving_task = asyncio.create_task(receive_handler(websocket))
sending_task = asyncio.create_task(sending_handler(websocket))
done, pending = await asyncio.wait([receiving_task, sending_task],
return_when=asyncio.FIRST_COMPLETED)
# return_when=asyncio.FIRST_EXCEPTION)
for task in pending:
print(task)
task.cancel()
async def gather_handler(websocket):
await asyncio.gather(
sending_handler(websocket),
receive_handler(websocket),
)
# using asyncio.wait
async def main_1(url=URL):
async with websockets.connect(url) as websocket:
await full_duplex_handler(websocket)
# using asyncio.gather
# async def main_2(url=URL):
# async with websockets.connect(url) as websocket:
# await gather_handler(websocket)
if __name__ == "__main__":
asyncio.run(main_1())
# asyncio.run(main_2())
`
Server code:
`
import asyncio
import websockets
msgs = []
sent = []
async def handle_send(websocket, message):
await websocket.send(message)
msgs.append(message)
async def handle_recv(websocket):
message = await websocket.recv()
sent.append(message)
return f"echo {message}"
async def handler(websocket):
while True:
try:
message = await handle_recv(websocket)
await handle_send(websocket, message)
except Exception as e:
print(e)
print(msgs)
print(sent)
break
async def main():
async with websockets.serve(handler, "localhost", 8001):
await asyncio.Future()
if __name__ == "__main__":
print("starting the server now")
asyncio.run(main())
`
After sending some messages, all sent and received messages should be written to a file,
but only sent messages are received and processed.
TL;DR
I've put a sleep statement:
await asyncio.sleep(0.02)
in the sending_handler while loop, and it resolved the problem,
apparently the issue was that the sender is way faster than the receiver, that it keeps locking the resources for its use, while the receiver is being blocked.
Any shorter sleep durations can't solve this problem.
final while loop:
async def sending_handler(websocket):
while True:
await asyncio.sleep(0.02)
try:
message = input("send message:>")
await websocket.send(message)
await update_sent(message)
except Exception as e:
print("Sender: connection closed due to Exception", e)
break
Hope this answer helps anyone else who faces the same problem
In the minimal snippet bellow with both client producer/consumers and server, I force an error on the client's send method:
import asyncio
import websockets
async def producer(websocket):
for i in range(11):
await websocket.send(i) # Invalid type, fails silently
# await websocket.send(str(i)) # Works
async def consumer(websocket):
async for msg in websocket:
i = int(msg)
print(i)
if i == 10:
return
async def echo(websocket, path):
try:
async for msg in websocket:
await websocket.send(msg)
except websockets.exceptions.ConnectionClosedOK:
print("Client disconnect.")
async def client():
async with websockets.connect("ws://localhost:8765") as websocket:
consumer_task = consumer(websocket)
producer_task = producer(websocket)
await asyncio.wait([consumer_task, producer_task])
loop = asyncio.get_event_loop()
loop.run_until_complete(websockets.serve(echo, "localhost", 8765))
loop.run_until_complete(client())
In the invalid type case, the code hangs indefinitely, seemingly as it had "failed silently". Shouldn't the send method raise an exception due to the invalid int type (since it only accepts bytes or str)?
The problem was actually lying in the asyncio.wait method, as hinted by Thomas' comments (thanks). Switching asyncio.wait for the higher-level asyncio.gather solved it.
I have this basic exchange monitor script. I'm trying to create one thread per symbol, apart from the main thread which is handling other work, and have them listen to public Gemini websocket endpoints. I'm getting the first thread running and printing exchange data to the console, but not the second one. I had expected to see data from both threads being printed at approximately the same time. I've tried using the threading library instead of asyncio and encountered the same situation.
I realize my two public API MarketWebsocket classes could be combined to be cleaner, I'm still trying to work out a way to easily add other symbols to the list. Thanks for any nudges in the right direction!
import asyncio
from websockets import connect
symbols_to_watch = [
"BTCUSD",
"ETHUSD"
]
class BTCMarketWebsocket:
disable = False
async def __aenter__(self):
symbol = symbols_to_watch[0]
self._conn = connect("wss://api.gemini.com/v1/marketdata/{}".format(symbol))
self.websocket = await self._conn.__aenter__()
return self
async def __aexit__(self, *args, **kwargs):
await self._conn.__aexit__(*args, **kwargs)
async def receive(self):
return await self.websocket.recv()
class ETHMarketWebsocket:
disable = False
async def __aenter__(self):
symbol = symbols_to_watch[1]
self._conn = connect("wss://api.gemini.com/v1/marketdata/{}".format(symbol))
self.websocket = await self._conn.__aenter__()
return self
async def __aexit__(self, *args, **kwargs):
await self._conn.__aexit__(*args, **kwargs)
async def receive(self):
return await self.websocket.recv()
async def btcMarketWebsocket():
async with BTCMarketWebsocket() as btcMarketWebsocket:
while not btcMarketWebsocket.disable:
print(await btcMarketWebsocket.receive())
async def ethMarketWebsocket():
async with ETHMarketWebsocket() as ethMarketWebsocket:
while not ethMarketWebsocket.disable:
print(await ethMarketWebsocket.receive())
if __name__ == '__main__':
asyncio.run(btcMarketWebsocket())
asyncio.run(ethMarketWebsocket())
You can do
async def multiple_tasks():
Tasks =[]
Tasks.append(btcMarketWebsocket())
Tasks.append(ethMarketWebsocket())
await asyncio.gather(*Tasks, return_exceptions=True)
if __name__ == '__main__':
asyncio.get_event_loop().run_until_complete(multiple_tasks())
I'm programming a server in Python3, which takes screenshot and sends it over websockets. I have coroutine for handling connection and I would like to create another coroutine for taking screenshot at some interval. Screenshot coroutine will probably run in different thread and I will need to propagate the result to some shared variable with read-write lock, to be able to send it. My questions: (result should be multiplatform, if possible)
How is it possible to schedule tasks like this? I created server which runs forever, and I can create periodical coroutine, but somehow I can't put them together in one loop.
What is a good way to propagate the result from one thread (or coroutine, if server is single threaded) to another?
I found this piece of code similar to this and I can't get it to work (second coroutine doesn't execute). Can someone correct this with and without multithreading?
async def print_var():
global number
await asyncio.sleep(2)
print(number)
async def inc_var():
global number
await asyncio.sleep(5)
number += 1
number = 0
asyncio.get_event_loop().run_until_complete(print_var())
asyncio.async(inc_var)
asyncio.get_event_loop().run_forever()
Post-answer edit
In the end after more hours of googling, I actually got it to work on a single thread, so there's no danger of race condition. (But I'm still not sure what ensure_future does, and why it isn't called on event loop.)
users = set()
def register(websocket):
users.add(websocket)
def unregister(websocket):
users.remove(websocket)
async def get_screenshot():
global screenshot
while True:
screenshot = screenshot()
await asyncio.sleep(0.2)
async def server(websocket, path):
global screenshot
register(websocket)
try:
async for message in websocket:
respond(screenshot)
finally:
unregister(websocket)
def main():
asyncio.get_event_loop().run_until_complete(
websockets.serve(server, 'localhost', 6789))
asyncio.ensure_future(get_screenshot())
asyncio.get_event_loop().run_forever()
main()
In Python 3.7:
import asyncio
import websockets
CAPTURE_INTERVAL = 1
running = True
queues = set()
async def handle(ws, path):
queue = asyncio.Queue()
queues.add(queue)
while running:
data = await queue.get()
if not data:
break
await ws.send(data)
def capture_screen():
# Do some work here, preferably in C extension without holding the GIL
return b'screenshot data'
async def main():
global running
loop = asyncio.get_running_loop()
server = await websockets.serve(handle, 'localhost', 8765)
try:
while running:
data = await loop.run_in_executor(None, capture_screen)
for queue in queues:
queue.put_nowait(data)
await asyncio.sleep(CAPTURE_INTERVAL)
finally:
running = False
for queue in queues:
queue.put_nowait(None)
server.close()
await server.wait_closed()
if __name__ == '__main__':
asyncio.run(main())
Please note, this is only for demonstrating the producer-consumer fan-out pattern. The queues are not essential - you can simply send data to all server.sockets in main() directly, while in handle() you should worry about incoming websocket messages. For example, client may control image compression rate like this:
import asyncio
import websockets
CAPTURE_INTERVAL = 1
DEFAULT = b'default'
qualities = {}
async def handle(ws, path):
try:
async for req in ws:
qualities[ws] = req
finally:
qualities.pop(ws, None)
def capture_screen():
# Do some work here, preferably in C extension without holding the GIL
return {
DEFAULT: b'default screenshot data',
b'60': b'data at 60% quality',
b'80': b'data at 80% quality',
}
async def main():
loop = asyncio.get_running_loop()
server = await websockets.serve(handle, 'localhost', 8765)
try:
while True:
data = await loop.run_in_executor(None, capture_screen)
for ws in server.sockets:
quality = qualities.get(ws, DEFAULT)
if quality not in data:
quality = DEFAULT
asyncio.create_task(ws.send(data[quality]))
await asyncio.sleep(CAPTURE_INTERVAL)
finally:
server.close()
await server.wait_closed()
if __name__ == '__main__':
asyncio.run(main())
I am currently developing a server program in Python that uses the websockets and asyncio packages.
I got a basic script handling websockets working (Exhibit A). This script locks when waiting for input, which is not what I want.
The solution for this that I imagine is I can start two asynchronous tasks - one that handles inputs and one that handles outputs - and start them in a secondary event loop. I had to do some research about coroutines, and I came up with Exhibit B as a proof of concept for running two things simultaneously in an event loop.
Now what I'm stuck on is Exhibit C. When I attempted to use this in a practical scenario with the websockets package, I found that websocket.recv() never finishes (or the coroutine never un-pauses - I'm not sure what's going on exactly). In exhibit A it works fine, and I've determined that the coroutine definitely runs at least up until that point.
Any ideas?
Exhibit A:
#!/usr/bin/python3
import asyncio
import websockets
import time
# This works great!
async def hello(websocket, path):
while True:
# This line waits for input from socket
name = await websocket.recv()
print("< {}".format(name))
# "echo... echo... echo... echo... echo..."
greeting = ''.join(name + "... " for x in range(5))
await websocket.send(greeting)
print("> {}".format(greeting))
time.sleep(0.1);
start_server = websockets.serve(hello, '', 26231)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Exhibit B:
#!/usr/bin/python3
import asyncio
import time
class Thing:
def __init__(self):
pass
async def test(self):
for x in range(10):
print("Hello"+str(x))
await asyncio.sleep(0)
def run(self):
# Add the task to the event loop twice
asyncio.ensure_future(self.test())
asyncio.ensure_future(self.test())
t = Thing()
t.run()
loop = asyncio.get_event_loop();
loop.run_forever()
Exhibit C:
#!/usr/bin/python3
import asyncio
import websockets
import time
class WebsocketRequest:
def __init__(self, websocket):
self.websocket = websocket
# Works great
async def handle_oputs(self):
# This works fine - sends a message
# every 10 seconds to the client
while True:
print("sending...")
await self.websocket.send("Hello")
print("> {}".format("Hello"))
time.sleep(10)
# Doesn't work
async def handle_iputs(self):
# This stops at the await and never receives
# any input from the client :/
while True:
try:
print("receiving...")
# This is the line that causes sadness
data = await self.websocket.recv()
print("< {}".format(data))
except:
# This doesn't happen either
print("Listener is dead")
async def run(self):
# This is the part where the coroutine for
# a client get split off into two of them
# to handle input and output separately.
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
asyncio.ensure_future(self.handle_iputs())
asyncio.ensure_future(self.handle_oputs())
loop.run_forever()
class WebsocketServer:
def __init__(self, address):
self.ip = address[0]
self.port = address[1]
async def hello(self, websocket, path):
req = WebsocketRequest(websocket)
await req.run()
def run(self):
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
start_server = websockets.serve(self.hello, self.ip, self.port)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Maybe the module websocket (unlike websockets) can help you.
The use of WebsocketApp is very simple:
from websocket import WebSocketApp
class ExampleClass(object):
def __init__(self):
websocket.enableTrace(True)
self.ws = websocket.WebSocketApp("ws://echo.websocket.org",
on_message=on_message,
on_error=on_error,
on_close=on_close)
def on_message(ws, msg):
print "Message Arrived:" + msg
def on_error(ws, error):
print error
def on_close(ws):
print "Connection Closed"
def on_open(ws):
ws.send("Hello!")
To download this module: https://pypi.python.org/pypi/websocket-client
time.sleep() is a blocking operation, so any other tasks cannot interrupt and are not scheduled. Use await asyncio.sleep() instead.