I always get errors in closing handshake of Websockets in Python - python

I've coded a little server and client with Python3, Asyncio and Websockets.
I generate just for testing random numbers from a array, parse them into json and send them as websocket to the server. But often I get many errors because of closing handshakes.
Here is the code:
Server:
import asyncio
import websockets
import json
async def receiver(websocket, path):
ws = await websocket.recv()
print("< {}".format(ws))
start_server = websockets.serve(receiver, 'localhost', 8765)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
Client:
import asyncio
import websockets
import json
from array import *
import random
async def randomNumbers():
while True:
numbers = array('i', [1, 2, 3, 4, 5, 6, 7, 8, 9])
random.shuffle(numbers)
await sendWebsocket(numbers[0])
await asyncio.sleep(1)
async def sendWebsocket(number):
async with websockets.connect('ws://localhost:8765') as websocket:
ws = json.dumps({"number": number})
await websocket.send(ws)
print("> {}".format(ws))
loop = asyncio.get_event_loop()
try:
asyncio.ensure_future(randomNumbers())
loop.run_forever()
finally:
print("Client1 closed")
loop.close()
Error:
Error in closing handshake
Traceback (most recent call last):
File "C:\Users\dgred\AppData\Local\Programs\Python\Python36-32\lib\site-packages\websockets\server.py", line 145, in handler
yield from self.close()
File "C:\Users\dgred\AppData\Local\Programs\Python\Python36-32\lib\site-packages\websockets\protocol.py", line 370, in close
self.timeout, loop=self.loop)
File "C:\Users\dgred\AppData\Local\Programs\Python\Python36-32\lib\asyncio\tasks.py", line 358, in wait_for
return fut.result()
File "C:\Users\dgred\AppData\Local\Programs\Python\Python36-32\lib\site-packages\websockets\protocol.py", line 642, in write_frame
"in the {} state".format(self.state.name))
websockets.exceptions.InvalidState: Cannot write to a WebSocket in the CLOSING state
Can anyone help me here?
Thanks!

Here is a slight modification of your server side:
import asyncio
import websockets
import json
import signal
async def receiver(websocket, path):
while True:
try:
ws = await websocket.recv()
print("< {}".format(ws))
except websockets.ConnectionClosed:
print("Connection closed")
break
async def simple_server(stop):
async with websockets.serve(receiver, 'localhost', 8765):
await stop
loop = asyncio.get_event_loop()
stop = asyncio.Future()
loop.add_signal_handler(signal.SIGTERM, stop.set_result, None)
loop.run_until_complete(simple_server(stop))
There are more official examples available on websocket's GitHub

Related

Create another thread for an await function

I'm working with Webserver for the first time, I had worked with socket and parallelism before but it was very different and simple, it didn't use Async as parallelism.
My goal is simple, I have my server and my client. In my client I want to create a separate thread to receive the messages that the server will send and in the previous thread do some other things, as in the code example (client.py):
from typing import Dict
import websockets
import asyncio
import json
URL = "my localhost webserver"
connection = None
async def listen() -> None:
global connection
input("Press enter to connect.")
async with websockets.connect(URL) as ws:
connection = ws
msg_initial: Dict[str,str] = get_dict()
await ws.send(json.dumps(msg_initial))
## This i want to be in another thread
await receive_msg()
print("I`m at listener`s thread")
# do some stuffs
async def recieve_msg() -> None:
while True:
msg = await connection.recv()
print(f"Server: {msg}")
asyncio.get_event_loop().run_until_complete(listen())
For me to get a message I need to use await in recv() but I don't know how to create a separate thread for that. I've already tried using threading to create a separate thread but it didn't work.
Does anyone know how to do this and if it is possible to do this?
It's not clear what you want to do can be done in the exact way you propose. In the following example I am connecting to an echo server. The most straightforward way of implementing what you are suggesting directly is to create a new thread to which the connection is passed. But this does not quite work:
import websockets
import asyncio
from threading import Thread
URL = "ws://localhost:4000"
async def listen() -> None:
async with websockets.connect(URL) as ws:
# pass connection:
t = Thread(target=receiver_thread, args=(ws,))
t.start()
# Generate some messages to be echoed back:
await ws.send('msg1')
await ws.send('msg2')
await ws.send('msg3')
await ws.send('msg4')
await ws.send('msg5')
def receiver_thread(connection):
print("I`m at listener`s thread")
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(receive_msg(connection))
async def receive_msg(connection) -> None:
while True:
msg = await connection.recv()
print(f"Server: {msg}")
asyncio.get_event_loop().run_until_complete(listen())
Prints:
I`m at listener`s thread
Server: msg1
Server: msg2
Server: msg3
Server: msg4
Server: msg5
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Program Files\Python38\lib\threading.py", line 932, in _bootstrap_inner
self.run()
File "C:\Program Files\Python38\lib\threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "C:\Ron\test\test.py", line 22, in receiver_thread
loop.run_until_complete(receive_msg(connection))
File "C:\Program Files\Python38\lib\asyncio\base_events.py", line 616, in run_until_complete
return future.result()
File "C:\Ron\test\test.py", line 29, in receive_msg
msg = await connection.recv()
File "C:\Program Files\Python38\lib\site-packages\websockets\legacy\protocol.py", line 404, in recv
await asyncio.wait(
File "C:\Program Files\Python38\lib\asyncio\tasks.py", line 424, in wait
fs = {ensure_future(f, loop=loop) for f in set(fs)}
File "C:\Program Files\Python38\lib\asyncio\tasks.py", line 424, in <setcomp>
fs = {ensure_future(f, loop=loop) for f in set(fs)}
File "C:\Program Files\Python38\lib\asyncio\tasks.py", line 667, in ensure_future
raise ValueError('The future belongs to a different loop than '
ValueError: The future belongs to a different loop than the one specified as the loop argument
The messages are received okay but the problem occurs in function receiver_thread on the statement:
loop.run_until_complete(receive_msg(connection))
By necessity the started thread has no running event loop and cannot use the event loop being used by function listen and so must create a new event loop. That would be fine if this thread/event loop were not using any resources (i.e. the connection) from a difference event loop:
import websockets
import asyncio
from threading import Thread
URL = "ws://localhost:4000"
async def listen() -> None:
async with websockets.connect(URL) as ws:
t = Thread(target=receiver_thread)
t.start()
def receiver_thread():
print("I`m at listener`s thread")
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(receive_msg())
async def receive_msg() -> None:
await asyncio.sleep(2)
print('I just slept for 2 seconds')
asyncio.get_event_loop().run_until_complete(listen())
Prints:
I`m at listener`s thread
I just slept for 2 seconds
I can see no real need to be running anything in threads based on the minimal code you have showed but assuming you omitted showing some processing of the received message for which asyncio alone is not sufficient, then perhaps all you need to do is receive the messages in the current running loop (in function listen) and use threading just for the processing of the message:
from typing import Dict
import websockets
import asyncio
import json
from threading import Thread
URL = "my localhost webserver"
async def listen() -> None:
input("Press enter to connect.")
async with websockets.connect(URL) as ws:
msg_initial: Dict[str,str] = get_dict()
await ws.send(json.dumps(msg_initial))
while True:
msg = await ws.recv()
print(f"Server: {msg}")
# Non-daemon threads so program will not end until these threads terminate:
t = Thread(target=process_msg, args=(msg,))
t.start()
asyncio.get_event_loop().run_until_complete(listen())
Update
Based on your last comment to my answer concerning creating a chat program, you should either implement this using pure multithreading or pure asyncio. Here is a rough outline using asyncio:
import websockets
import asyncio
import aioconsole
URL = "my localhost webserver"
async def receiver(connection):
while True:
msg = await connection.recv()
print(f"\nServer: {msg}")
async def sender(connection):
while True:
msg = await aioconsole.ainput('\nEnter msg: ')
await connection.send(msg)
async def chat() -> None:
async with websockets.connect(URL) as ws:
await asyncio.gather(
receiver(ws),
sender(ws)
)
asyncio.get_event_loop().run_until_complete(chat())
However, you may be limited in the type of user input you can do with asyncio. I would think, therefore, that multithreading might be a better approach.

How can I fix my asyncio server from returning [Win 10022]?

I've been writing code for an asyncio server on my raspberry pi 4. When I was coding the Client, and did a test run with my Server class, it returns:
Traceback (most recent call last):
File "C:\Users\####\Documents\rpiswr\client.py", line 23, in <module>
asyncio.run(client_run())
File "C:\Users\####\AppData\Local\Programs\Python\Python39\lib\asyncio\runners.py", line 44, in run
return loop.run_until_complete(main)
File "C:\Users\####\AppData\Local\Programs\Python\Python39\lib\asyncio\base_events.py", line 642, in run_until_complete
return future.result()
File "C:\Users\####\Documents\rpiswr\client.py", line 18, in client_run
await client.connect(("192.168.0.11", 56500))
File "C:\Users\####\Documents\rpiswr\aserver\__init__.py", line 11, in connect
print(f"Connecting as {self.s.getsockname()}")
OSError: [WinError 10022] An invalid argument was supplied
I've looked up the error, and there is no substantial result. That's why I'm asking this. I'll list down my code below:
init.py(in directory aserver):
import asyncio, socket
class Client:
def __init__(self):
pass
async def connect(self, addr):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.setblocking(False)
print(f"Connecting as {self.s.getsockname()}")
self.s.connect(addr)
async def recv(self,buf_size):
data = buf_size
print(f"Recieved {data.decode()} in byte data")
async def close_conn(self):
print(f"Closed connection with server")
self.s.close()
async def send_bytes(self, data:bytes):
self.s.sendall(data)
print(f"Sent {data.decode()} in byte format")
class Server:
"""Implement handle function or decorate a function with #Server.handler(<Server>)"""
def __init__(self, addr):
self.addr = addr
async def start_server(self, handler_method):
self.server = await asyncio.start_server(handler_method, self.addr[0], self.addr[1])
self.addr = self.server.sockets[0].getsockname()
print(f"Serving on {self.addr}")
async with self.server:
await self.server.serve_forever()
client.py(problems occur here):
from aserver import Client
import asyncio
client = Client()
async def client_run():
await client.connect(("192.168.0.11", 56500))
await client.send_bytes(b'Hi Echo Server!')
await client.recv(1024)
await client.close_conn()
asyncio.run(client_run())
server.py(runs perfectly well):
import aserver as srv
from aserver import Server
import asyncio
server = srv.Server(["192.168.0.11", 56500])
async def echo_server(reader, writer):
data = await reader.read(1024)
message = data.decode()
addr = writer.get_extra_info('peername')
print(f"Recieved {message} from addr")
print(f"Echoing {message} back")
writer.write(data)
await writer.drain()
print(f"Closing connection with {addr}")
writer.close()
asyncio.run(server.start_server(echo_server))
Why is this error occurring? How do I fix this? Any answer will be appreciated. I would love if you could fix my code where you see the problem.
Edit:
Answered by #user4815162342, but thanks everyone else who contributed!

Streaming list of dictionaries in Python with asyncio

I am working on Python server/client application where the server receives some data from the client and based on this data it collects a list of dictionaries from an embedded k/v store and streams it back.
I put here a code that reproduces the error. There is a reason why I put everything into separate functions on server side (clients send different requests).
The problem is that the server sends faster than the client can consume and the client reads several responses at a time, sometimes it is just a part of the message which has been truncated. I thought writelines/readline pair will read from the socket appropriately, but I think I missed something. write/drain also overloads the socket and once multiple results are read the client failes because chunked serialized dictionary is read to orjson.loads.
What is the proper way to solve this problem? Thank you in advance!
Server:
import orjson
async def getResult(cnt : int):
await asyncio.sleep(0)
result = []
for i in range(cnt):
result.append({"key" : i})
return result
async def send(writer, list_of_dict):
for r in list_of_dict:
print(f"\nSending: {r}")
writer.writelines([orjson.dumps(r)])
await writer.drain()
# sending END signal
writer.writelines([orjson.dumps("END")])
await writer.drain()
async def handleClient(reader, writer):
addr = writer.get_extra_info('peername')
print(f"Connection from {addr}")
data = await reader.readline()
message = orjson.loads(data)
print(f"Received {message} from {addr}")
counter = message["send_me"]
responses = await getResult(counter)
await send(writer, responses)
print("Close the client socket")
writer.close()
loop = asyncio.get_event_loop()
coro = asyncio.start_server(handleClient, '127.0.0.1', 4000, loop=loop)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
Client
import asyncio
import orjson
async def async_client(loop):
reader, writer = await asyncio.open_connection('127.0.0.1', 4000, loop=loop)
counter = 5
print(f"Request counter: {counter}")
# in real life the message is a complex dictionary
msg = {"send_me" : counter}
writer.writelines([orjson.dumps(msg)])
#without write_eof the server reader.readline() waits for data and blocks
if writer.can_write_eof():
writer.write_eof()
while True:
data = await reader.readline()
if data:
print(data)
r = orjson.loads(data)
print(f"Received: {r}")
if r == "END":
print("server completed")
break
else:
await asyncio.sleep(0.1)
print('Close the socket')
writer.close()
loop = asyncio.get_event_loop()
loop.run_until_complete(async_client(loop))
loop.close()
Error:
>python echo_client.py
Request counter: 5
b'{"key":0}{"key":1}{"key":2}{"key":3}{"key":4}"END"'
Traceback (most recent call last):
File "echo_client.py", line 32, in <module>
loop.run_until_complete(async_client(loop))
File "C:\Program Files (x86)\Anaconda\lib\asyncio\base_events.py", line 587, in run_until_complete
return future.result()
File "echo_client.py", line 21, in async_client
r = orjson.loads(data)
orjson.JSONDecodeError: trailing characters at line 1 column 10: line 1 column 1 (char 0)
I think the problem is much simpler: writelines doesn't do what you think it does. It doesn't insert newline characters, it just writes any data you give it. This is why the readline() by your client picks up the payload and "END" concatenated together. This is also why you need write_eof in the other direction.
If you want to write a line, then just write a newline character (byte) after your payload. You can abstract that in a function that handles it for you:
async def write_msg(writer, msg):
writer.write(orjson.dumps(msg))
writer.write('\n')
await writer.drain()
async def read_msg(reader):
line = await reader.readline()
return orjson.loads(line)
You can use these on both the client and the server to communicate.
On an aside note, you should probably switch to the newer asyncio.run() API which creates and correctly tears down the event loop with a single async entry point. Your server setup would look like this:
async def main():
await asyncio.start_server(handleClient, '127.0.0.1', 4000)
await server.wait_closed()
asyncio.run(main())

asyncio server client terminates early without output

I'm learning to play around with the asyncio library in py3.5 syntax with async def and await, and trying to write a simple server/client architecture.
For some reason, the client never receives the message and terminates early:
Client
IP = ''
PORT = 8880
import asyncio
import multiprocessing
import ssl
async def start_client(loop):
reader, writer = await asyncio.open_connection(IP, PORT, loop=loop)
writer.write("Gimme gimme gimme".encode())
writer.close()
data = await reader.read()
print(data.decode())
loop = asyncio.get_event_loop()
loop.run_until_complete(start_client(loop))
loop.close()
Server
IP = ''
PORT = 8880
import asyncio
import requests
import json
async def handle_echo(reader, writer):
data = await reader.read()
response = await whatsup()
print(response)
writer.write(response.encode())
writer.write_eof()
await writer.drain()
writer.close()
async def whatsup():
return "Hello there!"
loop = asyncio.get_event_loop()
server = asyncio.start_server(handle_echo,
IP,
PORT,
loop=loop)
server = loop.run_until_complete(server)
try:
loop.run_forever()
except:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
What I observe is that the server was able to print out the "Hello world!", and return successfully, but from what I can gather, the client reads a total of 0 bytes and just exits.
What I tried already
I tried doing:
while not reader.at_eof():
data = await reader.read(100)
print(data.decode())
But it evaluates at_eof() to be true and exits early as well.
OK I found the solution:
read() will read until eof marker. We need to do writer.write_eof() on both sides for the read() to be read.
Here's the solution code:
async def handle_echo(reader, writer):
data = await reader.read()
response = await whatsup()
print(response)
writer.write(response.encode())
writer.write_eof()
await writer.drain()
writer.close()
async def start_client(loop):
reader, writer = await asyncio.open_connection(IP, PORT, loop=loop)
writer.write("Gimme gimme gimme".encode())
writer.write_eof() # crucial here
writer.close()
data = await reader.read()
print(data.decode())

Web Sockets client throwing an exception once it starts receiving data

I am trying to establish a Web-Socket connection to a server and enter into receive mode.Once the client starts receiving the data, it immediately closes the connection with below exception
webSoc_Received = await websocket.recv()
File "/root/envname/lib/python3.6/site-packages/websockets/protocol.py", line 319, in recv
raise ConnectionClosed(self.close_code, self.close_reason)
websockets.exceptions.ConnectionClosed: WebSocket connection is closed: code = 1007, no reason.
Client-side Code Snippet :
import asyncio
import websockets
async def connect_ws():
print("websockets.client module defines a simple WebSocket client API::::::")
async with websockets.client.connect(full_url,extra_headers=headers_conn1) as websocket:
print ("starting")
webSoc_Received = await websocket.recv()
print ("Ending")
Decode_data = zlib.decompress(webSoc_Received)
print(Decode_data)
asyncio.get_event_loop().run_until_complete(connect_ws())
Any thoughts on this?
You use run_until_complete() which completes once you started process. Instead, you should use .run_forever(). It will keep your socket open, until you close it.
EDIT:
You can do something like this:
loop = asyncio.get_event_loop()
loop.call_soon(connect_ws) # Calls connect_ws once the event loop starts
loop.run_forever()
Or:
loop = asyncio.get_event_loop()
loop.run_until_complete(connect_ws())
loop.run_forever()
Or if previous examples didn't succeed, you can try with following code:
import asyncio
#asyncio.coroutine
def periodic():
while True:
print("websockets.client module defines a simple WebSocket client API::::::")
with websockets.client.connect(full_url,extra_headers=headers_conn1) as websocket:
print ("starting")
webSoc_Received = websocket.recv()
print ("Ending")
Decode_data = zlib.decompress(webSoc_Received)
print(Decode_data)
def stop():
task.cancel()
task = asyncio.Task(periodic())
loop = asyncio.get_event_loop()
loop.call_later(5, stop)
try:
loop.run_until_complete(task)
except asyncio.CancelledError:
pass
From what I can tell, your current code will exit after it receives its first message.
Try changing your code to a consumer pattern, as mentioned in the websocket docs here:
https://websockets.readthedocs.io/en/stable/intro.html#common-patterns
import asyncio
import websockets
async def connect_ws():
print("websockets.client module defines a simple WebSocket client API::::::")
async with websockets.client.connect(full_url,extra_headers=headers_conn1) as websocket:
while True:
print ("starting")
webSoc_Received = await websocket.recv()
print ("Ending")
Decode_data = zlib.decompress(webSoc_Received)
print(Decode_data)

Categories