I have a simple echo server function. Everything works fine if I launch pytest with a single test, but if I launch it with two tests then the second one hangs on waiting for the server to start, and I can't understand why. Here's a complete code.
The server:
async def handle_echo(reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print(f"SERVER: Received {message!r} from {addr!r}")
writer.write(data)
await writer.drain()
print(f"SERVER: Sent: {message!r}")
writer.close()
print("SERVER: Closed the connection")
Test setup:
HOST = "localhost"
#pytest.fixture()
def event_loop():
return asyncio.get_event_loop()
async def _async_wait_for_server(event_loop, addr, port):
while True:
a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
await event_loop.sock_connect(a_socket, (addr, port))
return
except ConnectionRefusedError:
await asyncio.sleep(0.001)
finally:
a_socket.close()
#pytest.fixture()
def server(event_loop):
unused_tcp_port = 65432
cancel_handle = asyncio.ensure_future(main(unused_tcp_port), loop=event_loop)
event_loop.run_until_complete(asyncio.wait_for(
_async_wait_for_server(event_loop, HOST, unused_tcp_port), 5.0))
try:
yield unused_tcp_port
finally:
cancel_handle.cancel()
async def main(port):
server = await asyncio.start_server(handle_echo, HOST, port)
addr = server.sockets[0].getsockname()
print(f'SERVER: Serving on {addr[0:2]}')
async with server:
await server.serve_forever()
Tests:
#pytest.mark.asyncio
async def test_something(server):
message = "Foobar!"
reader, writer = await asyncio.open_connection(HOST, server)
print(f'CLIENT: Sent {message!r}')
writer.write(message.encode())
await writer.drain()
data = await reader.read(100)
print(f'CLIENT: Received {data.decode()!r}')
print('CLIENT: Close the connection')
writer.close()
await writer.wait_closed()
#pytest.mark.asyncio
async def test_another_thing(server):
# Literally the same
Pytest output:
================================================================== test session starts ===================================================================
platform win32 -- Python 3.7.0, pytest-6.2.5, py-1.10.0, pluggy-1.0.0 -- C:\Users\hurrd.virtualenvs\chatServer-iuEQ-ghN\Scripts\python.exe
cachedir: .pytest_cache
rootdir: C:\Users\hurrd\PycharmProjects\chatServer
plugins: asyncio-0.16.0
collected 2 items
tests/unit/test_themes.py::test_something PASSED [ 50%]
tests/unit/test_themes.py::test_another_thing
Related
I'm using asyncio to run a straightforward server and a client. The server is a simple echo server with two "special" commands sent by the client, "quit" and "timer". The quit command closes the connection, and the timer command starts a timer that will print a message in the console (server and client) every second. The timer will stop when the client sends the "quit" command.
Unfortunately, I'm having some problems with the timer. It blocks the server and the client.
How can I solve this problem?
Server
import asyncio
import time
HOST = '127.0.0.1'
PORT = 9999
async def timer():
while True:
print('tick')
await asyncio.sleep(1)
async def handle_echo(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
'''Handle the echo protocol.'''
data = None
while True:
if(data == b'quit'):
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
if(data == b'timer'):
timertask = asyncio.create_task(timer())
await timertask #<-- This line freezes the server and the client
else:
data = await reader.read(1024) # Read 256 bytes from the reader. Size of the message
msg = data.decode() # Decode the message
addr, port = writer.get_extra_info('peername') # Get the address of the client
print(f"Received {msg!r} from {addr}:{port!r}")
send_message = 'Message received: ' + msg
writer.write(send_message.encode()) # Echo the data back to the client
await writer.drain() # This will wait until everything is clear to move to the next thing.
async def run_server() -> None:
# Our awaitable callable.
# This callable is ran when the server recieves some data
# Question: Does it run when a client connects?
server = await asyncio.start_server(handle_echo, HOST, PORT)
async with server:
await server.serve_forever()
if __name__ == '__main__':
loop = asyncio.new_event_loop() # new_event_loop() is for python 3.10. For older versions, use get_event_loop()
loop.run_until_complete(run_server())
Client
import asyncio
import time
HOST = '127.0.0.1'
PORT = 9999
async def run_client() -> None:
# It's a coroutine. It will wait until the connection is established
reader, writer = await asyncio.open_connection(HOST, PORT)
while True:
message = input('Enter a message: ')
writer.write(message.encode())
await writer.drain()
data = await reader.read(1024)
if not data:
raise Exception('Socket not communicating with the client')
print(f"Received {data.decode()!r}")
if(message == 'quit'):
writer.write(b"quit")
writer.close()
await writer.wait_closed()
exit(2)
#break # Don't know if this is necessary
if __name__ == '__main__':
loop = asyncio.new_event_loop()
loop.run_until_complete(run_client())
I slightly updated your server code: removed the await timertask line (this task never ends, can be only canceled):
Server.py
import asyncio
import time
HOST = "127.0.0.1"
PORT = 9999
async def timer():
while True:
print("tick")
await asyncio.sleep(1)
async def handle_echo(
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
) -> None:
# timer task that the client can start:
timer_task = None
while True:
data = await reader.read(1024)
msg = data.decode()
addr, port = writer.get_extra_info("peername")
print(f"Received {msg!r} from {addr}:{port!r}")
send_message = "Message received: " + msg
writer.write(send_message.encode())
await writer.drain()
if data == b"quit":
# cancel the timer_task (if any)
if timer_task:
timer_task.cancel()
await timer_task
writer.close()
await writer.wait_closed()
elif data == b"timer" and timer_task is None:
timer_task = asyncio.create_task(timer())
async def run_server() -> None:
server = await asyncio.start_server(handle_echo, HOST, PORT)
async with server:
await server.serve_forever()
if __name__ == "__main__":
loop = asyncio.new_event_loop()
loop.run_until_complete(run_server())
Client.py
import asyncio
import time
HOST = "127.0.0.1"
PORT = 9999
async def run_client() -> None:
# It's a coroutine. It will wait until the connection is established
reader, writer = await asyncio.open_connection(HOST, PORT)
while True:
message = input("Enter a message: ")
writer.write(message.encode())
await writer.drain()
data = await reader.read(1024)
if not data:
raise Exception("Socket not communicating with the client")
print(f"Received {data.decode()!r}")
if message == "quit":
writer.write(b"quit")
await writer.drain()
writer.close()
await writer.wait_closed()
exit(2)
if __name__ == "__main__":
loop = asyncio.new_event_loop()
loop.run_until_complete(run_client())
The server waits for a client's messages. If message timer arrives, starts a new timer task and continues to read other messages.
When quit arrives, server cancels the timer_task (if exists) and exits.
I'm trying to run a server using asyncio in a pytest fixture
#pytest.fixture(autouse=True)
#pytest.mark.asyncio
async def start_endpoints(
endpoint1: ServerEndpoint,
endpoint2: ServerEndpoint
):
pool = ThreadPoolExecutor(max_workers=2)
loop = asyncio.get_running_loop()
await loop.run_in_executor(pool, endpoint1.start)
await loop.run_in_executor(pool, endpoint2.start)
The start method is like the following
async def start(self):
try:
server = await asyncio.start_server(self.handle_req, self.addr, self.port)
addr = server.sockets[0].getsockname()
print(f'{self.name}: serving on {addr}')
async with server:
await server.serve_forever()
Whereas the test prints this error once it tries to open a connection with the server
self = <_WindowsSelectorEventLoop running=False closed=False debug=False>
fut = <Future finished exception=ConnectionRefusedError(10061, "Connect call failed ('127.0.0.1', 9000)")>
sock = <socket.socket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6>
address = ('127.0.0.1', 9000)
def _sock_connect_cb(self, fut, sock, address):
if fut.cancelled():
return
try:
err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
# Jump to any except clause below.
> raise OSError(err, f'Connect call failed {address}')
E ConnectionRefusedError: [Errno 10061] Connect call failed ('127.0.0.1', 9000)
EDIT:
The problem is that the event loop is closed right after so I tried to mark all my fixture with (scope="module") but now I get
ScopeMismatch: You tried to access the 'function' scoped fixture 'event_loop' with a 'module' scoped request object, involved factories
test\e2e\test_peer.py:380: def start_endpoints
EDIT2:
So I added the event_loop fixture
#pytest.fixture(scope="module")
def event_loop():
loop = asyncio.get_event_loop()
yield loop
loop.close()
that should override the default loop for each fixture using #pytest.mark.asyncio.
#pytest.fixture(autouse=True, scope="module")
#pytest.mark.asyncio
async def start_endpoints(
event_loop,
endpoint1: ServerEndpoint,
endpoint2: ServerEndpoint
):
pool = ThreadPoolExecutor(max_workers=2)
await event_loop.run_in_executor(pool, endpoint1.start)
await event_loop.run_in_executor(pool, endpoint2.start)
By debugging inside my test the event_loop is equal to the loop that I'm storing inside the ServerEndpoint (that is asyncio.get_running_loop()) but I'm still getting the ConnectionRefusedError
Probably it should works like this:
async def handle(reader, writer):
data = await reader.read(100)
message = data.decode()
print(f"SERVER: Received {message!r}")
writer.write(data)
await writer.drain()
print(f"SERVER: Sent: {message!r}")
writer.close()
print("SERVER: Closed the connection")
async def start():
server = await asyncio.start_server(handle, host, port)
addr = server.sockets[0].getsockname()
print(f'Server is running on {addr[0:2]}')
async with server:
await server.serve_forever()
async def _async_wait_for_server(event_loop, host, port):
while True:
a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
await event_loop.sock_connect(a_socket, (host, port))
return
except ConnectionRefusedError:
await asyncio.sleep(0.01)
finally:
a_socket.close()
#pytest.fixture()
def server(event_loop, host, port):
cancel_handle = asyncio.ensure_future(start(host, port), loop=event_loop)
event_loop.run_until_complete(
asyncio.wait_for(_async_wait_for_server(event_loop, host, port), 5.0)
)
try:
yield
finally:
cancel_handle.cancel()
But I recommend you make functional tests in another way:
create docker image for server and image for tests
create docker-compose.yml file (use depends_on and healthcheck for tests container)
run tests after starting server
I'm running this server which receives data. However I want it to update every second. This Asyncio loop says it's running forever but it only receives data once.
What loops can I execute to update message retrieval every n seconds and where should I place these loops? I've tried Threading, For/While Loops etc but I may have been placing them in the wrong places.
What should I do?
import asyncio
import websockets
import socket
UDP_IP = socket.gethostname()
UDP_PORT = 5225
sock = socket.socket(socket.AF_INET, # Internet
socket.SOCK_DGRAM) # UDP
sock.bind((UDP_IP, UDP_PORT))
while True:
data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
#print(str(data))
x = 1
async def echo(websocket, path):
async for message in websocket:
await asyncio.sleep(1)
await websocket.send(str(data)) #FontWeight Value
print(bytes(data))
start_server = websockets.serve(echo, "localhost", 9090)
asyncio.get_event_loop().run_until_complete(start_server)
asyncio.get_event_loop().run_forever()
#loop.run_forever(start_server)
You can't use ordinary sockets in asyncio because their blocking recv stalls the event loop. You need to use something like this:
data = None
class ServerProtocol(asyncio.Protocol):
def data_received(self, newdata):
global data
data = newdata
async def serve_udp():
loop = asyncio.get_running_loop()
server = await loop.create_server(ServerProtocol, UDP_IP, UDP_PORT)
async with server:
await server.serve_forever()
Then you to integrate it with the websocket serving code. For example:
async def ws_echo(websocket, path):
async for message in websocket:
await asyncio.sleep(1)
await websocket.send(str(data))
async def main():
asyncio.create_task(serve_udp())
await websockets.serve(ws_echo, "localhost", 9090)
await asyncio.Event().wait() # prevent main() from returning
asyncio.run(main())
I'm learning to play around with the asyncio library in py3.5 syntax with async def and await, and trying to write a simple server/client architecture.
For some reason, the client never receives the message and terminates early:
Client
IP = ''
PORT = 8880
import asyncio
import multiprocessing
import ssl
async def start_client(loop):
reader, writer = await asyncio.open_connection(IP, PORT, loop=loop)
writer.write("Gimme gimme gimme".encode())
writer.close()
data = await reader.read()
print(data.decode())
loop = asyncio.get_event_loop()
loop.run_until_complete(start_client(loop))
loop.close()
Server
IP = ''
PORT = 8880
import asyncio
import requests
import json
async def handle_echo(reader, writer):
data = await reader.read()
response = await whatsup()
print(response)
writer.write(response.encode())
writer.write_eof()
await writer.drain()
writer.close()
async def whatsup():
return "Hello there!"
loop = asyncio.get_event_loop()
server = asyncio.start_server(handle_echo,
IP,
PORT,
loop=loop)
server = loop.run_until_complete(server)
try:
loop.run_forever()
except:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
What I observe is that the server was able to print out the "Hello world!", and return successfully, but from what I can gather, the client reads a total of 0 bytes and just exits.
What I tried already
I tried doing:
while not reader.at_eof():
data = await reader.read(100)
print(data.decode())
But it evaluates at_eof() to be true and exits early as well.
OK I found the solution:
read() will read until eof marker. We need to do writer.write_eof() on both sides for the read() to be read.
Here's the solution code:
async def handle_echo(reader, writer):
data = await reader.read()
response = await whatsup()
print(response)
writer.write(response.encode())
writer.write_eof()
await writer.drain()
writer.close()
async def start_client(loop):
reader, writer = await asyncio.open_connection(IP, PORT, loop=loop)
writer.write("Gimme gimme gimme".encode())
writer.write_eof() # crucial here
writer.close()
data = await reader.read()
print(data.decode())
I am using python asyncio streams to connect to several socket servers, but when the server is down, my code can't auto reconnect.
What I need is that, when the server is down, my script will try to reconnect every 5 seconds, until connected and start to parse the data again.
import asyncio
server1 = {'host': '192.168.1.51', 'port': 11110}
server2 = {'host': '192.168.1.52', 'port': 11110}
async def tcp_client(host, port, loop):
print('connect to server {} {}'.format(host, str(port)))
reader, writer = await asyncio.open_connection(host, port, loop=loop)
while True:
data = await reader.read(100)
print('raw data received: {}'.format(data))
await asyncio.sleep(0.1)
loop = asyncio.get_event_loop()
try:
for server in [server1, server2]:
loop.run_until_complete(tcp_client(server['host'], server['port'], loop))
print('task added: connect to server {} {}'.format(server['host'], server['port']))
finally:
loop.close()
print('loop closed')
You can handle reconnection by simply looping over a try/except statement.
Additionally, asyncio.wait_for can be used to set a timeout on the read operation.
Consider this working example:
import asyncio
async def tcp_client(host, port):
reader, writer = await asyncio.open_connection(host, port)
try:
while not reader.at_eof():
data = await asyncio.wait_for(reader.read(100), 3.0)
print('raw data received: {}'.format(data))
finally:
writer.close()
async def tcp_reconnect(host, port):
server = '{} {}'.format(host, port)
while True:
print('Connecting to server {} ...'.format(server))
try:
await tcp_client(host, port)
except ConnectionRefusedError:
print('Connection to server {} failed!'.format(server))
except asyncio.TimeoutError:
print('Connection to server {} timed out!'.format(server))
else:
print('Connection to server {} is closed.'.format(server))
await asyncio.sleep(2.0)
async def main():
servers = [('localhost', 8888), ('localhost', 9999)]
coros = [tcp_reconnect(host, port) for host, port in servers]
await asyncio.gather(*coros)
if __name__ == '__main__':
loop = asyncio.get_event_loop()
loop.run_until_complete(main())
loop.close()