We Keep Coding

python asyncio server freezes after receiving the initial packet (Only on production mode!!!)

I'm new into python asyncio library and I'm using it to create a tcp sockets, I've created a very simple messaging client/server script that works perfectly on my computer, but for some reason it doesn't works on the production.
By production i mean when i'm trying to make the connection from the client script on my computer, to the server port on my online VPS.
The problem occurs only when the first packet have been sent, the server received it and responded to it. It just stuck there, the client cannot read anything, nor the server. both returning timeouts.
client.py:
import asyncio, datetime, time
async def main():
reader, writer = await asyncio.open_connection('mydomain.com', 4444)
while True:
msg = 'New message: ' + datetime.datetime.now().strftime('%H:%M:%S')
writer.write( msg.encode('utf8') )
print('sent: ', msg)
try:
response = await asyncio.wait_for( reader.read(1024), timeout=5 )
except asyncio.exceptions.TimeoutError:
print('timeout error ..')
break
print('received:', response)
await writer.drain()
time.sleep(2)
if __name__ == '__main__':
try:
asyncio.run(main())
except KeyboardInterrupt:
pass
server.py:
import asyncio, datetime
from asyncio import StreamReader, StreamWriter
async def echo_server(reader: StreamReader, writer: StreamWriter):
while True:
try:
data = await asyncio.wait_for( reader.read(1024), timeout=5 )
except asyncio.exceptions.TimeoutError:
print('timeout error ..')
break
print('Received: {}'.format( data ))
current_dtime = datetime.datetime.now().strftime('%H:%M:%S')
writer.write( current_dtime.encode('utf8') )
await writer.drain()
async def main(host: str, port: int):
server = await asyncio.start_server(echo_server, host, port)
await server.serve_forever()
if __name__ == '__main__':
try:
asyncio.run(main('0.0.0.0', 4444))
except KeyboardInterrupt:
pass
I'm new to asyncio but i'm not new to python and debugging, i am sure that the problem is not about misconfiguration on the server.
So what's up? what is the reason and how to solve it? anything to consider?

Asyncio Server with Timer in python

I'm using asyncio to run a straightforward server and a client. The server is a simple echo server with two "special" commands sent by the client, "quit" and "timer". The quit command closes the connection, and the timer command starts a timer that will print a message in the console (server and client) every second. The timer will stop when the client sends the "quit" command.
Unfortunately, I'm having some problems with the timer. It blocks the server and the client.
How can I solve this problem?
Server
import asyncio
import time
HOST = '127.0.0.1'
PORT = 9999
async def timer():
while True:
print('tick')
await asyncio.sleep(1)
async def handle_echo(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
'''Handle the echo protocol.'''
data = None
while True:
if(data == b'quit'):
writer.close() # Close the connection
await writer.wait_closed() # Wait for the connection to close
if(data == b'timer'):
timertask = asyncio.create_task(timer())
await timertask #<-- This line freezes the server and the client
else:
data = await reader.read(1024) # Read 256 bytes from the reader. Size of the message
msg = data.decode() # Decode the message
addr, port = writer.get_extra_info('peername') # Get the address of the client
print(f"Received {msg!r} from {addr}:{port!r}")
send_message = 'Message received: ' + msg
writer.write(send_message.encode()) # Echo the data back to the client
await writer.drain() # This will wait until everything is clear to move to the next thing.
async def run_server() -> None:
# Our awaitable callable.
# This callable is ran when the server recieves some data
# Question: Does it run when a client connects?
server = await asyncio.start_server(handle_echo, HOST, PORT)
async with server:
await server.serve_forever()
if __name__ == '__main__':
loop = asyncio.new_event_loop() # new_event_loop() is for python 3.10. For older versions, use get_event_loop()
loop.run_until_complete(run_server())
Client
import asyncio
import time
HOST = '127.0.0.1'
PORT = 9999
async def run_client() -> None:
# It's a coroutine. It will wait until the connection is established
reader, writer = await asyncio.open_connection(HOST, PORT)
while True:
message = input('Enter a message: ')
writer.write(message.encode())
await writer.drain()
data = await reader.read(1024)
if not data:
raise Exception('Socket not communicating with the client')
print(f"Received {data.decode()!r}")
if(message == 'quit'):
writer.write(b"quit")
writer.close()
await writer.wait_closed()
exit(2)
#break # Don't know if this is necessary
if __name__ == '__main__':
loop = asyncio.new_event_loop()
loop.run_until_complete(run_client())

I slightly updated your server code: removed the await timertask line (this task never ends, can be only canceled):
Server.py
import asyncio
import time
HOST = "127.0.0.1"
PORT = 9999
async def timer():
while True:
print("tick")
await asyncio.sleep(1)
async def handle_echo(
reader: asyncio.StreamReader, writer: asyncio.StreamWriter
) -> None:
# timer task that the client can start:
timer_task = None
while True:
data = await reader.read(1024)
msg = data.decode()
addr, port = writer.get_extra_info("peername")
print(f"Received {msg!r} from {addr}:{port!r}")
send_message = "Message received: " + msg
writer.write(send_message.encode())
await writer.drain()
if data == b"quit":
# cancel the timer_task (if any)
if timer_task:
timer_task.cancel()
await timer_task
writer.close()
await writer.wait_closed()
elif data == b"timer" and timer_task is None:
timer_task = asyncio.create_task(timer())
async def run_server() -> None:
server = await asyncio.start_server(handle_echo, HOST, PORT)
async with server:
await server.serve_forever()
if __name__ == "__main__":
loop = asyncio.new_event_loop()
loop.run_until_complete(run_server())
Client.py
import asyncio
import time
HOST = "127.0.0.1"
PORT = 9999
async def run_client() -> None:
# It's a coroutine. It will wait until the connection is established
reader, writer = await asyncio.open_connection(HOST, PORT)
while True:
message = input("Enter a message: ")
writer.write(message.encode())
await writer.drain()
data = await reader.read(1024)
if not data:
raise Exception("Socket not communicating with the client")
print(f"Received {data.decode()!r}")
if message == "quit":
writer.write(b"quit")
await writer.drain()
writer.close()
await writer.wait_closed()
exit(2)
if __name__ == "__main__":
loop = asyncio.new_event_loop()
loop.run_until_complete(run_client())
The server waits for a client's messages. If message timer arrives, starts a new timer task and continues to read other messages.
When quit arrives, server cancels the timer_task (if exists) and exits.

Running an asyncio server from a pytest fixture

I'm trying to run a server using asyncio in a pytest fixture
#pytest.fixture(autouse=True)
#pytest.mark.asyncio
async def start_endpoints(
endpoint1: ServerEndpoint,
endpoint2: ServerEndpoint
):
pool = ThreadPoolExecutor(max_workers=2)
loop = asyncio.get_running_loop()
await loop.run_in_executor(pool, endpoint1.start)
await loop.run_in_executor(pool, endpoint2.start)
The start method is like the following
async def start(self):
try:
server = await asyncio.start_server(self.handle_req, self.addr, self.port)
addr = server.sockets[0].getsockname()
print(f'{self.name}: serving on {addr}')
async with server:
await server.serve_forever()
Whereas the test prints this error once it tries to open a connection with the server
self = <_WindowsSelectorEventLoop running=False closed=False debug=False>
fut = <Future finished exception=ConnectionRefusedError(10061, "Connect call failed ('127.0.0.1', 9000)")>
sock = <socket.socket [closed] fd=-1, family=AddressFamily.AF_INET, type=SocketKind.SOCK_STREAM, proto=6>
address = ('127.0.0.1', 9000)
def _sock_connect_cb(self, fut, sock, address):
if fut.cancelled():
return
try:
err = sock.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
if err != 0:
# Jump to any except clause below.
> raise OSError(err, f'Connect call failed {address}')
E ConnectionRefusedError: [Errno 10061] Connect call failed ('127.0.0.1', 9000)
EDIT:
The problem is that the event loop is closed right after so I tried to mark all my fixture with (scope="module") but now I get
ScopeMismatch: You tried to access the 'function' scoped fixture 'event_loop' with a 'module' scoped request object, involved factories
test\e2e\test_peer.py:380: def start_endpoints
EDIT2:
So I added the event_loop fixture
#pytest.fixture(scope="module")
def event_loop():
loop = asyncio.get_event_loop()
yield loop
loop.close()
that should override the default loop for each fixture using #pytest.mark.asyncio.
#pytest.fixture(autouse=True, scope="module")
#pytest.mark.asyncio
async def start_endpoints(
event_loop,
endpoint1: ServerEndpoint,
endpoint2: ServerEndpoint
):
pool = ThreadPoolExecutor(max_workers=2)
await event_loop.run_in_executor(pool, endpoint1.start)
await event_loop.run_in_executor(pool, endpoint2.start)
By debugging inside my test the event_loop is equal to the loop that I'm storing inside the ServerEndpoint (that is asyncio.get_running_loop()) but I'm still getting the ConnectionRefusedError

Probably it should works like this:
async def handle(reader, writer):
data = await reader.read(100)
message = data.decode()
print(f"SERVER: Received {message!r}")
writer.write(data)
await writer.drain()
print(f"SERVER: Sent: {message!r}")
writer.close()
print("SERVER: Closed the connection")
async def start():
server = await asyncio.start_server(handle, host, port)
addr = server.sockets[0].getsockname()
print(f'Server is running on {addr[0:2]}')
async with server:
await server.serve_forever()
async def _async_wait_for_server(event_loop, host, port):
while True:
a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
await event_loop.sock_connect(a_socket, (host, port))
return
except ConnectionRefusedError:
await asyncio.sleep(0.01)
finally:
a_socket.close()
#pytest.fixture()
def server(event_loop, host, port):
cancel_handle = asyncio.ensure_future(start(host, port), loop=event_loop)
event_loop.run_until_complete(
asyncio.wait_for(_async_wait_for_server(event_loop, host, port), 5.0)
)
try:
yield
finally:
cancel_handle.cancel()
But I recommend you make functional tests in another way:
create docker image for server and image for tests
create docker-compose.yml file (use depends_on and healthcheck for tests container)
run tests after starting server

Socket leak with wait_for in asyncio with python3.6 and python3.7

In some cases asyncio.wait_for can lead to socket leak.
Condensed example:
async def _create_connection(timeout=60, ssl_obj):
loop = asyncio.get_event_loop()
connector = loop.create_connection(MyEchoClientProtocol, '127.0.0.1', 5000, ssl=ssl_obj)
connector = asyncio.ensure_future(connector)
tr, pr = await asyncio.wait_for(connector, timeout=timeout, loop=loop)
return tr, pr
async def main():
...
res = await asyncio.wait_for(_acquire_impl(), timeout=timeout, loop=loop)
If my understanding is correct wait_for should work in exactly 2 ways
the inner task is completed and the outer task will receive the result – transport and protocol in this case
The inner task is cancelled and no connection was established
I provided source code for client and server so the problem can be easily reproduced on your system.
certificate and key can be easily generated with minica
I found out that if I catch CancelledError and add a done_callback to the inner task, like so:
try:
tr, pr = await asyncio.wait_for(connector, timeout=timeout, loop=loop)
return tr, pr
except asyncio.CancelledError as e:
connector.add_done_callback(_done_callback)
raise e
then inside of _done_callback I can access the transport and protocol object and close the transport manually to prevent leaking:
Client
import asyncio
import io
import struct
import functools
import ssl as ssl_module
import socket
import collections
import time
import traceback
class MyEchoClientProtocol(asyncio.Protocol):
def connection_made(self, transport):
print('connection_made', transport)
query = 'hello world'
transport.write(query.encode('latin-1'))
def data_received(self, data):
print('data_received', data)
def connection_lost(self, exc):
print('connection_lost', exc)
async def create_connection(ssl_obj, timeout=60):
loop = asyncio.get_event_loop()
connector = loop.create_connection(MyEchoClientProtocol, '127.0.0.1', 5000, ssl=ssl_obj)
connector = asyncio.ensure_future(connector)
tr, pr = await asyncio.wait_for(connector, timeout=timeout, loop=loop)
return tr, pr
async def main(timeout, ssl_obj):
async def _acquire_impl():
try:
proxy = await create_connection(ssl_obj)
except Exception:
raise
else:
return proxy
res = await asyncio.wait_for(_acquire_impl(), timeout=timeout, loop=loop)
return res
async def test_cancel():
sc = ssl_module.create_default_context(ssl_module.Purpose.SERVER_AUTH, cafile='localhostc.crt')
sc.check_hostname = False
sc.verify_mode = ssl_module.CERT_NONE
for i in range(10): # try 50 times
timeout = 0.003
try:
tr, pr = await main(
timeout=timeout, ssl_obj=sc
)
tr.close()
except asyncio.TimeoutError as e:
print('timeouterror', repr(e))
await asyncio.sleep(600)
import asyncio
loop = asyncio.get_event_loop()
loop.run_until_complete(test_cancel())
loop.run_forever()
loop.close()
Server
import asyncio
import ssl
async def handle_echo(reader, writer):
data = await reader.read(100)
message = data.decode()
addr = writer.get_extra_info('peername')
print("Received %r from %r" % (message, addr))
print("Send: %r" % message)
writer.write(data)
await writer.drain()
loop = asyncio.get_event_loop()
sc = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
sc.load_cert_chain('localhost.crt', 'localhost.key')
coro = asyncio.start_server(handle_echo, '127.0.0.1', 5000, loop=loop, ssl=sc)
server = loop.run_until_complete(coro)
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
I run netstat -a | grep 5000 | grep ESTAB | awk '{ print $5 }' | sort | uniq -c | grep 5000 after the script is done and there are many unclosed connections.
The output depends on your hardware so you might need to tweak the timeout parameter
The Questions
Why can this happen?
Is it an expected behaviour? Maybe I'm missing something fundamental about asyncio and python.
What is correct method to prevent leaking sockets while using timeout?
Environment
OS: Ubuntu 16.04 LTS
python: Python 3.6.6 or Python 3.7.3

asyncio server client terminates early without output

I'm learning to play around with the asyncio library in py3.5 syntax with async def and await, and trying to write a simple server/client architecture.
For some reason, the client never receives the message and terminates early:
Client
IP = ''
PORT = 8880
import asyncio
import multiprocessing
import ssl
async def start_client(loop):
reader, writer = await asyncio.open_connection(IP, PORT, loop=loop)
writer.write("Gimme gimme gimme".encode())
writer.close()
data = await reader.read()
print(data.decode())
loop = asyncio.get_event_loop()
loop.run_until_complete(start_client(loop))
loop.close()
Server
IP = ''
PORT = 8880
import asyncio
import requests
import json
async def handle_echo(reader, writer):
data = await reader.read()
response = await whatsup()
print(response)
writer.write(response.encode())
writer.write_eof()
await writer.drain()
writer.close()
async def whatsup():
return "Hello there!"
loop = asyncio.get_event_loop()
server = asyncio.start_server(handle_echo,
IP,
PORT,
loop=loop)
server = loop.run_until_complete(server)
try:
loop.run_forever()
except:
pass
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
What I observe is that the server was able to print out the "Hello world!", and return successfully, but from what I can gather, the client reads a total of 0 bytes and just exits.
What I tried already
I tried doing:
while not reader.at_eof():
data = await reader.read(100)
print(data.decode())
But it evaluates at_eof() to be true and exits early as well.

OK I found the solution:
read() will read until eof marker. We need to do writer.write_eof() on both sides for the read() to be read.
Here's the solution code:
async def handle_echo(reader, writer):
data = await reader.read()
response = await whatsup()
print(response)
writer.write(response.encode())
writer.write_eof()
await writer.drain()
writer.close()
async def start_client(loop):
reader, writer = await asyncio.open_connection(IP, PORT, loop=loop)
writer.write("Gimme gimme gimme".encode())
writer.write_eof() # crucial here
writer.close()
data = await reader.read()
print(data.decode())