We Keep Coding

Python socket.recv() hangs after connection drop

I have implemented a server in Python 3.8.6 using the socket library (running on a Raspberry Pi 4 w/ 8G RAM & Raspberry Pi OS: Buster) which, by design, only handles a single connection at a time. When a connection is gained, the server spawns two threads, one for sending data, and one for receiving it. When the connection is lost (in theory, it will never be intentionally closed), these threads are terminated and a new thread is spawned to accept a new connection. The problem is, the receiving thread never actually terminates, instead getting stuck infinitely on socket.recv(). This persists even after calling socket.shutdown(SHUT_RDWR) and socket.close(). The code is essentially as follows (I've paired it down to only the pertinent bits and left the heartbeat management code, as I've found this is the only way I can consistently tell that the connection has dropped, as the errors that are supposed to be thrown don't always):
import dataclasses
import logging
import pickle
import queue
import socket
import threading
import time
import traceback
log = logging.getLogger()
class Server(threading.Thread):
#dataclasses.dataclass
class Heartbeat:
failures: int = 0
reciprocate: bool = True
time: float = 0
__CLIENT_IP = '10.1.1.58'
__EOF = "\r\nEOF\r\n".encode()
__PORT = 33368
def __init__(self, kill_flag:threading.Event) -> None:
self.__kill_flag: threading.Event = kill_flag
self.__server = socket.create_server(("", self.__PORT), backlog=0, reuse_port=True)
self.__server.settimeout(5)
self.__client = None
self.__queue_out = queue.Queue(-1)
self.__queue_in = queue.Queue(-1)
self._connected = False
self.__connecting_lock = threading.Lock()
self.__receive_thread = None
self.__transmit_thread = None
threading.Thread(target=self.__connect).start()
def __connect(self) -> None:
with self.__connecting_lock:
if self._connected:
return
addr = ("", 0)
if self.__client:
try:
self.__client.shutdown(socket.SHUT_RDWR)
except OSError:
pass
finally:
self.__client.close()
self.__client = None
while (self.__receive_thread and self.__receive_thread.is_alive()) or (self.__transmit_thread and self.__transmit_thread.is_alive()):
log.debug("connected: %s, recv alive: %s, trans alive: %s", self._connected, self.__receive_thread.is_alive(), self.__transmit_thread.is_alive())
time.sleep(1)
log.info("Waiting for connection...")
while addr[0] != self.__CLIENT_IP and not self.__kill_flag.is_set():
try:
self.__client, addr = self.__server.accept()
if addr[0] != self.__CLIENT_IP:
self.__client.close()
log.warning("Connection from %s:%s - Rejected", addr[0], addr[1])
time.sleep(1)
except socket.timeout:
continue
if self.__kill_flag.is_set():
self.__del__()
else:
log.info("Connection from %s:%s - Accepted", addr[0], addr[1])
self.heartbeat = self.Heartbeat()
self._connected = True
self.__receive_thread = threading.Thread(target=self.__receive)
self.__transmit_thread = threading.Thread(target=self.__transmit)
self.__queue_in.queue.clear()
self.__receive_thread.start()
self.__transmit_thread.start()
def get_package(self) -> tuple:
return self.__queue_out.get(True)
def queue_message(self, content:tuple = ()) -> None:
self.__queue_in.put_nowait(content)
def __receive(self) -> None:
buffer = bytearray()
while self._connected and not self.__kill_flag.is_set():
try:
if self.__EOF in buffer:
payload = pickle.loads(buffer.split(self.__EOF, maxsplit=1)[0])
self.__queue_out.put_nowait(payload)
buffer = bytearray(self.__EOF.join(buffer.split(self.__EOF)[1:]))
else:
try:
log.debug("looping into recv")
data = self.__client.recv(1024)
log.debug("looping past recv")
if len(data) == 0:
raise ConnectionResetError
else:
buffer += data
except OSError as error:
log.error(f"Receive error: {error}")
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
except AttributeError:
break
except socket.timeout:
continue
except Exception as error:
log.error(f"Receive error: {error}")
traceback.print_tb(error.__traceback__)
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
def __transmit(self) -> None:
while self._connected and not self.__kill_flag.is_set():
try:
payload = self.__queue_in.get(True, 5)
except queue.Empty:
if self.heartbeat.reciprocate:
payload = (time.time(),)
self.heartbeat.reciprocate = False
elif time.time() - 7 >= self.heartbeat.time:
self.heartbeat.failures += 1
log.debug("Heartbeat failure")
if self.heartbeat.failures >= 3:
log.warning("Lost connection to client: No heartbeat detected")
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
self.heartbeat.reciprocate = True
continue
else:
continue
self.heartbeat.time = time.time()
try:
self.__client.sendall(pickle.dumps(payload))
log.debug("Package sent: %s", payload)
except BrokenPipeError:
log.warning("Lost connection to client: Broken pipe")
self.__queue_in.put_nowait(payload)
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
except AttributeError:
break
def __del__(self) -> None:
if self.__client:
self.__client.close()
self.__server.close()
After the connection drops, the __receive method/thread hangs specifically on this line: data = self.__client.recv(1024).
Perhaps the strangest part is that the client, whose __transmit and __receive methods are virtually identical, does not have this problem.
Any insight anyone can offer into as why this is happening, and how to remedy it, would be greatly appreciated.
Many thanks!

Calling a class function inside another class function

I am trying to call a function inside another class function in Python. However, I am getting the error that global name is not defined. I don't know what to do as I can't figure out what's the issue.
The error is:
NameError: global name 'sendCmdWaitForRaffleResponse' is not defined
Here is the code I have written:
Class T:
def connect(self,ipAddr,portNum):
try:
print "Check1"
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ipAddr,portNum))
s.settimeout(1000*30)
response=s.recv(8192)
print response
cmd="some command"
s.send(cmd)
print 'check'
response=s.recv(1024)
print response
#time.sleep(5)
except Exception as e:
print e
def Send(self,command):
result ="ok"
try:
print "start"
if command.startswith("#"):
#print result
return result
elif not command:
return result
print 'yes'
result=sendCmdWaitResponse(command)
except Exception as e:
print (e)
return result
def sendCmdWaitResponse(self,cmd):
print "Running"
s.send(cmd)
while true:
response=s.recv(1024)
response=+ '\n'
print response
print "Leaving"
return sb.getvalue()
if __name__ == "__main__":
test=T()
a='some command'
test.Connect(ipaddr,port)
test.Send(a)

You need self.
result=self.sendCmdWaitResponse(command)

Change
result=sendCmdWaitResponse(command)
to
result=self.sendCmdWaitResponse(command)

Your code seems to have many typo: This is the fix I have done. I don't know what is your expected output. Please add the valid code:
import socket
class T():
def __init__(self, ipAddr, portNum):
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def connect(self,ipAddr,portNum):
try:
print "Check1"
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((ipAddr,portNum))
s.settimeout(1000*30)
response=s.recv(8192)
print response
cmd="some command"
s.send(cmd)
print 'check'
response=s.recv(1024)
print response
#time.sleep(5)
except Exception as e:
print e
def Send(self, command):
result ="ok"
try:
print "start"
if command.startswith("#"):
#print result
return result
elif not command:
return result
print 'yes'
result= self.sendCmdWaitResponse(command)
except Exception as e:
print (e)
return result
def sendCmdWaitResponse(self,cmd):
print "Running"
self.s.send(cmd)
while True:
response= self.s.recv(1024)
response=+ '\n'
print response
print "Leaving"
return sb.getvalue()
if __name__ == "__main__":
a='some command'
ipAddr = ""
port = 1000
test = T(ipAddr, port)
test.connect(ipAddr,port)
test.Send(a)

Modify server's variable from client's thread (threading, python)

I implemented a simple network 'game' in Python - server draws a random number, and then the client tries to guess it. My application works great, when the client guesses the number, it disconnects from server (it is handled on client's side).
However, after the proper guess, the number is still the same. I would like to modify the application, such that when the client guesses the number, the server should then rand a new number, so other clients should guess the new one. How can I do this?
Some template, just to draw an attention to the problem:
#!/usr/bin/env python
from random import randint
import socket, select
from time import gmtime, strftime
import threading
import sys
class Handler(threading.Thread):
def __init__(self, connection, randomnumber):
threading.Thread.__init__(self)
self.connection = connection
self.randomnumber = randomnumber
def run(self):
while True:
try:
data = self.connection.recv(1024)
if data:
print data
try:
num = int(data)
if Server.guess(num) :
msg = "You won! This is the right number!"
self.connection.send(msg)
break
else :
msg = "Try again!"
self.connection.send(msg)
except ValueError, e:
msg = "%s" % e
self.connection.send(msg)
else:
msg = "error"
self.connection.send(msg)
except socket.error:
self.connection.close()
break
self.connection.close()
class Server:
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.address = (self.ip, self.port)
self.server_socket = None
self.randnum = randint(1, 100)
#classmethod
def guess(cls, no):
if cls.randnum == no:
cls.randnum = randint(1, 1000)
result = True
else:
result = False
return reslut
def run(self):
try:
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(10)
print 'Num is %s' % self.randnum
while True:
connection, (ip, port) = self.server_socket.accept()
c = Handler(connection, self.randnum)
c.start()
except socket.error, e:
if self.server_socket:
self.server_socket.close()
sys.exit(1)
if __name__ == '__main__':
s = Server('127.0.0.1', 1234)
s.run()

Generate the random number that is shared between both server and all the client, there should be only instance of this, hence this should be class attribute.
Add a class function guess which return False upon incorrect guess and upon correct guess changes the randnum and returns True
class Server:
randnum = randint(1, 1000) # class attribute created
#classmethod
def guess(cls, no): # To be used "guess" if `no` attribute if the same as `cls.randnum`
if cls.randnum == no:
cls.randnum = randint(1, 1000)
result = True
else:
result = False
return result
def __init__(self, ip, port):
# ...
The client should call this Server.guess function each time.

Actually your issue comes from the fact that you create randnum as an instance method (see your self.randnum) as #shanmuga explained, if you simply declare it as being a class attribute, and remove the instance method it solves your issue (i.e. declaring it in the class directly).
As a side issue (not being an expert on socket), when you send message to the client, you might want to encode them as a byte object (in the run method of Handler, I changed self.connection.send(msg) to self.connection.send(msg.encode())). Also note that I used Python 3.6 (which mainly change the style of print statements)
See the code below:
#!/usr/bin/env python
from random import randint
import socket, select
from time import gmtime, strftime
import threading
import sys
class Handler(threading.Thread):
def __init__(self, connection, randomnumber):
threading.Thread.__init__(self)
self.connection = connection
self.randomnumber = randomnumber
def run(self):
while True:
try:
data = self.connection.recv(1024)
if data:
print(data)
try:
num = int(data)
if Server.guess(num) :
msg = "You won! This is the right number!"
self.connection.send(msg.encode())
break
else :
msg = "Try again!"
self.connection.send(msg.encode())
except ValueError as e:
msg = "%s" % e
self.connection.send(msg.encode())
else:
msg = "error"
self.connection.send(msg.encode())
except socket.error:
self.connection.close()
break
self.connection.close()
class Server:
randnum = randint(1,100)
def __init__(self, ip, port):
self.ip = ip
self.port = port
self.address = (self.ip, self.port)
self.server_socket = None
#classmethod
def guess(cls, no):
if cls.randnum == no:
cls.randnum = randint(1, 1000)
print("New number is ", cls.randnum )
result = True
else:
result = False
return result
def run(self):
try:
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.ip, self.port))
self.server_socket.listen(10)
print('Num is %s' % self.randnum)
while True:
connection, (ip, port) = self.server_socket.accept()
c = Handler(connection, self.randnum)
c.start()
except socket.error as e:
if self.server_socket:
self.server_socket.close()
sys.exit(1)
if __name__ == '__main__':
s = Server('127.0.0.1', 1234)
s.run()

TCP client not able to write to socket but receive works fine

I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?

You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.

asyncores's handle_close keeps firing continuously

self.handlers.append(ConnHandler(sock, self.handlers))I'm new to python and I tried to write a simple socket server to test stuff out and get to know the language better.
import asyncore
import socket
import json
class ConnHandler(asyncore.dispatcher_with_send):
def __init__(self, conn, handlerlist):
asyncore.dispatcher_with_send.__init__(self, conn)
self.handlers = handlerlist
def handle_close(self):
self.close()
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
data += self.recv(1024)
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message
class TestServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
self.handlers = []
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
self.handlers.append(ConnHandler(sock, self.handlers))
def sendToAll(self, string):
for h in self.handlers:
h.send(string + '\n')
server = TestServer('localhost', 55555)
asyncore.loop()
My problem is the following.
If I connect to the server with telnet and then quit telnet to close the connection, python just spams 'Socket closed' in the terminal. How can I fix this? Are there obvious beginner mistakes in the above code?

handle_close is not meant to detect if peer is disconnected, you get this information in handle_read if .recv call returns an empty string. Then you can close the socket, and then handle_close is called.
Here is a modified version of your code:
def handle_close(self):
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
new_data = self.recv(1024)
if not new_data:
self.close()
return
else:
data += new_data
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message