We Keep Coding

Python import async tcp server as a module: data exchange not working, Multithreading with modules

I am working on the following project: I created a async tcp tcp ip client/ server connection. The client (c# wpf GUI) writes messages to the server (Raspberry pi, Python) and waits for sensor data. The server receive the message from the client and sends back the current sensor data in a loop.
The server/ client application standalone works fine. In the next step I want to use modules in Python. Here are the main parts shown that I want to implement in a seperate Python file:
Main.py
start the async tcp server and run it in the background (thread)
print the received data from the client
read sensor data (thread)
evaluation (optional)
hand over the sensor data to the async server
Here is the code in python that I have so fare (start the async tcp server):
Main.py
import thread
import logging
import socket
import asyncore
#Import async server
from AsyncServerPi_Test import Server
if __name__=='__main__':
logging.basicConfig(level=logging.DEBUG,format='%(name)s: %(message)s',)
log = logging.getLogger('main')
#parameter
ip = socket.gethostname()
port = 12345
address = (ip, port) # port 0 means the kernel gives port
dataFromPi= 'SensorData'
server = Server(address,dataFromPi) #issue with parameter dataFromPI
#dataFromClient = server.getDataFromClient() Code is not working!
#log.debug(dataFromGUI)
asyncore.loop(timeout=1) # checks all client's readable/writable every second
AsyncServerPi_Test.py
import asyncore
import logging
import time
import socket
# Create one or more network channels -- instances of class asyncore.dispatcher.
# These channels are automatically added to a global map, used by the loop() function.
class Server(asyncore.dispatcher):
"""Receives connections and establishes handlers for each client."""
def __init__(self, address, dataFromPi):
self.dataFromPi = dataFromPi
self.logger = logging.getLogger('Server')
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(address)
self.address = self.socket.getsockname()
self.logger.debug('binding to %s', self.address)
self.listen(1)
def timein(self):
self.start_time = time.time()
self.logger.debug('init start_time -> %e', self.start_time)
def handle_accept(self):
# Called when a client connects to our socket
client_info = self.accept()
if client_info is not None:
# start the timeout clock right away
self.timein()
self.logger.debug('handle_accept() -> %s', client_info[1])
Handler(client_info[0], self.start_time, self.dataFromPi)
def handle_close(self):
self.logger.debug('server_handle_close()')
self.close()
class Handler(asyncore.dispatcher):
"""Handles echoing messages from a single client. """
def __init__(self, sock, start_time, dataFromPi, chunk_size=1024):
self.dataFromPi = dataFromPi
self.start_time = start_time
# send data per 5 seconds
self.timeout = 5
self.chunk_size = chunk_size
self.logger = logging.getLogger('Handler%s' % str(sock.getsockname()))
asyncore.dispatcher.__init__(self, sock=sock)
self.data_to_write = []
def timeout_check(self):
#Time event
delta_t = time.time() - self.start_time
if delta_t > self.timeout:
self.logger.debug('timeout! -> %e %e', delta_t, self.timeout)
return True
else:
self.logger.debug('no timeout -> %e %e', delta_t, self.timeout)
return False
def trigger_close(self):
return self.timeout_check()
def writable(self):
"""We want to write if we have received data and when sendSensor data is on."""
# the trigger_close here is a hack
response = bool(self.data_to_write) or self.trigger_close()
self.logger.debug('writable() -> %s', response)
return response
def handle_write(self):
"""Write as much as possible of the reversed recent message we received."""
self.logger.debug('write data to GUI')
if self.trigger_close(): # hack to timeout socket
self.start_time = time.time()
#data = self.data_to_write.pop()
sent = self.send(dataFromPi)
return
data = self.data_to_write.pop()
sent = self.send(dataFromPi)
self.logger.debug('Send data to GUI:'+ str(sent))
#if sent < len(data)
#remaining = data[sent:]
#self.data_to_write.append(remaining)
#self.logger.debug('handle_write() -> (%d) "%s"', sent, data[:sent])
#if not self.writable(): # empty recv
#self.handle_close()
def reverse(self, s):
s = list(s)
s.reverse()
s = "".join(s)
return s
def handle_read(self):
"""Read an incoming message from the client and put it into our outgoing queue."""
data = self.recv(self.chunk_size)
self.logger.debug('handle_read() -> (%d) "%s"', len(data), data)
data = self.reverse(data)
self.data_to_write.insert(0, data)
#return self.data --> didnt work!
def handle_close(self):
"""server close only gets called if client decides or after timeout"""
self.logger.debug('handle_close()')
self.close()
I have some issues with the parameter dataFromPi and dataFromClient. dataFromPi is a input parameter that I handover the instance of the server (server = Server(address,dataFromPi)). I get the following error when I execute the main.py file:
error: uncaptured python exception, closing channel <AsyncServerPi_Test.Handler connected 192.168.0.108:51028 at 0x31ba7d8> (<type 'exceptions.NameError'>:global name 'dataFromPi' is not defined [C:\Python27\lib\asyncore.py|write|91] [C:\Python27\lib\asyncore.py|handle_write_event|468] [C:\Python27\lib\AsyncServerPi_Test.py|handle_write|75])
Handler('192.168.0.108', 12345): handle_close()
I checked the handover of dataFromPi in the AsyncServerPi_Test.py file but I couldnt find a solution.
The next issue I have is that I am not able to print the dataFromClient in the main.py file. Within the AsyncServerPi_Test.py file I can read the dataFromClient (handle_read). I tried to return the received data in handle_read, but I think the return of the data must happen in the class server(). How can I archieve that the server class returns the data from the client so I can print the received data in main.py?
Furthermore the server should run in the background so that I can run other operations parallel (read sensor data from ultrasonic, evaluation...). As an example I want that the main.py should look similar like that:
import thread
import logging
import socket
import asyncore
#Import US
import Ultrasonic
#Import async tcp ip server
from AsyncServerPi_Test import Server
def main():
#Logging
logging.basicConfig(level=logging.DEBUG,format='%(name)s: %(message)s',)
log = logging.getLogger('main')
#Parameter for server
ip = socket.gethostname()
port = 12345
address = (ip, port) # port 0 means the kernel gives port
ultrasonicTest = Ultrasonic() # run in a seperate thread?
dataFromSensor = ultrasonicTest.readSensorData
serverTest = Server(address,dataFromSensor) # run in a seperate thread?
dataFromClient = serverTest.receivedData()
asyncore.loop(timeout=1) # checks all client's readable/writable every second
if __name__=='__main__':
main()
In this context I have the following questions:
The instance of the server (serverTest = Server(address,dataFromSensor)) must run in a seperate thread?
Create a new thread for the sensor (ultrasonicTest = Ultrasonic()):
Is it better to run threads within a module (e.g. start threading within the ultrasonic.py and not in the main.py)
Synchronize the threads?
I am relatie new in python and I dont have much experience in multithreading. It would be helpful if someone has done some similar projects in python and could give me some tips or reference to similar projects.

Python Socket connection class

I'm trying to create a small program that will log information output from a device via TCP
Basically this just streams data out, that i want to capture, and dump into a database for dealing with later
but the device reboots so i need to be able to reconnect when the socket closes with out any human interference
so this is what i have so far
import socket, time, logging, sys, smtplib # Import socket module
logging.basicConfig(filename='Tcplogger.log',level=logging.DEBUG,format='%(asctime)s : %(levelname)s : %(message)s')
logging.info('|--------------------------------------|')
logging.info('|--------------- TCP Logger Starting---|')
logging.info('|--------------------------------------|')
host = '127.0.0.01' # host or Ip address
port = 12345 # output port
retrytime = 1 # reconnect time
reconnectattemps = 10 # Number of time to try and reconnect
class TPCLogger:
def __init__(self):
logging.debug('****Trying connection****')
print('****Trying connection****')
self.initConnection()
def initConnection(self):
s = socket.socket()
try:
s.connect((host, port))
logging.debug('****Connected****')
except IOError as e:
while 1:
reconnectcount = 0;
logging.error(format(e.errno)+' : '+format(e.strerror))
while 1:
reconnectcount = reconnectcount + 1
logging.error('Retrying connection to Mitel attempt : '+str(reconnectcount))
try:
s.connect((host, port))
connected = True
logging.debug('****Connected****')
except IOError as e:
connected = False
logging.error(format(e.errno)+' : '+format(e.strerror))
if reconnectcount == reconnectattemps:
logging.error('******####### Max Reconnect attempts reached logger will Terminate ######******')
sys.exit("could Not connect")
time.sleep(retrytime)
if connected == True:
break
break
while 1:
s.recv(1034)
LOGGER= TCPLogger()
Which all works fine on start up if a try to connect and its not there it will retry the amount of times set by reconnectattemps
but he is my issue
while 1:
s.recv(1034)
when this fails i want to try to reconnect
i could of course type out or just copy my connection part again but what i want to be able todo is call a function that will handle the connection and retry and hand me back the connection object
for example like this
class tcpclient
#set some var
host, port etc....
def initconnection:
connect to socket and retry if needed
RETURN SOCKET
def dealwithdata:
initconnection()
while 1:
try:
s.recv
do stuff here copy to db
except:
log error
initconnection()
I think this is possible but im really not geting how the class/method system works in python so i think im missing something here
FYI just in case you didn't notice iv very new to python. any other comments on what i already have are welcome too :)
Thanks
Aj

Recommendation
For this use-case I would recommend something higher-level than sockets. Why? Controlling all these exceptions and errors for yourself can be irritating when you just want to retrieve or send data and maintain connection.
Of course you can achieve what you want with your plain solution, but you mess with code a bit more, methinks. Anyway it'll look similarly to class amustafa wrote, with handling socket errors to close/reconnect method, etc.
Example
I made some example for easier solution using asyncore module:
import asyncore
import socket
from time import sleep
class Client(asyncore.dispatcher_with_send):
def __init__(self, host, port, tries_max=5, tries_delay=2):
asyncore.dispatcher.__init__(self)
self.host, self.port = host, port
self.tries_max = tries_max
self.tries_done = 0
self.tries_delay = tries_delay
self.end = False # Flag that indicates whether socket should reconnect or quit.
self.out_buffer = '' # Buffer for sending.
self.reconnect() # Initial connection.
def reconnect(self):
if self.tries_done == self.tries_max:
self.end = True
return
print 'Trying connecting in {} sec...'.format(self.tries_delay)
sleep(self.tries_delay)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.connect((self.host, self.port))
except socket.error:
pass
if not self.connected:
self.tries_done += 1
print 'Could not connect for {} time(s).'.format(self.tries_done)
def handle_connect(self):
self.tries_done = 0
print 'We connected and can get the stuff done!'
def handle_read(self):
data = self.recv(1024)
if not data:
return
# Check for terminator. Can be any action instead of this clause.
if 'END' in data:
self.end = True # Everything went good. Shutdown.
else:
print data # Store to DB or other thing.
def handle_close(self):
print 'Connection closed.'
self.close()
if not self.end:
self.reconnect()
Client('localhost', 6666)
asyncore.loop(timeout=1)
reconnnect() method is somehow core of your case - it's called when connection is needed to be made: when class initializes or connection brokes.
handle_read() operates any recieved data, here you log it or something.
You can even send data using buffer (self.out_buffer += 'message'), which will remain untouched after reconnection, so class will resume sending when connected again.
Setting self.end to True will inform class to quit when possible.
asyncore takes care of exceptions and calls handle_close() when such events occur, which is convenient way of dealing with connection failures.

You should look at the python documentation to understand how classes and methods work. The biggest difference between python methods and methods in most other languages is the addition of the "self" tag. The self represents the instance that a method is called against and is automatically fed in by the python system. So:
class TCPClient():
def __init__(self, host, port, retryAttempts=10 ):
#this is the constructor that takes in host and port. retryAttempts is given
# a default value but can also be fed in.
self.host = host
self.port = port
self.retryAttempts = retryAttempts
self.socket = None
def connect(self, attempt=0):
if attempts<self.retryAttempts:
#put connecting code here
if connectionFailed:
self.connect(attempt+1)
def diconnectSocket(self):
#perform all breakdown operations
...
self.socket = None
def sendDataToDB(self, data):
#send data to db
def readData(self):
#read data here
while True:
if self.socket is None:
self.connect()
...
Just make sure you properly disconnect the socket and set it to None.

Building asynchronous socket tutorial: how do I connect to listening socket in python?

python 2.6
Windows 7
I am trying to put together an as simple as possible tutorial of how to write cooperative multitasking programs. As an example application I've written a chat server with python's asyncore backend. I think this will be a valuable resource for the community. However, I have not gotten it to work yet, hence this post.
The structure is as follows. An instance of ChatServer runs on a remote computer. It's socket listens on REMOTE_PORT. When it detects an incoming connection, it spawns an instance of ChatHandler to mediate communication with that connection. Now, who is that connection? On the user's local machine we run an instance of ChatDaemon. This guy listens on LOCAL_PORT. When you connect to him like this
import socket
s = socket.socket()
s.connect(('localhost',LOCAL_PORT))
he detects the connection and spawns two things, a LocalListener and a Connection. The Connection connects to the server, answering our question from above. The LocalListener simply waits for data coming in from the user. If you send data
s.send("Hello, world!")
the LocalListener picks it up, and gives it to the Connection, which then sends it to the ChatServer. The server then puts the data in each ChatHandler's buffer to be sent out to all connected clients. When the Connection receives that data, it passes it to the Daemon who prints it to the screen.
(The Daemon layer seems overly complex but without it you have to do other complicated things to prevent hot loops in asyncore's select() loop whilst keeping latency for the user sending data low. I don't want to go down that road.)
The problem is that the connection to the Daemon does not seem to be made. My exact steps are
In one python session
d = ChatDaemon('localhost')
d.start()
When I do this I see the message "Chat daemon binding to 'localhost: 7668' as expected.
In another python session
import socket
s = socket.socket()
s.connect(('localhost',7668))
When I do this, I do not see the "Got new local connection" line printed.
I have edited my etc/hosts file to map 'localhost' to 127.0.0.1, and I installed the Microsoft Loopback adapter.
EDIT: I have found and fixed the problem. The code below should now be acceptable as a very simple chat implementation using asyncore.
Here is the source
import asyncore
import socket
import sys
LOCAL_HOST = 'localhost'
LOCAL_PORT = 7668
REMOTE_HOST = 'localhost'
REMOTE_PORT = 7667
class LocalListener(asyncore.dispatcher):
"""Receive data from user, putting into cxn's buffer"""
def __init__(self, sock, cxn):
self.cxn = cxn
asyncore.dispatcher.__init__(self, sock)
def writable(self):
return False
def readable(self):
return True
def handle_read(self):
data = self.recv(4096)
if data:
self.cxn.buf = self.cxn.buf + data
class Connection(asyncore.dispatcher):
"""Mediates between user and server"""
def __init__(self, host, port, master):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host,port))
self.buf=""
def writable(self):
return len(self.buf) > 0
def readable(self):
return True
def handle_read(self):
data = self.recv(4096)
if data:
self.master.newMessage(data)
def handle_write(self):
sent = self.send(self.buf)
self.buf = self.buf[sent:]
class ChatDaemon(asyncore.dispatcher):
"""Listen for local connections and dispatch in/out data"""
ADDRESS_FAMILY = socket.AF_INET
SOCKET_TYPE = socket.SOCK_STREAM
def __init__(self, remoteHost, remotePort=REMOTE_PORT,
localHost=LOCAL_HOST, localPort=LOCAL_PORT):
self.remoteHost = remoteHost
self.remotePort = remotePort
self.localHost = localHost
self.localPort = localPort
self.buffer = ""
asyncore.dispatcher.__init__(self)
def writable(self):
return False
def readable(self):
return True
def newMessage(self, data):
print data
def start(self):
"""Listen for user connection on local port"""
self.create_socket(self.ADDRESS_FAMILY, self.SOCKET_TYPE)
print("Chat deamon binding to '%s': %s"%(self.localHost,self.localPort))
self.bind((self.localHost,self.localPort))
self.listen(1)
asyncore.loop()
def handle_accept(self):
"""Spawn local reader and remote reader/writer"""
print "Got new local connection"
(connSock, localAddress) = self.accept()
print("New connection address is %s"%localAddress)
#Make a server connection
cxn = Connection(self.remoteHost, self.remotePort, self)
#Connect to local user
LocalListener(connSock, cxn)
### SERVER ###
class ChatHandler(asyncore.dispatcher):
def __init__(self, sock, map, server):
self.server = server
self.buffer = ''
asyncore.dispatcher.__init__(self, sock, map)
def writable(self):
return len(self.buffer) > 0
def readable(self):
return True
def handle_read(self):
"""Notify server of any new incoming data"""
data = self.recv(4096)
if data:
self.server.newMessage(data, self)
def handle_write(self):
"""send some amount of buffer"""
sent = self.send(self.buffer)
self.buffer = self.buffer[sent:]
class ChatServer(asyncore.dispatcher):
"""Receive and forward chat messages
When a new connection is made we spawn a dispatcher for that
connection.
"""
ADDRESS_FAMILY = socket.AF_INET
SOCKET_TYPE = socket.SOCK_STREAM
def __init__(self, host=REMOTE_HOST, port=REMOTE_PORT):
self.map = {}
self.address = (host,port)
self.clients = []
asyncore.dispatcher.__init__(self, map=self.map)
def serve(self):
"""Bind to socket and start asynchronous loop"""
self.create_socket(self.ADDRESS_FAMILY, self.SOCKET_TYPE)
self.bind(self.address)
self.listen(1)
asyncore.loop(map=self.map)
def writable(self):
return False
def readable(self):
return True
def newMessage(self, data, fromWho):
"""Put data in all clients' buffers"""
for client in self.clients:
client.buf = client.buf + data
def handle_accept(self):
"""Deal with newly accepted connection"""
print 'got new connection'
(connSock, clientAddress) = self.accept()
self.clients.append(ChatHandler(connSock, self.map, self))

The problem was that in ChatDaemon I forget the "return" keywords in readable and writable

Need help creating a TCP relay between two sockets

I have the following situation:
SomeServer(S) <-> (C)MyApp(S) <-> (C)User
(S) represents a server socket
(C) represents a client socket
Essentially, MyApp initiates communication with SomeServer (SomeServer(S) <-> (C)MyApp) and once some authentication routines are successful MyApp(S) starts waiting for (C)User to connect. As soon as User connects, MyApp relays data from SomeServer to User. This happens in both directions.
I have SomeServer(S) <-> (C)MyApp working perfectly, but I'm not able to get MyApp(S) <-> (C)User working. I get as far as User connecting to MyApp(S), but can't get data relayed!
Ok, I hope that's some what clear ;) Now let me show my code for MyApp. Btw the implementation of SomeServer and User are not relevant for solving my question, as neither can be modified.
I have commented my code indicating where I'm experiencing issues. Oh, I should also mention that I have no problem scrapping the whole "Server Section" for some other code if necessary. This is a POC, so my main focus is getting the functionality working rather than writing efficient code. Thanks for you time.
''' MyApp.py module '''
import asyncore, socket
import SSL
# Client Section
# Connects to SomeServer
class MyAppClient(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
connectionPhase = 1
def handle_read(self):
print "connectionPhase =", self.connectionPhase
# The following IF statements may not make sense
# as I have removed code irrelevant to this question
if self.connectionPhase < 3: # authentication phase
data = self.recv(1024)
print 'Received:', data
# Client/Server authentication is handled here
# Everything from this point on happens over
# an encrypted socket using SSL
# Start the RelayServer listening on localhost 8080
# self.socket is encrypted and is the socket communicating
# with SomeServer
rs = RelayServer(('localhost', 8080), self.socket)
print 'RelayServer started'
# connectionPhase = 3 when this IF loop is done
elif self.connectionPhase == 3: # receiving data for User
data = self.recv(1024)
print 'Received data - forward to User:', data
# Forward this data to User
# Don't understand why data is being read here
# when the RelayServer was instantiated above
# Server Section
# Connects to User
class RelayConnection(asyncore.dispatcher):
def __init__(self, client, sock):
asyncore.dispatcher.__init__(self)
self.client = client
print "connecting to %s..." % str(sock)
def handle_connect(self):
print "connected."
# Allow reading once the connection
# on the other side is open.
self.client.is_readable = True
# For some reason this never runs, i.e. data from SomeServer
# isn't read here, but instead in MyAppClient.handle_read()
# don't know how to make it arrive here instead as it should
# be relayed to User
def handle_read(self):
self.client.send(self.recv(1024))
class RelayClient(asyncore.dispatcher):
def __init__(self, server, client, sock):
asyncore.dispatcher.__init__(self, client)
self.is_readable = False
self.server = server
self.relay = RelayConnection(self, sock)
def handle_read(self):
self.relay.send(self.recv(1024))
def handle_close(self):
print "Closing relay..."
# If the client disconnects, close the
# relay connection as well.
self.relay.close()
self.close()
def readable(self):
return self.is_readable
class RelayServer(asyncore.dispatcher):
def __init__(self, bind_address, MyAppClient_sock):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(bind_address)
self.MyAppClient_sock = MyAppClient_sock
print self.MyAppClient_sock
self.listen(1)
def handle_accept(self):
conn, addr = self.accept()
RelayClient(self, conn, self.MyAppClient_sock)
if __name__ == "__main__":
# Connect to host
# First connection stage
connectionPhase = 1
c = MyAppClient('host', port) # SomeServer's host and port
asyncore.loop()
EDIT:
#samplebias I replaced my complete module with your code (not shown) and I have re-added all the bits and pieces that I need for authentication etc.
At this point I'm getting the same result, as with my own code above. What I mean is that MyApp (or Server in your code) is connected to SomeServer and passing data back and forth. Everything is fine thus far. When User (or client application) connects to localhost 8080, this code is run:
if not self.listener:
self.listener = Listener(self.listener_addr, self)
BUT, this is not run
# if user is attached, send data
elif self.user:
print 'self.user'
self.user.send(data)
So, Server is not relaying data to User. I added print statements throughout the User class to see what is run and init is the only thing. handle_read() never runs.
Why is this?

The code is a bit hard to follow, and I'm sure there are a few bugs. For
example in handle_read() you're passing MyAppClient's raw socket self.socket to
RelayServer. You end up with both MyAppClient and RelayConnection working on the same socket.
Rather than attempt to suggest bug fixes to the original code I put together
an example which does what your code intents and is cleaner and easier to follow.
I've tested it talking to an IMAP server and it works, but omits some
things for brevity (error handling, proper close() handling in all cases, etc).
Server initiates the connection to "someserver". Once it connects
it starts the Listener.
Listener listens on port 8080 and accepts only 1 connection, creates a User,
and passes it a reference to Server. Listener rejects all other
client connections while User is active.
User forwards all data to Server, and vice versa. The comments
indicate where the authentication should be plugged in.
Source:
import asyncore
import socket
class User(asyncore.dispatcher_with_send):
def __init__(self, sock, server):
asyncore.dispatcher_with_send.__init__(self, sock)
self.server = server
def handle_read(self):
data = self.recv(4096)
# parse User auth protocol here, authenticate, set phase flag, etc.
# if authenticated, send data to server
if self.server:
self.server.send(data)
def handle_close(self):
if self.server:
self.server.close()
self.close()
class Listener(asyncore.dispatcher_with_send):
def __init__(self, listener_addr, server):
asyncore.dispatcher_with_send.__init__(self)
self.server = server
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind(listener_addr)
self.listen(1)
def handle_accept(self):
conn, addr = self.accept()
# this listener only accepts 1 client. while it is serving 1 client
# it will reject all other clients.
if not self.server.user:
self.server.user = User(conn, self.server)
else:
conn.close()
class Server(asyncore.dispatcher_with_send):
def __init__(self, server_addr, listener_addr):
asyncore.dispatcher_with_send.__init__(self)
self.server_addr = server_addr
self.listener_addr = listener_addr
self.listener = None
self.user = None
def start(self):
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect(self.server_addr)
def handle_error(self, *n):
self.close()
def handle_read(self):
data = self.recv(4096)
# parse SomeServer auth protocol here, set phase flag, etc.
if not self.listener:
self.listener = Listener(self.listener_addr, self)
# if user is attached, send data
elif self.user:
self.user.send(data)
def handle_close(self):
if self.user:
self.user.server = None
self.user.close()
self.user = None
if self.listener:
self.listener.close()
self.listener = None
self.close()
self.start()
if __name__ == '__main__':
app = Server(('someserver', 143), ('localhost', 8080))
app.start()
asyncore.loop()

Start a TCPServer with ThreadingMixIn again in code directly after shutting it down. (Gives `Address already in use`)

I try to program a TCPServer with threads (ThreadingMixIn) in Python. The problem is that I can't shut it down properly as I get the socket.error: [Errno 48] Address already in use when I try to run it again. This is a minimal example of the python code that triggers the problem:
import socket
import threading
import SocketServer
class FakeNetio230aHandler(SocketServer.BaseRequestHandler):
def send(self,message):
self.request.send(message+N_LINE_ENDING)
def handle(self):
self.request.send("Hello\n")
class FakeNetio230a(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
def __init__(self, server_address, RequestHandlerClass):
self.allow_reuse_address = True
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
if __name__ == '__main__':
for i in range(2):
fake_server = FakeNetio230a(("", 1234), FakeNetio230aHandler)
server_thread = threading.Thread(target=fake_server.serve_forever)
server_thread.setDaemon(True)
server_thread.start()
# might add some client connection here
fake_server.shutdown()
All the main code should do is to start the server, shut it down and run it again. But it triggers the error stated above because the socket has not been released after the first shutdown.
I thought that setting self.allow_reuse_address = True could solve the problem, but that did not work. When the python program finishes I can run it again straight away and it can start the server once (but again not twice).
However the problem is gone when I randomize the port (replace 1234 by 1234+i for example) as no other server is listening on that address.
There is a similar SO Q Shutting down gracefully from ThreadingTCPServer but the solution (set allow_reuse_address to True does not work for my code and I don't use ThreadingTCPServer).
How do I have to modify my code in order to be able to start the server twice in my code?
Some more information: The reason why I'm doing this is that I want to run some unit tests for my python project. This requires to provide a (fake) server that my software should to connect to.
edit:
I just found the most correct answer to my problem: I have to add fake_server.server_close() at the end of my main execution code (right after fake_server.shutdown()). I found it in the source file of the TCPServer implementation. All it does is self.socket.close().

Somehow, fake_server doesn't unbind when you assign to it (in first line in for statement).
To fix that, just remove fake_server at the end of loop:
del fake_server # force server to unbind

This post helped me get over the un-closed socket problem.
I had the same problem and wanted to post here my simple implementation for TCP server class (and client method).
I made a TCPThreadedServer class. In order to use it is needed to be inherited, and the method process(msg) must be overridden. the overridden method invokes every time the server gets a message msg, and if it returns a not None object, it will be returned as string to the connected client.
from SocketServer import TCPServer, StreamRequestHandler, ThreadingMixIn
import threading
class TCPThreadedServer(TCPServer, ThreadingMixIn):
class RequstHandler(StreamRequestHandler):
def handle(self):
msg = self.rfile.readline().strip()
reply = self.server.process(msg)
if reply is not None:
self.wfile.write(str(reply) + '\n')
def __init__(self, host, port, name=None):
self.allow_reuse_address = True
TCPServer.__init__(self, (host, port), self.RequstHandler)
if name is None: name = "%s:%s" % (host, port)
self.name = name
self.poll_interval = 0.5
def process(self, msg):
"""
should be overridden
process a message
msg - string containing a received message
return - if returns a not None object, it will be sent back
to the client.
"""
raise NotImplemented
def serve_forever(self, poll_interval=0.5):
self.poll_interval = poll_interval
self.trd = threading.Thread(target=TCPServer.serve_forever,
args = [self, self.poll_interval],
name = "PyServer-" + self.name)
self.trd.start()
def shutdown(self):
TCPServer.shutdown(self)
TCPServer.server_close(self)
self.trd.join()
del self.trd
I found it quite easy to use:
class EchoServerExample(TCPThreadedServer):
def __init__(self):
TCPThreadedServer.__init__(self, "localhost", 1234, "Server")
def process(self, data):
print "EchoServer Got: " + data
return str.upper(data)
for i in range(10):
echo = EchoServerExample()
echo.serve_forever()
response = client("localhost", 1234, "hi-%i" % i)
print "Client received: " + response
echo.shutdown()
I used the method:
import socket
def client(ip, port, msg, recv_len=4096,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
msg = str(msg)
response = None
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((ip, port))
if timeout != socket._GLOBAL_DEFAULT_TIMEOUT:
sock.settimeout(timeout)
sock.send(msg + "\n")
if recv_len > 0:
response = sock.recv(recv_len)
finally:
sock.close()
return response
Enjoy it!

Change your FakeNetio230a definition to this:
class FakeNetio230a(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
def __init__(self, server_address, RequestHandlerClass):
self.allow_reuse_address = True
SocketServer.TCPServer.__init__(self,
server_address,
RequestHandlerClass,
False) # do not implicitly bind
Then, add these two lines in your entry point below your FakeNetio230a instantiation:
fake_server.server_bind() # explicitly bind
fake_server.server_activate() # activate the server
Here's an example:
if __name__ == '__main__':
for i in range(2):
fake_server = FakeNetio230a(("", 1234), FakeNetio230aHandler)
fake_server.server_bind() # explicitly bind
fake_server.server_activate() # activate the server
server_thread = threading.Thread(target=fake_server.serve_forever)
server_thread.setDaemon(True)
server_thread.start()
# might add some client connection here
fake_server.shutdown()