We Keep Coding

Python import async tcp server as a module: data exchange not working, Multithreading with modules

I am working on the following project: I created a async tcp tcp ip client/ server connection. The client (c# wpf GUI) writes messages to the server (Raspberry pi, Python) and waits for sensor data. The server receive the message from the client and sends back the current sensor data in a loop.
The server/ client application standalone works fine. In the next step I want to use modules in Python. Here are the main parts shown that I want to implement in a seperate Python file:
Main.py
start the async tcp server and run it in the background (thread)
print the received data from the client
read sensor data (thread)
evaluation (optional)
hand over the sensor data to the async server
Here is the code in python that I have so fare (start the async tcp server):
Main.py
import thread
import logging
import socket
import asyncore
#Import async server
from AsyncServerPi_Test import Server
if __name__=='__main__':
logging.basicConfig(level=logging.DEBUG,format='%(name)s: %(message)s',)
log = logging.getLogger('main')
#parameter
ip = socket.gethostname()
port = 12345
address = (ip, port) # port 0 means the kernel gives port
dataFromPi= 'SensorData'
server = Server(address,dataFromPi) #issue with parameter dataFromPI
#dataFromClient = server.getDataFromClient() Code is not working!
#log.debug(dataFromGUI)
asyncore.loop(timeout=1) # checks all client's readable/writable every second
AsyncServerPi_Test.py
import asyncore
import logging
import time
import socket
# Create one or more network channels -- instances of class asyncore.dispatcher.
# These channels are automatically added to a global map, used by the loop() function.
class Server(asyncore.dispatcher):
"""Receives connections and establishes handlers for each client."""
def __init__(self, address, dataFromPi):
self.dataFromPi = dataFromPi
self.logger = logging.getLogger('Server')
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(address)
self.address = self.socket.getsockname()
self.logger.debug('binding to %s', self.address)
self.listen(1)
def timein(self):
self.start_time = time.time()
self.logger.debug('init start_time -> %e', self.start_time)
def handle_accept(self):
# Called when a client connects to our socket
client_info = self.accept()
if client_info is not None:
# start the timeout clock right away
self.timein()
self.logger.debug('handle_accept() -> %s', client_info[1])
Handler(client_info[0], self.start_time, self.dataFromPi)
def handle_close(self):
self.logger.debug('server_handle_close()')
self.close()
class Handler(asyncore.dispatcher):
"""Handles echoing messages from a single client. """
def __init__(self, sock, start_time, dataFromPi, chunk_size=1024):
self.dataFromPi = dataFromPi
self.start_time = start_time
# send data per 5 seconds
self.timeout = 5
self.chunk_size = chunk_size
self.logger = logging.getLogger('Handler%s' % str(sock.getsockname()))
asyncore.dispatcher.__init__(self, sock=sock)
self.data_to_write = []
def timeout_check(self):
#Time event
delta_t = time.time() - self.start_time
if delta_t > self.timeout:
self.logger.debug('timeout! -> %e %e', delta_t, self.timeout)
return True
else:
self.logger.debug('no timeout -> %e %e', delta_t, self.timeout)
return False
def trigger_close(self):
return self.timeout_check()
def writable(self):
"""We want to write if we have received data and when sendSensor data is on."""
# the trigger_close here is a hack
response = bool(self.data_to_write) or self.trigger_close()
self.logger.debug('writable() -> %s', response)
return response
def handle_write(self):
"""Write as much as possible of the reversed recent message we received."""
self.logger.debug('write data to GUI')
if self.trigger_close(): # hack to timeout socket
self.start_time = time.time()
#data = self.data_to_write.pop()
sent = self.send(dataFromPi)
return
data = self.data_to_write.pop()
sent = self.send(dataFromPi)
self.logger.debug('Send data to GUI:'+ str(sent))
#if sent < len(data)
#remaining = data[sent:]
#self.data_to_write.append(remaining)
#self.logger.debug('handle_write() -> (%d) "%s"', sent, data[:sent])
#if not self.writable(): # empty recv
#self.handle_close()
def reverse(self, s):
s = list(s)
s.reverse()
s = "".join(s)
return s
def handle_read(self):
"""Read an incoming message from the client and put it into our outgoing queue."""
data = self.recv(self.chunk_size)
self.logger.debug('handle_read() -> (%d) "%s"', len(data), data)
data = self.reverse(data)
self.data_to_write.insert(0, data)
#return self.data --> didnt work!
def handle_close(self):
"""server close only gets called if client decides or after timeout"""
self.logger.debug('handle_close()')
self.close()
I have some issues with the parameter dataFromPi and dataFromClient. dataFromPi is a input parameter that I handover the instance of the server (server = Server(address,dataFromPi)). I get the following error when I execute the main.py file:
error: uncaptured python exception, closing channel <AsyncServerPi_Test.Handler connected 192.168.0.108:51028 at 0x31ba7d8> (<type 'exceptions.NameError'>:global name 'dataFromPi' is not defined [C:\Python27\lib\asyncore.py|write|91] [C:\Python27\lib\asyncore.py|handle_write_event|468] [C:\Python27\lib\AsyncServerPi_Test.py|handle_write|75])
Handler('192.168.0.108', 12345): handle_close()
I checked the handover of dataFromPi in the AsyncServerPi_Test.py file but I couldnt find a solution.
The next issue I have is that I am not able to print the dataFromClient in the main.py file. Within the AsyncServerPi_Test.py file I can read the dataFromClient (handle_read). I tried to return the received data in handle_read, but I think the return of the data must happen in the class server(). How can I archieve that the server class returns the data from the client so I can print the received data in main.py?
Furthermore the server should run in the background so that I can run other operations parallel (read sensor data from ultrasonic, evaluation...). As an example I want that the main.py should look similar like that:
import thread
import logging
import socket
import asyncore
#Import US
import Ultrasonic
#Import async tcp ip server
from AsyncServerPi_Test import Server
def main():
#Logging
logging.basicConfig(level=logging.DEBUG,format='%(name)s: %(message)s',)
log = logging.getLogger('main')
#Parameter for server
ip = socket.gethostname()
port = 12345
address = (ip, port) # port 0 means the kernel gives port
ultrasonicTest = Ultrasonic() # run in a seperate thread?
dataFromSensor = ultrasonicTest.readSensorData
serverTest = Server(address,dataFromSensor) # run in a seperate thread?
dataFromClient = serverTest.receivedData()
asyncore.loop(timeout=1) # checks all client's readable/writable every second
if __name__=='__main__':
main()
In this context I have the following questions:
The instance of the server (serverTest = Server(address,dataFromSensor)) must run in a seperate thread?
Create a new thread for the sensor (ultrasonicTest = Ultrasonic()):
Is it better to run threads within a module (e.g. start threading within the ultrasonic.py and not in the main.py)
Synchronize the threads?
I am relatie new in python and I dont have much experience in multithreading. It would be helpful if someone has done some similar projects in python and could give me some tips or reference to similar projects.

Run independent file from main

I have a run.py that looks something like this:
def main():
# Tested and working code here
if __name__ == '__main__':
main()
Then I have another file that runs a TCP Socket Server, bup.py:
import socket
import os
from threading import Thread
# PMS Settings
TCP_IP = ''
TCP_PORT = 8080
my_ID = '105032495291981824'.encode()
my_dir = os.path.dirname(os.path.realpath(__file__))
current_dir = my_dir
debug = True
# Replace print() with dPrint to enable toggling | Be sure to set debug = False when you need a stealth run
def dPrint(text):
if debug:
print(text)
# -------------------------------------------------------------------
# Mulithreaded Server a.k.a. PMS
class ClientThread(Thread):
def __init__(self, ip, port):
Thread.__init__(self)
self.ip = ip
self.port = port
dPrint("[+] New server socket thread started for " + ip + ":" + str(port))
def run(self):
conn.send(current_dir.encode())
while True:
try:
data = conn.recv(2048).decode()
if "$exec " in data:
data = data.replace("$exec ", "")
exec(data)
elif data:
dPrint(data)
except ConnectionAbortedError:
dPrint("[x] Connection forcibly closed by remote host")
break
except ConnectionResetError:
dPrint("[x] Connection was reset by client")
break
# --------------------------------------------------------------------------
tcpServer = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcpServer.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcpServer.bind((TCP_IP, TCP_PORT))
threads = []
while True:
tcpServer.listen(5)
(conn, (ip, port)) = tcpServer.accept()
newThread = ClientThread(ip, port)
newThread.start()
threads.append(newThread)
for t in threads:
t.join()
I want bup.py executed from main() as an independent file. Also, it has to run either in the background or in an invisible window. Is this even possible? bup.py is a server script so it doesn't return anything and it has to be completely detached from run.py.

You can use subprocess.
import subprocess
def main()
# do your work
subprocess.Popen(["python","bup.py"])
This should run in the background if your current process doesn't depend on the output of the started process.
Alternatively you can reorganise bup.py as a python module and use multiprocessing:
import bup
from multiprocessing import Process
def runServer(name):
# assuming this would start the loop in bup
bup.startServerLoop();
if __name__ == '__main__':
p = Process(target=f)
p.start()
# do all other work
# close the server process
p.join()

If you just want to run bup.py as a separate file, maybe you can define that main in your bup.py and run that file using python bup.py. I am not exactly sure what bup.py need to be bound to a run.py, did I miss anything?

issues with socket programming - python

I am doing a client-server project for my college project,
we have to allocate the login to the client.
Client system will request its status for every 2 seconds(to check whether the client is locked or unlocked). and server will accept the client request and reply the client status to the system.
But the problem is server thread is not responding to the client request.
CLIENT THREAD:
def checkPort():
while True:
try:
s = socket.socket()
s.connect((host, port))
s.send('pc1') # send PC name to the server
status = s.recv(1024) # receive the status from the server
if status == "unlock":
disableIntrrupts() # enable all the functions of system
else:
enableInterrupts() # enable all the functions of system
time.sleep(5)
s.close()
except Exception:
pass
SERVER THREAD:
def check_port():
while True:
try:
print "hello loop is repeating"
conn, addr = s.accept()
data = conn.recv(1024)
if exit_on_click == 1:
break
if (any(sublist[0] == data for sublist in available_sys)):
print "locked"
conn.send("lock")
elif (any(sublist[0] == data for sublist in occupied_sys)):
conn.send("unlock")
print "unlocked"
else:
print "added to gui for first time"
available_sys.append([data,addr[0],nameText,usnText,branchText])
availSysList.insert('end',data)
except Exception:
pass
But my problem is server thread is not executing more than 2 time,
So its unable to accept client request more than one time.
can't we handle multiple client sockets using single server socket?
How to handle multiple client request from server ?
Thanks for any help !!

Its because your server, will block waiting for a new connection on this line
conn, addr = s.accept()
This is because calls like .accept and .read are blocking calls that hold the process
You need to consider an alternative design, where in you either.
Have one process per connection (this idea is stupid)
One thread per connection (this idea is less stupid than the first but still mostly foolish)
Have a non blocking design that allows multiple clients and read/write without blocking execution.
To achieve the first, look at multiprocessing, the second is threading the third is slightly more complicated to get your head around but will yield the best results, the go to library for event driven code in Python is twisted but there are others like
gevent
tulip
tornado
And so so many more that I haven't listed here.

here's an full example of implementing a threaded server. it's fully functional and comes with the benefit of using SSL as well. further, i use threaded event objects to signal another class object after storing my received data in a database.
please note, _sni and _cams_db are additional modules purely of my own. if you want to see the _sni module (provides SNI support for pyOpenSSL), let me know.
what follows this, is a snippet from camsbot.py, there's a whole lot more that far exceeds the scope of this question. what i've built is a centralized message relay system. it listens to tcp/2345 and accepts SSL connections. each connection passes messages into the system. short lived connections will connect, pass message, and disconnect. long lived connections will pass numerous messages after connecting. messages are stored in a database and a threading.Event() object (attached to the DB class) is set to tell the bot to poll the database for new messages and relay them.
the below example shows
how to set up a threaded tcp server
how to pass information from the listener to the accept handler such as config data and etc
in addition, this example also shows
how to employ an SSL socket
how to do some basic certificate validations
how to cleanly wrap and unwrap SSL from a tcp socket
how to use poll() on the socket instead of select()
db.pending is a threading.Event() object in _cams_db.py
in the main process we start another thread that waits on the pending object with db.pending.wait(). this makes that thread wait until another thread does db.pending.set(). once it is set, our waiting thread immediately wakes up and continues to work. when our waiting thread is done, it calls db.pending.clear() and goes back to the beginning of the loop and starts waiting again with db.pending.wait()
while True:
db.pending.wait()
# after waking up, do code. for example, we wait for incoming messages to
# be stored in the database. the threaded server will call db.pending.set()
# which will wake us up. we'll poll the DB for new messages, relay them, clear
# our event flag and go back to waiting.
# ...
db.pending.clear()
snippet from camsbot.py:
import sys, os, sys, time, datetime, threading, select, logging, logging.handlers
import configparser, traceback, re, socket, hashlib
# local .py
sys.path.append('/var/vse/python')
import _util, _webby, _sni, _cams_db, _cams_threaded_server, _cams_bot
# ...
def start_courier(config):
# default values
host = '::'
port = 2345
configp = config['configp']
host = configp.get('main', 'relay msp hostport')
# require ipv6 addresses be specified in [xx:xx:xx] notation, therefore
# it is safe to look for :nnnn at the end
if ':' in host and not host.endswith(']'):
port = host.split(':')[-1]
try:
port = int(port, 10)
except:
port = 2345
host = host.split(':')[:-1][0]
server = _cams_threaded_server.ThreadedTCPServer((host, port), _cams_threaded_server.ThreadedTCPRequestHandler, config)
t = threading.Thread(target=server.serve_forever, name='courier')
t.start()
_cams_threaded_server.py:
import socket, socketserver, select, datetime, time, threading
import sys, struct
from OpenSSL.SSL import SSLv23_METHOD, SSLv3_METHOD, TLSv1_METHOD, OP_NO_SSLv2
from OpenSSL.SSL import VERIFY_NONE, VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, Context, Connection
from OpenSSL.SSL import FILETYPE_PEM
from OpenSSL.SSL import WantWriteError, WantReadError, WantX509LookupError, ZeroReturnError, SysCallError
from OpenSSL.crypto import load_certificate
from OpenSSL import SSL
# see note at beginning of answer
import _sni, _cams_db
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self, server_address, HandlerClass, config):
socketserver.BaseServer.__init__(self, server_address, HandlerClass)
self.address_family = socket.AF_INET6
self.connected = []
self.logger = config['logger']
self.config = config
self.socket = socket.socket(self.address_family, self.socket_type)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sc = Context(TLSv1_METHOD)
sc.set_verify(VERIFY_PEER|VERIFY_FAIL_IF_NO_PEER_CERT, _sni.verify_cb)
sc.set_tlsext_servername_callback(_sni.pick_certificate)
self.sc = sc
self.server_bind()
self.server_activate()
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
config = self.server.config
logger = self.server.logger
connected = self.server.connected
sc = self.server.sc
try:
self.peer_hostname = socket.gethostbyaddr(socket.gethostbyname(self.request.getpeername()[0]))[0]
except:
self.peer_hostname = '!'+self.request.getpeername()[0]
logger.info('peer: {}'.format(self.peer_hostname))
ssl_s = Connection(sc, self.request)
ssl_s.set_accept_state()
try:
ssl_s.do_handshake()
except:
t,v,tb = sys.exc_info()
logger.warn('handshake failed {}'.format(v))
ssl_s.setblocking(True)
self.ssl_s = ssl_s
try:
peercert = ssl_s.get_peer_certificate()
except:
peercert = False
t,v,tb = sys.exc_info()
logger.warn('SSL get peer cert failed: {}'.format(v))
if not peercert:
logger.warn('No peer certificate')
else:
acl = config['configp']['main'].get('client cn acl', '').split(' ')
cert_subject = peercert.get_subject().CN
logger.info('Looking for {} in acl: {}'.format(cert_subject,acl))
if cert_subject in acl:
logger.info('{} is permitted'.format(cert_subject))
else:
logger.warn('''client CN not approved''')
# it's ok to block here, every socket has its own thread
ssl_s.setblocking(True)
self.db = config['db']
msgcount = 0
p = select.poll()
# don't want writable, just readable
p.register(self.request, select.POLLIN|select.POLLPRI|select.POLLERR|select.POLLHUP|select.POLLNVAL)
peername = ssl_s.getpeername()
x = peername[0]
if x.startswith('::ffff:'):
x = x[7:]
peer_ip = x
try:
host = socket.gethostbyaddr(x)[0]
except:
host = peer_ip
logger.info('{}/{}:{} connected'.format(host, peer_ip, peername[1]))
connected.append( [host, peername[1]] )
if peercert:
threading.current_thread().setName('{}/port={}/CN={}'.format(host, peername[1], peercert.get_subject().CN))
else:
threading.current_thread().setName('{}/port={}'.format(host, peername[1]))
sockclosed = False
while not sockclosed:
keepreading = True
#logger.debug('starting 30 second timeout for poll')
pe = p.poll(30.0)
if not pe:
# empty list means poll timeout
# for SSL sockets it means WTF. we get an EAGAIN like return even if the socket is blocking
continue
logger.debug('poll indicates: {}'.format(pe))
#define SSL_NOTHING 1
#define SSL_WRITING 2
#define SSL_READING 3
#define SSL_X509_LOOKUP 4
while keepreading and not sockclosed:
data,sockclosed,keepreading = self._read_ssl_data(2, head=True)
if sockclosed or not keepreading:
time.sleep(5)
continue
plen = struct.unpack('H', data)[0]
data,sockclosed,keepreading = self._read_ssl_data(plen)
if sockclosed or not keepreading:
time.sleep(5)
continue
# send thank you, ignore any errors since we appear to have gotten
# the message
try:
self.ssl_s.sendall(b'ty')
except:
pass
# extract the timestamp
message_ts = data[0:8]
msgtype = chr(data[8])
message = data[9:].decode()
message_ts = struct.unpack('d', message_ts)[0]
message_ts = datetime.datetime.utcfromtimestamp(message_ts).replace(tzinfo=datetime.timezone.utc)
self.db.enqueue(config['group'], peer_ip, msgtype, message, message_ts)
self.db.pending.set()
# we're recommended to use the return socket object for any future operations rather than the original
try:
s = ssl_s.unwrap()
s.close()
except:
pass
connected.remove( [host, peername[1]] )
t_name = threading.current_thread().getName()
logger.debug('disconnect: {}'.format(t_name))
def _read_ssl_data(self, wantsize=16384, head=False):
_w = ['WANT_NOTHING','WANT_READ','WANT_WRITE','WANT_X509_LOOKUP']
logger = self.server.logger
data = b''
sockclosed = False
keepreading = True
while len(data) < wantsize and keepreading and not sockclosed:
rlen = wantsize - len(data)
try:
w,wr = self.ssl_s.want(),self.ssl_s.want_read()
#logger.debug(' want({}) want_read({})'.format(_w[w],wr))
x = self.ssl_s.recv(rlen)
#logger.debug(' recv(): {}'.format(x))
if not ( x or len(x) ):
raise ZeroReturnError
data += x
if not (len(x) == len(data) == wantsize):
logger.info(' read={}, len(data)={}, plen={}'.format(len(x),len(data),wantsize))
except WantReadError:
# poll(), when ready, read more
keepreading = False
logger.info(' got WantReadError')
continue
except WantWriteError:
# poll(), when ready, write more
keepreading = False
logger.info(' got WantWriteError')
continue
except ZeroReturnError:
# socket got closed, a '0' bytes read also means the same thing
keepreading = False
sockclosed = True
logger.info(' ZRE, socket closed normally')
continue
except SysCallError:
keepreading = False
sockclosed = True
t,v,tb = sys.exc_info()
if v.args[0] == -1: # normal EOF
logger.info(' EOF found, keepreading=False')
else:
logger.info('{} terminated session abruptly while reading plen'.format(self.peer_hostname))
logger.info('t: {}'.format(t))
logger.info('v: {}'.format(v))
continue
except:
t,v,tb = sys.exc_info()
logger.warning(' fucked? {}'.format(v))
raise
if not head and not len(data) == wantsize:
logger.warn(' short read {} of {}'.format(len(data), wantsize))
return data,sockclosed,keepreading

let's start with a bare bones threaded tcp server.
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self, server_address, HandlerClass):
socketserver.BaseServer.__init__(self, server_address, HandlerClass)
self.address_family = socket.AF_INET
self.socket = socket.socket(self.address_family, self.socket_type)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_bind()
self.server_activate()
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
# self.request is your accepted socket, do all your .read() and .wirte() on it
s = self.request
request = s.read(1024)
# decide locked or unlocked. this example arbitrarily writes back 'locked'
s.write('locked')
# we're done, close the socket and exit with a default return of None
s.close()
ok, start your threaded server with this in your main() function:
server = threading.ThreadedTCPServer(('127.0.0.1', 1234), ThreadedTCPRequestHandler)
t = threading.Thread(target=server.serve_forever, name='optional_name')
t.start()
now you can let the threading module handle the semantics of concurrency and not worry about it.

You might want to take a look at 0MQ and concurrent.futures. 0MQ has a Tornado event loop in the library and it reduces the complexity of socket programming. concurrent.futures is a high level interface over threading or multiprocessing.

You can see different concurrent server approaches at
https://bitbucket.org/arco_group/upper/src
These will help you to choose the better way for you.
Cheers

Why does gevent.socket break multiprocessing.connection's auth

I have an application that uses both grequests and multiprocessing.managers for a combination of IPC communication and Asynchronous RESTful communications over HTTP.
It seems that grequests, in using gevent.monkey's patch_all() method, breaks the multiprocessing.connection module used by the multiprocessing.manager.SyncManager class and its derivatives.
This is apparently not an isolated issue, but affects any use case that implements multiprocessing.connetion, such as multiprocessing.pool, for example.
Drilling down into the code in gevent/monkey.py, I found that the swapping of the stdlib socket module with gevent.socket is what causes the breakage.
This can be found at line 115 in gevent/monkey.py under the patch_socket() function:
def patch_socket(dns=True, aggressive=True):
"""Replace the standard socket object with gevent's cooperative sockets.
...
_socket.socket = socket.socket # This line breaks multiprocessing.connection!
...
My question is then why does this swappage break multiprocessing.connection, and what advantages are derived from using gevent.socket instead of the stdlib's socket module? That is, what performance loss, if any, will I incur from not patching the socket module?
Traceback
Traceback (most recent call last):
File "clientWithGeventMonkeyPatch.py", line 49, in <module>
client = GetClient(host, port, authkey)
File "clientWithGeventMonkeyPatch.py", line 39, in GetClient
client.connect()
File "/usr/lib/python2.7/multiprocessing/managers.py", line 500, in connect
conn = Client(self._address, authkey=self._authkey)
File "/usr/lib/python2.7/multiprocessing/connection.py", line 175, in Client
answer_challenge(c, authkey)
File "/usr/lib/python2.7/multiprocessing/connection.py", line 414, in answer_challenge
response = connection.recv_bytes(256) # reject large message
IOError: [Errno 11] Resource temporarily unavailable
code to reproduce the error
(on ubuntu server 11.10, python2.7.3, with gevent, greenlet, and grequests installed)
manager.py
## manager.py
import multiprocessing
import multiprocessing.managers
import datetime
class LocalManager(multiprocessing.managers.SyncManager):
def __init__(self, *args, **kwargs):
multiprocessing.managers.SyncManager.__init__(self, *args, **kwargs)
self.__type__ = 'LocalManager'
def GetManager(host, port, authkey):
def getdatetime():
return '{}'.format(datetime.datetime.now())
LocalManager.register('getdatetime', callable = getdatetime)
manager = LocalManager(address = (host, port), authkey = authkey)
manager.start()
return manager
if __name__ == '__main__':
# define our manager connection parameters
port = 55555
host = 'localhost'
authkey = 'auth1234'
# start a manager
man = GetManager(host, port, authkey)
# wait for user input to shut down
raw_input('return to shutdown')
man.shutdown()
client.py
## client.py -- this one works
import time
import multiprocessing.managers
class RemoteClient(multiprocessing.managers.SyncManager):
def __init__(self, *args, **kwargs):
multiprocessing.managers.SyncManager.__init__(self, *args, **kwargs)
self.__type__ = 'RemoteClient'
def GetClient(host, port, authkey):
RemoteClient.register('getdatetime')
client = RemoteClient(address = (host, port), authkey = authkey)
client.connect()
return client
if __name__ == '__main__':
# define our client connection parameters
port = 55555
host = 'localhost'
authkey = 'auth1234'
# start a manager
client = GetClient(host, port, authkey)
print 'connected', client
print 'client.getdatetime()', client.getdatetime()
# wait a couple of seconds, then do it again
time.sleep(2)
print 'client.getdatetime()', client.getdatetime()
# exit...
clientWithGeventMonkeyPatch.py
## clientWithGeventMonkeyPatch.py -- breaks, depending on patch_all() parameters
import time
import multiprocessing.managers
# this part is copied from grequests
# bear in mind that it doesn't actually do anything in this module.
try:
import gevent
from gevent import monkey as curious_george
from gevent.pool import Pool
except ImportError:
raise RuntimeError('Gevent is required for grequests.')
# this line causes breakage of the multiprocessing.manager connection auth method:
# Monkey-patch.
# patch_all() parameters with default values: socket=True, dns=True, time=True, select=True, thread=True, os=True, ssl=True, aggressive=True
curious_george.patch_all(thread=False, select=False) # breaks
#~ curious_george.patch_all(thread=False, select=False, socket = False) # works!
#~ curious_george.patch_all(thread=False, select=False, socket = True, aggressive = True, dns = True) # same as (thread=False, select=False); breaks
#~ curious_george.patch_all(thread=False, select=False, socket = True, aggressive = True, dns = False) # breaks
#~ curious_george.patch_all(thread=False, select=False, socket = True, aggressive = False, dns = True) # breaks
#~ curious_george.patch_all(thread=False, select=False, socket = True, aggressive = False, dns = False) # breaks
class RemoteClient(multiprocessing.managers.SyncManager):
def __init__(self, *args, **kwargs):
multiprocessing.managers.SyncManager.__init__(self, *args, **kwargs)
self.__type__ = 'RemoteClient'
def GetClient(host, port, authkey):
RemoteClient.register('getdatetime')
client = RemoteClient(address = (host, port), authkey = authkey)
client.connect()
return client
if __name__ == '__main__':
# define our client connection parameters
port = 55555
host = 'localhost'
authkey = 'auth1234'
# start a manager
client = GetClient(host, port, authkey)
print 'connected', client
print 'client.getdatetime()', client.getdatetime()
# wait a couple of seconds, then do it again
time.sleep(2)
print 'client.getdatetime()', client.getdatetime()
# exit...

If you don't patch the socket module, gevent's ability to not block on network operations won't be available, and thus most of the benefit of using gevent in the first place won't be available.
gevent and multiprocessing aren't really designed to play nicely with one another - gevent mostly assumes that you're doing your network connections through it, and not bypassing the highest level Python socket interfaces (which multiprocessing does).

Very simple python client/server working, but strange delays

I am trying to learn how to use sockets and a useful asynchronous backend. I've started in python with asyncore. After reading various online posts I've written a very simple chat server and connection client, reproduced below.
It seems to work. I open a python interactive session and type
> import chatserver
> server = chatserver.EchoServer('localhost', 7667)
> server.serve()
Then I open another IPython interactive session and type
> import chatserver
> cxn = chatserver.Connection()
> cxn._connect('localhost', 7667)
When I do that, I get a log output in the server window indicating that a connection has been made. Good. Then I type
> cxn.say('hi')
Nothing happens for a while, and then log messages show up for the server and client as expected.
Why is this delay ocurring?
Am I using the log functionality correctly?
I used threading to make it so that I could use the interactive session while the asyncore loop does it's thing for the Connection. Did I do this in a reasonable way?
(optional) If I don't include the line self.out_buffer="" in the Connection._connect function I get an error saying that .out_buffer does not exist. What's up with this?
import asyncore
import socket
import logging
import threading
logging.basicConfig(level=logging.DEBUG, format="%(created)-15s %(msecs)d %(levelname)8s %(thread)d %(name)s %(message)s")
log = logging.getLogger(__name__)
class Connection(asyncore.dispatcher_with_send):
def __init__(self):
asyncore.dispatcher.__init__(self)
def _connect(self, host, port, timeout=5, password=None):
self.host = host
self.port = port
self.out_buffer=""
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
#Run the asyncore loop in its own thread so that we can use the interactive session
self.loop = threading.Thread(target=asyncore.loop)
self.loop.daemon = True
self.loop.start()
def say(self, msg):
self.out_buffer = msg
def handle_read(self):
data = self.recv(4096)
log.debug('Received %s'%data)
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
log.debug("handle_read")
data = self.recv(1024)
log.debug("after recv")
if data:
log.debug("got data: %s"%data)
self.out_buffer = data
else:
log.debug("got null data")
class EchoServer(asyncore.dispatcher):
SOCKET_TYPE = socket.SOCK_STREAM
ADDRESS_FAMILY = socket.AF_INET
def __init__(self, host, port):
self.address = (host,port)
asyncore.dispatcher.__init__(self)
self.create_socket(self.ADDRESS_FAMILY, self.SOCKET_TYPE)
log.debug("bind address=%s %s"%(host,port))
self.bind(self.address)
self.listen(1)
def fileno(self):
return self.socket.fileno()
def serve(self):
asyncore.loop()
#Start asyncore loop in new thread
# self.loop = threading.Thread(target=asyncore.loop)
# self.loop.daemon = True
# self.loop.start()
def handle_accept(self):
"""Deal with a newly accepted client"""
(connSock, clientAddress) = self.accept()
log.info("conn made: clientAddress=%s %s"%(clientAddress[0], clientAddress[1]))
#Make a handler for this connection
EchoHandler(connSock)
def handle_close(self):
self.close()

Looking at the asyncore docs you are relying on asyncore.dispatcher_with_send to call send() and the default timeout for asyncore.loop() is 30 seconds. This may explain the delay.

It turns out the problem was as Eero suggested.
I made two changes:
In EchoServer
asyncore.loop() to asyncore.loop(timeout=0.1)
In Connection
self.loop = threading.Thread(target=asyncore.loop) to self.loop = threading.Thread(target=asyncore.loop, kwargs={'timeout':0.1})
The response is now much faster. This seems like a hack though so if someone can explain a way to get the same effect in a proper way please contribute.