We Keep Coding

Callback function does not see correct values in instance

I'm having a strange phenomena in Python with callback functions and handlers.
I use ZMQ to handle my communication and use a stream for the socket. I have the base class:
import multiprocessing
import zmq
from concurrent.futures import ThreadPoolExecutor
from zmq.eventloop import ioloop, zmqstream
from zmq.utils import jsonapi as json
# Types of messages
TYPE_A = 'type_a'
TYPE_B = 'type_b'
class ZmqProcess(multiprocessing.Process):
def __init__(self):
super(ZmqProcess, self).__init__()
self.context = None
self.loop = None
self.handle_stream = None
def setup(self):
self.context = zmq.Context()
self.loop = ioloop.IOLoop.instance()
def send(self, msg_type, msg, host, port):
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://%s:%s' % (host, port))
sock.send_json([msg_type, msg])
def stream(self, sock_type, addr):
sock = self.context.socket(sock_type)
if isinstance(addr, str):
addr = addr.split(':')
host, port = addr if len(addr) == 2 else (addr[0], None)
if port:
sock.bind('tcp://%s:%s' % (host, port))
else:
port = sock.bind_to_random_port('tcp://%s' % host)
stream = zmqstream.ZMQStream(sock, self.loop)
return stream, int(port)
class MessageHandler(object):
def __init__(self, json_load=-1):
self._json_load = json_load
self.pool = ThreadPoolExecutor(max_workers=10)
def __call__(self, msg):
i = self._json_load
msg_type, data = json.loads(msg[i])
msg[i] = data
if msg_type.startswith('_'):
raise AttributeError('%s starts with an "_"' % msg_type)
getattr(self, msg_type)(*msg)
And I have a class that inherits from it:
import zmq
import zmq_base
class ZmqServerMeta(zmq_base.ZmqProcess):
def __init__(self, bind_addr, handlers):
super(ZmqServerMeta, self).__init__()
self.bind_addr = bind_addr
self.handlers = handlers
def setup(self):
super(ZmqServerMeta, self).setup()
self.handle_stream, _ = self.stream(zmq.PAIR, self.bind_addr)
self.handle_stream.on_recv(StreamHandler(self.handle_stream, self.stop,
self.handlers))
def run(self):
self.setup()
self.loop.start()
def stop(self):
self.loop.stop()
class StreamHandler(zmq_base.MessageHandler):
def __init__(self, handle_stream, stop, handlers):
super(StreamHandler, self).__init__()
self._handle_stream = handle_stream
self._stop = stop
self._handlers = handlers
def type_a(self, data):
if zmq_base.TYPE_A in self._handlers:
if self._handlers[zmq_base.TYPE_A]:
for handle in self._handlers[zmq_base.TYPE_A]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def type_b(self, data):
if zmq_base.TYPE_B in self._handlers:
if self._handlers[zmq_base.TYPE_B]:
for handle in self._handlers[zmq_base.TYPE_B]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def endit(self):
self._stop()
Additionally, I have a class that I want to use as storage. And here is where the trouble starts:
import threading
import zmq_server_meta as server
import zmq_base as base
class Storage:
def __init__(self):
self.list = []
self.list_lock = threading.RLock()
self.zmq_server = None
self.host = '127.0.0.1'
self.port = 5432
self.bind_addr = (self.host, self.port)
def setup(self):
handlers = {base.TYPE_A: [self. remove]}
self.zmq_server = server.ZmqServerMeta(handlers=handlers, bind_addr=self.bind_addr)
self.zmq_server.start()
def add(self, data):
with self.list_lock:
try:
self.list.append(data)
except:
print "Didn't work"
def remove(self, msg):
with self.list_lock:
try:
self.list.remove(msg)
except:
print "Didn't work"
The idea is that that class stores some global information that it receives.
It is all started in a file to test:
import sys
import time
import storage
import zmq_base as base
import zmq_server_meta as server
def printMsg(msg):
print msg
store = storage.Storage()
store.setup()
handlers = {base.TYPE_B: [printMsg]}
client = server.ZmqServerMeta(handlers=handlers, bind_addr=('127.0.0.1', 5431))
client.start()
message = "Test"
store.add(message)
client.send(base.TYPE_A, message, '127.0.0.1', 5432)
I simplified it to reduce clutter. Instead of just adding it, it is usually send and then a response comes back. The response, the client sending, should be processed by the correct callback, remove(), and it should remove something out of the list. The problem that occurs is, that the remove() function sees an empty list, although there should be an element in the list. If I check from the testing file, I can see the element after it was added, and if I call remove() from there, I see a non-empty list and can remove it. My question is, why does the callback sees an empty list and how can I make sure it does see the correct elements in the list?
Kind regards
Patrick

I believe the problem lays in the fact that the ZmqProcess class inherits from multiprocessing.Process. Multiprocessing does not allow to share objects among different processes, except by using a shared memory map using Value or Array ( as can be seen in the documentation: https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes )
If you want to use your custom object, you can use a Server process / proxy object, which can be found in on the same page of the documentation.
So you can, for instance, define a manager in the init function of the Storage class like: self.manager = Manager() Afterwards you put self.list = self.manager.list(). This should do the trick.

Asynchronous client in Twisted not sending / receiving request (using NetStringReceiver)

I am trying to do asynchronous programming using Twisted. What I'm trying to do is create a client which will pass query parameters (in my case, hash type and hash value) and wait for the server's response. I am currently using NetstringReceiver for the parameters. However, am having these problems:
The client is not able to send its request to the server, and
The client hangs forever when I ran it. It seems that there is a callback that's not returning.
Below are the codes for the client and server.
This code is actually based on this tutorial by Dave Peticolas.
Client Code
import os, sys, argparse
from twisted.internet import defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.protocols.basic import NetstringReceiver
class QueryProtocol(Protocol):
response = ''
def dataReceived(self, data):
self.response = data
def connectionLost(self, reason):
self.responseReceived(self.response)
def responseReceived(self, response):
self.factory.response_finished(response)
class QueryFactory(ClientFactory):
protocol = QueryProtocol
def __init__(self, deferred):
self.deferred = deferred
def response_finished(self, response):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionFailed(self, connector, reason):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
class QueryNetProtocol(NetstringReceiver):
def connectionMade(self):
self.sendRequest(self.factory.hash_type, self.factory.hash_value)
def sendRequest(self, hash_type, hash_value):
self.sendString(hash_type + '.' + hash_value)
def stringReceived(self, s):
self.transport.loseConnection()
self.responseReceived(s)
def responseReceived(self, response):
self.factory.handleResponse(response)
class QueryNetFactory(ClientFactory):
protocol = QueryNetProtocol
def __init__(self, hash_type, hash_value):
self.hash_type = hash_type
self.hash_value = hash_value
self.deferred = defer.Deferred()
def handleResponse(self, response):
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionLost(self, _, reason):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
clientConnectionFailed = clientConnectionLost
class QueryProxy(object):
def __init__(self, host, port):
self.host = host
self.port = port
def query(self, hash_type, hash_value):
factory = QueryNetFactory(hash_type, hash_value)
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, factory)
return factory.deferred
def perform_query(host, port):
d = defer.Deferred()
from twisted.internet import reactor
factory = QueryFactory(d)
reactor.connectTCP(host, port, factory)
return d
def main(options):
done = False
query_result = ""
host = options.host
port = int(options.port)
sha1 = options.sha1
proxy = QueryProxy(host, port)
from twisted.internet import reactor
def process_query_result(response):
d = proxy.query('sha1', sha1)
def fail(err):
print "Problem in processing response : %s" % err
return response
return d.addErrback(fail)
def query_ok(response):
query_result = response
done = True
def query_failed(err):
print "Problem in query : %s" % err
done = True
def query_done(_):
if done == True: reactor.stop()
d = perform_query(host, port)
d.addCallback(process_query_result)
d.addCallbacks(query_ok, query_failed)
d.addBoth(query_done)
reactor.run()
print "The result of the query is : %s" % query_result
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("host", help="server host/ip")
parser.add_argument("port", help="server port number to listen to")
parser.add_argument("-sha1", help="sha1 value to be queried")
options = parser.parse_args()
main(options)
Server Code
import os, sys, argparse
from twisted.internet.protocol import ServerFactory
from twisted.protocols.basic import NetstringReceiver
class GridQueryService(object):
def query(self, hash_type, hash_value):
print "this is the query service. Type is %s and value is %s" % (hash_type, hash_value)
return hash_value
class GridQueryProtocol(NetstringReceiver):
def stringReceived(self, request):
print >>sys.stderr, request
if '.' not in request:
self.transport.loseConnection()
return
hash_type, hash_value = request.split('.')
self.formRequestReceived(hash_type, hash_value)
def formRequestReceived(self, hash_type, hash_value):
found_flag = self.factory.query(hash_type, hash_value)
if found_flag: self.sendString(str(found_flag))
self.transport.loseConnection()
class GridQueryFactory(ServerFactory):
protocol = GridQueryProtocol
def __init__(self, service):
self.service = service
def query(self, hash_type, hash_value):
return self.service.query(hash_type, hash_value)
def main(options):
grid_query_service = GridQueryService()
grid_query_factory = GridQueryFactory(grid_query_service)
from twisted.internet import reactor
port = reactor.listenTCP(int(options.port), grid_query_factory, interface=options.host)
print "Serving GRID query service on %s" % str(port.getHost())
reactor.run()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("host", help="server host/ip")
parser.add_argument("port", help="server port number to listen to")
options = parser.parse_args()
main(options)
Any ideas on how to resolve this? Thanks in advance. Help is greatly appreciated!

Your server may use netstrings and your client may contain code for using netstrings, but your client doesn't use netstrings.
main calls perform_query. perform_query makes a QueryFactory which connects a QueryProtocol which never sends any data and doesn't have any logic related to handling netstrings even if it did.

I've updated my client. Revised client code is below.
I am now able to send request. Here are the outputs:
Sent request!
String received!
Handling response!
I'm called!
Connection lost! - QueryNetFactory
As you can see, the callback process_query_result callback has been fired. However, the other callbacks were not and I can't still receive the data/result using the QueryFactory/QueryProtocol. I have a suspect that it's about the deferred and callbacks, but I'm quite confused.
In initializing the QueryFactory class, should I really create a new deferred? If yes, how will it realize/know the callbacks added to the original deferred (the one that was returned by the perform_query method)? Now if in case I should not create a new deferred, how can I fire the callbacks added in the original deferred in the 'responseReceived' method of the QueryProtocol?
Client Code
class QueryProtocol(Protocol):
response = ''
def dataReceived(self, data):
print "Data received!"
self.response = data
def connectionLost(self, reason):
print "Connection lost!"
self.responseReceived(self.response)
def responseReceived(self, response):
print "Response received!"
self.factory.response_finished(response)
class QueryFactory(ClientFactory):
protocol = QueryProtocol
def __init__(self):
self.deferred = defer.Deferred()
def response_finished(self, response):
print "Response finished!"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionFailed(self, connector, reason):
print "Client connection failed! - QueryFactory"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
class QueryNetProtocol(NetstringReceiver):
def connectionMade(self):
self.sendRequest(self.factory.hash_type, self.factory.hash_value)
def sendRequest(self, hash_type, hash_value):
print "Sent request!"
self.sendString(hash_type + '.' + hash_value)
def stringReceived(self, s):
print "String received!"
self.transport.loseConnection()
self.responseReceived(s)
def responseReceived(self, response):
self.factory.handleResponse(response)
class QueryNetFactory(ClientFactory):
protocol = QueryNetProtocol
def __init__(self, deferred, hash_type, hash_value):
self.hash_type = hash_type
self.hash_value = hash_value
self.deferred = deferred
def handleResponse(self, response):
print "Handling response!"
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionLost(self, _, reason):
print "Connection lost! - QueryNetFactory"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
clientConnectionFailed = clientConnectionLost
class QueryProxy(object):
def __init__(self, host, port):
self.host = host
self.port = port
def query(self):
factory = QueryFactory()
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, factory)
return factory.deferred
def perform_query(host, port, hash_type, hash_value):
d = defer.Deferred()
from twisted.internet import reactor
factory = QueryNetFactory(d, hash_type, hash_value)
reactor.connectTCP(host, port, factory)
return d
def main(options):
done = False
query_result = ""
host = options.host
port = int(options.port)
sha1 = options.sha1
proxy = QueryProxy(host, port)
from twisted.internet import reactor
def process_query_result(response):
print "I'm called!"
d = proxy.query()
def fail(err):
print "Process query result failure : %s" % err
return d.addErrback(fail)
def query_ok(response):
print "query ok!"
query_result = response
done = True
def query_failed(err):
print "Problem in query : %s" % err
done = True
def query_done(_):
if done == True: reactor.stop()
d = perform_query(host, port, "sha1", sha1)
d.addCallback(process_query_result)
d.addCallbacks(query_ok, query_failed)
d.addBoth(query_done)
reactor.run()
print "The result of the query is : %s" % query_result
Again, any help is greatly appreciated! Thanks!

TCP client not able to write to socket but receive works fine

I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?

You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.

zmq PUBhandler could not be used in __init__() method

I have a class MyLogger for sending messages to log server by using PUBhandler.
An exception gets raised when MyLogger is instanced in LogWorker.init() method (like version 1), however, it is ok if MyLogger is instanced in LogWorker.log_worker() method (version 2).
Any suggestions would be appreciated.
import logging
from multiprocessing import Process
import os
import random
import sys
import time
import zmq
from zmq.log.handlers import PUBHandler
class MyLogger(object):
''''''
def __init__(self, port, handler=None):
self.port = port
self.handler = handler or self._construct_sock_handler()
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
if not self.logger.handlers:
self.logger.addHandler(self.handler)
def _construct_sock_handler(self):
context = zmq.Context()
log_sock = context.socket(zmq.PUB)
log_sock.connect("tcp://127.0.0.1:%i" % self.port)
time.sleep(0.1)
handler = PUBHandler(log_sock)
return handler
def get_logger(self):
return self.logger
def sub_logger(port, level=logging.DEBUG):
ctx = zmq.Context()
sub = ctx.socket(zmq.SUB)
sub.bind('tcp://127.0.0.1:%i' % port)
sub.setsockopt(zmq.SUBSCRIBE, "")
logging.basicConfig(level=level)
while True:
level, message = sub.recv_multipart()
if message.endswith('\n'):
# trim trailing newline, which will get appended again
message = message[:-1]
log = getattr(logging, level.lower())
log(message)
class LogWorker(object):
def __init__(self):
- pass # version 1
+ self.logger = MyLogger(port).get_logger() # version 2
def log_worker(self, port):
- self.logger = MyLogger(port).get_logger() # version 1
print "starting logger at %i with level=%s" % (os.getpid(), logging.DEBUG)
while True:
level = logging.INFO
self.logger.log(level, "Hello from %i!" % os.getpid())
time.sleep(1)
if __name__ == '__main__':
if len(sys.argv) > 1:
n = int(sys.argv[1])
else:
n = 2
port = 5555
workers = [Process(target=LogWorker().log_worker, args=(port,)) for _ in range(n)]
[w.start() for w in workers]
try:
sub_logger(port)
except KeyboardInterrupt:
pass
finally:
[ w.terminate() for w in workers ]

answer from pyzmq owner minrk:
You cannot pass zmq contexts or sockets across the fork boundary that happens when you instantiate a subprocess with multiprocessing. You have to make sure that you create your Context after you are in the subprocess.
solution:
def work():
worker = LogWorker(port)
worker.log_worker()
workers = [ Process(target=work) for _ in range(n) ]

PyQT4 Asynchronous QTcpServer (How to create event loop)

Trying to convert a server written in C++ into Python. The server was written to be Asynchronous/Non Blocking. What works in C++ doesn't seem to want to work for me in Python
I am using PyQT4. I read Python you have to create the event loop or something along those lines any ideas are greatly appreciated
I should mention what seems to not work is that the incomingConnection function in Class Server is never called.
*cheers
import sys
from PyQt4.QtCore import *
from PyQt4.QtNetwork import *
class Client(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
QThreadPool.globalInstance().setMaxThreadCount(15)
def SetSocket(self, Descriptor):
self.socket = QTcpSocket(self)
self.connect(self.socket, SIGNAL("connected()"), SLOT(self.connected()))
self.connect(self.socket, SIGNAL("disconnected()"), SLOT(self.disconnected()))
self.connect(self.socket, SIGNAL("readyRead()"), SLOT(self.readyRead()))
self.socket.setSocketDescriptor(Descriptor)
print "Client Connected from IP %s" % self.socket.peerAddress().toString()
def connected(self):
print "Client Connected Event"
def disconnected(self):
print "Client Disconnected"
def readyRead(self):
msg = self.socket.readAll()
print msg
class Server(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
def incomingConnection(self, handle):
print "incoming"
self.client = Client(self)
self.client.SetSocket(handle)
def StartServer(self):
self.server = QTcpServer()
if self.server.listen(QHostAddress("0.0.0.0"), 8888):
print "Server is awake"
else:
print "Server couldn't wake up"
def main():
app = QCoreApplication(sys.argv)
Server().StartServer()
sys.exit(app.exec_())
if __name__ == '__main__':
main()

incomingConnection is not called because QTcpServer's base implementation of the function is called. as incomingConnection is a vitual function, you just have to assign your's to QTcpServer's incomingConnection attribute, like this:
class Server(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
def incomingConnection(self, handle):
print "incoming"
self.client = Client(self)
self.client.SetSocket(handle)
def StartServer(self):
self.server = QTcpServer()
self.server.incomingConnection = self.incomingConnection
if self.server.listen(QHostAddress("0.0.0.0"), 8888):
print "Server is awake"
else:
print "Server couldn't wake up"
you can check out PySide's documentation, as it's much more pythonic than PyQt's, currently hosted only here:
http://srinikom.github.com/pyside-docs/