I'm having a strange phenomena in Python with callback functions and handlers.
I use ZMQ to handle my communication and use a stream for the socket. I have the base class:
import multiprocessing
import zmq
from concurrent.futures import ThreadPoolExecutor
from zmq.eventloop import ioloop, zmqstream
from zmq.utils import jsonapi as json
# Types of messages
TYPE_A = 'type_a'
TYPE_B = 'type_b'
class ZmqProcess(multiprocessing.Process):
def __init__(self):
super(ZmqProcess, self).__init__()
self.context = None
self.loop = None
self.handle_stream = None
def setup(self):
self.context = zmq.Context()
self.loop = ioloop.IOLoop.instance()
def send(self, msg_type, msg, host, port):
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://%s:%s' % (host, port))
sock.send_json([msg_type, msg])
def stream(self, sock_type, addr):
sock = self.context.socket(sock_type)
if isinstance(addr, str):
addr = addr.split(':')
host, port = addr if len(addr) == 2 else (addr[0], None)
if port:
sock.bind('tcp://%s:%s' % (host, port))
else:
port = sock.bind_to_random_port('tcp://%s' % host)
stream = zmqstream.ZMQStream(sock, self.loop)
return stream, int(port)
class MessageHandler(object):
def __init__(self, json_load=-1):
self._json_load = json_load
self.pool = ThreadPoolExecutor(max_workers=10)
def __call__(self, msg):
i = self._json_load
msg_type, data = json.loads(msg[i])
msg[i] = data
if msg_type.startswith('_'):
raise AttributeError('%s starts with an "_"' % msg_type)
getattr(self, msg_type)(*msg)
And I have a class that inherits from it:
import zmq
import zmq_base
class ZmqServerMeta(zmq_base.ZmqProcess):
def __init__(self, bind_addr, handlers):
super(ZmqServerMeta, self).__init__()
self.bind_addr = bind_addr
self.handlers = handlers
def setup(self):
super(ZmqServerMeta, self).setup()
self.handle_stream, _ = self.stream(zmq.PAIR, self.bind_addr)
self.handle_stream.on_recv(StreamHandler(self.handle_stream, self.stop,
self.handlers))
def run(self):
self.setup()
self.loop.start()
def stop(self):
self.loop.stop()
class StreamHandler(zmq_base.MessageHandler):
def __init__(self, handle_stream, stop, handlers):
super(StreamHandler, self).__init__()
self._handle_stream = handle_stream
self._stop = stop
self._handlers = handlers
def type_a(self, data):
if zmq_base.TYPE_A in self._handlers:
if self._handlers[zmq_base.TYPE_A]:
for handle in self._handlers[zmq_base.TYPE_A]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def type_b(self, data):
if zmq_base.TYPE_B in self._handlers:
if self._handlers[zmq_base.TYPE_B]:
for handle in self._handlers[zmq_base.TYPE_B]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def endit(self):
self._stop()
Additionally, I have a class that I want to use as storage. And here is where the trouble starts:
import threading
import zmq_server_meta as server
import zmq_base as base
class Storage:
def __init__(self):
self.list = []
self.list_lock = threading.RLock()
self.zmq_server = None
self.host = '127.0.0.1'
self.port = 5432
self.bind_addr = (self.host, self.port)
def setup(self):
handlers = {base.TYPE_A: [self. remove]}
self.zmq_server = server.ZmqServerMeta(handlers=handlers, bind_addr=self.bind_addr)
self.zmq_server.start()
def add(self, data):
with self.list_lock:
try:
self.list.append(data)
except:
print "Didn't work"
def remove(self, msg):
with self.list_lock:
try:
self.list.remove(msg)
except:
print "Didn't work"
The idea is that that class stores some global information that it receives.
It is all started in a file to test:
import sys
import time
import storage
import zmq_base as base
import zmq_server_meta as server
def printMsg(msg):
print msg
store = storage.Storage()
store.setup()
handlers = {base.TYPE_B: [printMsg]}
client = server.ZmqServerMeta(handlers=handlers, bind_addr=('127.0.0.1', 5431))
client.start()
message = "Test"
store.add(message)
client.send(base.TYPE_A, message, '127.0.0.1', 5432)
I simplified it to reduce clutter. Instead of just adding it, it is usually send and then a response comes back. The response, the client sending, should be processed by the correct callback, remove(), and it should remove something out of the list. The problem that occurs is, that the remove() function sees an empty list, although there should be an element in the list. If I check from the testing file, I can see the element after it was added, and if I call remove() from there, I see a non-empty list and can remove it. My question is, why does the callback sees an empty list and how can I make sure it does see the correct elements in the list?
Kind regards
Patrick
I believe the problem lays in the fact that the ZmqProcess class inherits from multiprocessing.Process. Multiprocessing does not allow to share objects among different processes, except by using a shared memory map using Value or Array ( as can be seen in the documentation: https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes )
If you want to use your custom object, you can use a Server process / proxy object, which can be found in on the same page of the documentation.
So you can, for instance, define a manager in the init function of the Storage class like: self.manager = Manager() Afterwards you put self.list = self.manager.list(). This should do the trick.
Related
I have following code for a Publisher, which instantiates a few class instances and publishes some messages.
However, I don't receive anything at Subscriber side.
Publisher
import zmq
import time
from multiprocessing import Process
class SendData:
def __init__(self, msg, port):
self.msg = msg
self.port = port
ctx = zmq.Context()
self.sock = ctx.socket(zmq.PUB)
self.sock.bind('tcp://127.0.0.1:'+str(self.port))
time.sleep(1)
def sender(self):
self.sock.send_json(self.msg)
def main():
for device, port in zip(['2.2.2.2', '5.5.5.5'],[5001, 5002]):
msg = {device:'Some random message'}
instance = SendData(device, port)
Process(target=instance.sender).start()
if __name__ == "__main__":
main()
Subscriber
import zmq
ctx = zmq.Context()
recv_sock1 = ctx.socket(zmq.SUB)
recv_sock1.connect('tcp://127.0.0.1:5001')
recv_sock1.setsockopt(zmq.SUBSCRIBE, '')
recv_sock2 = ctx.socket(zmq.SUB)
recv_sock2.connect('tcp://127.0.0.1:5002')
recv_sock2.setsockopt(zmq.SUBSCRIBE, '')
while True:
if recv_sock1.poll(10):
msg = recv_sock1.recv_json()
print msg
if recv_sock2.poll(10):
msg = recv_sock2.recv_json()
print msg
I had subscribers started before publishers could publish anything. Also, I can see TCP connections are in Established so connections are made.
pyzmq version 16.0.0
python version 2.7
Q1: Are 0mq publishers supported from class instances?
Q2: What am I missing?
As was said before, trying to share the ZeroMQ context between processes is the problem here and the solution by user3666197 will work.
However, I would suggest subclassing multiprocessing.Process in this case. That way, it is much clearer what part of the code is executed in which process. It also makes your code more readable and reusable.
The following code creates one sender process per device. The sender processes can be reused during the runtime of your program to send more data:
import multiprocessing
import queue
import zmq
import time
class Sender(multiprocessing.Process):
def __init__(self, port):
super(Sender, self).__init__()
self._port = port
self._messages = multiprocessing.Queue()
self._do_stop = multiprocesing.Event()
def run(self):
"""
This code is executed in a new process.
"""
ctx = zmq.Context()
sock = ctx.socket(zmq.PUB)
sock.bind("tcp://127.0.0.1:" + str(self._port))
while not self._do_stop.is_set():
try:
msg = self._message.get_nowait()
sock.send_json(msg)
except queue.Empty:
time.sleep(0.01)
def stop(self):
self._do_stop.set()
def send_message(self, msg):
self._messages.put(msg)
def main():
data = zip(['2.2.2.2', '5.5.5.5'], [5001, 5002])
# create senders
senders = {device: Sender(port) for device, port in data}
# start senders
for device in senders:
senders[device].start()
# send messages
for device, port in zip(['2.2.2.2', '5.5.5.5'],[5001, 5002]):
msg = {device: 'Some random message'}
senders[device].send_message(msg)
# do more stuff here....
# ... e.g. send more messages
# ...
# once you are finished, stop the subprocesses
for device in senders:
senders[device].stop()
I hope this helps solving your problem.
A1: Yes, they are.
A2: Conflicts of scope-of-use v/s Zero-sharing, one of ZeroMQ maxims
Once your original Publisher code is being executed in main(), the class instantiation process creates ( i.e. inside the main()-process scope-of-use ), via the .__init__() constructor-method, it's own Context() -instance, which thus belongs ( incl. all of it's derived child-objects ( sockets et al ) ) to this main()-process.
Next, the call to the Process(...) initiates another few processes, that receive the class-instances ( the pity is that these have already created ZeroMQ non-shareable toys ) from the main()-scope-of-use.
Solution?
A possible dirty quick hack could be to defer the ZeroMQ Context() instantiation -- yes, just move it from .__init__() to some .aDeferredSETUP() that will be executed specifically under different scope-of-use inside each of the spinned-of Process()-process, different from main()-process and you ought be done, as Zero-sharing is safely obeyed.
class SendData:
def __init__(self, msg, port):
self.msg = msg
self.port = port
self.NotSETUP = True
self.ctx = None
self.sock = None
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ L8R
# ctx = zmq.Context()
# self.sock = ctx.socket( zmq.PUB )
# self.sock.bind( 'tcp://127.0.0.1:' + str( self.port ) )
# time.sleep( 1 )
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ L8R
def sender( self ):
if self.NotSETUP:
self.aDeferredSETUP()
self.sock.send_json( self.msg )
def aDeferredSETUP( self ): # create I/O-threads in Process(), not main()
self.ctx = zmq.Context()
self.sock = self.ctx.socket( zmq.PUB )
self.sock.bind( 'tcp://127.0.0.1:' + str( self.port ) )
time.sleep( 1 )
How to mock a TCPSocket wrapper for the socket from the Python's standard libary via the mock library (unittest.mock in case of Python 3)?
This is my wrapper:
import socket
import utils
class TCPSocket:
def __init__(self):
self.buf = ''
def __enter__(self):
pass
def __exit__(self, exc_type, exc_val, exc_tb):
self.close()
def connect(self, host, port):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((host, port))
def close(self):
self.sock.close()
def send(self, data):
self.sock.send(data)
def sendall(self, data):
self.sock.sendall(data)
# For best match with hardware and network realities,
# the value of limit should be a relatively small power of 2, for example, 4096
def recv_some(self, limit=4096):
return self.sock.recv(limit)
def recv_bytes(self, count):
while len(self.buf) < count:
self.buf += self.recv_some()
parts = utils.split_first_n(self.buf, count)
self.buf = parts[1]
return parts[0]
def recv_until(self, delim):
while delim not in self.buf:
self.buf += self.recv_some()
parts = self.buf.split(delim, maxsplit=1)
self.buf = parts[1]
return parts[0]
I want to test whether functions like recv_until and recv_bytes do what they really need.
It can be as simple as
import mock # or from unittest import mock
mock_socket = mock.Mock()
mock_socket.recv.return_value = data
then use mock_socket where you would use the real socket. You can also mock whatever creates the socket to return a mock value like the one configured here, depending on your needs.
For your case, you can mock socket.socket so that it returns something whose method you can configure. Note that mock_socket in this example is a function that returns a Socket object, not a Socket object itself.
with mock.patch('socket.socket') as mock_socket:
mock_socket.return_value.recv.return_value = some_data
t = TCPSocket()
t.connect('example.com', 12345) # t.sock is a mock object, not a Socket
self.assertEqual(t.recv_bytes(), whatever_you_expect)
t.sock.connect.assert_called_with(('example.com', 12345))
I'm trying to implement this code, in which each "node" is a self-contained "actor".
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor, defer
from twisted.protocols.basic import LineReceiver
class ChatProtocol(LineReceiver):
def __init__(self, factory):
self.factory = factory
self.name = None
self.state = "REGISTER"
def connectionMade(self):
self.sendLine("What's your name?")
def connectionLost(self, reason):
if self.name in self.factory.users:
del self.factory.users[self.name]
self.broadcastMessage("{} has left the channel.".format(self.name))
def lineReceived(self, line):
if self.state == "REGISTER":
self.handle_REGISTER(line)
else:
self.handle_CHAT(line)
def handle_REGISTER(self, name):
if name in self.factory.users:
self.sendLine("Name taken, please choose another!")
return
self.sendLine("Welcome, {}".format(name))
self.broadcastMessage("{} has joined the channel.".format(name))
self.name = name
self.factory.users[name] = self
self.state = "CHAT"
def handle_CHAT(self, message):
message = "[%s]>> %s" % (self.name, message)
self.broadcastMessage(message)
def broadcastMessage(self, message):
for name, protocol in self.factory.users.iteritems():
if protocol != self:
protocol.sendLine(message)
class ChatFactory(Factory):
"""Handle all the nodes' connection"""
def __init__(self):
self.users = {}
def buildProtocol(self, addr):
return ChatProtocol(self)
class Node:
def __init__(self, stop=None):
self.Factory = ChatFactory
self.reactor = reactor
self.d = defer.Deferred()
# with `stop` the node is bound to die
if stop:
self.reactor.callLater(stop, self.stop)
def listen(self, port):
self.reactor.listenTCP(port, self.Factory())
def run(self):
self.reactor.run()
def stop(self):
self.reactor.stop()
class Organization:
"""
An organization consists of several nodes, with one node as a leader
"""
def __init__(self):
self.nodes = []
def create_leader(self):
# create first node now with intentionally kill the leader's reactor after 5 seconds
leader_node = Node(5)
leader_node.listen(8000)
self.nodes.append(leader_node)
def create_more_nodes(self):
node_1 = Node()
node_2 = Node()
self.nodes.append(node_1)
self.nodes.append(node_2)
def activate(self):
self.nodes[1].listen(8001)
self.nodes[2].listen(8002)
"""
now leader_node listens at 8000
node_1 listens at 8001
node_2 listens at 8002
"""
# run each node's reactor
for n in self.nodes:
n.run()
if __name__ == '__main__':
org = Organization()
org.create_leader()
org.create_more_nodes()
org.activate()
After 5 seconds, the leader_node's reactor is stopped via Node.stop() that gets deferred. However, I have no idea why node_1 and node_2 listening at 8001 and 8002 were also stopped. If anyone with more experience with Twisted could point this out that'd be great!
reactor.run() means "run the entire program". While it does not forcibly terminate (the function does return), it only does so in order to allow you to clean up some state before exiting. So you should only ever run one reactor per process and exit shortly after it is finished.
If you want to have self-contained services which can shut down all of their incoming and outgoing connections as well as their listening ports, you have to track those connections in connectionMade and connectionLost. You also have to keep track of your listening port so that you can stopListening.
As a side note, listenTCP is a very low-level API that you probably shouldn't be calling directly; instead, use the high-level Endpoints API, which is much more flexible.
Here's a version of your code that tracks inbound and connections and listening ports and shuts them down as you'd like, while sharing a reactor between all the nodes.
from twisted.internet.protocol import Factory
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.protocols.basic import LineReceiver
from twisted.internet.defer import Deferred
class ChatProtocol(LineReceiver):
def __init__(self, factory):
self.factory = factory
self.name = None
self.state = "REGISTER"
def connectionMade(self):
self.factory.node.activeTransports.append(self.transport)
self.sendLine("What's your name?")
def connectionLost(self, reason):
self.factory.node.activeTransports.remove(self.transport)
if self.name in self.factory.users:
del self.factory.users[self.name]
self.broadcastMessage("{} has left the channel.".format(self.name))
def lineReceived(self, line):
if self.state == "REGISTER":
self.handle_REGISTER(line)
else:
self.handle_CHAT(line)
def handle_REGISTER(self, name):
if name in self.factory.users:
self.sendLine("Name taken, please choose another!")
return
self.sendLine("Welcome, {}".format(name))
self.broadcastMessage("{} has joined the channel.".format(name))
self.name = name
self.factory.users[name] = self
self.state = "CHAT"
def handle_CHAT(self, message):
message = "[%s]>> %s" % (self.name, message)
self.broadcastMessage(message)
def broadcastMessage(self, message):
for name, protocol in self.factory.users.iteritems():
if protocol != self:
protocol.sendLine(message)
class ChatFactory(Factory):
"""Handle all the nodes' connection"""
def __init__(self, node):
self.users = {}
self.node = node
def buildProtocol(self, addr):
return ChatProtocol(self)
class Node:
def __init__(self, endpoint, clock, stop=None):
self.Factory = ChatFactory
self._endpoint = endpoint
self._listenStarting = None
self._listeningPort = None
self.activeTransports = []
if stop is not None:
print("Scheduling stop.", stop)
clock.callLater(stop, self.stop)
def listen(self):
self._listenStarting = self._endpoint.listen(self.Factory(self))
def setPort(port):
self._listeningPort = port
def clear(whatever):
self._listenStarting = None
return whatever
self._listenStarting.addCallback(setPort).addBoth(clear)
def stop(self):
if self._listenStarting is not None:
self._listenStarting.cancel()
if self._listeningPort is not None:
self._listeningPort.stopListening()
for transport in self.activeTransports[:]:
transport.abortConnection()
class Organization:
def __init__(self, reactor):
self.reactor = reactor
self.nodes = []
def port(self, number):
return TCP4ServerEndpoint(self.reactor, number)
def create_leader(self):
leader_node = Node(self.port(8000), self.reactor, 5)
leader_node.listen()
self.nodes.append(leader_node)
def create_more_nodes(self):
node_1 = Node(self.port(8001), self.reactor)
node_2 = Node(self.port(8002), self.reactor)
self.nodes.append(node_1)
self.nodes.append(node_2)
def activate(self):
self.nodes[1].listen()
self.nodes[2].listen()
def main(reactor):
org = Organization(reactor)
org.create_leader()
org.create_more_nodes()
org.activate()
return Deferred()
if __name__ == '__main__':
from twisted.internet.task import react
react(main)
With Python 2.7, I have extended the BaseHTTPServer.BaseHTTPRequestHandler to support a do_POST method. I would like to give the request handler a queue, so that it can put the posted data on a queue to be processed by another thread.
Here is a stripped down version of my class:
import BaseHTTPServer
import json
class PostHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.end_headers()
length = int(self.headers['Content-Length'])
self.post_data = self.rfile.read(length)
try:
if self.headers['Content-Type'] == 'application/json':
self.post_data = json.loads(self.post_data)
self.log_message(json.dumps(self.post_data))
### WANT TO PUT self.post_data ON A QUEUE HERE ###
except KeyError as error:
self.log_message('No Content-Type header')
except ValueError as error:
self.log_message("%s" % error)
Since then handler gets created by the BaseHTTPServer, I don't think I can alter the init method to pass in a queue.
I'd like my main() to look something like this:
def main():
import logging
import Queue
import signal
import threading
import traceback
try:
# set stoprequest to accept Ctrl+c
stoprequest = threading.Event()
signal.signal(signal.SIGINT, lambda signal, frame: stoprequest.set())
args = _get_main_args()
### HERE IS THE QUEUE, HOW TO I GIVE A REFERENCE TO THE HANDLER??? ###
data_queue = Queue.Queue()
handler = PostHTTPRequestHandler
server = BaseHTTPServer.HTTPServer((args.address, args.port), handler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
while not stoprequest.is_set():
try:
data = data_queue.get(False)
### I WANT TO PROCESS THE DATA HERE ###
except Queue.Empty:
pass
server.shutdown()
#logging.debug("Exiting with return code 0")
return 0
except Exception:
sys.stderr.write("%s" % traceback.format_exc())
return 1
I now see that the init for the BaseRequestHandler looks like this:
class BaseRequestHandler:
def __init__(self, request, client_address, server):
self.request = request
self.client_address = client_address
self.server = server
self.setup()
try:
self.handle()
finally:
self.finish()
So I will extend the BaseHTTPServer to contain a queue, then it will be available to the handler.
class QueuingHTTPServer(BaseHTTPServer.HTTPServer):
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.data_queue = Queue.Queue()
So now the handler looks like this:
class PostHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.end_headers()
length = int(self.headers['Content-Length'])
self.post_data = self.rfile.read(length)
try:
if self.headers['Content-Type'] == 'application/json':
self.post_data = json.loads(self.post_data)
self.log_message(json.dumps(self.post_data))
try:
self.server.data_queue.put(self.post_data)
except Queue.Full:
pass
except KeyError as error:
self.log_message('No Content-Type header')
except ValueError as error:
self.log_message("%s" % error)
I modified the code submitted by Josh to use a shared Queue reference instead of creating one during instantiation.
class QueuingHTTPServer(BaseHTTPServer.HTTPServer):
def __init__(self, server_address, RequestHandlerClass, data_queue, bind_and_activate=True):
BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.data_queue = queue
I'm trying to start a data queue server under a managing process (so that it can later be turned into a service), and while the data queue server function works fine in the main process, it does not work in a process created using multiprocessing.Process.
The dataQueueServer and dataQueueClient code is based on the code from the multiprocessing module documentation here.
When run on its own, dataQueueServer works well. However, when run using a multiprocessing.Process's start() in mpquueue, it doesn't work (when tested with the client). I am using the dataQueueClient without changes to test both cases.
The code does reach the serve_forever in both cases, so I think the server is working, but something is blocking it from communicating back to the client in the mpqueue case.
I have placed the loop that runs the serve_forever() part under a thread, so that it can be stoppable.
Here is the code:
mpqueue # this is the "manager" process trying to spawn the server in a child process
import time
import multiprocessing
import threading
import dataQueueServer
class Printer():
def __init__(self):
self.lock = threading.Lock()
def tsprint(self, text):
with self.lock:
print text
class QueueServer(multiprocessing.Process):
def __init__(self, name = '', printer = None):
multiprocessing.Process.__init__(self)
self.name = name
self.printer = printer
self.ml = dataQueueServer.MainLoop(name = 'ml', printer = self.printer)
def run(self):
self.printer.tsprint(self.ml)
self.ml.start()
def stop(self):
self.ml.stop()
if __name__ == '__main__':
printer = Printer()
qs = QueueServer(name = 'QueueServer', printer = printer)
printer.tsprint(qs)
printer.tsprint('starting')
qs.start()
printer.tsprint('started.')
printer.tsprint('Press Ctrl-C to quit')
try:
while True:
time.sleep(60)
except KeyboardInterrupt:
printer.tsprint('\nTrying to exit cleanly...')
qs.stop()
printer.tsprint('stopped')
dataQueueServer
import time
import threading
from multiprocessing.managers import BaseManager
from multiprocessing import Queue
HOST = ''
PORT = 50010
AUTHKEY = 'authkey'
## Define some helper functions for use by the main process loop
class Printer():
def __init__(self):
self.lock = threading.Lock()
def tsprint(self, text):
with self.lock:
print text
class QueueManager(BaseManager):
pass
class MainLoop(threading.Thread):
"""A thread based loop manager, allowing termination signals to be sent
to the thread"""
def __init__(self, name = '', printer = None):
threading.Thread.__init__(self)
self._stopEvent = threading.Event()
self.daemon = True
self.name = name
if printer is None:
self.printer = Printer()
else:
self.printer = printer
## create the queue
self.queue = Queue()
## Add a function to the handler to return the queue to clients
self.QM = QueueManager
self.QM.register('get_queue', callable=lambda:self.queue)
self.queue_manager = self.QM(address=(HOST, PORT), authkey=AUTHKEY)
self.queue_server = self.queue_manager.get_server()
def __del__(self):
self.printer.tsprint( 'closing...')
def run(self):
self.printer.tsprint( '{}: started serving'.format(self.name))
self.queue_server.serve_forever()
def stop(self):
self.printer.tsprint ('{}: stopping'.format(self.name))
self._stopEvent.set()
def stopped(self):
return self._stopEvent.isSet()
def start():
printer = Printer()
ml = MainLoop(name = 'ml', printer = printer)
ml.start()
return ml
def stop(ml):
ml.stop()
if __name__ == '__main__':
ml = start()
raw_input("\nhit return to stop")
stop(ml)
And a client:
dataQueueClient
import datetime
from multiprocessing.managers import BaseManager
n = 0
N = 10**n
HOST = ''
PORT = 50010
AUTHKEY = 'authkey'
def now():
return datetime.datetime.now()
def gen(n, func, *args, **kwargs):
k = 0
while k < n:
yield func(*args, **kwargs)
k += 1
class QueueManager(BaseManager):
pass
QueueManager.register('get_queue')
m = QueueManager(address=(HOST, PORT), authkey=AUTHKEY)
m.connect()
queue = m.get_queue()
def load(msg, q):
return q.put(msg)
def get(q):
return q.get()
lgen = gen(N, load, msg = 'hello', q = queue)
t0 = now()
while True:
try:
lgen.next()
except StopIteration:
break
t1 = now()
print 'loaded %d items in ' % N, t1-t0
t0 = now()
while queue.qsize() > 0:
queue.get()
t1 = now()
print 'got %d items in ' % N, t1-t0
So it seems like the solution is simple enough: Don't use serve_forever(), and use manager.start() instead.
According to Eli Bendersky, the BaseManager (and it's extended version SyncManager) already spawns the server in a new process (and looking at the multiprocessing.managers code confirms this). The problem I have been experiencing stems from the form used in the example, in which the server is started under the main process.
I still don't understand why the current example doesn't work when run under a child process, but that's no longer an issue.
Here's the working (and much simplified from OP) code to manage multiple queue servers:
Server:
from multiprocessing import Queue
from multiprocessing.managers import SyncManager
HOST = ''
PORT0 = 5011
PORT1 = 5012
PORT2 = 5013
AUTHKEY = 'authkey'
name0 = 'qm0'
name1 = 'qm1'
name2 = 'qm2'
description = 'Queue Server'
def CreateQueueServer(HOST, PORT, AUTHKEY, name = None, description = None):
name = name
description = description
q = Queue()
class QueueManager(SyncManager):
pass
QueueManager.register('get_queue', callable = lambda: q)
QueueManager.register('get_name', callable = name)
QueueManager.register('get_description', callable = description)
manager = QueueManager(address = (HOST, PORT), authkey = AUTHKEY)
manager.start() # This actually starts the server
return manager
# Start three queue servers
qm0 = CreateQueueServer(HOST, PORT0, AUTHKEY, name0, description)
qm1 = CreateQueueServer(HOST, PORT1, AUTHKEY, name1, description)
qm2 = CreateQueueServer(HOST, PORT2, AUTHKEY, name2, description)
raw_input("return to end")
Client:
from multiprocessing.managers import SyncManager
HOST = ''
PORT0 = 5011
PORT1 = 5012
PORT2 = 5013
AUTHKEY = 'authkey'
def QueueServerClient(HOST, PORT, AUTHKEY):
class QueueManager(SyncManager):
pass
QueueManager.register('get_queue')
QueueManager.register('get_name')
QueueManager.register('get_description')
manager = QueueManager(address = (HOST, PORT), authkey = AUTHKEY)
manager.connect() # This starts the connected client
return manager
# create three connected managers
qc0 = QueueServerClient(HOST, PORT0, AUTHKEY)
qc1 = QueueServerClient(HOST, PORT1, AUTHKEY)
qc2 = QueueServerClient(HOST, PORT2, AUTHKEY)
# Get the queue objects from the clients
q0 = qc0.get_queue()
q1 = qc1.get_queue()
q2 = qc2.get_queue()
# put stuff in the queues
q0.put('some stuff')
q1.put('other stuff')
q2.put({1:123, 2:'abc'})
# check their sizes
print 'q0 size', q0.qsize()
print 'q1 size', q1.qsize()
print 'q2 size', q2.qsize()
# pull some stuff and print it
print q0.get()
print q1.get()
print q2.get()
Adding an additional server to share a dictionary with the information of the running queue servers so that consumers can easily tell what's available where is easy enough using that model. One thing to note, though, is that the shared dictionary requires slightly different syntax than a normal dictionary: dictionary[0] = something will not work. You need to use dictionary.update([(key, value), (otherkey, othervalue)]) and dictionary.get(key) syntax, which propagates across to all other clients connected to this dictionary..