We Keep Coding

Python Multiprocessing remote manager: failure to connect when using BaseManager.start() instead of .server().serve_forever()

I'm following an example from the official python documentation here:
I'm trying to make it so that I spin up a BaseManager at localhost:50000 which registers a queue, then a bunch of workers that read from that queue. I can get it to work if I use the method in the official python docs that has three files (one server, one put client, one get client), but I can't get it to work all in one file where I spawn the clients via multiprocessing.Process(target=...).
Here is my full code. The issue is that when the clients attempt to connect they get a ConnectionRefused (stack trace below)
from typing import Dict, Optional, Any, List
from multiprocessing.managers import BaseManager, SyncManager
import time
import multiprocessing as mp
import argparse
import queue
q = queue.Queue()
def parse_args() -> argparse.Namespace:
a = argparse.ArgumentParser()
a.add_argument("--n-workers", type=int, default=2)
return a.parse_args()
def run_queue_server(args: argparse.Namespace) -> None:
class QueueManager(BaseManager): pass
QueueManager.register("get_queue", lambda: q)
m = QueueManager(address=('', 50000), authkey=b'abracadabra')
m.start()
def _worker_process(worker_uid: str) -> None:
class QueueManager(BaseManager): pass
QueueManager.register("get_queue")
m = QueueManager(address=('', 50000), authkey=b'abracadabra')
# <-- This line fails with ConnectionRefused -->
m.connect()
queue: queue.Queue = m.get_queue()
def spawn_workers(args: argparse.Namespace) -> None:
time.sleep(2)
worker_procs = dict()
for i in range(args.n_workers):
print(f"Spawning worker process {i}..")
p = mp.Process(target=_worker_process, args=[str(i)])
p.start()
worker_procs[str(i)] = p
def main():
args = parse_args()
run_queue_server(args)
spawn_workers(args)
while True:
time.sleep(1)
if __name__ == '__main__':
main()
The error is here
$ python minimal.py
Spawning worker process 0..
Spawning worker process 1..
Process Process-2:
Process Process-3:
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/process.py", line 313, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "minimal.py", line 26, in _worker_process
m.connect()
File "/usr/lib/python3.8/multiprocessing/managers.py", line 548, in connect
conn = Client(self._address, authkey=self._authkey)
File "/usr/lib/python3.8/multiprocessing/connection.py", line 502, in Client
c = SocketClient(address)
File "/usr/lib/python3.8/multiprocessing/connection.py", line 629, in SocketClient
s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
Traceback (most recent call last):
File "/usr/lib/python3.8/multiprocessing/process.py", line 313, in _bootstrap
self.run()
File "/usr/lib/python3.8/multiprocessing/process.py", line 108, in run
self._target(*self._args, **self._kwargs)
File "minimal.py", line 26, in _worker_process
m.connect()
File "/usr/lib/python3.8/multiprocessing/managers.py", line 548, in connect
conn = Client(self._address, authkey=self._authkey)
File "/usr/lib/python3.8/multiprocessing/connection.py", line 502, in Client
c = SocketClient(address)
File "/usr/lib/python3.8/multiprocessing/connection.py", line 629, in SocketClient
s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
However, If I spawn another process that targets the manager creatoin step and run m.get_server().serve_forever() then I do not get the connection-refused error, see this code below which works
from typing import Dict, Optional, Any, List
from multiprocessing.managers import BaseManager, SyncManager
import time
import multiprocessing as mp
import argparse
import queue
q = queue.Queue()
def parse_args() -> argparse.Namespace:
a = argparse.ArgumentParser()
a.add_argument("--n-workers", type=int, default=2)
return a.parse_args()
def run_queue_server(args: argparse.Namespace) -> None:
class QueueManager(BaseManager): pass
QueueManager.register("get_queue", lambda: q)
m = QueueManager(address=('', 50000), authkey=b'abracadabra')
#m.start()
# This works!!
m.get_server().serve_forever()
def _worker_process(worker_uid: str) -> None:
class QueueManager(BaseManager): pass
QueueManager.register("get_queue")
m = QueueManager(address=('', 50000), authkey=b'abracadabra')
m.connect()
queue: queue.Queue = m.get_queue()
print(f"Gotten queue: {queue}")
def spawn_workers(args: argparse.Namespace) -> None:
time.sleep(2)
worker_procs = dict()
for i in range(args.n_workers):
print(f"Spawning worker process {i}..")
p = mp.Process(target=_worker_process, args=[str(i)])
p.start()
worker_procs[str(i)] = p
def main():
args = parse_args()
#run_queue_server(args)
# I don't want to run this in another process?
mp.Process(target=run_queue_server, args=(args,)).start()
spawn_workers(args)
while True:
time.sleep(1)
if __name__ == '__main__':
main()
The thing is, I don't want to have to start another process to be the manager.. why can't it just be this process?

Edit - I'm an idiot who was programming too late into the night. The issue is that my run_queue_server when calling m.start() and returning was... then losing the reference to the QueueManager which I'm sure caused it to be garbage collected.
All I did was change
def run_queue_server(args: argparse.Namespace) -> None:
class QueueManager(BaseManager): pass
QueueManager.register("get_queue", lambda: q)
m = QueueManager(address=('', 50000), authkey=b'abracadabra')
m.start()
return m
and change the caller to accept the return value and everything works..

ProcessPoolExectur and Ctrl C

I'm using ProcessPoolExecutor on Windows 10. Python version is 3.9.5.
When I press Ctrl+C twice the program hangs endless, even if I set a timeout.
with concurrent.futures.ProcessPoolExecutor() as executor:
results = executor.map(Worker, iterable, timeout=5)
try:
for result in results:
DoSomething(result)
except Exception as exc:
print(exc)
executor.shutdown(wait=True, cancel_futures=True)
Error message is:
...
File "C:\Python\foo.py", line 162, in FooFunc
executor.shutdown(wait=True, cancel_futures=True)
File "C:\Python\_envs\Python39\lib\concurrent\futures\_base.py", line 636, in __exit__
self.shutdown(wait=True)
File "C:\Python\_envs\Python39\lib\concurrent\futures\process.py", line 740, in shutdown
self._executor_manager_thread.join()
File "C:\Python\_envs\Python39\lib\threading.py", line 1033, in join
self._wait_for_tstate_lock()
File "C:\Python\_envs\Python39\lib\threading.py", line 1049, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt

I've found a solution. A workaround. It is a bit strange, because I would expect that the Python standard implementation is already able to deal with Ctrl+C.
The solution is to add to the main process:
def handler(signum, frame):
print('Signal handler called with signal', signum)
if __name__ == "__main__":
import signal
signal.signal(signal.SIGINT, handler)

Sending socket through multiprocessing Queue Python 3.6

I am trying to implement a generic "timeout" function which allows me to send a function to be run, and if it doesn't complete after a certain amount of time, kill it. Here is the current implementation:
from multiprocessing import Process, Queue
import multiprocessing as mp
mp.allow_connection_pickling()
from fn.monad import Full, Empty
import traceback
def timeout(timeout, func, args=()):
'''
Calls function, and if it times out returns an Empty()
:param timeout: int | The amount of time to wait for the function
:param func: () => Any | The function to call (must take no arguments)
:param queue: Queue | The multiprocessing queue to put the result into
:return Option | Full(Result) if we get the result, Empty() if it times out
'''
queue = Queue()
p = Process(target=_helper_func, args=(func, queue, args,))
p.daemon = True
p.start()
p.join(timeout)
if p.is_alive() or queue.empty():
p.terminate()
return Empty()
else:
out = queue.get()
# if 'rebuild_handle' in dir(out):
# out.rebuild_handle()
return Full(out)
def _helper_func(func, queue, args):
try:
func(*args, queue)
except Exception as e:
pass
The function must put the "return value" into the multiprocessing queue. However, when timeout is run, and a socket is put into the queue, I get the following error.
Traceback (most recent call last):
File "socket_test.py", line 27, in <module>
print(q.get())
File "/usr/lib/python3.6/multiprocessing/queues.py", line 113, in get
return _ForkingPickler.loads(res)
File "/usr/lib/python3.6/multiprocessing/reduction.py", line 239, in _rebuild_socket
fd = df.detach()
File "/usr/lib/python3.6/multiprocessing/resource_sharer.py", line 57, in detach
with _resource_sharer.get_connection(self._id) as conn:
File "/usr/lib/python3.6/multiprocessing/resource_sharer.py", line 87, in get_connection
c = Client(address, authkey=process.current_process().authkey)
File "/usr/lib/python3.6/multiprocessing/connection.py", line 487, in Client
c = SocketClient(address)
File "/usr/lib/python3.6/multiprocessing/connection.py", line 614, in SocketClient
s.connect(address)
ConnectionRefusedError: [Errno 111] Connection refused
I have tried various previous stack overflow posts, such as the following: Python3 Windows multiprocessing passing socket to process
Please let me know if you know of a solution to this issue, as it is throwing a giant wrench into my code. Thanks!

Error happen python3.6 while using tornado in multi threading

I just simply use the tornado application together with threading as the following code:
def MakeApp():
return tornado.web.Application([(r"/websocket", EchoWebSocket), ])
def run_tornado_websocket():
app = MakeApp()
http_server = tornado.httpserver.HTTPServer(app, ssl_options={
"certfile": os.path.join(os.path.abspath("."), "server.crt"),
"keyfile": os.path.join(os.path.abspath("."), "server_no_passwd.key"),
})
http_server.listen(options.port)
tornado.ioloop.IOLoop.current().start()
if __name__ == '__main__':
threads = []
t = threading.Thread(target=run_tornado_websocket, args=())
threads.append(t)
for t in threads:
t.start()
It works fine on python3.5.But it fails on python3.6 and the lasted tornado.It gets the error:
Exception in thread Thread-1:
Traceback (most recent call last):
File "D:\python3\lib\threading.py", line 916, in _bootstrap_inner
self.run()
File "D:\python3\lib\threading.py", line 864, in run
self._target(*self._args, **self._kwargs)
File "D:\ssl\ws_server.py", line 49, in run_tornado_websocket
http_server.listen(options.port)
File "D:\python3\lib\site-packages\tornado\tcpserver.py", line 145, in listen
self.add_sockets(sockets)
File "D:\python3\lib\site-packages\tornado\tcpserver.py", line 159, in add_sockets
sock, self._handle_connection)
File "D:\python3\lib\site-packages\tornado\netutil.py", line 219, in add_accept_handler
io_loop = IOLoop.current()
File "D:\python3\lib\site-packages\tornado\ioloop.py", line 282, in current
loop = asyncio.get_event_loop()
File "D:\python3\lib\asyncio\events.py", line 694, in get_event_loop
return get_event_loop_policy().get_event_loop()
File "D:\python3\lib\asyncio\events.py", line 602, in get_event_loop
% threading.current_thread().name)
RuntimeError: There is no current event loop in thread 'Thread-1'.
I think there are some changes in IOLOOP in python3.6.But I don't know how to fix this and really want to know the reason.

Starting in Tornado 5.0, the asyncio event loop is used by default. asyncio has some extra restrictions because starting event loops on threads other than the main thread is an uncommon pattern and is often a mistake. You must tell asyncio that you want to use an event loop in your new thread with asyncio.set_event_loop(asyncio.new_event_loop()), or use asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy()) to disable this restriction.

How to wait for task created by create_task() to complete?

I wrote a test program to try out using create_task() that needs to wait until the created task completes.
I tried using loop.run_until_complete() to wait for task completion, but it results in an error with a traceback.
/Users/jason/.virtualenvs/xxx/bin/python3.5 /Users/jason/asyncio/examples/hello_coroutine.py
Traceback (most recent call last):
Test
File "/Users/jason/asyncio/examples/hello_coroutine.py", line 42, in <module>
Hello World, is a task
loop.run_until_complete(test.greet_every_two_seconds())
File "/Users/jason/asyncio/asyncio/base_events.py", line 373, in run_until_complete
return future.result()
File "/Users/jason/asyncio/asyncio/futures.py", line 274, in result
raise self._exception
File "/Users/jason/asyncio/asyncio/tasks.py", line 240, in _step
result = coro.send(None)
File "/Users/jason/asyncio/examples/hello_coroutine.py", line 33, in greet_every_two_seconds
self.a()
File "/Users/jason/asyncio/examples/hello_coroutine.py", line 26, in a
t = loop.run_until_complete(self.greet_every_one_seconds(self.db_presell))
File "/Users/jason/asyncio/asyncio/base_events.py", line 361, in run_until_complete
self.run_forever()
File "/Users/jason/asyncio/asyncio/base_events.py", line 326, in run_forever
raise RuntimeError('Event loop is running.')
RuntimeError: Event loop is running.
The test code is as follows. The function a() must not be a coroutine,
How can I wait for the task until complete?
import asyncio
class Test(object):
def __init__(self):
print(self.__class__.__name__)
pass
#asyncio.coroutine
def greet_every_one_seconds(self, value):
print('Hello World, one second.')
fut = asyncio.sleep(1,result=value)
a = yield from fut
print(a)
def a(self):
loop = asyncio.get_event_loop()
task=loop.run_until_complete(self.greet_every_one_seconds(4))
#How can i wait for the task until complete?
#asyncio.coroutine
def greet_every_two_seconds(self):
while True:
self.a()
print('Hello World, two seconds.')
yield from asyncio.sleep(2)
if __name__ == '__main__':
test = Test()
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(test.greet_every_two_seconds())
finally:
loop.close()

How can i wait for the task until complete?
loop.run_until_complete(task) in an ordinary function. Or await task in a coroutine.
Here's a complete code example that demonstrates both cases:
#!/usr/bin/env python3
import asyncio
async def some_coroutine(loop):
task = loop.create_task(asyncio.sleep(1)) # just some task
await task # wait for it (inside a coroutine)
loop = asyncio.get_event_loop()
task = loop.create_task(asyncio.sleep(1)) # just some task
loop.run_until_complete(task) # wait for it (outside of a coroutine)
loop.run_until_complete(some_coroutine(loop))