We Keep Coding

How to use concurrent.future.wait?

I'm studying python, here I got some problem on the concurrent.futures.wait() -- Here's the details--
I want to make the main process hold until all the child processes completed. So I used wait() to block the main process. But I always got error , please kind help.
def child_process(args):
pid=os.getpid();
while (args.len() > 0 ):
task=args.pop(0)
time.sleep(1+ random.random()*5) #simulate the worker time
print("Process "+str(pid)+" : "+task[0]+" "+task[1])
return
if (__name__ == "__main__") :
mgr = multiprocessing.Manager()
tasks=mgr.list()
tasks=[[1,10],[2,20],[3,30],[4,40],[5,50],[6,60]]
#executor=ProcessPoolExecutor(max_workers=3)
f=[]
with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:
f.append(executor.submit(child_process,tasks))
f.append(executor.submit(child_process,tasks))
f.append(executor.submit(child_process,tasks))
# wait(future,timeout=0,return_when=concurrent.futures.ALL_COMPLETED)
concurrent.futures.wait(f[0])
concurrent.futures.wait(f[1])
concurrent.futures.wait(f[2])
executor.shutdown()
The error is --
C:\Work\python\src\test>python test.py
Traceback (most recent call last):
File "C:\Work\python\src\test\test.py", line 70, in <module>
concurrent.futures.wait(f[0])
File "C:\tools\Python310\lib\concurrent\futures\_base.py", line 290, in wait
fs = set(fs)
TypeError: 'Future' object is not iterable
This puzzles me most -- is that f[0] not a future object returned by submit()?
Then I tried with --
wait(f,timeout=0,return_when=concurrent.futures.ALL_COMPLETED)
the new error is --
C:\Work\python\src\test>python test.py
C:\Work\python\src\test\test.py:68: RuntimeWarning: coroutine 'wait' was never awaited
wait(f,timeout=0,return_when=concurrent.futures.ALL_COMPLETED)
RuntimeWarning: Enable tracemalloc to get the object allocation traceback
I really don't know how to fix it. Please kidn advise. Thanks
Regards
Eisen

Few things to point out:
Wrapping expression with parenthesis in while statement is redundant.
>>> a = 0
>>> while a < 10:
... a += 1
Error message is saying "Future object is not iterable" - which means, f[0] you passed is indeed Future Object which is not wait method was expecting.
>>> from concurrent import futures
>>> help(futures.wait)
Help on function wait in module concurrent.futures._base:
wait(fs, timeout=None, return_when='ALL_COMPLETED')
Wait for the futures in the given sequence to complete.
Args:
fs: The sequence of Futures (possibly created by different Executors) to
wait upon.
# ...
Here we can see argument fs actually expect you the Sequence of Futures.
So instead of this:
concurrent.futures.wait(f[0])
concurrent.futures.wait(f[1])
concurrent.futures.wait(f[2])
You probably want this:
concurrent.futures.wait(f)
Which is still not required since with block wait until all processes stops.
Here's demonstration:
"""
Demo codes for https://stackoverflow.com/q/71458088/10909029
Waiting for child process to complete
"""
import os
import math
import queue
import multiprocessing as mp
from concurrent import futures
def child_process(task_queue: mp.Queue):
# If this doesn't work, save this function in other file. REPL or jupyter especially.
pid = os.getpid()
print(f"[{pid}] Started!")
processed_count = 0
while True:
try:
item = task_queue.get_nowait()
except queue.Empty:
# task done
break
# else continue on
# some workload
try:
print(f"[{pid}] {item}! = {math.factorial(item)}")
finally:
# tell queue we processed the item.
task_queue.task_done()
processed_count += 1
print(f"[{pid}] Task done!")
def main():
# just merely rapping codes in function namespace makes codes tiny bit faster
mp_manager = mp.Manager()
task_queue = mp_manager.Queue()
# populate queue
for n in range(100):
task_queue.put_nowait(n)
# start pool
with futures.ProcessPoolExecutor() as executor:
future_list = [executor.submit(child_process, task_queue) for _ in range(5)]
# can use executor.shutdown(wait=True) instead
# not required since all executor wait for all process to stop when exiting `with` block.
# hence, also no need to manually call executor.shutdown().
futures.wait(future_list)
if __name__ == '__main__':
main()
Output:
[18412] Started!
[18412] 0! = 1
[4680] Started!
[18412] 1! = 1
[2664] Started!
[18412] 2! = 2
[18412] 3! = 6
[17900] Started!
[18412] 4! = 24
[18412] 5! = 120
[4680] 6! = 720
[4680] 7! = 5040
[18412] 8! = 40320
[17900] 9! = 362880
[4680] 10! = 3628800
[18412] 11! = 39916800
...
[17900] 21! = 51090942171709440000
[4680] 22! = 1124000727777607680000
[2664] 23! = 25852016738884976640000
[16792] Started!
[18412] 24! = 620448401733239439360000
[17900] 25! = 15511210043330985984000000
...
[17900] 99! = 933262154439441526816992388562667004907159682643816214685929638952175999932299156089414639761565182862536979208272237582511852109168640000000000000000000000
[18412] Task done!
[17900] Task done!
[16792] Task done!
[2664] Task done!
[4680] Task done!

Stopping a python thread while the queue is not empty

I have some code which I hopefully boiled down to a correct MWE.
My goal is to stop the (multiple) threads if a list within the thread has a specific length.
In contrast to the MWE it is not known how many iterations are needed:
from queue import Queue
from threading import Thread
def is_even(n):
return n % 2 == 0
class MT(Thread):
def __init__(self, queue):
super().__init__()
self.queue = queue
self.output = []
def run(self):
while len(self.output) < 4:
task = self.queue.get()
if is_even(task):
self.output.append(task)
self.queue.task_done()
else:
self.queue.task_done()
print(self.output)
print('done')
queue = Queue(10)
threads = 1
thr = []
for th in range(threads):
thr.append(MT(queue))
for th in thr:
th.start()
for i in range(100):
queue.put(i)
queue.join()
for th in thr:
th.join()
print('finished')
This code wil not hit finish...

To quote the documentation,
Queue.join()
Blocks until all items in the queue have been gotten and processed.
You have placed 100 items in the queue. The thread pulls 4 items, and completes. There are still 96 unprocessed items, and nobody is going to pull them. Therefore, queue.join() never returns.

Why doesn't the producer-consumer stop?

I've found an example representing producer-consumer with two threads. But, when I send a signal to the process to stop, it doesn't. It expects second signal e.g. SIGKILL to completely stop. I thought the problem is with task_done() but it seems not.
import time
import queue
import threading
import random
class Producer(threading.Thread):
"""
Produces random integers to a list
"""
def __init__(self, queue):
"""
Constructor.
#param queue queue synchronization object
"""
threading.Thread.__init__(self)
self.queue = queue
def run(self):
"""
Thread run method. Append random integers to the integers
list at random time.
"""
while True:
integer = random.randint(0, 256)
self.queue.put(integer)
print('%d put to queue by %s' % (integer, self.name))
time.sleep(1)
class Consumer(threading.Thread):
"""
Consumes random integers from a list
"""
def __init__(self, queue):
"""
Constructor.
#param integers list of integers
#param queue queue synchronization object
"""
threading.Thread.__init__(self)
self.queue = queue
def run(self):
"""
Thread run method. Consumes integers from list
"""
while True:
integer = self.queue.get()
print('%d popped from list by %s' % (integer, self.name))
self.queue.task_done()
def main():
q = queue.Queue()
t1 = Producer(q)
t2 = Consumer(q)
t1.start()
t2.start()
t1.join()
t2.join()
if __name__ == '__main__':
main()
Output:
210 put to queue by Thread-1
210 popped from list by Thread-2
Traceback (most recent call last):
File "/Users/abc/PycharmProjects/untitled1/ssid.py", line 74, in <module>
main()
File "/Users/abc/PycharmProjects/untitled1/ssid.py", line 69, in main
t1.join()
File "/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py", line 1056, in join
self._wait_for_tstate_lock()
File "/usr/local/Cellar/python3/3.6.3/Frameworks/Python.framework/Versions/3.6/lib/python3.6/threading.py", line 1072, in _wait_for_tstate_lock
elif lock.acquire(block, timeout):
KeyboardInterrupt
244 put to queue by Thread-1
244 popped from list by Thread-2
85 put to queue by Thread-1
85 popped from list by Thread-2
160 put to queue by Thread-1
160 popped from list by Thread-2

It's because only the main-thread get's stopped by the KeyboardInterrupt. You can watch this by letting your child threads print threading.enumerate() which returns all alive threads + the main thread.
import time
import queue
import threading
import random
class Producer(threading.Thread):
def __init__(self, queue):
super().__init__()
self.queue = queue
def run(self):
while True:
integer = random.randint(0, 256)
self.queue.put(integer)
print(f'{integer} put to queue by {self.name} '
f'threads: {threading.enumerate()}')
time.sleep(1)
class Consumer(threading.Thread):
def __init__(self, queue):
super().__init__()
self.queue = queue
def run(self):
while True:
integer = self.queue.get()
print(f'{integer} popped from list by {self.name} '
f'threads:{threading.enumerate()}')
self.queue.task_done()
def main():
q = queue.Queue()
t1 = Producer(q)
t2 = Consumer(q)
# t1.daemon = True
# t2.daemon = True
t1.start()
t2.start()
t1.join()
t2.join()
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('got KeyboardInterrupt')
Example Output with KeyboardInterrupt. Note the MainThread listed as 'stopped' after the KeyboardInterrupt:
97 put to queue by Thread-1 threads: [<_MainThread(MainThread, started
139810293606208)>, <Producer(Thread-1, started 139810250913536)>,
<Consumer(Thread-2, started 139810242520832)>]
97 popped from list by Thread-2 threads:[<_MainThread(MainThread, started
139810293606208)>, <Producer(Thread-1, started 139810250913536)>,
<Consumer(Thread-2, started 139810242520832)>]
got KeyboardInterrupt
92 put to queue by Thread-1 threads: [<_MainThread(MainThread, stopped
139810293606208)>, <Producer(Thread-1, started 139810250913536)>,
<Consumer(Thread-2, started 139810242520832)>]
92 popped from list by Thread-2 threads:[<_MainThread(MainThread, stopped
139810293606208)>, <Producer(Thread-1, started 139810250913536)>,
<Consumer(Thread-2, started 139810242520832)>]
You could make the child-threads daemons to let them exit with the main-thread. But that should be only considered in case your threads don't hold any resources:
Note Daemon threads are abruptly stopped at shutdown. Their resources (such as open files, database transactions, etc.) may not be released properly. If you want your threads to stop gracefully, make them non-daemonic and use a suitable signalling mechanism such as an Event docs.
The better way would be to catch the KeyboardInterrupt like in the code above and send a sentinel value over the queue to the child-threads to let them know they should finish, allowing them to do clean-up before exit.

Multi-threading in Python

I am facing some issues while implementing multi-threading in python. The issue is very specific to my use case. Having gone through numerous posts on the same, I deployed the most widely suggested/used method for doing so.
I start by defining my thread class as follows.
class myThread(Thread):
def __init__(self, graphobj, q):
Thread.__init__(self)
self.graphobj = graphobj
self.q = q
def run(self):
improcess(self.graphobj, self.q)
Post which I define my function that does all the processing required.
def improcess(graphobj, q):
while not exitFlag:
queueLock.acquire()
if not q.empty():
photo_id = q.get()
queueLock.release()
# Complete processing
else:
queueLock.release()
Now comes the part where I am stuck. I am able to run the below mentioned code exactly as it is without any issues. However if I try and wrap the same in a function as such it breaks down.
def train_control(graphobj, photo_ids):
workQueue = Queue(len(photo_ids))
for i in range(1,5):
thread = myThread(graphobj=graphobj, q=workQueue)
thread.start()
threads.append(thread)
queueLock.acquire()
for photo_id in photo_ids:
workQueue.put(photo_id)
queueLock.release()
while not workQueue.empty():
pass
exitFlag = 1
for t in threads:
t.join()
By breaking down I mean that the threads complete their work but they don't stop waiting i.e. the exitFlag is never set to 1. I am unsure as to how to make this work.
Unfortunately the design of our systems is such that this piece of codes needs to be wrapped in a function which can be invoked by another module, so pulling it out is not really an option.
Looking forward to hearing from experts on this. Thanks in advance.
Edit : Forgot to mention this in the first draft. I globally initialize exitFlag and set its value to 0.
Below is the minimum, verifiable code snippet that I created to capture this problem:
import threading
import Queue
globvar01 = 5
globvar02 = 7
exitFlag = 0
globlist = []
threads = []
queueLock = threading.Lock()
workQueue = Queue.Queue(16)
class myThread(threading.Thread):
def __init__(self, threadID, q):
threading.Thread.__init__(self)
self.threadID = threadID
self.q = q
def run(self):
print "Starting thread " + str(self.threadID)
myfunc(self.threadID, self.q)
print "Exiting thread " + str(self.threadID)
def myfunc(threadID, q):
while not exitFlag:
queueLock.acquire()
if not workQueue.empty():
thoughtnum = q.get()
queueLock.release()
print "Processing thread " + str(threadID)
if (thoughtnum < globvar01):
globlist.append([1,2,3])
elif (thoughtnum < globvar02):
globlist.append([2,3,4])
else:
queueLock.release()
def controlfunc():
for i in range(1,5):
thread = myThread(i, workQueue)
thread.start()
threads.append(thread)
queueLock.acquire()
for i in range(1,11):
workQueue.put(i)
queueLock.release()
# Wait for queue to empty
while not workQueue.empty():
pass
exitFlag = 1
# Wait for all threads to complete
for t in threads:
t.join()
print "Starting main thread"
controlfunc()
print "Exiting Main Thread"

From your MCVE, the only thing missing is:
while not workQueue.empty():
pass
global exitFlag # Need this or `exitFlag` is a local variable only.
exitFlag = 1
You could eliminate the queueLock and the exitFlag, however, by using a sentinel value in the Queue to shut down the worker threads, and it eliminates the spin-waiting. Worker threads will sleep on a q.get() and the main thread won't have to spin-wait for an empty queue:
#!python2
from __future__ import print_function
import threading
import Queue
debug = 1
console = threading.Lock()
def tprint(*args,**kwargs):
if debug:
name = threading.current_thread().getName()
with console:
print('{}: '.format(name),end='')
print(*args,**kwargs)
globvar01 = 5
globvar02 = 7
globlist = []
threads = []
workQueue = Queue.Queue(16)
class myThread(threading.Thread):
def __init__(self, threadID, q):
threading.Thread.__init__(self)
self.threadID = threadID
self.q = q
def run(self):
tprint("Starting thread " + str(self.threadID))
myfunc(self.threadID, self.q)
tprint("Exiting thread " + str(self.threadID))
def myfunc(threadID, q):
while True:
thoughtnum = q.get()
tprint("Processing thread " + str(threadID))
if thoughtnum is None:
break
elif thoughtnum < globvar01:
globlist.append([1,2,3])
elif thoughtnum < globvar02:
globlist.append([2,3,4])
def controlfunc():
for i in range(1,5):
thread = myThread(i, workQueue)
thread.start()
threads.append(thread)
for i in range(1,11):
workQueue.put(i)
# Wait for all threads to complete
for t in threads:
workQueue.put(None)
for t in threads:
t.join()
tprint("Starting main thread")
controlfunc()
tprint("Exiting Main Thread")
Output:
MainThread: Starting main thread
Thread-1: Starting thread 1
Thread-2: Starting thread 2
Thread-3: Starting thread 3
Thread-4: Starting thread 4
Thread-1: Processing thread 1
Thread-2: Processing thread 2
Thread-3: Processing thread 3
Thread-4: Processing thread 4
Thread-1: Processing thread 1
Thread-2: Processing thread 2
Thread-3: Processing thread 3
Thread-4: Processing thread 4
Thread-1: Processing thread 1
Thread-2: Processing thread 2
Thread-3: Processing thread 3
Thread-4: Processing thread 4
Thread-1: Processing thread 1
Thread-2: Processing thread 2
Thread-3: Exiting thread 3
Thread-4: Exiting thread 4
Thread-1: Exiting thread 1
Thread-2: Exiting thread 2
MainThread: Exiting Main Thread

You need to make sure exitFlag is set to 0 (False) before spawning any threads otherwise in impprocess() they won't do anything and the queue will remain non-empty.
This problem could happen if you have exitFlag as a global and it's not cleared from a previous run.

Is it possible to kill the parent thread from within a child thread in python?

I am using Python 3.5.2 on windows.
I would like to run a python script but guarantee that it will not take more than N seconds. If it does take more than N seconds, an exception should be raised, and the program should exit. Initially I had thought I could just launch a thread at the beginning that waits for N seconds before throwing an exception, but this only manages to throw an exception to the timer thread, not to the parent thread. For example:
import threading
import time
def may_take_a_long_time(name, wait_time):
print("{} started...".format(name))
time.sleep(wait_time)
print("{} finished!.".format(name))
def kill():
time.sleep(3)
raise TimeoutError("No more time!")
kill_thread = threading.Thread(target=kill)
kill_thread.start()
may_take_a_long_time("A", 2)
may_take_a_long_time("B", 2)
may_take_a_long_time("C", 2)
may_take_a_long_time("D", 2)
This outputs:
A started...
A finished!.
B started...
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Program Files\Python35\lib\threading.py", line 914, in _bootstrap_inner
self.run()
File "C:\Program Files\Python35\lib\threading.py", line 862, in run
self._target(*self._args, **self._kwargs)
File "timeout.py", line 11, in kill
raise TimeoutError("No more time!")
TimeoutError: No more time!
B finished!.
C started...
C finished!.
D started...
D finished!.
Is this even remotely possible? I realize I could do something like this:
import threading
import time
def may_take_a_long_time(name, wait_time, thread):
if not thread.is_alive():
return
print("{} started...".format(name))
time.sleep(wait_time)
print("{} finished!.".format(name))
def kill():
time.sleep(3)
raise TimeoutError("No more time!")
kill_thread = threading.Thread(target=kill)
kill_thread.start()
may_take_a_long_time("A", 2, kill_thread)
may_take_a_long_time("B", 2, kill_thread)
may_take_a_long_time("C", 2, kill_thread)
may_take_a_long_time("D", 2, kill_thread)
But this method fails if, for example, may_take_a_long_time("B", 60, kill_thread) was called.
So I guess my TL;DR question is, what's the best way to put a time limit on the main thread itself?

You can use _thread.interrupt_main (this module is called thread in Python 2.7):
import time, threading, _thread
def long_running():
while True:
print('Hello')
def stopper(sec):
time.sleep(sec)
print('Exiting...')
_thread.interrupt_main()
threading.Thread(target = stopper, args = (2, )).start()
long_running()

If the parent thread is the root thread, you may want to try os._exit(0).
import os
import threading
import time
def may_take_a_long_time(name, wait_time):
print("{} started...".format(name))
time.sleep(wait_time)
print("{} finished!.".format(name))
def kill():
time.sleep(3)
os._exit(0)
kill_thread = threading.Thread(target=kill)
kill_thread.start()
may_take_a_long_time("A", 2)
may_take_a_long_time("B", 2)
may_take_a_long_time("C", 2)
may_take_a_long_time("D", 2)

The cleanest way to do this is to have a queue that the parent thread checks on a regular basis. If the child wants to kill the parent, it sends a message (e.g. "DIE") to the queue. The parent checks the queue and if it sees the message it dies.
q = queue.Queue()
bc = queue.Queue() # backchannel
def worker():
while True:
key = q.get()
try:
process_key(key)
except ValueError as e:
bc.put('DIE')
q.task_done()
# Start the threads
for i in range(args.threads):
threading.Thread(target=worker, daemon=True).start()
for obj in object_farm():
q.put(obj)
try:
back = bc.get(block=False)
except queue.Empty:
pass
else:
print("Data received on backchannel:",back)
if back=='DIE':
raise RuntimeError("time to die")