I'm making remote API calls using threads, using no join so that the program could make the next API call without waiting for the last to complete.
Like so:
def run_single_thread_no_join(function, args):
thread = Thread(target=function, args=(args,))
thread.start()
return
The problem was I needed to know when all API calls were completed. So I moved to code that's using a cue & join.
Threads seem to run in serial now.
I can't seem to figure out how to get the join to work so that threads execute in parallel.
What am I doing wrong?
def run_que_block(methods_list, num_worker_threads=10):
'''
Runs methods on threads. Stores method returns in a list. Then outputs that list
after all methods in the list have been completed.
:param methods_list: example ((method name, args), (method_2, args), (method_3, args)
:param num_worker_threads: The number of threads to use in the block.
:return: The full list of returns from each method.
'''
method_returns = []
# log = StandardLogger(logger_name='run_que_block')
# lock to serialize console output
lock = threading.Lock()
def _output(item):
# Make sure the whole print completes or threads can mix up output in one line.
with lock:
if item:
print(item)
msg = threading.current_thread().name, item
# log.log_debug(msg)
return
# The worker thread pulls an item from the queue and processes it
def _worker():
while True:
item = q.get()
if item is None:
break
method_returns.append(item)
_output(item)
q.task_done()
# Create the queue and thread pool.
q = Queue()
threads = []
# starts worker threads.
for i in range(num_worker_threads):
t = threading.Thread(target=_worker)
t.daemon = True # thread dies when main thread (only non-daemon thread) exits.
t.start()
threads.append(t)
for method in methods_list:
q.put(method[0](*method[1]))
# block until all tasks are done
q.join()
# stop workers
for i in range(num_worker_threads):
q.put(None)
for t in threads:
t.join()
return method_returns
You're doing all the work in the main thread:
for method in methods_list:
q.put(method[0](*method[1]))
Assuming each entry in methods_list is a callable and a sequence of arguments for it, you did all the work in the main thread, then put the result from each function call in the queue, which doesn't allow any parallelization aside from printing (which is generally not a big enough cost to justify thread/queue overhead).
Presumably, you want the threads to do the work for each function, so change that loop to:
for method in methods_list:
q.put(method) # Don't call it, queue it to be called in worker
and change the _worker function so it calls the function that does the work in the thread:
def _worker():
while True:
method, args = q.get() # Extract and unpack callable and arguments
item = method(*args) # Call callable with provided args and store result
if item is None:
break
method_returns.append(item)
_output(item)
q.task_done()
Related
I am preparing a Python multiprocessing tool where I use Process and Queue commands. The queue is putting another script in a process to run in parallel. As a sanity check, in the queue, I want to check if there is any error happing in my other script and return a flag/message if there was an error (status = os.system() will run the process and status is a flag for error). But I can't output errors from the queue/child in the consumer process to the parent process. Following are the main parts of my code (shortened):
import os
import time
from multiprocessing import Process, Queue, Lock
command_queue = Queue()
lock = Lock()
p = Process(target=producer, args=(command_queue, lock, test_config_list_path))
for i in range(consumer_num):
c = Process(target=consumer, args=(command_queue, lock))
consumers.append(c)
p.daemon = True
p.start()
for c in consumers:
c.daemon = True
c.start()
p.join()
for c in consumers:
c.join()
if error_flag:
Stop_this_process_and_send_a_message!
def producer(queue, lock, ...):
for config_path in test_config_list_path:
queue.put((config_path, process_to_be_queued))
def consumer(queue, lock):
while True:
elem = queue.get()
if elem is None:
return
status = os.system(elem[1])
if status:
error_flag = 1
time.sleep(3)
Now I want to get that error_flag and use it in the main code to handle things. But seems I can't output error_flag from the consumer (child) part to the main part of the code. I'd appreciate it if someone can help with this.
Given your update, I also pass an multiprocessing.Event instance to your to_do process. This allows you to simply issue a call to wait on the event in the main process, which will block until a call to set is called on it. Naturally, when to_do or one of its threads detects a script error, it would call set on the event after setting error_flag.value to True. This will wake up the main process who can then call method terminate on the process, which will do what you want. On a normal completion of to_do, it still is necessary to call set on the event since the main process is blocking until the event has been set. But in this case the main process will just call join on the process.
Using a multiprocessing.Value instance alone would have required periodically checking its value in a loop, so I think waiting on a multiprocessing.Event is better. I have also made a couple of other updates to your code with comments, so please review them:
import multiprocessing
from ctypes import c_bool
...
def to_do(event, error_flag):
# Run the tests
wrapper_threads.main(event, error_flag)
# on error or normal process completion:
event.set()
def git_pull_change(path_to_repo):
repo = Repo(path)
current = repo.head.commit
repo.remotes.origin.pull()
if current == repo.head.commit:
print("Repo not changed. Sleep mode activated.")
# Call to time.sleep(some_number_of_seconds) should go here, right?
return False
else:
print("Repo changed. Start running the tests!")
return True
def main():
while True:
status = git_pull_change(git_path)
if status:
# The repo was just pulled, so no point in doing it again:
#repo = Repo(git_path)
#repo.remotes.origin.pull()
event = multiprocessing.Event()
error_flag = multiprocessing.Value(c_bool, False, lock=False)
process = multiprocessing.Process(target=to_do, args=(event, error_flag))
process.start()
# wait for an error or normal process completion:
event.wait()
if error_flag.value:
print('Error! breaking the process!!!!!!!!!!!!!!!!!!!!!!!')
process.terminate() # Kill the process
else:
process.join()
break
You should always tag multiprocessing questions with the platform you are running on. Since I do not see your process-creating code within a if __name__ == '__main__': block, I have to assume you are running on a platform that uses OS fork calls to create new processes, such as Linux.
That means your newly created processes inherit the value of error_flag when they are created but for all intents and purposes, if a process modifies this variable, it is modifying a local copy of this variable that exists in an address space that is unique to that process.
You need to create error_flag in shared memory and pass it as an argument to your process:
from multiprocessing import Value
from ctypes import c_bool
...
error_flag = Value(c_bool, False, lock=False)
for i in range(consumer_num):
c = Process(target=consumer, args=(command_queue, lock, error_flag))
consumers.append(c)
...
if error_flag.value:
...
#Stop_this_process_and_send_a_message!
def consumer(queue, lock, error_flag):
while True:
elem = queue.get()
if elem is None:
return
status = os.system(elem[1])
if status:
error_flag.value = True
time.sleep(3)
But I have a questions/comments for you. You have in your original code the following statement:
if error_flag:
Stop_this_process_and_send_a_message!
But this statement is located after you have already joined all the started processes. So what processes are there to stop and where are you sending a message to (you have potentially multiple consumers any of which might be setting the error_flag -- by the way, no need to have this done under a lock since setting the value True is an atomic action). And since you are joining all your processes, i.e. waiting for them to complete, I am not sure why you are making them daemon processes. You are also passing a Lock instance to your producer and consumers, but it is not being used at all.
Your consumers return when they get a None record from the queue. So if you have N consumers, the last N elements of test_config_path need to be None.
I also see no need for having the producer process. The main process could just as well write all the records to the queue either before or even after it starts the consumer processes.
The call to time.sleep(3) you have at the end of function consumer is unreachable.
So the above code summary is the inner process to run some tests in parallel. I removed the def function part from it, but just assume that is the wrapper_threads in the following code summary. Here I'll add the parent process which is checking a variable (let's assume a commit in my git repo). The following process is meant to run indefinitely and when there is a change it will trigger the multiprocess in the main question:
def to_do():
# Run the tests
wrapper_threads.main()
def git_pull_change(path_to_repo):
repo = Repo(path)
current = repo.head.commit
repo.remotes.origin.pull()
if current == repo.head.commit:
print("Repo not changed. Sleep mode activated.")
return False
else:
print("Repo changed. Start running the tests!")
return True
def main():
process = None
while True:
status = git_pull_change(git_path)
if status:
repo = Repo(git_path)
repo.remotes.origin.pull()
process = multiprocessing.Process(target=to_do)
process.start()
if error_flag.value:
print('Error! breaking the process!!!!!!!!!!!!!!!!!!!!!!!')
os.system('pkill -U user XXX')
break
Now I want to propagate that error_flag from the child process to this process and stop process XXX. The problem is that I don't know how to bring that error_flag to this (grand)parent process.
I want to run multiple threads in parallel. Each thread picks up a task from a task queue and executes that task.
from threading import Thread
from Queue import Queue
import time
class link(object):
def __init__(self, i):
self.name = str(i)
def run_jobs_in_parallel(consumer_func, jobs, results, thread_count,
async_run=False):
def consume_from_queue(jobs, results):
while not jobs.empty():
job = jobs.get()
try:
results.append(consumer_func(job))
except Exception as e:
print str(e)
results.append(False)
finally:
jobs.task_done()
#start worker threads
if jobs.qsize() < thread_count:
thread_count = jobs.qsize()
for tc in range(1,thread_count+1):
worker = Thread(
target=consume_from_queue,
name="worker_{0}".format(str(tc)),
args=(jobs,results,))
worker.start()
if not async_run:
jobs.join()
def create_link(link):
print str(link.name)
time.sleep(10)
return True
def consumer_func(link):
return create_link(link)
# create_link takes a while to execute
jobs = Queue()
results = list()
for i in range(0,10):
jobs.put(link(i))
run_jobs_in_parallel(consumer_func, jobs, results, 25, async_run=False)
Now what is happening is, let say we have 10 link objects in jobs queue, while the threads are running in parallel, multiple threads are executing same task. How can I prevent this from happening?
Note - the above sample code does not have the problem describe above, but i have exactly same code except create_link method does some complex stuff.
I think what you need is a lock object (docs,tutorial+examples). If you create an instance of such an object you can 'lock' some parts of your code, ensuring that only one thread executes this part at a time.
I guess in your case you want to lock the line job = jobs.get().
First you have to create the lock in a scope where all threads have access to it. (You don't want a lock for every thread but a single lock for all your threads. That means creating the lock within your thread just before acquiring it won't work)
import threading
lock = threading.Lock()
then you can use it on your line like:
lock.acquire()
job = jobs.get()
lock.release()
or
with lock:
job = jobs.get()
The first thread to reach acquire() will lock the lock. other threads that try to acquire() the lock will pause until the lock gets unlocked again by calling release().
I am writing a multithreading class. The class has a parallel_process() function that is overridden with the parallel task. The data to be processed is put in the queue. The worker() function in each thread keeps calling parallel_process() until the queue is empty. Results are put in the results Queue object. The class definition is:
import threading
try:
from Queue import Queue
except ImportError:
from queue import Queue
class Parallel:
def __init__(self, pkgs, common=None, nthreads=1):
self.nthreads = nthreads
self.threads = []
self.queue = Queue()
self.results = Queue()
self.common = common
for pkg in pkgs:
self.queue.put(pkg)
def parallel_process(self, pkg, common):
pass
def worker(self):
while not self.queue.empty():
pkg = self.queue.get()
self.results.put(self.parallel_process(pkg, self.common))
self.queue.task_done()
return
def start(self):
for i in range(self.nthreads):
t = threading.Thread(target=self.worker)
t.daemon = False
t.start()
self.threads.append(t)
def wait_for_threads(self):
print('Waiting on queue to empty...')
self.queue.join()
print('Queue processed. Joining threads...')
for t in self.threads:
t.join()
print('...Thread joined.')
def get_results(self):
results = []
print('Obtaining results...')
while not self.results.empty():
results.append(self.results.get())
return results
I use it to create a parallel task:
class myParallel(Parallel): # return square of numbers in a list
def parallel_process(self, pkg, common):
return pkg**2
p = myParallel(range(50),nthreads=4)
p.start()
p.wait_for_threads()
r = p.get_results()
print('FINISHED')
However all threads do not join every time the code is run. Sometimes only 2 join, sometimes no thread joins. I do not think I am blocking the threads from finishing. What reason could there be for join() to not work here?
This statement may lead to errors:
while not self.queue.empty():
pkg = self.queue.get()
With multiple threads pulling items from the queue, there's no guarantee that self.queue.get() will return a valid item, even if you check if the queue is empty beforehand. Here is a possible scenario
Thread 1 checks the queue and the queue is not empty, control proceeds into the while loop.
Control passes to Thread 2, which also checks the queue, finds it is not empty and enters the while loop. Thread 2 gets an item from the loop. The queue is now empty.
Control passes back to Thread 1, it gets an item from the queue, but the queue is now empty, an Empty Exception should be raised.
You should just use a try/except to get an item from the queue
try:
pkg = self.queue.get_nowait()
except Empty:
pass
#Brendan Abel identified the cause. I'd like to suggest a different solution: queue.join() is usually a Bad Idea too. Instead, create a unique value to use as a sentinel:
class Parallel:
_sentinel = object()
At the end of __init__(), add one sentinel to the queue for each thread:
for i in range(nthreads):
self.queue.put(self._sentinel)
Change the start of worker() like so:
while True:
pkg = self.queue.get()
if pkg is self._sentinel:
break
By the construction of the queue, it won't be empty until each thread has seen its sentinel value, so there's no need to mess with the unpredictable queue.size().
Also remove the queue.join() and queue.task_done() cruft.
This will give you reliable code that's easy to modify for fancier scenarios. For example, if you want to add more work items while the threads are running, fine - just write another method to say "I'm done adding work items now", and move the loop adding sentinels into that.
My code:
def create_rods(folder="./", kappas=10, allowed_kappa_error=.3,
radius_correction_ratio=0.1):
"""
Create one rod for each rod_data and for each file
returns [RodGroup1, RodGroup2, ...]
"""
names, files = import_files(folder=folder)
if len(files) == 0:
print "No files to import."
raise ValueError
states = [None for dummy_ in range(len(files))]
processes = []
states_queue = mp.Queue()
for index in range(len(files)):
process = mp.Process(target=create_rods_process,
args=(kappas, allowed_kappa_error,
radius_correction_ratio, names,
files, index, states_queue))
processes.append(process)
run_processes(processes) #This part seem to take a lot of time.
try:
while True:
[index, state] = states_queue.get(False)
states[index] = state
except Queue.Empty:
pass
return names, states
def create_rods_process(kappas, allowed_kappa_error,
radius_correction_ratio, names,
files, index, states_queue):
"""
Process of method.
"""
state = SystemState(kappas, allowed_kappa_error,
radius_correction_ratio, names[index])
data = import_data(files[index])
for dataline in data:
parameters = tuple(dataline)
new_rod = Rod(parameters)
state.put_rod(new_rod)
state.check_rods()
states_queue.put([index, state])
def run_processes(processes, time_out=None):
"""
Runs all processes using all cores.
"""
running = []
cpus = mp.cpu_count()
try:
while True:
#for cpu in range(cpus):
next_process = processes.pop()
running.append(next_process)
next_process.start()
except IndexError:
pass
if not time_out:
try:
while True:
for process in running:
if not process.is_alive():
running.remove(process)
except TypeError:
pass
else:
for process in running:
process.join(time_out)
I expect processes to end but I get a process stucked. I don't know if there is a problem with run_processes() method or with create_rods() method. With join cpus are freed, but program doesn't go on.
From Python's multiprocessing guidelines.
Joining processes that use queues
Bear in mind that a process that has put items in a queue will wait before terminating until all the buffered items are fed by the “feeder” thread to the underlying pipe. (The child process can call the Queue.cancel_join_thread method of the queue to avoid this behaviour.)
This means that whenever you use a queue you need to make sure that all items which have been put on the queue will eventually be removed before the process is joined. Otherwise you cannot be sure that processes which have put items on the queue will terminate. Remember also that non-daemonic processes will be joined automatically.
Joining processes before draining their Queues results in a deadlock. You need to be sure the queues are emptied before joining the processes.
This script used to have the Queue as a global object that could be accessed where the threads were being instantiated and in the threaded function itself, but to make things cleaner I refactored things in a more "acceptable" way and instead decided to pass in the Queue's get() and task_done() methods to the threaded function on its instantiation to get rid of the global; however, I'm noticing that join() hangs indefinitely now as a result, whereas before when the Queue was global, it would always run to completion and terminate appropriately.
For full context, here is the source: http://dpaste.com/0PD6SFX
However, here are what I think are the only relevant snippets of code; init is my main method essentially and the class it belongs to possesses the Queue, while run is the method of class Transcode being run (the rest of the Transcode class details aren't relevant, I feel):
def __init__(self, kargs):
self.params = kargs
# will store generated Transcode objects
self.transcodes = []
# Queue to thread ffmpeg processes with
self.q = Queue.Queue()
# generate transcodes with self.params for all rasters x formats
self.generate_transcodes()
# give the Queue the right number of task threads to make
for i in range(self.get_num_transcodes()):
self.q.put(i)
for transcode in self.transcodes:
transcode.print_commands()
# testing code to be sure command strings are generating appropriately
for transcode in self.transcodes:
self.q.put(transcode)
# kick off all transcodes by creating a new daemon (terminating on program close) thread;
# the thread is given each Transcode's run() method, which is dynamically created within
# the Transcode constructor given its command strings
for transcode in self.transcodes:
t = threading.Thread(target=transcode.run, args=(self.q.get, self.q.task_done))
t.daemon = True
t.start()
print("Transcoding in progress...")
# go through each transcode and print which process is currently underway, then sleep
# 1 = first pass, 2 = second pass, 3 = complete
while True:
still_running = False
for transcode in self.transcodes:
if not transcode.complete:
still_running = True
print('Transcode %s still running!' % transcode.filename)
if transcode.current_proc in range(3):
print(os.path.basename(transcode.filename) + ': pass %s' % transcode.current_proc)
else:
print(os.path.basename(transcode.filename) + ': complete!')
print(transcode.complete)
if not still_running:
break
time.sleep(2)
print('poll')
# block until all items in queue are gotten and processed
print('About to join...')
self.q.join()
print('End of transcode batch!')
'''
Executes sequentially the command strings given to this Transcode via subprocess; it will
first execute one, then the next, as the second command relies on the first being completed.
It will take in a get() and done() method that are Queue methods and call them at the right
moments to signify to an external Queue object that the worker thread needed for this instance
is ready to start and finish.
#param get A Queue get() function to be called at run()'s start.
#param done A Queue done() function to be called at run()'s end.
#return none
'''
def run(self, get, done):
# get a thread from the queue
get()
# convert command lists to command strings
for i in range(len(self.commands)):
self.commands[i] = ' '.join(self.commands[i])
# show that we're working with our first command string
self.current_proc = 1
# assign our current proc the first command string subprocess
self.proc = Popen(self.commands[0], stdout=PIPE, stderr=PIPE, shell=True)
# execute process until complete
self.proc.communicate()
print('Transcode %s first pass complete' % self.identifier)
# run second command string if exists
if len(self.commands) > 1:
# show that we're working with second command string
self.current_proc = 2
# spawn second process and parse output line by line as before
self.proc = Popen(self.commands[1], stdout=PIPE, stderr=PIPE, shell=True)
# execute process until complete
self.proc.communicate()
print('Transcode %s second pass complete' % self.identifier)
# delete log files when done
if self.logfile and os.path.exists(self.logfile + '-0.log'):
os.remove(self.logfile + '-0.log')
if self.logfile and os.path.exists(self.logfile + '-0.log.mbtree'):
os.remove(self.logfile + '-0.log.mbtree')
if self.logfile and os.path.exists(self.logfile + '-0.log.temp'):
os.remove(self.logfile + '-0.log.temp')
if self.logfile and os.path.exists(self.logfile + '-0.log.mbtree.temp'):
os.remove(self.logfile + '-0.log.mbtree.temp')
self.complete = True
print('Transcode %s complete' % self.identifier)
# assign value of 3 to signify completed task
self.current_proc = 3
# finish up with Queue task
done()
The solution was very silly: entries were being added to the Queue twice for each of the transcodes taking place.
# give the Queue the right number of task threads to make
for i in range(self.get_num_transcodes()):
self.q.put(i)
# code omitted...
# testing code to be sure command strings are generating appropriately
for transcode in self.transcodes:
self.q.put(transcode)
Twas a vestige from refactoring. The Queue therefore thought it needed twice as many worker threads as it actually did and was waiting for its remaining entries to be processed.