I have a mono threaded function that I would like to parallelize. The code is a bit too complex to show you, but here is a modelization of its behaviours
R = list(range(4))
def compute(val):
res = sum(val)
if res%2 == 0: #first condition on res
sleep(0.5) #expansive operation
if res%4 == 0: #second condition on res
sleep(0.5) #expansive operation
return 2
else:
return 1
else:
return 1
def f_single(idx, val):
if idx == len(R):
return 1
else:
val = val + [R[idx]]
ret = compute(val)
if ret == 1:
return f_single(idx+1, val)
else:
#DISJUNCT
return f_single(idx+1, val) + f_single(idx+1, val)
Basically, each recursion performs an update of the variable val, and I may need to perform a double call on certain conditions depending on the result of compute(val), which is an expansive computation.
(Sidenote: this implementation does not scale up to large lists, since I will StackOverflow quite rapidly; the multiprocessing effort is also an excuse to rewrite this code).
Ideally, I would like to spawn a new process for computing the new call to f_single.
I started refactoring the code like this:
# list of indices
def process_idxs():
return list(range(len(R)))
def are_two_path(idx, val):
val = val + [R[idx]]
ret = compute(val)
if ret == 1:
return False #simulate a "only one path"
return True #simulate a "two path available"
if __name__ == '__main__':
ret = f_single(0,[])
now = time()
idxs = process_idxs()
# start a job when the job queue is not full
# when the job is complete, return the results (solvers with call stacks)
# add the new results to the job queue
# program terminates when the job queue is done
# TODO: how to do this?
with ProcessPoolExecutor(max_workers=12) as executor:
for idx in idxs:
f = executor.submit(are_two_path, idx, val)
print(f.result())
print("multi proc: ", time()-now, "s")
I don't know how to write my parallelization routine to obtain the same return value than f_single (the last few lines are a tentative to do just that).
When looking in concurrent.future and multiprocessing, I did not found an easy way to collect the results of computation for the current index, conditionally spawn the process and perform to the next recursion, while passing the updated value of val.
I don't have any shared state, except R which is read-only so it shouldn't be an issue here.
Do you have any suggestions or guides on how to convert f_single to a multiprocessing function?
A possible way to do it is to do the following:
import os
from time import time, sleep
from multiprocessing import Queue, Process
from queue import Empty
R = list(range(16))
NUMBER_OF_PROCESSES = 32
TIMEOUT = 1
def compute(val):
res = sum(val)
if res%2 == 0: #first condition on res
sleep(1) #expansive operation
if res%4 == 0: #second condition on res
sleep(1) #expansive operation
return 2
else:
return 1
else:
return 1
def are_two_path(idx, val):
val = val + [R[idx]]
ret = compute(val)
if ret == 1:
return False #simulate a "only one path"
return True #simulate a "two path available"
def worker(q, r, start_val, start_idx):
"""Worker spawned in a new process, in charge of
going through the list iteratively.
Sends a new job to the tasks queue if two path are available
"""
val = start_val
for idx in range(start_idx, len(R)+1):
if idx == len(R):
r.put(1)
else:
result = are_two_path(idx, val)
if result:
q.put((idx+1, val+[R[idx]]))
val = val + [R[idx]]
def overseer():
"""Running in the initial process,
this function create tasks and results queues,
maintain the number of current running processes
and spawn new processes when there is enough room
"""
tasks = Queue()
results = Queue()
init_p = Process(target=worker,
args=(tasks, results, [], 0))
init_p.start()
working = 1
completed_last_cycle = 0
while True:
completed_tasks = results.qsize()
if working < NUMBER_OF_PROCESSES:
# if there is enough room in the working queue,
# spawn a new process and add it
try:
(idx, val) = tasks.get(timeout=5)
except Empty:
break
p = Process(target=worker, args=(tasks, results, val, idx))
p.start()
working += 1
if completed_tasks > completed_last_cycle:
# if some processes terminated during last cycle,
# update the working counter
working -= (completed_tasks - completed_last_cycle)
completed_last_cycle = completed_tasks
tasks.close()
tasks.join_thread()
results.close()
results.join_thread()
return results
def test():
res = overseer()
print("Number of case splits: ", res.qsize())
if __name__ == '__main__':
now = time()
test()
print("multi proc: ", time()-now, "s")
Related
Good afternoon,
I am trying to parallelize a linear programming solving scheme, code is partially reproduced below. The solving method make use of the PuLP library, which uses subprocesses to run solver
commands.
from collections import OrderedDict
from time import time
from multiprocessing import Queue, Process
from queue import Empty
from os import getpid, path, mkdir
import sys
SOLVER = None
NUMBER_OF_PROCESSES = 12
# other parameters
def choose_solver():
"""Choose an initial solver"""
if SOLVER == "CHOCO":
solver = plp.PULP_CHOCO_CMD()
elif SOLVER == "GLPK":
solver = plp.GLPK_CMD(msg=0)
elif SOLVER == "GUROBI":
solver = plp.GUROBI_CMD(msg=0)
else:
solver = plp.PULP_CBC_CMD(msg=0)
return solver
# other functions that are not multiprocess relevant
def is_infeasible(status):
"""Wrapper around PulP infeasible status"""
return status in (plp.LpStatusInfeasible, plp.LpStatusUndefined)
def feasible_problems(input_var, output_var, initial_problem, solver):
"""Perform LP solving on a initial
problem, return the feasible ones"""
input_gt = input_var - TOL >= 0
input_lt = input_var + TOL <= 0
output_eq_input = (output_var - input_var == 0)
output_eq_zero = (output_var == 0)
problem_a = initial_problem.deepcopy()
problem_a += input_gt
problem_a += output_eq_input
problem_b = initial_problem.deepcopy()
problem_b += input_lt
problem_b += output_eq_zero
problem_a.solve(solver)
problem_b.solve(solver)
status_act = problem_a.status
status_inact = problem_b.status
if is_infeasible(status_act):
return (problem_b,)
else:
if is_infeasible(status_inact):
return (problem_a,)
else:
return (problem_a, problem_b)
def worker(q, r, start_problem, start_idx, to_check):
"""Worker spawned in a new process.
Iterates over the neuron expression list.
Sends a new job to the tasks queue if two activations are available.
"""
problem = start_problem
solver = choose_solver()
for idx in range(start_idx, len(to_check) + 1):
if idx == len(to_check):
r.put_nowait(problem)
else:
output_var, input_var = to_check[idx]
pbs = feasible_problems(input_var, output_var, problem, solver)
if len(pbs) == 1:
problem = pbs[0]
elif len(pbs) == 2:
q.put_nowait((idx+1, pbs[0]))
problem = pbs[1]
def overseer(init_prob, neuron_exprs):
"""Running in the initial process,
this function create tasks and results queues,
maintain the number of current running processes
and spawn new processes when there is enough resources
for them to run.
"""
tasks = Queue()
results = Queue()
working_processes = {}
init_p = Process(target=worker,
args=(tasks, results, init_prob, 0, neuron_exprs))
init_p.start()
working_processes[init_p.pid] = init_p
res_list = []
while len(working_processes) > 0:
if len(working_processes) <= NUMBER_OF_PROCESSES:
# if there is enough room in the working queue,
# spawn a new process and add it
try:
(idx, problem) = tasks.get(timeout=1)
except Empty:
break
proc = Process(target=worker, args=(tasks,
results, problem, idx, neuron_exprs))
proc.start()
working_processes[proc.pid] = proc
to_del = []
for pid in working_processes:
pwork = working_processes[pid]
pwork.join(timeout=0)
if pwork.exitcode is not None:
to_del.append(pid)
for pid in to_del:
#deleting working process
del working_processes[pid]
results.join_thread()
for i in range(results.qsize()):
elt = results.get()
res_list.append(elt)
return res_list
def test_multi(init_prob, neuron_exprs):
print("Testing multi process mode")
now = time()
init_prob, exprs = #some function that calculate those
res = overseer(init_prob, exprs)
print("Time spent: {:.4f}s".format(time()-now))
for idx, problem in enumerate(res):
if not path.exists("results"):
mkdir("results")
problem.writeLP("results/"+str(idx))
if __name__ == '__main__':
torch_model = read_model(MODEL_PATH)
print("Number of neurons: ", count_neurons(torch_model))
print("Expected number of facets: ",
theoretical_number(torch_model, DIM_INPUT))
prob, to_check, hp, var_dict = init_problem(torch_model)
test_multi(prob, to_check)
In my worker, I perform some costly calculations that may result in two different problems;
if that happens, I send one problem to a tasks queue while keeping the other for the current worker process. My overseer take a task in the queue and launch a process when it can.
to_check is a list of PuLP expressions,
What I want to do is to fill the working_processes dictionnary with processes that are actually running, then look for their results at each iteration and remove those who have finished. The expected behaviour would be to keep spawning new processes when old ones terminates, which does not seem to be the case. However here I am indefinitely hanging: I successfully take the tasks in the queue, but my program hangs when I spawn more than NUMBER_OF_PROCESSES.
I'm quite new to multiprocessing, so there is maybe something wrong with how I'm doing it. Does anyone have any idea?
Take a look at the ProcessPoolExecutor from concurrent.futures.
Executor objects allow you to specify a pool of workers with a capped size. You can submit all your jobs simultaneously and the executors run through them picking up new jobs as old ones are completed.
I need to use a pool to asynchronously parse results coming from an extraction method and send those results to a write queue.
I have tried this: but it seems to just run iteratively... one process after the other.
process_pool = Pool(processes=30, maxtasksperchild=1)
while True:
filepath = read_queue.get(True)
if filepath is None:
break
res = process_pool.apply_async(func=process.run, args=(filepath, final_path), callback=write_queue.put)
results.append(res)
for result in results:
result.wait()
process_pool.close()
process_pool.join()
I have also tried just waiting on each result, but that does the same thing as the above:
process_pool = Pool(processes=30, maxtasksperchild=1)
while True:
filepath = read_queue.get(True)
if filepath is None:
break
res = process_pool.apply_async(func=process.run, args=(filepath, final_path), callback=write_queue.put)
res.wait()
process_pool.close()
process_pool.join()
I also tried just scheduling processes and letting the pool block itself if it's out of workers to spawn:
process_pool = Pool(processes=30, maxtasksperchild=1)
while True:
filepath = read_queue.get(True)
if filepath is None:
break
process_pool.apply_async(func=process.run, args=(filepath, final_path), callback=write_queue.put)
process_pool.close()
process_pool.join()
This doesn't work, and just runs through the processes over and over, not actually running any sort of function and I'm not sure why. It seems I have to do something with the AsyncResult for the pool to actually schedule the process.
I need it to work like this:
When there is a result waiting in the queue, spawn a new process in the pool with that specific argument from the queue.
On callback, put that processed result in the write queue.
However, I can't seem to get it to work asynchronously correctly. It will only work iteratively because I have to do something with result to actually get the task to schedule properly. Whether that is a .get, .wait, whatever.
# write.py
def write(p_list):
outfile = Path('outfile.txt.bz2')
for data in p_list:
if Path.exists(outfile):
mode = 'ab'
else:
mode = 'wb'
with bz2.open(filename=outfile, mode=mode, compresslevel=9) as output:
temp = (str(data) + '\n').encode('utf-8')
output.write(temp)
print('JSON files written', flush=True)
class Write(Process):
def __init__(self, write_queue: Queue):
Process.__init__(self)
self.write_queue = write_queue
def run(self):
while True:
try:
p_list = self.write_queue.get(True, 900)
except Empty:
continue
if p_list is None:
break
write(p_list)
-
# process.py
def parse(data: int):
global json_list
time.sleep(.1) # simulate parsing the json
json_list.append(data)
def read(data: int):
time.sleep(.1)
parse(data)
def run(data: int):
global json_list
json_list = []
read(data)
return json_list
if __name__ == '__main__':
global output_path, json_list
-
# main.py
if __name__ == '__main__':
read_queue = Queue()
write_queue = Queue()
write = Write(write_queue=write_queue)
write.daemon = True
write.start()
for i in range(0, 1000000):
read_queue.put(i)
read_queue.put(None)
process_pool = Pool(processes=30, maxtasksperchild=1)
while True:
data = read_queue.get(True)
if data is None:
break
res = process_pool.apply_async(func=process.run, args=(data,), callback=write_queue.put)
write_queue.put(None)
process_pool.close()
process_pool.join()
write.join()
print('process done')
So, the problem is that there is no problem. I'm just stupid. If you define a max task per worker of 1, the processes will schedule very quickly and it will look like nothing is happening (or maybe im the only one who thought that).
Here's a reasonable way to use an asynchronous process pool correctly within a while loop with a maxtasksperchild of 1
if __name__ == '__main__':
def func(elem):
time.sleep(0.5)
return elem
def callback(elem):
# do something with processed data
pass
queue = multiprocessing.Queue()
for i in range(0, 10000):
queue.put(i)
process_pool = multiprocessing.Pool(processes=num_processes, maxtasksperchild=1)
results = []
while True:
data = queue.get(True)
if data is None:
break
res = process_pool.apply_async(func=func, args=(data,), callback=callback)
results.append(res)
flag = False
for i, res in enumerate(results):
try:
res.wait(600)
# do some logging
results[i] = None
except TimeoutError:
flag = True
# do some logging
process_pool.close()
if flag:
process_pool.terminate()
process_pool.join()
# done!
Suppose I have the following multiprocessing structure:
import multiprocessing as mp
def worker(working_queue, output_queue):
while True:
if working_queue.empty() == True:
break
else:
picked = working_queue.get()
res_item = "Number " + str(picked)
output_queue.put(res_item)
return
if __name__ == '__main__':
static_input = xrange(100)
working_q = mp.Queue()
output_q = mp.Queue()
results_bank = []
for i in static_input:
working_q.put(i)
processes = [mp.Process(target=worker,args=(working_q, output_q)) for i in range(2)]
for proc in processes:
proc.start()
for proc in processes:
proc.join()
results_bank = []
while True:
if output_q.empty() == True:
break
results_bank.append(output_q.get_nowait())
if len(results_bank) == len(static_input):
print "Good run"
else:
print "Bad run"
My question: How would I 'batch' write my results to a single file while the working_queue is still 'working' (or at least, not finished)?
Note: My actual data structure is not sensitive to unordered results relative to inputs (despite my example using integers).
Also, I think that batch/set writing from the output queue is best practice rather than from the growing results bank object. However, I am open to solutions relying on either approach. I am new to multiprocessing so unsure of best practice or most efficient solution(s) to this question.
If you wish to use mp.Processes and mp.Queues, here is a way to process the results in batches. The main idea is in the writer function, below:
import itertools as IT
import multiprocessing as mp
SENTINEL = None
static_len = 100
def worker(working_queue, output_queue):
for picked in iter(working_queue.get, SENTINEL):
res_item = "Number {:2d}".format(picked)
output_queue.put(res_item)
def writer(output_queue, threshold=10):
result_length = 0
items = iter(output_queue.get, SENTINEL)
for batch in iter(lambda: list(IT.islice(items, threshold)), []):
print('\n'.join(batch))
result_length += len(batch)
state = 'Good run' if result_length == static_len else 'Bad run'
print(state)
if __name__ == '__main__':
num_workers = 2
static_input = range(static_len)
working_q = mp.Queue()
output_q = mp.Queue()
writer_proc = mp.Process(target=writer, args=(output_q,))
writer_proc.start()
for i in static_input:
working_q.put(i)
processes = [mp.Process(target=worker, args=(working_q, output_q))
for i in range(num_workers)]
for proc in processes:
proc.start()
# Put SENTINELs in the Queue to tell the workers to exit their for-loop
working_q.put(SENTINEL)
for proc in processes:
proc.join()
output_q.put(SENTINEL)
writer_proc.join()
When passed two arguments, iter expects a callable and a sentinel:
iter(callable, sentinel). The callable (i.e. a function) gets called repeatedly until it returns a value equal to the sentinel. So
items = iter(output_queue.get, SENTINEL)
defines items to be an iterable which, when iterated over, will return items from output_queue
until output_queue.get() returns SENTINEL.
The for-loop:
for batch in iter(lambda: list(IT.islice(items, threshold)), []):
calls the lambda function repeatedly until an empty list is returned. When called, the lambda function returns a list of up to threshold number of items from the iterable items. Thus, this is an idiom for "grouping by n items without padding". See this post for more on this idiom.
Note that it is not a good practice to test working_q.empty(). It could lead to a race condition. For example, suppose we have the 2 worker processes on these lines when the working_q has only 1 item left in it:
def worker(working_queue, output_queue):
while True:
if working_queue.empty() == True: <-- Process-1
break
else:
picked = working_queue.get() <-- Process-2
res_item = "Number " + str(picked)
output_queue.put(res_item)
return
Suppose Process-1 calls working_queue.empty() while there is still one item in the queue. So it returns False. Then Process-2 calls working_queue.get() and obtains the last item. Then Process-1 gets to line picked = working_queue.get() and hangs because there are no more items in the queue.
Therefore, use sentinels (as shown above) to concretely signal when a for-loop
or while-loop should stop instead of checking queue.empty().
There is no operation like "batch q.get". But it is a good practice to put/pop a batch of items instead of items one by one.
Which is exactly what multiprocessing.Pool.map is doing with its parameter chunksize :)
For writing output as soon as possible there is Pool.imap_unordered which returns an iterable instead of list.
def work(item):
return "Number " + str(item)
import multiprocessing
static_input = range(100)
chunksize = 10
with multiprocessing.Pool() as pool:
for out in pool.imap_unordered(work, static_input, chunksize):
print(out)
I modified the example on Joinable queues on this link https://pymotw.com/2/multiprocessing/communication.html to run a function I wrote instead of a Task object. The modified code is listed below. The problem I am getting is that the consumers get poisoned without putting None in the tasks queue. They exit before completing the tasks. So I removed the check on None (as shown below) from the run function and I caught this exception:
'NoneType' object is not callable
I am sure that the None is not passed yet since the message "Poisoning Consumers" is not yet printed
import multiprocessing as mp
import MyLib
# Subclass of Process
class Consumer(mp.Process):
def __init__(self, task_queue, result_queue):
mp.Process.__init__(self)
self.task_queue = task_queue
self.result_queue = result_queue
self.daemon = True
# A method that defines the behavior of the process
def run(self):
proc_name = self.name
while True:
try:
next_task = self.task_queue.get()
# if next_task is None:
# # Poison pill means shutdown
# print('%s: Exiting' % proc_name)
# self.task_queue.task_done()
# break
mxR, disC = next_task()
self.task_queue.task_done()
self.result_queue.put((mxR, disC))
except Exception as e:
print(e)
return
if __name__ == '__main__':
# Establish communication queues
tasks = mp.JoinableQueue()
results = mp.Queue()
# Start consumers
num_consumers = mp.cpu_count() * 2
print('Creating %d consumers' % num_consumers)
consumers = [ Consumer(tasks, results)
for i in range(num_consumers) ]
for w in consumers:
w.start()
# Enqueue jobs
trials = 10
Tx_Range = 50
prnts = 4
for tx in list(range(30, 200, 20)):
file_name = 'output_{}_{}.txt'.format(tx,prnts)
output_file = open(file_name,'a')
output_file.write('Nodes\tTx_Range\tAvg_Rings\tAvg_Disc\n')
for n in list(range(50, 101, 50)):
ring_sum, disc_sum = 0, 0
for i in range (0, trials):
tasks.put(MyLib.GBMR_mp(1000, 1000, n, prnts, tx, False, results))
print('Done putting jobs')
for i in range (0, trials):
mxR, discN = results.get()
ring_sum += mxR
disc_sum += discN
avg_ring = ring_sum/trials
avg_disc = disc_sum/trials
print('Done Collecting Results, avg_disc = ', avg_disc,' and avg_rings = ', avg_ring)
s = '{}\t\t{}\t\t{}\t\t{}\n'.format(n,tx,avg_ring,avg_disc)
print('Nodes', n, 'is Done for Tx_range', tx)
output_file.write(s)
output_file.close()
# Add a poison pill for each consumer
print('Poisoning Consumers')
for i in range(num_consumers):
tasks.put(None)
# Wait for all of the tasks to finish
tasks.join()
What could be the cause of this problem? Could it be the queue.get() is returning None?
Thanks in advance
Since I did not get any suggestions, I tried to solve the problem and I came up with a solution (a bad one I believe), where I simply just forgot about consumers and started each function run in a Process. I kept the number of concurrent process limited by checking the number of started ones as shown below. But I am sure I am doing something wrong because the performance of this solution is much worse than not using multiprocesses. With multiprocessing the inner for loop on "n" takes about 2 minutes, but without multiprocessing it takes a few seconds. I am still a noob, can anyone point me in the right direction? here is the code:
import multiprocessing as mp
import MyLib
if __name__ == '__main__':
results = mp.Queue()
num_consumers = mp.cpu_count()
trials = 500
prnts = 4
num_of_proc = 0
consumers = []
joined = 0
for tx in list(range(30, 200, 20)):
file_name = 'Centered_BS_output_{}_{}.txt'.format(tx,prnts)
output_file = open(file_name,'a')
output_file.write('Nodes\tTx_Range\tAvg_Rings\tAvg_Disc\n')
for n in list(range(30, 1030, 30)):
consumers.clear()
ring_sum, disc_sum, joined, i, num_of_proc = 0, 0, 0, 0, 0
#for i in range (0, trials):
while i < trials:
if num_of_proc < num_consumers:
consumers.append(mp.Process(target=MyLib.GBMR_mp, args=(1000, 1000, n, prnts, tx, False, results)))
consumers[i].daemon = True
consumers[i].start()
num_of_proc += 1
i += 1
else:
consumers[joined].join()
num_of_proc -= 1
joined += 1
print('Done putting jobs')
for i in range (0, trials):
mxR, discN = results.get()
ring_sum += mxR
disc_sum += discN
avg_ring = ring_sum/trials
avg_disc = disc_sum/trials
print('Done Collecting Results, avg_disc = ', avg_disc,' and avg_rings = ', avg_ring)
s = '{}\t\t{}\t\t{}\t\t{}\n'.format(n,tx,avg_ring,avg_disc)
print('Nodes', n, 'is Done for Tx_range', tx)
output_file.write(s)
output_file.close()
I would like to construct an algorithm that whittles a queue down to one element but may (temporarily) empty the queue. I tried to implement this with a multinumber gcd algorithm, but it isn't working at all. I suspect the program is with the while loop, but I am not sure how to fix it.
Any help would be appreciated. I'm new to programming, so apologies if this is too basic a question.
import threading
import Queue
import time, random
#Worker Class
class Worker(threading.Thread):
def __init__(self, queue, flag):
self.__queue = queue
self.__flag = flag
threading.Thread.__init__(self)
def run(self):
while (queue.qsize() > 1):
a = self.__queue.get()
b = self.__queue.get()
#Worker task
g = gcd(a, b)
if g == 1:
flag = False
queue.put(g)
def quit(self):
return
#gcd algorithm
def gcd(a, b):
while b:
a, b = b, a%b
return a
#kill all workers
def killall(workers):
for worker in workers:
worker.quit()
#queue starts empty
queue = Queue.Queue(0)
#input thread number
WORKERS = int(raw_input("Type in the number of threads (int > 0):"))
#input list length
k = int(raw_input("Length of integer list (int > 0):"))
#random list of integers generated
list = [random.randint(1,1000) for i in range(k)]
while (len(list)):
queue.put(list.pop())
#list of workers
wrkrs = []
#flag to kill if 1 is found
flag = True
#Master function
for i in range(WORKERS):
w = Worker(queue, flag)
wrkrs.append(w)
w.start()
#kill process if 1 is found
if flag == False:
killall(w)
print "gcd is: 1"
#if this worked, answer would be the only remaining element
if queue.qsize() == 1:
print "gcd is:", queue.get()