I want to start 4 process which put an integer in queue when counter is divisible by 100.Same time another process continuously read it and print it.Please correct my code to run...I am getting an error ['Queue' object is not iterable]
from multiprocessing import Lock, Process, Queue, current_process
import time
import queue
def doFirstjob(process_Queue):
i=0
while True:
if i%100==0:
process_Queue.put(i)
else:
i+=1
def doSecondjob(process_Queue):
while(1):
if not process_Queue.Empty:
task = process_Queue.get()
print("task: ",task)
else:
time.sleep(0.2)
def main():
number_of_processes = 4
process_Queue = Queue()
processes = []
process_Queue.put(1)
q = Process(target=doSecondjob, args=(process_Queue))
q.start()
for w in range(number_of_processes):
p = Process(target=doFirstjob, args=(process_Queue))
processes.append(p)
p.start()
if __name__ == '__main__':
main()
You were getting error because Process was expecting a list/tuple in arguments/args.
Also instead of Empty it should be empty.
change the code to below.
from multiprocessing import Lock, Process, Queue, current_process
import time
import queue
def doFirstjob(process_Queue):
i=0
while True:
print("foo")
if i%100==0:
process_Queue.put(i)
else:
i+=1
def doSecondjob(process_Queue):
while(1):
print("bar")
if not process_Queue.empty:
task = process_Queue.get()
print("task: ",task)
else:
time.sleep(0.2)
def main():
number_of_processes = 4
process_Queue = Queue()
processes = []
process_Queue.put(1)
q = Process(target=doSecondjob, args=(process_Queue,))
q.start()
for w in range(number_of_processes):
p = Process(target=doFirstjob, args=(process_Queue,))
processes.append(p)
p.start()
if __name__ == '__main__':
main()
Related
I'm trying to use python multiprocessing.Queue in my code:
import multiprocessing as mp
import datetime as dt
def function_to_get_from_q(Queue):
#while not Queue.empty():
print(Queue.get())
def collect(Queue):
for i in range(10000):
Queue.put([i, (dt.datetime.utcnow() + dt.timedelta(hours=5, minutes=30)).strftime('%H:%M:%S')])
if __name__ == "__main__":
Q = mp.Queue()
process1 = mp.Process(target=collect, args=(Q,))
process2 = mp.Process(target=function_to_get_from_q, args=(Q,))
process1.start()
process2.start()
I'm expecting that I get all the lists of number and current time. But I'm getting only first element ([0, '14:53:52']).
Where did I go wrong?
The while loop you have in your code is wrong because you don't want the function_to_get_from_q() process to quit everytime it checks the queue and it's empty. In the code below, a special value is put() into the queue to indicate that it's the last one.
import multiprocessing as mp
import datetime as dt
SENTINEL = 'stop'
def function_to_get_from_q(queue):
while (value := queue.get()) != SENTINEL:
print(value)
def collect(queue):
for i in range(10000):
t = dt.datetime.utcnow() + dt.timedelta(hours=5, minutes=30)
queue.put([i, t.strftime('%H:%M:%S')])
queue.put(SENTINEL) # Indicate end.
if __name__ == "__main__":
queue = mp.Queue()
process1 = mp.Process(target=collect, args=(queue,))
process2 = mp.Process(target=function_to_get_from_q, args=(queue,))
process1.start()
process2.start()
print('fini')
The solution using a special sentinel value offered by martineau, which I have upvoted, is one way. I just wanted to show a second method that uses a multiprocessing.JoinableQueue. In this case process2 would be a dameon process that will automatically terminate when the main process terminates since it loops forever trying to get items from the queue:
import multiprocessing as mp
import datetime as dt
def function_to_get_from_q(queue):
while True:
value = queue.get()
print(value)
queue.task_done()
def collect(queue):
for i in range(1000):
t = dt.datetime.utcnow() + dt.timedelta(hours=5, minutes=30)
queue.put([i, t.strftime('%H:%M:%S')])
# Wait for all work placed on queue to be completed:
queue.join()
if __name__ == "__main__":
queue = mp.JoinableQueue()
process1 = mp.Process(target=collect, args=(queue,))
# This needs to be a daemon process since it never terminates:
process2 = mp.Process(target=function_to_get_from_q, args=(queue,), daemon=True)
process1.start()
process2.start()
# We must explicitly wait for process1 to complete to ensure
# that our daemon process is not prematurely terminated:
process1.join()
print('fini')
If you don't wish to use a daemon thread, then we must explicitly terminate process2:
import multiprocessing as mp
import datetime as dt
def function_to_get_from_q(queue):
while True:
value = queue.get()
print(value)
queue.task_done()
def collect(queue):
for i in range(1000):
t = dt.datetime.utcnow() + dt.timedelta(hours=5, minutes=30)
queue.put([i, t.strftime('%H:%M:%S')])
# Wait for all work placed on queue to be completed:
queue.join()
if __name__ == "__main__":
queue = mp.JoinableQueue()
process1 = mp.Process(target=collect, args=(queue,))
# This needs to be a daemon process since it never terminates:
process2 = mp.Process(target=function_to_get_from_q, args=(queue,))
process1.start()
process2.start()
# We must explicitly wait for process1 to complete so that
# we know all work has been completed:
process1.join()
# Kill process2:
process2.terminate()
print('fini')
I would like to put output data into a queue in a multiprocessing computation. It seems that when the size of the return is too large, the program got stuck. To illustrate the problem, here is a minimal codes. Anyone can help to make this work?
from multiprocessing import Process, Queue
import numpy as np
def foo(q, qid):
x = np.random.randint(0,5,7)
y = np.random.random(100*10*10).reshape(100,10,10)
q.put([qid,x,y])
def main():
processes = []
q = Queue()
for qid in range(5):
p = Process(target=foo, args=(q, qid))
p.start()
processes.append(p)
for process in processes:
process.join()
for qid in range(5):
[_, x, y] = q.get()
print(x)
print(y)
if __name__ == '__main__':
main()
I figured out one solution is as below to switch the join and get. By default, the get method blocks.
from multiprocessing import Process, Queue
import numpy as np
def foo(q, qid):
x = np.random.randint(0,5,7)
y = np.random.random(100*10*10).reshape(100,10,10)
q.put([qid,x,y])
def main():
processes = []
q = Queue()
for qid in range(5):
p = Process(target=foo, args=(q, qid))
p.start()
processes.append(p)
for qid in range(5):
[_, x, y] = q.get()
print(x)
print(y)
for process in processes:
process.join()
if __name__ == '__main__':
main()
I have my demo code shown as below. I realize that all subprocesses have finished but they do not exit. Is there anything wrong with my code? Python version: 3.7.4, Operation system: win10
import multiprocessing as mp
res_queue = mp.Queue()
def runCalculation(i):
count_list = []
total_count = i
for k in range(100000):
total_count += k
count_list.append(total_count)
print('task {} finished calculation, putting results to queue'.format(i))
for item in count_list: res_queue.put(item)
print('task {} has put all results to queue'.format(i))
def initPool(res_queue_):
global res_queue
res_queue = res_queue_
def mainFunc():
p = mp.Pool(initializer=initPool, initargs=(res_queue,))
for i in range(20): p.apply_async(runCalculation, args=(i,))
print('Waiting for all subprocesses done...')
p.close()
p.join()
print('All subprocesses done.')
if __name__ == '__main__':
mainFunc()
I'm facing problems with the following example code:
from multiprocessing import Lock, Process, Queue, current_process
def worker(work_queue, done_queue):
for item in iter(work_queue.get, 'STOP'):
print("adding ", item, "to done queue")
#this works: done_queue.put(item*10)
done_queue.put(item*1000) #this doesnt!
return True
def main():
workers = 4
work_queue = Queue()
done_queue = Queue()
processes = []
for x in range(10):
work_queue.put("hi"+str(x))
for w in range(workers):
p = Process(target=worker, args=(work_queue, done_queue))
p.start()
processes.append(p)
work_queue.put('STOP')
for p in processes:
p.join()
done_queue.put('STOP')
for item in iter(done_queue.get, 'STOP'):
print(item)
if __name__ == '__main__':
main()
When the done Queue becomes big enough (a limit about 64k i think), the whole thing freezes without any further notice.
What is the general approach for such a situation when the queue becomes too big? is there some way to remove elements on the fly once they are processed? The Python docs recommend removing the p.join(), in a real application however i can not estimate when the processes have finished. Is there a simple solution for this problem besides infinite looping and using .get_nowait()?
This works for me with 3.4.0alpha4, 3.3, 3.2, 3.1 and 2.6. It tracebacks with 2.7 and 3.0. I pylint'd it, BTW.
#!/usr/local/cpython-3.3/bin/python
'''SSCCE for a queue deadlock'''
import sys
import multiprocessing
def worker(workerno, work_queue, done_queue):
'''Worker function'''
#reps = 10 # this worked for the OP
#reps = 1000 # this worked for me
reps = 10000 # this didn't
for item in iter(work_queue.get, 'STOP'):
print("adding", item, "to done queue")
#this works: done_queue.put(item*10)
for thing in item * reps:
#print('workerno: {}, adding thing {}'.format(workerno, thing))
done_queue.put(thing)
done_queue.put('STOP')
print('workerno: {0}, exited loop'.format(workerno))
return True
def main():
'''main function'''
workers = 4
work_queue = multiprocessing.Queue(maxsize=0)
done_queue = multiprocessing.Queue(maxsize=0)
processes = []
for integer in range(10):
work_queue.put("hi"+str(integer))
for workerno in range(workers):
dummy = workerno
process = multiprocessing.Process(target=worker, args=(workerno, work_queue, done_queue))
process.start()
processes.append(process)
work_queue.put('STOP')
itemno = 0
stops = 0
while True:
item = done_queue.get()
itemno += 1
sys.stdout.write('itemno {0}\r'.format(itemno))
if item == 'STOP':
stops += 1
if stops == workers:
break
print('exited done_queue empty loop')
for workerno, process in enumerate(processes):
print('attempting process.join() of workerno {0}'.format(workerno))
process.join()
done_queue.put('STOP')
if __name__ == '__main__':
main()
HTH
I want a long-running process to return its progress over a Queue (or something similar) which I will feed to a progress bar dialog. I also need the result when the process is completed. A test example here fails with a RuntimeError: Queue objects should only be shared between processes through inheritance.
import multiprocessing, time
def task(args):
count = args[0]
queue = args[1]
for i in xrange(count):
queue.put("%d mississippi" % i)
return "Done"
def main():
q = multiprocessing.Queue()
pool = multiprocessing.Pool()
result = pool.map_async(task, [(x, q) for x in range(10)])
time.sleep(1)
while not q.empty():
print q.get()
print result.get()
if __name__ == "__main__":
main()
I've been able to get this to work using individual Process objects (where I am alowed to pass a Queue reference) but then I don't have a pool to manage the many processes I want to launch. Any advise on a better pattern for this?
The following code seems to work:
import multiprocessing, time
def task(args):
count = args[0]
queue = args[1]
for i in xrange(count):
queue.put("%d mississippi" % i)
return "Done"
def main():
manager = multiprocessing.Manager()
q = manager.Queue()
pool = multiprocessing.Pool()
result = pool.map_async(task, [(x, q) for x in range(10)])
time.sleep(1)
while not q.empty():
print q.get()
print result.get()
if __name__ == "__main__":
main()
Note that the Queue is got from a manager.Queue() rather than multiprocessing.Queue(). Thanks Alex for pointing me in this direction.
Making q global works...:
import multiprocessing, time
q = multiprocessing.Queue()
def task(count):
for i in xrange(count):
q.put("%d mississippi" % i)
return "Done"
def main():
pool = multiprocessing.Pool()
result = pool.map_async(task, range(10))
time.sleep(1)
while not q.empty():
print q.get()
print result.get()
if __name__ == "__main__":
main()
If you need multiple queues, e.g. to avoid mixing up the progress of the various pool processes, a global list of queues should work (of course, each process will then need to know what index in the list to use, but that's OK to pass as an argument;-).