Current Code
import multiprocessing as mu
import time
global_array=[]
def add_array1(array):
while True:
time.sleep(2.5)
global_array.append(1)
print(global_array)
def add_array2(array):
while True:
time.sleep(3)
global_array.append(2)
print(global_array)
def runInParallel(*fns):
if __name__=='__main__':
proc = []
for fn in fns:
p = mu.Process(target=fn)
p.start()
proc.append(p)
for p in proc:
p.join()
runInParallel(
add_array1(global_array),
add_array2(global_array)
)
When running my code above only the first function add_array1() is appending the value to the array and printing instead of both functions providing the wrong output:
[1]
[1,1]
[1,1,1]
When the actual desired output for the following code is:
[1]
[1,2]
[1,2,1]
[1,2,1,2]
Your problem is that the function call
runInParallel( add_array1(global_array), add_array2(global_array))
executes the functions and provides the return value of the function calls as parameters to runInParallel. As add_array1 is an endless loop, it never returns from the execution. You need to provide your functions as functions - not the returnvalue of the functions as parameters to runInParallel(...)
Start with
runInParallel( add_array1, add_array2) # name of the functions, dont execute em
and change
def runInParallel(*fns):
proc = []
for fn in fns:
p = mu.Process(target=fn, args=(global_array,)) # provide param here
p.start()
proc.append(p)
for p in proc:
p.join()
and then fix the "not joining" problem due to your threaded functions never returning.
Example from the official documentation of multiprocessing.Process:
from multiprocessing import Process
import os
def info(title):
print(title)
print('module name:', __name__)
print('parent process:', os.getppid())
print('process id:', os.getpid())
# Function name is f
def f(name):
info('function f')
print('hello', name)
if __name__ == '__main__':
info('main line')
# f is provided, and args is provided - not f("bob")
p = Process(target=f, args=('bob',))
p.start()
p.join()
You can use the the code below to get your desired output:
import threading
import time
global_array = []
def add_array1():
while True:
time.sleep(2.5)
global_array.append(1)
print(global_array)
def add_array2():
while True:
time.sleep(3)
global_array.append(2)
print(global_array)
def runInParallel(*fns):
if __name__ == '__main__':
proc = []
for fn in fns:
p = threading.Thread(target=fn)
p.start()
proc.append(p)
for p in proc:
p.join()
if __name__ == '__main__':
runInParallel(
add_array1,
add_array2
)
you can use this simple code to get your desired output:
'''
import time
global_array = []
def add_array1(array):
while True:
time.sleep(2.5)
if len(global_array) % 2 == 0:
global_array.append(1)
print(global_array)
else:
global_array.append(2)
print(global_array)
add_array1(global_array)
Related
I found this code online:
import multiprocessing
import time
def foo(n):
for i in range(10000 * n):
print(i)
time.sleep(1)
if __name__ == '__main__':
p = multiprocessing.Process(target=foo, name="Foo", args=(10,))
p.start()
time.sleep(10)
p.terminate()
p.join()
Is it somehow possible to use that code inside a function? Something like:
def test():
def foo(n):
for i in range(10000 * n):
print(i)
time.sleep(1)
if __name__ == '__main__':
p = multiprocessing.Process(target=foo, name="Foo", args=(10,))
p.start()
time.sleep(10)
p.terminate()
p.join()
While I tried it there was always an error. But i'm still quite new to python and have most of the time no Idea what I'm really doing. Or is there maybe any alternative? (I'm using windows so SIGALRM is probably no alternative)
When running this code it just prints out a blank array at the end:
[]
So why is it not appending either the value a or the value b?
import multiprocessing as mu
array_values=[]
def a(array):
array.append('a')
def b(array):
array.append('b')
def runInParallel(*fns):
z=0
while z<6:
if __name__=='__main__':
proc = []
for fn in fns:
p = mu.Process(target=fn,args=(array_values,))
p.start()
proc.append(p)
for p in proc:
p.join()
z+=1
runInParallel(a,b)
print(array_values)
DESIRED FINAL OUTPUT OF FUNCTION:
['a','b','a','b','a','b','a','b','a','b','a','b']
Thanks in advance!
The reason it doesn't word is because multiprocessing doesn't use shared memory.
You can use the following code to get your desired output (it uses threading which uses shared memory):
import threading
array_values = []
def a(array):
array.append('a')
def b(array):
array.append('b')
def runInParallel(*fns):
z = 0
while z < 6:
if __name__ == '__main__':
proc = []
for fn in fns:
p = threading.Thread(target=fn, args=(array_values,))
p.start()
proc.append(p)
for p in proc:
p.join()
z += 1
runInParallel(a, b)
print(array_values)
Thanks to How to run functions in parallel? the following code works.
import time
from multiprocessing import Process
def worker():
time.sleep(2)
print("Working")
def runInParallel(*fns):
proc = []
for fn in fns:
p = Process(target=fn)
p.start()
proc.append(p)
for p in proc:
p.join()
if __name__ == '__main__':
start = time.time()
runInParallel(worker, worker, worker, worker)
print("Total time taken: ", time.time()-start)
However if I add argument to worker() it does not run in parallel anymore.
import time
from multiprocessing import Process
def worker(ii):
time.sleep(ii)
print("Working")
def runInParallel(*fns):
proc = []
for fn in fns:
p = Process(target=fn)
p.start()
proc.append(p)
for p in proc:
p.join()
if __name__ == '__main__':
start = time.time()
runInParallel(worker(2), worker(2), worker(2), worker(2))
print("Total time taken: ", time.time()-start)
What might be the reason for that?
You should modify runInParallel to do iterable unpacking.
import time
from multiprocessing import Process
def worker(ii):
time.sleep(ii)
print("Working")
def runInParallel(*fns):
proc = []
for fn in fns:
func, *args = fn
p = Process(target=func, args=args)
p.start()
proc.append(p)
for p in proc:
p.join()
if __name__ == '__main__':
start = time.time()
runInParallel((worker, 2), (worker, 3), (worker, 5), (worker, 2))
print("Total time taken: ", time.time()-start)
It's because of the difference between worker and worker(). The first is the function, and the latter is a function call. What is happening on the line runInParallel(worker(2), worker(2), worker(2), worker(2)) is that all four calls are run before the execution of runInParallel is even begun. If you add a print(fns) in beginning of runInParallel you will see some difference.
Quick fix:
def worker_caller():
worker(2)
and:
runInParallel(worker_caller, worker_caller, worker_caller, worker_caller)
That's not very convenient but it's mostly intended to show what the problem is. The problem is not in the function worker. The problem is that you're mixing up passing a function and passing a function call. If you changed your first version to:
runInParallel(worker(), worker(), worker(), worker())
then you would run into exactly the same issue.
But you can do this:
runInParallel(lambda:worker(2), lambda: worker(2), lambda: worker(2), lambda: worker(2))
Lambdas are very useful. Here is another version:
a = lambda:worker(2)
b = lambda:worker(4)
c = lambda:worker(3)
d = lambda:worker(1)
runInParallel(a, b, c, d)
To pass arguments, you need to pass them to the Process constructor:
p = Process(target=fn, args=(arg1,))
The Process constructor accepts args and kwargs parameters, which are then passed to the process when it is executed.
The documentation is quite clear about this.
So your code should be modified something like this:
def worker(ii):
time.sleep(ii)
print("Working")
def runInParallel(*fns):
proc = []
for fn in fns:
p = Process(target=fn, args=(2,))
p.start()
proc.append(p)
for p in proc:
p.join()
if __name__ == '__main__':
start = time.time()
runInParallel(worker, worker, worker, worker)
print("Total time taken: ", time.time()-start)
Of course parameters can be different for each process, you need to arrange that the right one is passed to each in args (or kwargs for keyword parameters).
This can be achieved by passing tuples such as runInParallel((worker,2), (worker,3), (worker,5), (worker,1) for example, and then processing the tuples inside runInParallel.
I want to start 4 process which put an integer in queue when counter is divisible by 100.Same time another process continuously read it and print it.Please correct my code to run...I am getting an error ['Queue' object is not iterable]
from multiprocessing import Lock, Process, Queue, current_process
import time
import queue
def doFirstjob(process_Queue):
i=0
while True:
if i%100==0:
process_Queue.put(i)
else:
i+=1
def doSecondjob(process_Queue):
while(1):
if not process_Queue.Empty:
task = process_Queue.get()
print("task: ",task)
else:
time.sleep(0.2)
def main():
number_of_processes = 4
process_Queue = Queue()
processes = []
process_Queue.put(1)
q = Process(target=doSecondjob, args=(process_Queue))
q.start()
for w in range(number_of_processes):
p = Process(target=doFirstjob, args=(process_Queue))
processes.append(p)
p.start()
if __name__ == '__main__':
main()
You were getting error because Process was expecting a list/tuple in arguments/args.
Also instead of Empty it should be empty.
change the code to below.
from multiprocessing import Lock, Process, Queue, current_process
import time
import queue
def doFirstjob(process_Queue):
i=0
while True:
print("foo")
if i%100==0:
process_Queue.put(i)
else:
i+=1
def doSecondjob(process_Queue):
while(1):
print("bar")
if not process_Queue.empty:
task = process_Queue.get()
print("task: ",task)
else:
time.sleep(0.2)
def main():
number_of_processes = 4
process_Queue = Queue()
processes = []
process_Queue.put(1)
q = Process(target=doSecondjob, args=(process_Queue,))
q.start()
for w in range(number_of_processes):
p = Process(target=doFirstjob, args=(process_Queue,))
processes.append(p)
p.start()
if __name__ == '__main__':
main()
Lets assume that i am starting a process in python with the following code:
from multiprocessing import Process
import time
def f(name):
print ('hello ', name)
if __name__ == '__main__':
p = Process(target=f,name = "Process-1", args=('bob',))
p.start()
Now,i want to terminate the process.I can simply do:
p.terminate()
However, i would like to terminate the process by its name.Is that possible?
To do that, you need to store a map between your process objects and their names. Using an helper function it makes your code even easier to read (IMO):
def terminate(procname):
return pmap[procname].terminate()
if __name__ == '__main__':
pmap = {}
pname = "process-1"
p = Process(target=f,name = pname, args=('bob',))
pmap[pname] = p
p.start()
Then to terminate:
terminate(pname)