how can i get variable in class which is override multiprocessing in python:
#!/usr/bin/env python
import multiprocessing
import os
class TestMultiprocess(multiprocessing.Process):
def __init__(self):
multiprocessing.Process.__init__(self)
self.myvar = ''
def myfunc(self):
return os.getpid()
def run(self):
self.myvar = self.myfunc()
mlist = []
for i in range(10):
t = TestMultiprocess()
mlist.append(t)
t.start()
for j in mlist:
t.join()
print t.myvar
i can not get value "myvar" from class TestMultiprocess, i just get blank. But i already override the run() function from Process.
sorry if my spell very bad ...
The run() will executed in a separate process; processes don't share memory, normally. multiprocessing does support shared variables, though, through the explicit Value class:
#!/usr/bin/env python
import multiprocessing
import os
class TestMultiprocess(multiprocessing.Process):
def __init__(self):
multiprocessing.Process.__init__(self)
self.myvar = multiprocessing.Value('i',0)
def myfunc(self):
return os.getpid()
def run(self):
self.myvar.value = self.myfunc()
mlist = []
for i in range(10):
t = TestMultiprocess()
mlist.append(t)
t.start()
for j in mlist:
j.join()
print j.myvar.value
replace t with j in the last loop
for j in mlist:
j.join() # t with j
print j.myvar # t with j
EDIT: and this will not solve your problem
by the way if you want to get the process pid you don't have to override the run() method just for that you can just do:
for j in mlist:
j.pid
Related
I have a series of classes that inherit from in a series and parallel manner and I need to use Python threading for all classes when possible. An example is below. The problem is that the Build class does not get its run function executed which is a method in the Thread class. Threading works fine in MyThread class though. Any idea how to make the Build class starts as a thread?
from threading import Thread
from random import randint
import time
class Build(Thread):
def __init__(self):
Thread.__init__(self)
def run(self):
# This run function currently not being executed
for i in range(20):
print('Second series %i in thread' % (i))
time.sleep(1)
class MyThread(Build, Thread):
def __init__(self, val):
''' Constructor. '''
Thread.__init__(self)
Build.__init__(self)
self.val = val
def run(self):
for i in range(1, self.val):
print('Value %d in thread %s' % (i, self.getName()))
# Sleep for random time between 1 ~ 3 second
secondsToSleep = randint(1, 5)
print('%s sleeping fo %d seconds...' % (self.getName(), secondsToSleep))
time.sleep(secondsToSleep)
# Run following code when the program starts
if __name__ == '__main__':
# Declare objects of MyThread class
myThreadOb1 = MyThread(4)
myThreadOb1.setName('Thread 1')
myThreadOb2 = MyThread(4)
myThreadOb2.setName('Thread 2')
# Start running the threads!
myThreadOb1.start()
myThreadOb2.start()
# Wait for the threads to finish...
myThreadOb1.join()
myThreadOb2.join()
print('Main Terminating...')`
FYI: Instead of subclassing threading.Thread the better way to achieve what you want is to make your class instances Callable and just pass them to the target keyword arg of the Thread class' constructor. The advantage of doing this is you can pass in additional arguments to each Thread instance.
going with your sample code.
class MyThread(Build):
def __init__(self):
''' Constructor. '''
Build.__init__(self)
self.val = val
# this allows your class to be a callable.
def __call__(self, val):
for i in range(1, val):
print('Value %d in thread %s' % (i, self.getName()))
# Sleep for random time between 1 ~ 3 second
secondsToSleep = randint(1, 5)
print('%s sleeping fo %d seconds...' % (self.getName(), secondsToSleep))
time.sleep(secondsToSleep)
# Run following code when the program starts
if __name__ == '__main__':
# Declare objects of MyThread class
myThreadObj1 = MyThread()
myThread1 = Thread(target=myThreadOb1, args=(4))
myThread1.start()
I want to use python's multiprocessing module in a class, which itself uses subprocesses to not block the main call.
The minimal example looks like this:
import multiprocessing as mp
class mpo():
def __init__(self):
cpu = mp.cpu_count()
self.Pool = mp.Pool(processes = 2)
self.alive = True
self.p = mp.Process(target = self.sub,args=())
def worker():
print 'Alive'
def sub(self):
print self.alive
for i in range(2):
print i
self.Pool.apply_async(self.worker, args=())
print 'done'
self.Pool.close()
# self.Pool.join()
I commented the last line out, as it raises an assertion Error (can only join a child process).
When I do:
m =mpo()
m.p.start()
The output is
True
0
1
done
My main question is, why the print statement in the worker thread never is reached?
Update:
The updated code looks like this.
import multiprocessing as mp
class mpo():
def __init__(self):
cpu = mp.cpu_count()
self.alive = True
self.p = mp.Process(target = self.sub,args=())
self.result=[]
def worker(self):
self.result.append(1)
print 'Alive'
def sub(self):
print self.alive
Pool = mp.Pool(processes = 2)
for i in range(2):
print i
Pool.apply_async(self.worker, args=())
print 'done'
Pool.close()
Pool.join()
The pool now doesn't have to be inherited as it is created in the subprocess. Instead of the print statement the result is appended to the calling object and the pool is properly joined. Nevertheless, there is no result showing up.
so I think this may correspond to a simple example of what you are looking for:
import multiprocessing as mp
def worker(arg):
#print 'Alive'+str(arg)
return "Alive and finished {0}".format(arg)
class mpo():
def __init__(self):
cpu = mp.cpu_count()
self.alive = True
self.pool = mp.Pool(processes = 2)
def sub(self,arguments):
self.results=self.pool.map_async(worker, arguments)
return self.results
if __name__=="__main__":
s=mpo()
s.sub(range(10))
print s.results.get()
Additionally you can call
self.results.ready()
to find out whether the processes have finished their work. You do not have to put this inside of another process because the map_async call does not block the rest of your program.
EDIT:
Concerning your comment, I do not really see the value of putting the calculation in a separate process, because the function is already running in separate processes (in the pool). You only add complexity by nesting it in another subprocess, but it is possible:
import multiprocessing as mp
def worker(arg):
#print 'Alive'+str(arg)
return "Alive and finished {0}".format(arg)
class mpo():
def __init__(self):
cpu = mp.cpu_count()
self.alive = True
self.pool = mp.Pool(processes = 2)
def sub(self,arguments):
self.results=self.pool.map_async(worker, arguments)
return self.results
def run_calculation(q):
s=mpo()
results=s.sub(range(10))
q.put(results.get())
queue=mp.Queue()
proc=mp.Process(target=run_calculation,args=(queue,))
proc.start()
proc.join()
queue.get()
I have the following situation process=Process(target=sample_object.run) I then would like to edit a property of the sample_object: sample_object.edit_property(some_other_object).
class sample_object:
def __init__(self):
self.storage=[]
def edit_property(self,some_other_object):
self.storage.append(some_other_object)
def run:
while True:
if len(self.storage) is not 0:
print "1"
#I know it's an infinite loop. It's just an example.
_______________________________________________________
from multiprocessing import Process
from sample import sample_object
from sample2 import some_other_object
class driver:
if __name__ == "__main__":
samp = sample_object()
proc = Process(target=samp.run)
proc.start()
while True:
some = some_other_object()
samp.edit_property(some)
#I know it's an infinite loop
The previous code never prints "1". How would I connect the Process to the sample_object so that an edit made to the object whose method Process is calling is recognized by the process? In other words, is there a way to get .run to recognize the change in sample_object ?
Thank you.
You can use multiprocessing.Manager to share Python data structures between processes.
from multiprocessing import Process, Manager
class A(object):
def __init__(self, storage):
self.storage = storage
def add(self, item):
self.storage.append(item)
def run(self):
while True:
if self.storage:
print 1
if __name__ == '__main__':
manager = Manager()
storage = manager.list()
a = A(storage)
p = Process(target=a.run)
p.start()
for i in range(10):
a.add({'id': i})
p.join()
I'm using Python's multiprocessing library to process a list of inputs with the built-in map() method. Here's the relevant code segment:
subp_pool = Pool(self.subprocesses)
cases = subp_pool.map(self.get_case, input_list)
return cases
The function to be run in parallel is self.get_case(), and the list of inputs is input_list.
I wish to print a progress prompt to the standard output in the following format:
Working (25/100 cases processed)
How can I update a local variable inside the class that contains the Pool, so that whenever a subprocess finishes, the variable is incremented by 1 (and then printed to the standard output)?
There's no way to do this using multiprocessing.map, because it doesn't alert the main process about anything until it's completed all its tasks. However, you can get similar behavior by using apply_async in tandem with the callback keyword argument:
from multiprocessing.dummy import Pool
from functools import partial
import time
class Test(object):
def __init__(self):
self.count = 0
self.threads = 4
def get_case(self, x):
time.sleep(x)
def callback(self, total, x):
self.count += 1
print("Working ({}/{}) cases processed.".format(self.count, total))
def do_async(self):
thread_pool = Pool(self.threads)
input_list = range(5)
callback = partial(self.callback, len(input_list))
tasks = [thread_pool.apply_async(self.get_case, (x,),
callback=callback) for x in input_list]
return [task.get() for task in tasks]
if __name__ == "__main__":
t = Test()
t.do_async()
Call the print_data() from the get_case() method and you are done.
from threading import Lock
Class A(object):
def __init__(self):
self.mutex = Lock()
self.count = 0
def print_data(self):
self.mutex.acquire()
try:
self.count += 1
print('Working (' + str(self.count) + 'cases processed)')
finally:
self.mutex.release()
I'm trying to make a simple thread that appends stuff to a global list and then print the results in the main thread after sleeping for a few seconds:
import time,threading
list_of_things = []
class MyThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def __run__(self):
global list_of_things
for i in range(0, 10):
list_of_things.append('hello ' + str(i))
if __name__ == "__main__":
mythread = MyThread()
mythread.start()
time.sleep(5)
print list_of_things
The list is apparently empty even though I declared it global in the thread.
Rename your __run__ method to run. And also instead of calling time.sleep(5), you should call .join() on thread to keep the program waiting till thread finishes its job.
import threading
list_of_things = []
class MyThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
global list_of_things
for i in range(0, 10):
list_of_things.append('hello ' + str(i))
if __name__ == "__main__":
mythread = MyThread()
mythread.start()
mythread.join()
print list_of_things