I'm trying to use the multiprocessing module in python 2.6, but apparently there is something I do not understand. I would expect the class below to add up the numbers sent to it by add() and return the sum in the get_result() method. The code below prints "0", I'd like it to print "2". What have I missed?
import multiprocessing
class AdderProcess(multiprocessing.Process):
def __init__(self):
multiprocessing.Process.__init__(self)
self.sum = 0
self.queue = multiprocessing.JoinableQueue(5)
self.daemon = True
self.start()
def run(self):
while True:
number = self.queue.get()
self.sum += number
self.queue.task_done()
def add(self, number):
self.queue.put(number)
def get_result(self):
self.queue.join()
return self.sum
p = AdderProcess()
p.add(1)
p.add(1)
print p.get_result()
PS. This problem has been solved. Thanks for the answers! Just to make it easier for any readers, here's the complete working version:
import multiprocessing
class AdderProcess(multiprocessing.Process):
def __init__(self):
multiprocessing.Process.__init__(self)
self.sum = multiprocessing.Value('d', 0.0)
self.queue = multiprocessing.JoinableQueue(5)
self.daemon = True
self.start()
def run(self):
while True:
number = self.queue.get()
self.sum.value += number
self.queue.task_done()
def add(self, number):
self.queue.put(number)
def get_result(self):
self.queue.join()
return self.sum.value
p = AdderProcess()
p.add(1)
p.add(1)
print p.get_result()
Change self.sum = 0 to self.sum = multiprocessing.Value('d', 0.0), and use self.sum.value to access or change the value.
class AdderProcess(multiprocessing.Process):
def __init__(self):
...
self.sum = multiprocessing.Value('d', 0.0)
...
def run(self):
while True:
number = self.queue.get()
self.sum.value += number # <-- use self.sum.value
self.queue.task_done()
def get_result(self):
self.queue.join()
return self.sum.value # <-- use self.sum.value
The problem is this: Once you call self.start() in __init__, the main process forks off a child process. All values are copied. Now there are two versions of p. In the main process, p.sum is 0. In the child process, the run method is called and p.sum is augmented to 2. But when the main process calls p.get_result(), its version of p still has p.sum equal to 0.
So 0 is printed.
When you want to share a float value between processes, you need to use a sharing mechanism, such as mp.Value.
See "Sharing state between processes" for more options on how to share values.
self.sum is 2... in that process:
def run(self):
while True:
number = self.queue.get()
print "got %s from queue" % number
print "Before adding - self.sum = %d" % self.sum
self.sum += number
print "After adding - self.sum = %d" % self.sum
self.queue.task_done()
[ 13:56 jon#host ~ ]$ ./mp.py
got 1 from queue
Before adding - self.sum = 0
After adding - self.sum = 1
got 1 from queue
Before adding - self.sum = 1
After adding - self.sum = 2
See multiprocessing 16.3.1.4. - Sharing state between processes on how to get self.sum to be the same in different processes.
Related
I used the threading.Lock() not to threads access to the shared resource at the same time. But, in my code case, it doesn't work.
I know that instead of using Writer(in my code), making this class as function, then threading lock works and results 0. But I want to know why my code doesn't work. It seems as same situation for me.
import threading
global lock
lock = threading.Lock()
class Counter:
def __init__(self):
self.count = 0
def increment(self, offset):
self.count += offset
class Writer(object):
def __init__(self, counter: Counter):
self.counter = counter
def write(self, value):
with lock:
self.counter.increment(value)
if __name__ == "__main__":
counter = Counter()
def run(loop, value):
writer = Writer(counter)
for _ in range(loop):
writer.write(value)
t1 = threading.Thread(target=run, args=(100000, 1))
t2 = threading.Thread(target=run, args=(100000, -1))
t1.start()
t2.start()
print(counter.count)
I expect result is 0. But Not 0.
I think it's because the threads are still running. if you try to pause for a second then it prints 0. Like this:
import threading
import time
global lock
lock = threading.Lock()
class Counter:
def __init__(self):
self.count = 0
def increment(self, offset):
self.count += offset
class Writer(object):
def __init__(self, counter: Counter):
self.counter = counter
def write(self, value):
with lock:
self.counter.increment(value)
if __name__ == "__main__":
counter = Counter()
def run(loop, value):
writer = Writer(counter)
for _ in range(loop):
writer.write(value)
t1 = threading.Thread(target=run, args=(100000, 1))
t2 = threading.Thread(target=run, args=(100000, -1))
t1.start()
t2.start()
time.sleep(1)
print(counter.count)
I want to create multiple file using multiple threads, and append data (after some operation is performed) to corresponding files from their respective threads.
I tried it, but data is getting messed between threads and correct data is not added to respective files.
import threading
import time
exitFlag = 0
class myThread (threading.Thread):
def __init__(self, threadID, name, counter):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
def run(self):
with open('file_'+count+'_logs.txt', 'a+') as result:
result.write("Starting " + self.name)
result.write("Exiting " + self.name)
print ("Starting " + self.name)
print_time(self.name, self.counter, 5)
print ("Exiting " + self.name)
def print_time(threadName, delay, counter):
while counter:
if exitFlag:
threadName.exit()
time.sleep(delay)
print ("%s: %s" % (threadName, time.ctime(time.time())))
counter -= 1
myList = ['string0', 'string1', 'string2', 'string3']
if __name__ == "__main__":
count = 0
for data in myList:
count += 1
mythread = myThread(count, "Thread-" + str(count), count)
mythread.start()
mythread.join()
I expect that 4 files shall be created from 4 threads, and data from thread 1 should be written to file_1_logs.txt and so on...
But while writing data, sometimes all data is written in a single file.
How do I write this data to file correctly?
Don't use higher-scope or global variables in threads. Every variable (that you want to modify) must be local to the thread.
That means you need to pass the initial values of everything to the Thread constructor. The same is true for functions like your print_time. Either everything the function needs to do its job is passed via the arguments – or you turn it into a class method.
Consider the following changes. Note how MyThread is completely self-contained:
from threading import Thread
from time import sleep
from datetime import datetime
class MyThread(Thread):
def __init__(self, threadID, name, delay, data):
Thread.__init__(self)
self.threadID = threadID
self.name = name
self.data = data
self.delay = delay
self.logfile = None
def run(self):
with open('file_%s_logs.txt' % self.threadID, 'a+') as logfile:
self.logfile = logfile
self.log("Starting")
self.print_time(5)
self.log("Exiting")
self.logfile = None
def print_time(self, repeat):
for c in range(repeat):
sleep(self.delay)
self.log(self.data)
def log(self, message):
now = datetime.now().isoformat()
formatted_line = "%s:%s:%s" % (now, self.name, message)
print(formatted_line)
if self.logfile:
self.logfile.write(formatted_line + '\n')
if __name__ == "__main__":
myList = ['string0', 'string1', 'string2', 'string3']
threads = []
# spawn threads
for idx, data in enumerate(myList):
thread = MyThread(idx, "Thread-%s" % idx, idx, data)
threads.append(thread)
thread.start()
# wait for threads to finish
while True:
if any(thread.is_alive() for thread in threads):
sleep(0.1)
else:
print("All done.")
break
I have 3 threads. 1 threads collects data and returns it
var1 = Thread1.start()
Thread 2 and thread 3 use this variable var1 to do routine.
I'm not sure If I do it right. because sometimes var1 is returned and it's not an empty list, I store it in variable of each thread, use list comprehension to extract data. In debugging logs I see elements that must be in thread 3, but debugging logger of that thread returns nothing.
In Thread 2 and thread 3 algoritm:
def __init__(self):
self.lock = threading.RLock()
def do_smth2(self,var1):
self.lock.acquire()
var1_2 = var1
self.lock.release()
def do_smth3(self,var3)
self.lock.acquire()
var1_3 = var1
self.lock.release()
in main
object = thread1.start()
thread2.start(object)
thread3.start(object)
Thread 2 and thread 3 run in the same time, and I use time.sleep(3) if var1_3 or var1_2 is None (this is a list type) or when len(var1_3) <0
EDIT
class Application:
def __init__(self):
self.logger = RootLogger()
self.logger.set_config(__name__, sys_log)
self.adapter = Adapter()
self.transit_listener = TransitListener()
def run(self):
#start listeners
transits_list = self.transit_listener.start()
self.adapter.start(transits_list)
# start REST service
RestWebService().run()
Thread 1
class TransitListener:
def __init__(self):
self.interval = session_interval
def _transits_data(self):
# while polling is running change interval after 1st cycle
while datetime.now() >= session_interval:
result = self.connector.query(self.statement,
fetch=True)
self.logger.debug(result)
# store result
self.transits_queue.put(result)
self.logger.debug(self.interval)
time.sleep(5)
# change interval
self._interval_granularity()
self.logger.debug(self.interval)
def start(self):
self.worker = Thread(target=self._transits_data)
self._configure()
self.logger.info("Starting thread 'transists listener'...")
try:
self.worker.start()
if self.worker.is_alive():
self.logger.info("Thread 'transits listener' started")
# return result from queue
return self.transits_queue.get()
Thread 2 and 3
class Adapter:
def __init__(self):
self.logger = RootLogger()
self.logger.set_config(name=__name__, logfile=epp_log)
self.lock = RLock()
self.threads = []
def _session_start(self, transits):
while datetime.now() >= session_interval:
self.lock.acquire()
transit_list = transits
self.lock.release()
self.logger.debug(f"ENTRIES {transit_list}")
def _session_stop(self, transits):
while datetime.now() >= session_interval:
self.lock.acquire()
transit_list = transits
self.lock.release()
self.logger.debug(f"EXITS{transit_list}")
def start(self, transits):
# prepare SQL tables
# define priority of threads
# 1st
session_start_thread = Thread(target=self._session_start, args=(transits,))
self.threads.append(session_start_thread)
# 2nd
session_stop_thread = Thread(target=self._session_stop, args=(transits,))
self.threads.append(session_stop_thread)
self.threads[0].start()
self.threads[1].start()
Am very much new to python. So what i might be asking may not be correct. What am looking to do is. Create a thread from mainss and start the thread. When the thread is started i want it to access the variable of the mainss class from where thread is created and Modify the variable values. And i want the execution of mainss to sleep until thread modifies one of its variable value. How can i achieve this? Here is the code i tried below. Comment in the code of mythread.py class is where I need to modify the value of count variable of mainss class
main.py
#!/usr/bin/python
import time
from myThread import myThread
class mainss():
def __init__(self):
print "s"
def callThread(self):
global count
count = 1
# Create new threads
thread1 = myThread(1, "Thread-1", 1, count)
thread1.start()
# time.sleep(10) until count value is changed by thread to 3
print "Changed Count value%s " % count
print "Exiting"
m = mainss()
m.callThread()
myThread.py
#!/usr/bin/python
import threading
import time
exitFlag = 0
class myThread (threading.Thread):
def __init__(self, threadID, name, counter, count):
threading.Thread.__init__(self)
self.threadID = threadID
self.name = name
self.counter = counter
self.count = count
def run(self):
print_time(self.name, 1, 5, self.count)
def print_time(threadName, delay, counter, count):
from main import mainss
while counter:
if exitFlag:
threadName.exit()
time.sleep(delay)
count = count + 1
print "count %s" % (count)
# here i want to modify count of mainss class
counter -= 1
Thanks in advance
Using Multiprocessing, a Manager dictionary is used to communicate between or to processes https://pymotw.com/3/multiprocessing/communication.html#managing-shared-state Note that the Manager dictionary can be changed while the process is running. Multiprocessing also has a wait feature https://pymotw.com/3/multiprocessing/communication.html#signaling-between-processes
I would use a threading.Event and a Queue
Something like this, (Please note that i didn't test this myself and obviously you gonna have to make some changes.)
main.py
import Queue
import threading
from myThread import myThread
class mainss:
def __init__(self):
self.queue = Queue.Queue()
self.event = threading.Event()
def callThread(self):
self.queue.put(1) # Put a value in the queue
t = myThread(self.queue, self.event)
t.start()
self.event.wait() # Wait for the value to update
count = self.queue.get()
print "Changed Count value %s" % count
if __name__ == '__main__':
m = mainss()
m.callThread()
myThread.py
import threading
class myThread(threading.Thread):
def __init__(self, queue, event):
super(myThread, self).__init__()
self.queue = queue
self.event = event
def run(self):
while True:
count = self.queue.get() # Get the value (1)
count += 1
print "count %s" % (count)
self.queue.put(count) # Put updated value
self.event.set() # Notify main thread
break
I know this question is asked a lot of time but i am still like to know.
def startMonitor(self,event):
selectedInterface = self.interfaces_cblist.GetValue()
Publisher().sendMessage(("test"),selectedInterface)
self.Close()
selectInterfaceStr = str(selectedInterface)
if len(selectedInterface) == 0:
noSelect_error = wx.MessageDialog(None,"Please select an interface","",wx.OK|wx.ICON_ERROR)
noSelect_error.ShowModal()
else:
monitorStarted = wx.MessageDialog(None,"Monitor on %s started"%selectInterfaceStr,"",wx.OK|wx.ICON_ERROR)
monitorStarted.ShowModal()
self.monitorInterface_button.Disable()
thread.start_new_thread(self.camtableDetection,(selectInterfaceStr,))
thread.start_new_thread(self.dhcpexhaustion,(selectInterfaceStr,))
how can i stop the threading?
You can have a stop method that assigns to a variable such as self.abort. Than, in the function you are threading, you should check for this variable regularly and stop the function(with return or something similar). Here's an example class that uses this technique for stopping the thread.
class PymineLogger:
def __init__(self):
self.file = open('server.log', 'a')
self.abort = False
self.log_queue = Queue.Queue()
threading.Thread(target=self.process_queue, args=()).start()
def error(self, i):
line = u'[%s] [ERROR] %s' % (str(time.time()), i)
self.log_queue.put(line)
def info(self, i):
line = u'[%s] [INFO] %s' % (str(time.time()), i)
self.log_queue.put(line)
def process_queue(self):
while not self.abort:
try:
log_line = self.log_queue.get(timeout=1)
print log_line
self.file.write("%s\n" % log_line)
self.file.flush()
except Queue.Empty:
pass
def stop(self):
self.abort = True
The stop method assigns the variable self.abort, which gets regularly checked by the thread.
Class source: pymine2 project