twisted - run in a thread - python

new to python
new to twisted
My team wants me to make some existing code run in a seperate thread.
I've come up with a fictional example:
from twisted.internet import threads, reactor
from twisted.internet.defer import inlineCallbacks
from time import sleep
class SomeClass(object):
def __init__(self):
self.working = False
def set_working(self, is_working):
self.working = is_working
print 'Flag set to {}'.format(is_working)
#inlineCallbacks
def do_worker_thread(self):
# I want to make this call on the main thread
self.set_working(True)
# I want to do all this garbage on a separate thread and keep trucking on the main thread
# This mimics some calls in the real code. There is a call to deferToThread and a try
# except block there.
def thread_proc():
try:
for i in range(0, 100):
print 'Step %d starting'.format(i)
self.execute_step(i)
except Exception:
print 'An exception happened'
reactor.callInThread(thread_proc)
# When the worker thread is done, I want to call back 'on_thread_work_done'
def execute_step(self, num):
sleep(num)
print 'Worker thread: %d'.format(num)
def on_thread_work_done(self):
"""I want to be called back when the worker thread is done"""
self.set_working(False)
#inlineCallbacks
def do_main_thread(self):
for c in range(ord('a'), ord('z')+1):
sleep(c)
print 'Main thread: {}'.format(c)
if __name__ == "__main__":
someClass = SomeClass()
result = someClass.do_worker_thread()
result.addCallback(someClass.do_main_thread())
reactor.run()
The stuff on do_worker_thread currently runs on the main thread. I put a comment in there were I want it to run in a seperate thread. It is important that do_worker_thread returns immediately.
I expect the output to look something like:
Flag set to True
Step 0 starting
Main thread: a
Worker thread: 0
Worker thread: 1
Main thread: b
Worker thread: 2
Main thread: c
...
How can I alter what's in do_worker_thread, such that my set_working calls are on the main thread and it isn't set to False until the worker thread is done its work?

Try giving this a shot. It uses callFromThread() to schedule the work on the main thread.
from twisted.internet import threads, reactor
from twisted.internet.defer import inlineCallbacks, returnValue
from time import sleep
class SomeClass(object):
def __init__(self):
self.working = False
def set_working(self, is_working):
self.working = is_working
print 'Flag set to {}'.format(is_working)
#inlineCallbacks
def do_worker_thread(self):
# I want to make this call on the main thread
self.set_working(True)
# I want to do all this garbage on a separate thread and keep trucking on the main thread
# This mimics some calls in the real code. There is a call to deferToThread and a try
# except block there.
def thread_proc():
try:
for i in xrange(0, 10):
print 'Step {} starting'.format(i)
self.execute_step(i)
except Exception:
print 'An exception happened'
yield threads.deferToThread(thread_proc)
# When the worker thread is done, I want to call back 'on_thread_work_done'
self.on_thread_work_done()
returnValue(17)
def execute_step(self, num):
sleep(1)
print 'Worker thread: {}'.format(num)
def on_thread_work_done(self):
"""I want to be called back when the worker thread is done"""
self.set_working(False)
def do_main_thread(self):
for i in [chr(x) for x in range(ord('a'), ord('z')+1)]:
print 'Main thread: {}'.format(i)
sleep(1)
def thread_done(self, result):
print 'Thread done: {}'.format(result)
if __name__ == "__main__":
someClass = SomeClass()
# Schedule the threaded work
result = someClass.do_worker_thread().addCallback(someClass.thread_done)
# Schedule the main thread work
reactor.callFromThread(someClass.do_main_thread)
reactor.run()

You can use either callFromThread or blockingCallFromThread.

Related

How can I create multiple threads

I am working on python 3 and my class is as below.
class MyClass():
def values(self):
***values***
i =0
def check_values(self):
for i in ValueList[i:i+1]:
self.server_connect()
new_value = self.update.values(i)
def run(self):
self.check_values()
if __name__ == "__main__"
format1 = "%(asctime)s: %(message)s"
logging.basicConfig(format=format1, level=logging.INFO,
datefmt="%H:%M:%S")
for i in range(4):
thread = threading.Thread(target=MyClass().run())
threads.append(thread)
i += 1
print("the %s thread is running", thread)
thread.start()
There are no threads getting created but code works.
I am not able to catch what I am doing wrong here.
EDIT
First, I would like to thank you for response and time given for the answer.
I have to update code and inherit other class as per new update from team as below.
class MyClass(MainServer):
Now, the server has it's own run function as below.
class MainServer(object):
***constructor***
***other functions ***
def run(self):
self.add_arguments()
self.parse_arguments()
self.check_values()
Now, without run(), my code is not properly running.
while including run() as below.
*** main ***
update_perform = MyClass()
for i range(4):
thread = threading.Thread(target=Myclass().run()) <-- code starts from here
threads.append(thread)
i += 1
print("the %s thread is running", thread)
thread.start() <-- not reaching till here
As per my knowledge I will require thread.start() to start threading. So I have tried below option
class MyClass(MainServer):
***code as above***
def check_values(self):
self.server_authenticate()
update_value = self.update.values()
def run(self):
self.server_connect()
i = 0
threads = list()
for i in ValueList[i:i+1]:
print("Updating the value = ", i)
thread = threading.Thread(target=check_values(), args=[i])
thread.start()
i += 1
print("Currently running thread", thread)
threads.append(thread)
for thread in threads:
thread.join()
Here thread is executing from start and in print I can see as below
for threading :-
Currently running threads = <Thread(Thread-8, stopped 14852)>
But for the value I can see only one is in process as below
for value :-
Updating the value = 10 <- first value
So, now threads may be getting created but the values are not getting executed in parallel.
Which I am not able to figure out.
modify the run function like this
def run(self):
self.check_values()

abortable sleep() in Python

I need a sleep() method which can be aborted (as described here or here).
My approach is to let a threading.Event.wait() timeout at the specified duration:
def abortable_sleep(secs, abort_event):
abort_event.wait(timeout=secs)
abort_event.clear()
After calling abortable_sleep(10, _abort) I can now (from another thread) call _event.set(_abort) to let abortable_sleep() terminate before the 10 seconds.
Example:
def sleeping_thread():
_start = time.perf_counter()
print("%f thread started" % (time.perf_counter() - _start))
abortable_sleep(5, _abort)
print("%f thread stopped" % (time.perf_counter() - _start))
if __name__ == '__main__':
_abort = threading.Event()
while True:
threading.Thread(target=sleeping_thread).start()
time.sleep(3)
_abort.set()
time.sleep(1)
Output:
0.000001 thread started
3.002668 thread stopped
0.000002 thread started
3.003014 thread stopped
0.000001 thread started
3.002928 thread stopped
0.000001 thread started
This code is working as expected but I still have some questions:
isn't there an easier way to have s.th. likea sleep() which can be aborted?
can this be done more elegant? E.g. this way I have to be careful with the Event instance which is not bound to an instance of abortable_sleep()
do I have to expect performance issues with high frequency loops like while True: abortable_sleep(0.0001)? How is the wait()-timeout implemented?
I have a wrapper class which basically slaps some sleep semantics on top of an Event. The nice thing is that you only have to pass around a Sleep object, which you can call sleep() on several times if you like (sleep() is not thread safe though) and that you can wake() from another thread.
from threading import Event
class Sleep(object):
def __init__(self, seconds, immediate=True):
self.seconds = seconds
self.event = Event()
if immediate:
self.sleep()
def sleep(self, seconds=None):
if seconds is None:
seconds = self.seconds
self.event.clear()
self.event.wait(timeout=seconds)
def wake(self):
self.event.set()
Usage example:
if __name__ == '__main__':
from threading import Thread
import time
import logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(created)d - %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.info("sleep")
s = Sleep(3)
logger.info("awake")
def wake_it(sleeper):
time.sleep(1)
logger.info("wakeup!")
sleeper.wake()
logger.info("sleeping again")
s = Sleep(60, immediate=False)
Thread(target=wake_it, args=[s]).start()
s.sleep()
logger.info("awake again")
The above might output something like this:
1423750549 - sleep
1423750552 - awake
1423750552 - sleeping again
1423750553 - wakeup!
1423750553 - awake again
Exactly what you did, but encapsulated in a class.
Due to race conditions, your solution is not always perfectly correct. You should use a threading.BoundedSemaphore() instead. Call aquire() immediately after creating it. When you want to sleep, call acquire() with a timeout, then call release() if the acquire() returned true. To abort the sleep early, call release() from a different thread; this will raise ValueError if there is no sleep in progress.
Using an event instead is problematic if the other thread calls set() at the wrong time (i.e. at any time other than when you are actually waiting on the event).
I'd wrap the sleep/abort function up in a new class:
class AbortableSleep():
def __init__(self):
self._condition = threading.Condition()
def __call__(self, secs):
with self._condition:
self._aborted = False
self._condition.wait(timeout=secs)
return not self._aborted
def abort(self):
with self._condition:
self._condition.notify()
self._aborted = True
I'd then also supply a Thread subclass to manage the sharing of the wakeup routine on a per-thread basis:
class ThreadWithWakeup(threading.Thread):
def __init__(self, *args, **kwargs):
self.abortable_sleep = AbortableSleep()
super(ThreadWithWakeup, self).__init__(*args, **kwargs)
def wakeup(self):
self.abortable_sleep.abort()
Any other thread with access to this thread can call wakeup() to abort the current abortable_sleep() (if one is in progress).
Using ThreadWithWakeup
You can create threads using the ThreadWithWakeup class, and use it like this:
class MyThread(ThreadWithWakeup):
def run(self):
print "Sleeper: sleeping for 10"
if self.abortable_sleep(10):
print "Sleeper: awoke naturally"
else:
print "Sleeper: rudely awoken"
t = MyThread()
t.start()
print "Main: sleeping for 5"
for i in range(5):
time.sleep(1)
print i + 1
print "Main: waking thread"
t.wakeup()
The output of which looks like:
Sleeper: sleeping for 10
Main: sleeping for 5
1
2
3
4
5
Main: waking thread
Sleeper: rudely awoken
Using AbortableSleep on its own
You can also use the AbortableSleep class on its own, which is handy if you can't use the ThreadWithWakeup class for some reason (maybe you're in the main thread, maybe something else creates the threads for you, etc.):
abortable_sleep = AbortableSleep()
def run():
print "Sleeper: sleeping for 10"
if abortable_sleep(10):
print "Sleeper: awoke naturally"
else:
print "Sleeper: rudely awoken"
threading.Thread(target=run).start()
print "Main: sleeping for 5"
for i in range(5):
time.sleep(1)
print i + 1
print "Main: aborting"
abortable_sleep.abort()

Controlling a python thread with a function

Thanks to those who helped me figure out I needed to use threading to run a loop in a control script I have run, I now have an issue to try and control the thread - by starting or stopping it based on a function:
I want to start a process to get a motor to cycle through a movement based on a 'start' parameter sent to the controlling function, also I want to send a 'stop' parameter to stop the thread too - here's where I got to:
def looper():
while True:
print 'forward loop'
bck.ChangeDutyCycle(10)
fwd.ChangeDutyCycle(0)
time.sleep(5)
print 'backwards loop'
bck.ChangeDutyCycle(0)
fwd.ChangeDutyCycle(20)
time.sleep(5)
def looper_control(state):
t = threading.Thread(target=looper)
if state == 'start':
t.start()
elif state == 'stop':
t.join()
print 'looper stopped!!'
This starts the thread okay when I call looper_control('start') but throws an error when looper_control('stop'):
File "/usr/lib/python2.7/threading.py", line 657, in join
raise RuntimeError("cannot join thread before it is started")
RuntimeError: cannot join thread before it is started
EDIT: looper_control called from here
if "motor" in tmp:
if tmp[-1:] == '0':
#stop both pin
MotorControl('fwd',0,0)
print 'stop motors'
looper_control('stop')
elif tmp[-1:] == '2':
#loop the motor
print 'loop motors'
looper_control('start')
UPDATE: Ive not been able to stop the thread using the method suggested - I thought I had it!
here's where I am:
class sliderControl(threading.Thread):
def __init__(self,stop_event):
super(sliderControl,self).__init__()
self.stop_event = stop_event
def run(self):
while self.stop_event:
print 'forward loop'
bck.ChangeDutyCycle(10)
fwd.ChangeDutyCycle(0)
time.sleep(5)
print 'backwards loop'
bck.ChangeDutyCycle(0)
fwd.ChangeDutyCycle(20)
time.sleep(5)
def looper_control(state,stop_event):
if state == 'start':
t = sliderControl(stop_event=stop_event)
t.start()
elif state == 'stop':
#time.sleep(3)
stop_event.set()
#t.join()
print 'looper stopped!!'
called via:
if tmp[-1:] == '0':
#stop both pin
MotorControl('fwd',0,0)
print 'stop motors'
#stop_thread_event = threading.Event()
print 'stopping thread'
print stop_thread_event
looper_control('stop',stop_thread_event)
elif tmp[-1:] == '2':
#loop the motor
print 'loop motors'
global stop_thread_event
stop_thread_event = threading.Event()
print stop_thread_event
looper_control('start', stop_thread_event)
It looked like a separate thread event was being called by loop and stop, so I thought a global would sort it out but its just not playing ball. When I start the loop - it runs, but when I try to stop it, I get looper stopped!! , but the process just keeps running
Your top-level thread routine will need to become an event handler that listens to a Queue object (as in from Queue import Queue) for messages, then handles them based on state. One of those messages can be a shutdown command, in which case the worker thread function simply exits, allowing the main thread to join it.
Instead of time.sleep, use threading.Timer with the body of the timer sending a message into your event queue.
This is a substantial refactoring. But especially if you plan on adding more conditions, you'll need it. One alternative is to use a package that handles this kind of thing for you, maybe pykka.
To stop a python thread you can use threading.Event()
try this:
class YourClass(threading.Thread):
def __init__(self, stop_event):
super(YourClass, self).__init__()
self.stop_event = stop_event
def run(self):
while not self.stop_event.is_set():
# do what you need here (what you had in looper)
def looper_control(state, stop_event):
if state == 'start':
t = YourClass(stop_event=stop_event)
t.start()
elif state == 'stop':
stop_event.set()
and call to looper_control:
stop_thread_event = threading.Event()
looper_control(state, stop_thread_event)
you only can "start" once a thread
but you can lock and unlock the thread.
the best way to stop and start a thread is with mutex, Example:
#!/usr/bin/python
import threading
from time import sleep
mutex2 = threading.Lock()
#This thread add values to d[]
class Hilo(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
mutex2.acquire()
#Add values to d[]
d.append("hi from Peru")
mutex2.release()
sleep(1)
d=[];
hilos = [Hilo()]
#Stop Thread
#If you have more threads you need make a mutex for every thread
mutex2.acquire()
#Start treades, but the thread is lock
for h in hilos:
h.start()
#so you need do
#unlock THREAD<
mutex2.release()
#>START THREAD
#Sleep for 4 seconds
sleep(4)
#And print d[]
print d
print "------------------------------------------"
#WAIT 5 SECONDS AND STOP THE THREAD
sleep(5)
try:
mutex2.acquire()
except Exception, e:
mutex2.release()
mutex2.acquire()
#AND PRINT d[]
print d
#AND NOW YOUR TRHEAD IS STOP#
#When the thread is lock(stop), you only need call: mutex2.release() for unlock(start)
#When your thread is unlock(start) and you want lock(stop):
#try:
# mutex2.acquire()
#except Exception, e:
# mutex2.release()
# mutex2.acquire()

How to let a Python thread finish gracefully

I'm doing a project involving data collection and logging. I have 2 threads running, a collection thread and a logging thread, both started in main. I'm trying to allow the program to be terminated gracefully when with Ctrl-C.
I'm using a threading.Event to signal to the threads to end their respective loops. It works fine to stop the sim_collectData method, but it doesn't seem to be properly stopping the logData thread. The Collection terminated print statement is never executed, and the program just stalls. (It doesn't end, just sits there).
The second while loop in logData is to make sure everything in the queue is logged. The goal is for Ctrl-C to stop the collection thread immediately, then allow the logging thread to finish emptying the queue, and only then fully terminate the program. (Right now, the data is just being printed out - eventually it's going to be logged to a database).
I don't understand why the second thread never terminates. I'm basing what I've done on this answer: Stopping a thread after a certain amount of time. What am I missing?
def sim_collectData(input_queue, stop_event):
''' this provides some output simulating the serial
data from the data logging hardware.
'''
n = 0
while not stop_event.is_set():
input_queue.put("DATA: <here are some random data> " + str(n))
stop_event.wait(random.randint(0,5))
n += 1
print "Terminating data collection..."
return
def logData(input_queue, stop_event):
n = 0
# we *don't* want to loop based on queue size because the queue could
# theoretically be empty while waiting on some data.
while not stop_event.is_set():
d = input_queue.get()
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
# if the stop event is recieved and the previous loop terminates,
# finish logging the rest of the items in the queue.
print "Collection terminated. Logging remaining data to database..."
while not input_queue.empty():
d = input_queue.get()
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
return
def main():
input_queue = Queue.Queue()
stop_event = threading.Event() # used to signal termination to the threads
print "Starting data collection thread...",
collection_thread = threading.Thread(target=sim_collectData, args=(input_queue, stop_event))
collection_thread.start()
print "Done."
print "Starting logging thread...",
logging_thread = threading.Thread(target=logData, args=(input_queue, stop_event))
logging_thread.start()
print "Done."
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
# stop data collection. Let the logging thread finish logging everything in the queue
stop_event.set()
main()
The problem is that your logger is waiting on d = input_queue.get() and will not check the event. One solution is to skip the event completely and invent a unique message that tells the logger to stop. When you get a signal, send that message to the queue.
import threading
import Queue
import random
import time
def sim_collectData(input_queue, stop_event):
''' this provides some output simulating the serial
data from the data logging hardware.
'''
n = 0
while not stop_event.is_set():
input_queue.put("DATA: <here are some random data> " + str(n))
stop_event.wait(random.randint(0,5))
n += 1
print "Terminating data collection..."
input_queue.put(None)
return
def logData(input_queue):
n = 0
# we *don't* want to loop based on queue size because the queue could
# theoretically be empty while waiting on some data.
while True:
d = input_queue.get()
if d is None:
input_queue.task_done()
return
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
def main():
input_queue = Queue.Queue()
stop_event = threading.Event() # used to signal termination to the threads
print "Starting data collection thread...",
collection_thread = threading.Thread(target=sim_collectData, args=(input_queue, stop_event))
collection_thread.start()
print "Done."
print "Starting logging thread...",
logging_thread = threading.Thread(target=logData, args=(input_queue,))
logging_thread.start()
print "Done."
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
# stop data collection. Let the logging thread finish logging everything in the queue
stop_event.set()
main()
I'm not an expert in threading, but in your logData function the first d=input_queue.get() is blocking, i.e., if the queue is empty it will sit an wait forever until a queue message is received. This is likely why the logData thread never terminates, it's sitting waiting forever for a queue message.
Refer to the [Python docs] to change this to a non-blocking queue read: use .get(False) or .get_nowait() - but either will require some exception handling for cases when the queue is empty.
You are calling a blocking get on your input_queue with no timeout. In either section of logData, if you call input_queue.get() and the queue is empty, it will block indefinitely, preventing the logging_thread from reaching completion.
To fix, you will want to call input_queue.get_nowait() or pass a timeout to input_queue.get().
Here is my suggestion:
def logData(input_queue, stop_event):
n = 0
while not stop_event.is_set():
try:
d = input_queue.get_nowait()
if d.startswith("DATA:"):
print "LOG: " + d
n += 1
except Queue.Empty:
time.sleep(1)
return
You are also signalling the threads to terminate, but not waiting for them to do so. Consider doing this in your main function.
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
stop_event.set()
collection_thread.join()
logging_thread.join()
Based on the answer of tdelaney I created an iterator based approach. The iterator exits when the termination message is encountered. I also added a counter of how many get-calls are currently blocking and a stop-method, which sends just as many termination messages. To prevent a race condition between incrementing and reading the counter, I'm setting a stopping bit there. Furthermore I don't use None as the termination message, because it can not necessarily be compared to other data types when using a PriorityQueue.
There are two restrictions, that I had no need to eliminate. For one the stop-method first waits until the queue is empty before shutting down the threads. The second restriction is, that I did not any code to make the queue reusable after stop. The latter can probably be added quite easily, while the former requires being careful about concurrency and the context in which the code is used.
You have to decide whether you want stop to also wait for all the termination messages to be consumed. I choose to put the necessary join there, but you may just remove it.
So this is the code:
import threading, queue
from functools import total_ordering
#total_ordering
class Final:
def __repr__(self):
return "∞"
def __lt__(self, other):
return False
def __eq__(self, other):
return isinstance(other, Final)
Infty = Final()
class IterQueue(queue.Queue):
def __init__(self):
self.lock = threading.Lock()
self.stopped = False
self.getters = 0
super().__init__()
def __iter__(self):
return self
def get(self):
raise NotImplementedError("This queue may only be used as an iterator.")
def __next__(self):
with self.lock:
if self.stopped:
raise StopIteration
self.getters += 1
data = super().get()
if data == Infty:
self.task_done()
raise StopIteration
with self.lock:
self.getters -= 1
return data
def stop(self):
self.join()
self.stopped = True
with self.lock:
for i in range(self.getters):
self.put(Infty)
self.join()
class IterPriorityQueue(IterQueue, queue.PriorityQueue):
pass
Oh, and I wrote this in python 3.2. So after backporting,
import threading, Queue
from functools import total_ordering
#total_ordering
class Final:
def __repr__(self):
return "Infinity"
def __lt__(self, other):
return False
def __eq__(self, other):
return isinstance(other, Final)
Infty = Final()
class IterQueue(Queue.Queue, object):
def __init__(self):
self.lock = threading.Lock()
self.stopped = False
self.getters = 0
super(IterQueue, self).__init__()
def __iter__(self):
return self
def get(self):
raise NotImplementedError("This queue may only be used as an iterator.")
def next(self):
with self.lock:
if self.stopped:
raise StopIteration
self.getters += 1
data = super(IterQueue, self).get()
if data == Infty:
self.task_done()
raise StopIteration
with self.lock:
self.getters -= 1
return data
def stop(self):
self.join()
self.stopped = True
with self.lock:
for i in range(self.getters):
self.put(Infty)
self.join()
class IterPriorityQueue(IterQueue, Queue.PriorityQueue):
pass
you would use it as
import random
import time
def sim_collectData(input_queue, stop_event):
''' this provides some output simulating the serial
data from the data logging hardware.
'''
n = 0
while not stop_event.is_set():
input_queue.put("DATA: <here are some random data> " + str(n))
stop_event.wait(random.randint(0,5))
n += 1
print "Terminating data collection..."
return
def logData(input_queue):
n = 0
# we *don't* want to loop based on queue size because the queue could
# theoretically be empty while waiting on some data.
for d in input_queue:
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
def main():
input_queue = IterQueue()
stop_event = threading.Event() # used to signal termination to the threads
print "Starting data collection thread...",
collection_thread = threading.Thread(target=sim_collectData, args=(input_queue, stop_event))
collection_thread.start()
print "Done."
print "Starting logging thread...",
logging_thread = threading.Thread(target=logData, args=(input_queue,))
logging_thread.start()
print "Done."
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
# stop data collection. Let the logging thread finish logging everything in the queue
stop_event.set()
input_queue.stop()
main()

Terminating a thread : Python

In the below example, if you execute the program multiple times, it spawns a new thread each time with a new ID.
1. How do I terminate all the threads on task completion ?
2. How can I assign name/ID to the threads ?
import threading, Queue
THREAD_LIMIT = 3
jobs = Queue.Queue(5) # This sets up the queue object to use 5 slots
singlelock = threading.Lock() # This is a lock so threads don't print trough each other
# list
inputlist_Values = [ (5,5),(10,4),(78,5),(87,2),(65,4),(10,10),(65,2),(88,95),(44,55),(33,3) ]
def DoWork(inputlist):
print "Inputlist received..."
print inputlist
# Spawn the threads
print "Spawning the {0} threads.".format(THREAD_LIMIT)
for x in xrange(THREAD_LIMIT):
print "Thread {0} started.".format(x)
# This is the thread class that we instantiate.
worker().start()
# Put stuff in queue
print "Putting stuff in queue"
for i in inputlist:
# Block if queue is full, and wait 5 seconds. After 5s raise Queue Full error.
try:
jobs.put(i, block=True, timeout=5)
except:
singlelock.acquire()
print "The queue is full !"
singlelock.release()
# Wait for the threads to finish
singlelock.acquire() # Acquire the lock so we can print
print "Waiting for threads to finish."
singlelock.release() # Release the lock
jobs.join() # This command waits for all threads to finish.
class worker(threading.Thread):
def run(self):
# run forever
while 1:
# Try and get a job out of the queue
try:
job = jobs.get(True,1)
singlelock.acquire() # Acquire the lock
print self
print "Multiplication of {0} with {1} gives {2}".format(job[0],job[1],(job[0]*job[1]))
singlelock.release() # Release the lock
# Let the queue know the job is finished.
jobs.task_done()
except:
break # No more jobs in the queue
def main():
DoWork(inputlist_Values)
How do I terminate all the threads on task completion?
You could put THREAD_LIMIT sentinel values (e.g., None) at the end of the queue and exit thread's run() method if a thread sees it.
On your main thread exit all non-daemon threads are joined so the program will keep running if any of the threads is alive. Daemon threads are terminated on your program exit.
How can I assign name/ID to the threads ?
You can assign name by passing it to the constructor or by changing .name directly.
Thread identifier .ident is a read-only property that is unique among alive threads. It maybe reused if one thread exits and another starts.
You could rewrite you code using multiprocessing.dummy.Pool that provides the same interface as multiprocessing.Pool but uses threads instead of processes:
#!/usr/bin/env python
import logging
from multiprocessing.dummy import Pool
debug = logging.getLogger(__name__).debug
def work(x_y):
try:
x, y = x_y # do some work here
debug('got %r', x_y)
return x / y, None
except Exception as e:
logging.getLogger(__name__).exception('work%r failed', x_y)
return None, e
def main():
logging.basicConfig(level=logging.DEBUG,
format="%(levelname)s:%(threadName)s:%(asctime)s %(message)s")
inputlist = [ (5,5),(10,4),(78,5),(87,2),(65,4),(10,10), (1,0), (0,1) ]
pool = Pool(3)
s = 0.
for result, error in pool.imap_unordered(work, inputlist):
if error is None:
s += result
print("sum=%s" % (s,))
pool.close()
pool.join()
if __name__ == "__main__":
main()
Output
DEBUG:Thread-1:2013-01-14 15:37:37,253 got (5, 5)
DEBUG:Thread-1:2013-01-14 15:37:37,253 got (87, 2)
DEBUG:Thread-1:2013-01-14 15:37:37,253 got (65, 4)
DEBUG:Thread-1:2013-01-14 15:37:37,254 got (10, 10)
DEBUG:Thread-1:2013-01-14 15:37:37,254 got (1, 0)
ERROR:Thread-1:2013-01-14 15:37:37,254 work(1, 0) failed
Traceback (most recent call last):
File "prog.py", line 11, in work
return x / y, None
ZeroDivisionError: integer division or modulo by zero
DEBUG:Thread-1:2013-01-14 15:37:37,254 got (0, 1)
DEBUG:Thread-3:2013-01-14 15:37:37,253 got (10, 4)
DEBUG:Thread-2:2013-01-14 15:37:37,253 got (78, 5)
sum=78.0
Threads don't stop unless you tell them to stop.
My recommendation is that you add a stop variable into your Thread subclass, and check whether this variable is True or not in your run loop (instead of while 1:).
An example:
class worker(threading.Thread):
def __init__(self):
self._stop = False
def stop(self):
self._stop = True
def run(self):
# run until stopped
while not self._stop:
# do work
Then when your program is quitting (for whatever reason) you have to make sure to call the stop method on all your working threads.
About your second question, doesn't adding a name variable to your Thread subclass work for you?

Categories