I'm writing a small concurrent program using Python 3.6. I have a question:
my program has a small Thread class (which simulates a thread);
this class has within it 3 methods that are executed as sub-threads:
class myThread(Thread):
def __init__(self, identifier):
super(myThread, self).__init__()
def fun1(self):
# broadcasts messages
def fun2(self):
# event that occurs when a message arrives
# do something
def fun3(self):
# event that occurs when a message arrives
# do something
def run(self):
t1 = Thread(target = self.fun1)
t2 = Thread(target = self.fun2)
t3 = Thread(target = self.fun3)
t1.start()
t2.start()
t3.start()
As you can see, fun1() sends broadcast messages (he sends objects) that the other 2 threads must receive. How can this thing be easily implemented in Python?
I have seen that the simplest way is to use Queue, but I have some doubts... where should I put this queue? How can a general method use the submitted object without emptying this queue (since the "broadcast" object must be used by the other methods)? How does a method perform its body every time a new object is added to the queue (as if it were an event)?
a good way to communicate between threads is using queue
it is better to use a designated queue for every thread
this is how you implement it in your code:
from queue import Queue
from threading import Thread
import time
# define some queues
fun2_q = Queue()
fun3_q = Queue()
class myThread(Thread):
def __init__(self, identifier):
super(myThread, self).__init__()
def fun1(self):
print('starting fun1')
# broadcasts messages
fun2_q.put('say something')
fun3_q.put('say something')
fun2_q.put('quit')
fun3_q.put('quit')
def fun2(self):
# event that occurs when a message arrives
# as a listener we should use infinite loop to monitor messages
# we will use non blocking way to read the queue using "if", also we can use fun2_q.get_nowait()
# instead of "if fun2_q.qsize() > 0:" statement
while True:
if fun2_q.qsize() > 0:
msg = fun2_q.get()
if msg == 'say something':
print('fun2 method saying hello')
elif msg == 'quit':
break # quit thread
# do other stuff below if no messages coming
time.sleep(0.1) # to stop while loop from abusing processor
print('fun2 terminating')
def fun3(self):
# event that occurs when a message arrives
# we will use a blocking way to read the queue
while True:
msg = fun3_q.get() # it will block here waiting for a message to come
if msg == 'say something':
print('fun3 method saying hello')
elif msg == 'quit':
break # quit thread
# can't do other stuff below if no messages coming, the loop will stuck waiting new message
# time.sleep(0.1) # no need for it since the loop will wait anyway
print('fun3 terminating')
def run(self):
t1 = Thread(target = self.fun1)
t2 = Thread(target = self.fun2)
t3 = Thread(target = self.fun3)
t1.start()
t2.start()
t3.start()
my_thread = myThread(1)
my_thread.run()
output:
starting fun1
fun2 method saying hello
fun3 method saying hello
fun3 terminating
fun2 terminating
Related
I am using RabbitMQ to get message from queue.
This messages being processed then sent to different queue in RabbitMQ.
here is how my program works:
I have consuming Thread for message consuming that puts the revised message in local Queue..
Another thread is listening and when a message arrives a nested Thread is created for analysis..
when the analysis is done the message is sent to RabbitMQ.
I AM TRYING TO ACK after this operation is done but since consuming Thread works faster my channel is closed. How can I ACK after finishing my analysis?
Here is my Python code:
import pandas as pd
import pickle
from queue import Queue
from threading import Thread
import time
class MyAnalysisThread (Thread):
def __init__(self, comingQuery,deliverTag):
Thread.__init__(self)
self.comingQuery = comingQuery
self.deliverTag = deliverTag
def run(self):
analysis(self.comingQuery,self.deliverTag)
def sendToRabbit(linkFeatures):
.....send
print(" [x] send to rabbitMQ ")
def analysis(comingQuery,deliverTag):
/////do analysis
sendToRabbit(message)
global ResiveChannel;
# I WANNA ACK HERE after analysis finish and sent
ResiveChannel.basic_ack(delivery_tag = deliverTag,multiple=False)
def analysisCall(messageQueue, consumerClose):
print('//////////////////////////// analysis started')
global sendChannel;
sendChannel.queue_declare(queue='send')
while True:
if(messageQueue.empty()==False):
#get the message
message=messageQueue.get()
comingQuery=message['comingQuery']
deliverTag=message['deliverTag']
messageQueue.task_done()
# each message will create different thread for analysis
analysisThread = MyAnalysisThread (comingQuery, deliverTag)
analysisThread.start()
analysisThread.join()
elif(consumerClose.empty()==False):
# when the consumer is stopped go out the loop
if(consumerClose.get()==True):
print('consumer stopped')
break;
else:
print('sleeping for .....1')
# wait some n sec before next iteration
time.sleep(1)
def consumeFromRabbit(messageQueue, consumerClose):
def callback(ch, method, properties, body):
comingQuery=json.loads(body)
message={'comingQuery':comingQuery,
'deliverTag':method.delivery_tag,
}
messageQueue.put(message)
global ResiveChannel;
ResiveChannel.basic_consume(queue='link_raw', on_message_callback=callback, auto_ack=False)
print(' [*] Waiting for messages. To exit press CTRL+C')
try:
ResiveChannel.start_consuming()
except KeyboardInterrupt:
consumerClose.put(True)
print("consumer stopped")
pass
def main():
print('this is main')
#create shared ques for passing messages :
messageQueue = Queue()
consumerClose=Queue()
consumerThread = Thread(target = consumeFromRabbit, args =(messageQueue, consumerClose))
analysisThreadCall = Thread(target = analysisCall, args =(messageQueue,consumerClose ))
consumerThread.start()
analysisThreadCall.start()
consumerThread.join()
analysisThreadCall.join()
if __name__ == '__main__':
#create connection for sending
Sendconnection = pika.BlockingConnection(pika.ConnectionParameters('host',,'/',credentials))
sendChannel= Sendconnection.channel()
ResiveConnection = pika.BlockingConnection( pika.ConnectionParameters(''host',,'/','/',credentials))
ResiveChannel = ResiveConnection.channel()
try:
main()
except KeyboardInterrupt:
print('Interrupted')
try:
sys.exit(0)
except SystemExit:
os._exit(0)
The error I got after few threads is :
Exception in thread Thread-38:
pika.exceptions.ChannelWrongStateError: Channel is closed.
I have this example code to explain my problem:
import threading
import time
class thread1(threading.Thread):
def __init__(self, lock):
threading.Thread.__init__(self)
self.daemon = True
self.start()
self.lock = lock
def run(self):
while True:
self.lock.acquire(True)
print ('write done by t1')
self.lock.release()
class thread2(threading.Thread):
def __init__(self, lock):
threading.Thread.__init__(self)
self.daemon = True
self.start()
self.lock = lock
def run(self):
while True:
self.lock.acquire(True)
print ('write done by t2')
self.lock.release()
if __name__ == '__main__':
lock = threading.Lock()
t1 = thread1(lock)
t2 = thread2(lock)
lock.acquire(True)
counter = 0
while True:
print("main...")
counter = counter + 1
if(counter==5 or counter==10):
lock.release() # Here I want to unlock both threads to run just one time and then wait until I release again
time.sleep(1)
t1.join()
t2.join()
What I'm having some issues is the following:
I want to have two threads (thread1 and thread2) that are launched at the beginning of the program, but they should wait until the main() counter reaches 5 or 10.
When the main() counter reaches 5 or 10, it should signal/trigger/unlock the threads, and both threads should run just once and then wait until a new unlock.
I was expecting the code to have the following output (Each line is 1 second running):
main...
main...
main...
main...
main...
write done by t1
write done by t2
main...
main...
main...
main...
main...
write done by t1
write done by t2
Instead I have a different behaviour, such as starting with:
write done by t1
write done by t1
write done by t1
write done by t1
(etc)
And after 5 seconds the
write done by t2
A lot of times...
Can someone help me explaining what is wrong and how can I improve this?
In __init__() of thread1 and thread2, start() is invoked before self.lock is assigned.
t1 and t2 are created before the main thread acquires the lock. That makes these two threads start printing before the main thread locks them. It is the reason your code print the first several lines of "write done by x".
After the counter reaches 5, the main thread releases the lock, but it never locks it again. That makes t1 and t2 keep running.
It never quits unless you kill it...
I suggest you to use Condition Object instead of Lock.
Here is an example based on your code.
import threading
import time
class Thread1(threading.Thread):
def __init__(self, condition_obj):
super().__init__()
self.daemon = True
self.condition_obj = condition_obj
self.start()
def run(self):
with self.condition_obj:
while True:
self.condition_obj.wait()
print('write done by t1')
class Thread2(threading.Thread):
def __init__(self, condition_obj):
super().__init__()
self.daemon = True
self.condition_obj = condition_obj
self.start()
def run(self):
with self.condition_obj:
while True:
self.condition_obj.wait()
print('write done by t2')
if __name__ == '__main__':
condition = threading.Condition()
t1 = Thread1(condition)
t2 = Thread2(condition)
counter = 0
while True:
print("main...")
counter += 1
if counter == 5 or counter == 10:
with condition:
condition.notify_all()
time.sleep(1)
t1.join()
t2.join()
I use Queue to provide tasks that threads can work on. After all work is done from Queue, I see the threads are still alive while I expected them being released. Here is my code. You can see the active threads number is increasing after a batch of task(in the same queue) increases from the console. How could I release the threads after a batch of work get done?
import threading
import time
from Queue import Queue
class ThreadWorker(threading.Thread):
def __init__(self, task_queue):
threading.Thread.__init__(self)
self.task_queue = task_queue
def run(self):
while True:
work = self.task_queue.get()
#do some work
# do_work(work)
time.sleep(0.1)
self.task_queue.task_done()
def get_batch_work_done(works):
task_queue = Queue()
for _ in range(5):
t = ThreadWorker(task_queue)
t.setDaemon(True)
t.start()
for work in range(works):
task_queue.put(work)
task_queue.join()
print 'get batch work done'
print 'active threads count is {}'.format(threading.activeCount())
if __name__ == '__main__':
for work_number in range(3):
print 'start with {}'.format(work_number)
get_batch_work_done(work_number)
Do a non blocking read in a loop and use the exception handling to terminate
def run(self):
try:
while True:
work = self.task_queue.get(True, 0.1)
#do some work
# do_work(work)
except Queue.Empty:
print "goodbye"
Thanks to those who helped me figure out I needed to use threading to run a loop in a control script I have run, I now have an issue to try and control the thread - by starting or stopping it based on a function:
I want to start a process to get a motor to cycle through a movement based on a 'start' parameter sent to the controlling function, also I want to send a 'stop' parameter to stop the thread too - here's where I got to:
def looper():
while True:
print 'forward loop'
bck.ChangeDutyCycle(10)
fwd.ChangeDutyCycle(0)
time.sleep(5)
print 'backwards loop'
bck.ChangeDutyCycle(0)
fwd.ChangeDutyCycle(20)
time.sleep(5)
def looper_control(state):
t = threading.Thread(target=looper)
if state == 'start':
t.start()
elif state == 'stop':
t.join()
print 'looper stopped!!'
This starts the thread okay when I call looper_control('start') but throws an error when looper_control('stop'):
File "/usr/lib/python2.7/threading.py", line 657, in join
raise RuntimeError("cannot join thread before it is started")
RuntimeError: cannot join thread before it is started
EDIT: looper_control called from here
if "motor" in tmp:
if tmp[-1:] == '0':
#stop both pin
MotorControl('fwd',0,0)
print 'stop motors'
looper_control('stop')
elif tmp[-1:] == '2':
#loop the motor
print 'loop motors'
looper_control('start')
UPDATE: Ive not been able to stop the thread using the method suggested - I thought I had it!
here's where I am:
class sliderControl(threading.Thread):
def __init__(self,stop_event):
super(sliderControl,self).__init__()
self.stop_event = stop_event
def run(self):
while self.stop_event:
print 'forward loop'
bck.ChangeDutyCycle(10)
fwd.ChangeDutyCycle(0)
time.sleep(5)
print 'backwards loop'
bck.ChangeDutyCycle(0)
fwd.ChangeDutyCycle(20)
time.sleep(5)
def looper_control(state,stop_event):
if state == 'start':
t = sliderControl(stop_event=stop_event)
t.start()
elif state == 'stop':
#time.sleep(3)
stop_event.set()
#t.join()
print 'looper stopped!!'
called via:
if tmp[-1:] == '0':
#stop both pin
MotorControl('fwd',0,0)
print 'stop motors'
#stop_thread_event = threading.Event()
print 'stopping thread'
print stop_thread_event
looper_control('stop',stop_thread_event)
elif tmp[-1:] == '2':
#loop the motor
print 'loop motors'
global stop_thread_event
stop_thread_event = threading.Event()
print stop_thread_event
looper_control('start', stop_thread_event)
It looked like a separate thread event was being called by loop and stop, so I thought a global would sort it out but its just not playing ball. When I start the loop - it runs, but when I try to stop it, I get looper stopped!! , but the process just keeps running
Your top-level thread routine will need to become an event handler that listens to a Queue object (as in from Queue import Queue) for messages, then handles them based on state. One of those messages can be a shutdown command, in which case the worker thread function simply exits, allowing the main thread to join it.
Instead of time.sleep, use threading.Timer with the body of the timer sending a message into your event queue.
This is a substantial refactoring. But especially if you plan on adding more conditions, you'll need it. One alternative is to use a package that handles this kind of thing for you, maybe pykka.
To stop a python thread you can use threading.Event()
try this:
class YourClass(threading.Thread):
def __init__(self, stop_event):
super(YourClass, self).__init__()
self.stop_event = stop_event
def run(self):
while not self.stop_event.is_set():
# do what you need here (what you had in looper)
def looper_control(state, stop_event):
if state == 'start':
t = YourClass(stop_event=stop_event)
t.start()
elif state == 'stop':
stop_event.set()
and call to looper_control:
stop_thread_event = threading.Event()
looper_control(state, stop_thread_event)
you only can "start" once a thread
but you can lock and unlock the thread.
the best way to stop and start a thread is with mutex, Example:
#!/usr/bin/python
import threading
from time import sleep
mutex2 = threading.Lock()
#This thread add values to d[]
class Hilo(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
while True:
mutex2.acquire()
#Add values to d[]
d.append("hi from Peru")
mutex2.release()
sleep(1)
d=[];
hilos = [Hilo()]
#Stop Thread
#If you have more threads you need make a mutex for every thread
mutex2.acquire()
#Start treades, but the thread is lock
for h in hilos:
h.start()
#so you need do
#unlock THREAD<
mutex2.release()
#>START THREAD
#Sleep for 4 seconds
sleep(4)
#And print d[]
print d
print "------------------------------------------"
#WAIT 5 SECONDS AND STOP THE THREAD
sleep(5)
try:
mutex2.acquire()
except Exception, e:
mutex2.release()
mutex2.acquire()
#AND PRINT d[]
print d
#AND NOW YOUR TRHEAD IS STOP#
#When the thread is lock(stop), you only need call: mutex2.release() for unlock(start)
#When your thread is unlock(start) and you want lock(stop):
#try:
# mutex2.acquire()
#except Exception, e:
# mutex2.release()
# mutex2.acquire()
I'm about to put this design into use in an application, but I'm fairly new to threading and Queue stuff in python. Obviously the actual application is not for saying hello, but the design is the same - i.e. there is a process which takes some time to set-up and tear down, but I can do multiple tasks in one hit. Tasks will arrive at random times, and often in bursts.
Is this a sensible and thread safe design?
class HelloThing(object):
def __init__(self):
self.queue = self._create_worker()
def _create_worker(self):
import threading, Queue
def worker():
while True:
things = [q.get()]
while True:
try:
things.append(q.get_nowait())
except Queue.Empty:
break
self._say_hello(things)
[q.task_done() for task in xrange(len(things))]
q = Queue.Queue()
n_worker_threads = 1
for i in xrange(n_worker_threads):
t = threading.Thread(target=worker)
t.daemon = True
t.start()
return q
def _say_hello(self, greeting_list):
import time, sys
# setup stuff
time.sleep(1)
# do some things
sys.stdout.write('hello {0}!\n'.format(', '.join(greeting_list)))
# tear down stuff
time.sleep(1)
if __name__ == '__main__':
print 'enter __main__'
import time
hello = HelloThing()
hello.queue.put('world')
hello.queue.put('cruel world')
hello.queue.put('stack overflow')
time.sleep(2)
hello.queue.put('a')
hello.queue.put('b')
time.sleep(2)
for i in xrange(20):
hello.queue.put(str(i))
#hello.queue.join()
print 'finish __main__'
The thread safety is handled by Queue implementation (also you must handle in your _say_hello implementation if it is required).
Burst handler problem: A burst should be handled by a single thread only.(ex: let's say your process setup/teardown takes 10 seconds; at second 1 all threads will be busy with burst from sec 0, on second 5 a new task(or burst) but no thread available to handle them/it). So a burst should be defined by max number of tasks (or maybe "infinite") for a specific time-window. An entry in queue should be a list of tasks.
How can you group burst tasks list?
I provide a solution as code, more easy to explain ...
producer_q = Queue()
def _burst_thread():
while True:
available_tasks = [producer_q.get()]
time.sleep(BURST_TIME_WINDOW)
available_tasks.extend(producer_q.get() # I'm the single consumer, so will be at least qsize elements
for i in range(producer_q.qsize()))
consumer_q.push(available_tasks)
If you want to have a maximum of messages in a burst, you just need to slice the available_tasks in multiple lists.