We Keep Coding

How to simulate broadcast message passing between Thread

I'm writing a small concurrent program using Python 3.6. I have a question:
my program has a small Thread class (which simulates a thread);
this class has within it 3 methods that are executed as sub-threads:
class myThread(Thread):
def __init__(self, identifier):
super(myThread, self).__init__()
def fun1(self):
# broadcasts messages
def fun2(self):
# event that occurs when a message arrives
# do something
def fun3(self):
# event that occurs when a message arrives
# do something
def run(self):
t1 = Thread(target = self.fun1)
t2 = Thread(target = self.fun2)
t3 = Thread(target = self.fun3)
t1.start()
t2.start()
t3.start()
As you can see, fun1() sends broadcast messages (he sends objects) that the other 2 threads must receive. How can this thing be easily implemented in Python?
I have seen that the simplest way is to use Queue, but I have some doubts... where should I put this queue? How can a general method use the submitted object without emptying this queue (since the "broadcast" object must be used by the other methods)? How does a method perform its body every time a new object is added to the queue (as if it were an event)?

a good way to communicate between threads is using queue
it is better to use a designated queue for every thread
this is how you implement it in your code:
from queue import Queue
from threading import Thread
import time
# define some queues
fun2_q = Queue()
fun3_q = Queue()
class myThread(Thread):
def __init__(self, identifier):
super(myThread, self).__init__()
def fun1(self):
print('starting fun1')
# broadcasts messages
fun2_q.put('say something')
fun3_q.put('say something')
fun2_q.put('quit')
fun3_q.put('quit')
def fun2(self):
# event that occurs when a message arrives
# as a listener we should use infinite loop to monitor messages
# we will use non blocking way to read the queue using "if", also we can use fun2_q.get_nowait()
# instead of "if fun2_q.qsize() > 0:" statement
while True:
if fun2_q.qsize() > 0:
msg = fun2_q.get()
if msg == 'say something':
print('fun2 method saying hello')
elif msg == 'quit':
break # quit thread
# do other stuff below if no messages coming
time.sleep(0.1) # to stop while loop from abusing processor
print('fun2 terminating')
def fun3(self):
# event that occurs when a message arrives
# we will use a blocking way to read the queue
while True:
msg = fun3_q.get() # it will block here waiting for a message to come
if msg == 'say something':
print('fun3 method saying hello')
elif msg == 'quit':
break # quit thread
# can't do other stuff below if no messages coming, the loop will stuck waiting new message
# time.sleep(0.1) # no need for it since the loop will wait anyway
print('fun3 terminating')
def run(self):
t1 = Thread(target = self.fun1)
t2 = Thread(target = self.fun2)
t3 = Thread(target = self.fun3)
t1.start()
t2.start()
t3.start()
my_thread = myThread(1)
my_thread.run()
output:
starting fun1
fun2 method saying hello
fun3 method saying hello
fun3 terminating
fun2 terminating

RabbitMQ Unack'ed messages not getting requeued

I'm facing an issue in running RabbitMQ consumers for a long time. Several of my messages end up in an unack'ed state.
My RabbitMQ version: 3.6.15
Pika version: 0.11.0b
import pika
import time
import sys
import threading
from Queue import Queue
rabbitmq_server = "<SERVER>"
queue = "<QUEUE>"
connection = None
def check_acknowledge(channel, connection, ack_queue):
delivery_tag = None
while(True):
try:
delivery_tag = ack_queue.get_nowait()
channel.basic_nack(delivery_tag=delivery_tag)
break
except:
connection.process_data_events()
time.sleep(1)
def process_message(body, delivery_tag, ack_queue):
print "Received %s" % (body)
print "Waiting for 600 seconds before receiving next ID\n"
start = time.time()
elapsed = 0
while elapsed < 10:
elapsed = time.time() - start
print "loop cycle time: %f, seconds count: %02d" %(time.clock(), elapsed)
time.sleep(1)
ack_queue.put(delivery_tag)
def callback(ch, method, properties, body):
global connection
ack_queue = Queue()
t = threading.Thread(target=process_message, args=(body, method.delivery_tag, ack_queue))
t.start()
check_acknowledge(ch, connection, ack_queue)
while True:
try:
connection = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq_server))
channel = connection.channel()
print ' [*] Waiting for messages. To exit press CTRL+C'
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback, queue=queue)
channel.start_consuming()
except KeyboardInterrupt:
break
channel.close()
connection.close()
exit(0)
Am I missing something here?

I used the following multi-threaded consumer to solve this problem.
import pika
import time
import sys
import threading
from Queue import Queue
rabbitmq_server = "<RABBITMQ_SERVER_IP>"
queue = "hello1"
connection = None
def check_acknowledge(channel, connection, ack_queue):
delivery_tag = None
while(True):
try:
delivery_tag = ack_queue.get_nowait()
channel.basic_ack(delivery_tag=delivery_tag)
break
except:
connection.process_data_events()
time.sleep(1)
def process_message(body, delivery_tag, ack_queue):
print "Received %s" % (body)
print "Waiting for 600 seconds before receiving next ID\n"
start = time.time()
elapsed = 0
while elapsed < 300:
elapsed = time.time() - start
print "loop cycle time: %f, seconds count: %02d" %(time.clock(), elapsed)
time.sleep(1)
ack_queue.put(delivery_tag)
def callback(ch, method, properties, body):
global connection
ack_queue = Queue()
t = threading.Thread(target=process_message, args=(body, method.delivery_tag, ack_queue))
t.start()
check_acknowledge(ch, connection, ack_queue)
while True:
try:
connection = pika.BlockingConnection(pika.ConnectionParameters(host=rabbitmq_server))
channel = connection.channel()
print ' [*] Waiting for messages. To exit press CTRL+C'
channel.basic_qos(prefetch_count=1)
channel.basic_consume(callback, queue=queue)
channel.start_consuming()
except KeyboardInterrupt:
break
channel.close()
connection.close()
exit(0)
The consumer callback function triggers a separate function check_acknowledge in the main thread itself. Due to this, connection and channel objects are retained in the same thread. Note that Pika is not thread-safe so we need to maintain these objects in the same thread.
The actual processing happens in a new thread spawned off the main.
Once process_message is done with its processing, it puts the delivery_tag in the queue.
check_acknowledge loops indefinitely till it finds the delivery_tag put in the queue by process_message. Once it does find, it acks the message and returns.
I have tested this implementation by running this consumer by sleeping for 5 min, 10 mins, 30 mins and an hour. This is working very well for me.

How to exit from a generator at some specific time?

I'm reading tweets from Twitter Streaming API. After connecting to the API, I'm getting a generator.
I'm looping through each tweet received but I want to exit from the iterator, say, at 18PM. After receiving each tweet, I'm checking if it's later than the specified timestamp and stopping.
The issue is that I'm not receiving tweets frequently enough. So, I could receive one at 17:50 and the next one at 19PM. That's when I'll find out that the time has passed and I need to stop.
Is there a way to force the stop at 18PM exactly?
Here's a high-level view of my code:
def getStream(tweet_iter):
for tweet in tweet_iter:
#do stuff
if time_has_passed():
return
tweet_iter = ConnectAndGetStream()
getStream(tweet_iter)

Create a separate thread for the producer and use a Queue to communicate. I also had to use a threading.Event for stopping the producer.
import itertools, queue, threading, time
END_TIME = time.time() + 5 # run for ~5 seconds
def time_left():
return END_TIME - time.time()
def ConnectAndGetStream(): # stub for the real thing
for i in itertools.count():
time.sleep(1)
yield "tweet {}".format(i)
def producer(tweets_queue, the_end): # producer
it = ConnectAndGetStream()
while not the_end.is_set():
tweets_queue.put(next(it))
def getStream(tweets_queue, the_end): # consumer
try:
while True:
tweet = tweets_queue.get(timeout=time_left())
print('Got', tweet)
except queue.Empty:
print('THE END')
the_end.set()
tweets_queue = queue.Queue() # you might wanna use the maxsize parameter
the_end = threading.Event()
producer_thread = threading.Thread(target=producer,
args=(tweets_queue, the_end))
producer_thread.start()
getStream(tweets_queue, the_end)
producer_thread.join()

Your problem could be resolved by splitting the functionality of your design into two separated processes:
A twitter process that acts as wrapper to Twitter API and
A monitor process that is able to terminate the twitter process when the exit time is reached.
The following piece of code prototypes the functionality described above using Python's multiprocessing module:
import multiprocessing as mp
import time
EXIT_TIME = '12:21' #'18:00'
def twitter():
while True:
print 'Twittttttttttt.....'
time.sleep(5)
def get_time():
return time.ctime().split()[3][:5]
if __name__ == '__main__':
# Execute the function as a process
p = mp.Process( target=twitter, args=() )
p.start()
# Monitoring the process p
while True:
print 'Checking the hour...'
if get_time() == EXIT_TIME:
p.terminate()
print 'Current time:', time.ctime()
print 'twitter process has benn terminated...'
break
time.sleep(5)
Of course you can use p.join(TIMEOUT) instead of using the while True loop presented in my example as pointed here.

Here is an example with threading and python scheduler:
import threading
import time
import os
import schedule
def theKillingJob():
print("Kenny and Cartman die!")
os._exit(1)
schedule.every().day.at("18:00").do(theKillingJob,'It is 18:00')
def getStream(tweet_iter):
for tweet in tweet_iter:
#do stuff
def kenny():
while True:
print("Kenny alive..")
schedule.run_pending()
time.sleep(1)
def cartman():
while True:
print("Cartman alive..")
tweet_iter = ConnectAndGetStream()
getStream(tweet_iter)
# You can change whenever you want to check for tweets by changing sleep time here
time.sleep(1)
if __name__ == '__main__':
daemon_kenny = threading.Thread(name='kenny', target=kenny)
daemon_cartman = threading.Thread(name='cartman', target=cartman)
daemon_kenny.setDaemon(True)
daemon_cartman.setDaemon(True)
daemon_kenny.start()
daemon_cartman.start()
daemon_kenny.join()
daemon_cartman.join()

Cannot to line up item from one queue to another

I deal with two python queues.
Short description of my issue:
Clients pass through the waiting queue(q1) and they (the clients) are served afterwards. The size of the waiting queue can't be greater than N (10 in my program). If waiting queue becomes full, clients pass to outside queue(q2, size 20). If outside queue becomes full, clients are rejected and not served.
Every client that left a waiting queue allows another client from outside queue to join the waiting queue.
Work with queues should be thread-safe.
Below I implemented approximately what I want. But I'm faced with the problem - enqueuing a client from outside queue (q1) to the waiting queue (q2) during execution serve function. I guess I lost or forgot something important. I think this statement q1.put(client) blocks permanently but don't know why.
import time
import threading
from random import randrange
from Queue import Queue, Full as FullQueue
class Client(object):
def __repr__(self):
return '<{0}: {1}>'.format(self.__class__.__name__, id(self))
def serve(q1, q2):
while True:
if not q2.empty():
client = q2.get()
print '%s leaved outside queue' % client
q1.put(client)
print '%s is in the waiting queue' % client
q2.task_done()
client = q1.get()
print '%s leaved waiting queue for serving' % client
time.sleep(2) # Do something with client
q1.task_done()
def main():
waiting_queue = Queue(10)
outside_queue = Queue(20)
for _ in range(2):
worker = threading.Thread(target=serve, args=(waiting_queue, outside_queue))
worker.setDaemon(True)
worker.start()
delays = [randrange(1, 5) for _ in range(100)]
# Every d seconds 10 clients enter to the waiting queue
for d in delays:
time.sleep(d)
for _ in range(10):
client = Client()
try:
waiting_queue.put_nowait(client)
except FullQueue:
print 'Waiting queue is full. Please line up in outside queue.'
try:
outside_queue.put_nowait(client)
except FullQueue:
print 'Outside queue is full. Please go out.'
waiting_queue.join()
outside_queue.join()
print 'Done'

Finally I found the solution. I check docs more attentive
If full() returns True it doesn’t guarantee that a subsequent call to get() will not block https://docs.python.org/2/library/queue.html#Queue.Queue.full
That's why q1.full() is not reliable in a few threads. I added mutex before inserting item to queues and checking queue is full:
class Client(object):
def __init__(self, ident):
self.ident = ident
def __repr__(self):
return '<{0}: {1}>'.format(self.__class__.__name__, self.ident)
def serve(q1, q2, mutex):
while True:
client = q1.get()
print '%s leaved waiting queue for serving' % client
time.sleep(2) # Do something with client
q1.task_done()
with mutex:
if not q2.empty() and not q1.full():
client = q2.get()
print '%s leaved outside queue' % client
q1.put(client)
print '%s is in the waiting queue' % client
q2.task_done()
def main():
waiting_queue = Queue(10)
outside_queue = Queue(20)
lock = threading.RLock()
for _ in range(2):
worker = threading.Thread(target=serve, args=(waiting_queue, outside_queue, lock))
worker.setDaemon(True)
worker.start()
# Every 1-5 seconds 10 clients enter to the waiting room
i = 1 # Used for unique <int> client's id
while True:
delay = randrange(1, 5)
time.sleep(delay)
for _ in range(10):
client = Client(i)
try:
lock.acquire()
if not waiting_queue.full():
waiting_queue.put(client)
else:
outside_queue.put_nowait(client)
except FullQueue:
# print 'Outside queue is full. Please go out.'
pass
finally:
lock.release()
i += 1
waiting_queue.join()
outside_queue.join()
print 'Done'
Now it works well.

Why the threads are not released after all work is consumed from python Queue

I use Queue to provide tasks that threads can work on. After all work is done from Queue, I see the threads are still alive while I expected them being released. Here is my code. You can see the active threads number is increasing after a batch of task(in the same queue) increases from the console. How could I release the threads after a batch of work get done?
import threading
import time
from Queue import Queue
class ThreadWorker(threading.Thread):
def __init__(self, task_queue):
threading.Thread.__init__(self)
self.task_queue = task_queue
def run(self):
while True:
work = self.task_queue.get()
#do some work
# do_work(work)
time.sleep(0.1)
self.task_queue.task_done()
def get_batch_work_done(works):
task_queue = Queue()
for _ in range(5):
t = ThreadWorker(task_queue)
t.setDaemon(True)
t.start()
for work in range(works):
task_queue.put(work)
task_queue.join()
print 'get batch work done'
print 'active threads count is {}'.format(threading.activeCount())
if __name__ == '__main__':
for work_number in range(3):
print 'start with {}'.format(work_number)
get_batch_work_done(work_number)

Do a non blocking read in a loop and use the exception handling to terminate
def run(self):
try:
while True:
work = self.task_queue.get(True, 0.1)
#do some work
# do_work(work)
except Queue.Empty:
print "goodbye"