I have a program that contains two parts, they work at the same time using threads ,they communicate using a Queue.
import kafka
import time
from queue import Queue
from threading import Thread
# A thread that consumes data from Kafka consumer,it will stop after 40s if there are no new messages.
def consumer(in_q):
consumer = kafka.KafkaConsumer('mytopic',bootstrap_servers=['myserver'],enable_auto_commit=True,group_id='30',auto_offset_reset='earliest',consumer_timeout_ms=40000)
for message in consumer:
messageStr=message.value.decode("utf-8")
in_q.put(messageStr)
print(messageStr)
print(message.offset)
print("consumer is closed ")
# A thread that modify data
def modifier (out_q):
while True:
if(out_q.empty()==False):
data=out_q.get()
print('data after some modification',data)
# Create the shared queue and launch both threads
message = Queue()
consumeMessgae = Thread(target = consumer, args =(message, ))
modifyMessage = Thread(target = modifier , args =(message, ))
consumeMessgae.start()
modifyMessage.start()
I want to update my modifier function to be able :
change the while loop because it is CPU consuming and instead keep listening to the Queue
I want to be able to close the modifier function when the consumer thread is closed (consumer function will automatically close after 40s if no new messages)
how can I achieve this ?
You can achieve that using a threading.Event to notify the modifier thread to abort execution. the final code would be as follows:
import kafka
import time
from queue import Queue, Empty
from threading import Thread, Event
# A thread that consumes data from Kafka consumer,it will stop after 40s if there are no new messages.
def consumer(in_q, event):
consumer = kafka.KafkaConsumer('mytopic', bootstrap_servers=['myserver'], enable_auto_commit=True, group_id='30',
auto_offset_reset='earliest', consumer_timeout_ms=40000)
try:
for message in consumer:
messageStr = message.value.decode("utf-8")
in_q.put(messageStr)
print(messageStr)
print(message.offset)
except StopIteration:
# This exception is raised by the kafka.KafkaConsumer iterator, after be waiting 40000 without any new message:
# Notify the modifier thread, that he should abort execution:
event.set()
print("consumer is closed ")
# A thread that modify data
def modifier(out_q, event):
while True:
try:
# Block in the queue only for 1 second waiting for a pending item,
# to be able to check if the event was signaled, at most after 1 second lapsed:
data = out_q.get(timeout=1)
except Empty:
print("Queue 'out_q' is empty")
else:
# Will be executed only if there were no Exception.
# Add you additional logic here, but take care of handle any possible Exception that could be generated
# by your data processing logic:
print('data after some modification', data)
# Wait for 1 second before next while loop iteration, but it will shot-circuit before 1 second lapsed, if the
# event is signaled by the consumer thread:
if event.wait(1):
# If we're here is because event was signaled by consumer thread, so we must abort execution:
break
print("modifier is closed ")
# Create the shared queue and launch both threads
message = Queue()
# Create an Event, it will be used by the consumer thread, to notify the modifier thread that he must abort execution:
alert_event = Event()
consumeMessgae = Thread(target=consumer, args=(message, alert_event,))
modifyMessage = Thread(target=modifier, args=(message, alert_event,))
consumeMessgae.start()
modifyMessage.start()
Related
I'm using pika to process RabitMQ message by small batch, and using a thread for each batch.
At the end of the function in the thread, I send acknowledgement of the messages through add_callback_threadsafe to the channel.
In parallele I'm catching SIGINT signals to stop the program properly, by waiting with thread.join() that all threads finish before stopping the channel consume and closing the connection.
But once the CtrlC is sent to generate the SIGINT, event if the program wait for all threads to finish, the acknowledgement will not be processed.
==> is there a way to force the channel/connection to process the waiting add_callback_threadsafe before closing the connection ?
# import packages
# connect to Rabbit MQ
import pika
# intercept stop signal
import signal
# print exception
import traceback
# threading
import functools
import threading
from queue import Queue
# logs time
import datetime
import time
# Function Message Acknowledgement
def ack_message(ch, delivery_tag):
"""Note that `ch` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
print(f'DEBUG ack_message : begining of ack_message function')
if ch.is_open:
ch.basic_ack(delivery_tag)
print(f'DEBUG ack_message : Acknowledgement delivered')
else:
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'ERROR Channel Closed when trying to Acknowledge')
pass
# Function Process multiple messages in separate thread
def block_process():
# list global variables to be changed
global channel
# init local variables
body_list = list()
tag_list = list()
print(f'DEBUG block_process : start of block_process function')
# cancel the timer if exist, as we will proces all elements in the queue here
if event and event.isAlive():
event.cancel()
# extract all queued messages fom internal python queue and rebuild individual listes body and tag from tupple
for i in range(list_Boby_Tag.qsize()):
myTuppleBodyTag = list_Boby_Tag.get()
body_list += [myTuppleBodyTag[0]]
tag_list += [myTuppleBodyTag[1]]
# that also empty the queue
# do something that take time with the block of nessage in body_list
time.sleep(10)
for body in body_list:
body_str = body.decode()
print(f'DEBUG block_process : message processed is {body_str}')
# acknowledging all tags in tag_list by using the channel thread safe function .connection.add_callback_threadsafe
for tag in tag_list:
print(f'DEBUG preprare delivering Acknowledgement from thread')
cb = functools.partial(ack_message, channel, tag)
channel.connection.add_callback_threadsafe(cb)
print(f'DEBUG block_process : end of block_process function')
return
# Function Process message by message and call
def process_message(ch, method, properties, body):
# list global variables to be changed
global list_Boby_Tag
global event
global threads
# do nothing if this flag is on, as the program is about to close
if PauseConsume == 1:
return
# cancel the timer if exist as we are going to process a block or restart a new timer
if event and event.isAlive():
event.cancel()
# put in the queue the data from the body and tag as tupple
list_Boby_Tag.put((body,method.delivery_tag))
# if a max queue size is reached (here 500), immmediately launch a new thread to process the queue
if list_Boby_Tag.qsize() == 500 :
#print(f'DEBUG thread count before {len(threads)}')
# keep in the threads list only the thread still running
threads = [x for x in threads if x.is_alive()]
#print(f'DEBUG thread count after {len(threads)}')
# start the inference in a separated thread
t = threading.Thread(target=block_process)
t.start()
# keep trace of the thread so it can be waited at the end if still running
threads.append(t)
#print(f'DEBUG thread count after add {len(threads)}')
elif list_Boby_Tag.qsize() > 0 :
# if the queue is not full create a thread with a timer to do the process after sometime, here 10 seconds for test purpose
event = threading.Timer(interval=10, function=block_process)
event.start()
# also add this thread to the list of threads
threads.append(event)
# PARAMETERS
RabbitMQ_host = '192.168.1.190'
RabbitMQ_port = 5672
RabbitMQ_queue = 'test_ctrlC'
RabbitMQ_cred_un = 'xxxx'
RabbitMQ_cred_pd = 'xxxx'
# init variables for batch process
list_Boby_Tag = Queue()
threads = list()
event = None
PauseConsume = 0
init_time = time.time()
# connect to RabbitMQ via Pika
cred = pika.credentials.PlainCredentials(RabbitMQ_cred_un,RabbitMQ_cred_pd)
connection = pika.BlockingConnection(pika.ConnectionParameters(host=RabbitMQ_host, port=RabbitMQ_port, credentials=cred))
channel = connection.channel()
channel.queue_declare(queue=RabbitMQ_queue,durable=True)
# tell rabbitMQ to don't dispatch a new message to a worker until it has processed and acknowledged the previous one :
channel.basic_qos(prefetch_count=1)
# define the comsumer
channel.basic_consume(queue=RabbitMQ_queue,
auto_ack=False, # false = need message acknowledgement : basic_ack in the callback
on_message_callback=process_message)
# empty queue and generate test data
channel.queue_purge(queue=RabbitMQ_queue)
# wait few second so the purge can be check in the RabbitMQ ui
print(f'DEBUG main : queue {RabbitMQ_queue} purged')
connection.sleep(10)
# generate 10 test messages
for msgId in range(10):
channel.basic_publish(exchange='',
routing_key=RabbitMQ_queue,
body=f'message{msgId}',
properties=pika.BasicProperties(
delivery_mode = pika.spec.PERSISTENT_DELIVERY_MODE
))
print(f'DEBUG main : test messages created in {RabbitMQ_queue}')
# Function clean stop of pika connection in case of interruption or exception
def cleanClose():
# tell the on_message_callback to do nothing
PauseConsume = 1
# Wait for all threads to complete
for thread in threads:
thread.join()
# stop pika connection after a short pause
connection.sleep(3)
channel.stop_consuming()
connection.close()
return
# Function handle exit signals
def exit_handler(signum, frame):
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'Exit signal received ({signum})')
cleanClose()
exit(0)
signal.signal(signal.SIGINT, exit_handler) # send by a CTRL+C or modified Docker Stop
#signal.signal(signal.SIGTSTP, exit_handler) # send by a CTRL+Z Docker Stop
print(' [*] Waiting for messages. To exit press CTRL+C')
try:
channel.start_consuming()
except Exception:
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'Exception received within start_consumming')
traceback.print_exc()
cleanClose()
A workaround has been found by Luke here :
https://github.com/lukebakken/pika-1402/blob/lukebakken/pika-1402/test_pika_blockthread.py
He has changed the sample code with :
Simplified it a bit by removing the "batch processing via Queue" code since it wasn't related to the current issue
Moved the Pika connection to its own thread
Instead of using a consume callback, move to a generator-style for loop which allows for checking if exiting is requested. This could also be accomplished via SelectConnection and a timer.
Sample code with the batch processing via Queue added back, matching the original code is here :
# from https://github.com/lukebakken/pika-1402/blob/lukebakken/pika-1402/test_pika_blockthread.py
# import packages
# connect to Rabbit MQ
import pika
import pika.credentials
import pika.spec
# intercept stop signal
import signal
# print exception
# import traceback
# threading
import functools
import threading
from queue import Queue
# logs time
import datetime
import time
# PARAMETERS
RabbitMQ_host = "192.168.1.190"
RabbitMQ_port = 5672
RabbitMQ_queue = "test_ctrlC"
RabbitMQ_cred_un = "xxxx"
RabbitMQ_cred_pd = "xxxx"
nbTest = 100000
nbBatch = 1000
nbPrefetch = 10000
# note, prefecth always >= nbBatch
timerSec = 60 # timer wait
workSec = 5 # nbr of sec for simulating batch work
# init variables for batch process
init_time = time.time()
exiting = False
work_threads = list()
event = None
list_Boby_Tag = Queue()
# Function Message Acknowledgement
def ack_message(ch, delivery_tag):
"""Note that `ch` must be the same pika channel instance via which
the message being ACKed was retrieved (AMQP protocol constraint).
"""
if ch.is_open:
ch.basic_ack(delivery_tag)
#print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG ack_message : begining of ack_message function, tag: {delivery_tag}")
else:
# Channel is already closed, so we can't ACK this message;
# log and/or do something that makes sense for your app in this case.
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),"Channel Closed when trying to Acknowledge")
pass
return
# Function Process multiple messages in separate thread
def do_work(channel, list_Boby_Tag):
# init local variables
body_list = list()
tag_list = list()
# cancel the timer if exist, as we will proces all elements in the queue here
if event and event.is_alive():
event.cancel()
# extract all queued messages fom internal python queue and rebuild individual listes body and tag from tupple
for i in range(list_Boby_Tag.qsize()):
myTuppleBodyTag = list_Boby_Tag.get()
body_list += [myTuppleBodyTag[0]]
tag_list += [myTuppleBodyTag[1]]
# that also empty the queue
# do something that take time with the block of nessage in body_list
time.sleep(workSec)
for body in body_list:
body_str = body.decode()
#print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f'DEBUG block_process : message processed is {body_str}')
# acknowledging all tags in tag_list by using the channel thread safe function .connection.add_callback_threadsafe
for tag in tag_list:
cb = functools.partial(ack_message, channel, tag)
channel.connection.add_callback_threadsafe(cb)
return
# Function Process message by message and call thread by block or timer
def process_message(ch, method, body):
global work_threads
global list_Boby_Tag
global event
# cancel the timer if exist as we are going to process a block or restart a new timer
if event and event.is_alive():
event.cancel()
# put in the queue the data from the body and tag as tupple
list_Boby_Tag.put((body,method.delivery_tag))
# if a max queue size is reached (here 500), immmediately launch a new thread to process the queue
if list_Boby_Tag.qsize() == nbBatch :
#print(f'DEBUG thread count before {len(threads)}')
# keep in the threads list only the thread still running
work_threads = [x for x in work_threads if x.is_alive()]
#print(f'DEBUG thread count after {len(threads)}')
# start the inference in a separated thread
t = threading.Thread(target=do_work, args=(ch, list_Boby_Tag))
t.start()
# keep trace of the thread so it can be waited at the end if still running
work_threads.append(t)
#print(f'DEBUG thread count after add {len(threads)}')
elif list_Boby_Tag.qsize() > 0 :
# if the queue is not full create a thread with a timer to do the process after sometime, here 10 seconds for test purpose
event = threading.Timer(interval=timerSec, function=do_work, args=(ch, list_Boby_Tag))
event.start()
# also add this thread to the list of threads
work_threads.append(event)
return
# Function to start pika channel and stopping it
def pika_runner():
# connect to RabbitMQ via Pika
cred = pika.credentials.PlainCredentials(RabbitMQ_cred_un, RabbitMQ_cred_pd)
connection = pika.BlockingConnection(
pika.ConnectionParameters(
host=RabbitMQ_host, port=RabbitMQ_port, credentials=cred
)
)
channel = connection.channel()
channel.queue_declare(queue=RabbitMQ_queue, durable=True)
# tell rabbitMQ to don't dispatch a new message to a worker until it has processed and acknowledged the previous one :
channel.basic_qos(prefetch_count=nbPrefetch)
# empty queue and generate test data
channel.queue_purge(queue=RabbitMQ_queue)
# wait few second so the purge can be check in the RabbitMQ ui
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG main : queue {RabbitMQ_queue} purged, sleeping 5 seconds")
connection.sleep(5)
# generate test messages
for msgId in range(nbTest):
channel.basic_publish(
exchange="",
routing_key=RabbitMQ_queue,
body=f"message-{msgId+1}",
properties=pika.BasicProperties(
delivery_mode=pika.spec.PERSISTENT_DELIVERY_MODE
),
)
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG main : test messages created in {RabbitMQ_queue}")
# loop forever to retrieve messahes
for method_frame, properties, body in channel.consume(
queue=RabbitMQ_queue, inactivity_timeout=1, auto_ack=False
):
if exiting:
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : stopping consuming")
#channel.stop_consuming()
channel.cancel()
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : joining work threads")
for thread in work_threads:
thread.join()
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : all work threads done, sleeping 5 seconds to let acks be delivered")
connection.sleep(5)
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),f"DEBUG : closing connections and channels")
channel.close()
connection.close()
else:
if method_frame is not None:
process_message(channel, method_frame, body)
return
# Function handle exit signals
def exit_handler(signum, _):
global exiting
if exiting:
return
exiting = True
print(datetime.datetime.now(),str(datetime.timedelta(seconds=time.time() - init_time)),"Exit signal received")
pika_thread.join()
exit(0)
# launch the thread that will connect and listen to Pika
pika_thread = threading.Thread(target=pika_runner)
pika_thread.start()
# catch interuption signal to exit gracefully
signal.signal(signal.SIGINT, exit_handler) # send by a CTRL+C or modified Docker Stop
print(" [*] Waiting for messages. To exit press CTRL+C")
# wait for all threads to finish
for thread in work_threads:
thread.join()
pika_thread.join()
In my Python application, I have a function that consumes message from Amazon SQS FIFO queue.
def consume_msgs():
sqs = boto3.client('sqs',
region_name='us-east-1',
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY)
print('STARTING WORKER listening on {}'.format(QUEUE_URL))
while 1:
response = sqs.receive_message(
QueueUrl=QUEUE_URL,
MaxNumberOfMessages=1,
WaitTimeSeconds=10,
)
messages = response.get('Messages', [])
for message in messages:
try:
print('{} > {}'.format(threading.currentThread().getName(), message.get('Body')))
body = json.loads(message.get('Body'))
sqs.delete_message(QueueUrl=QUEUE_URL, ReceiptHandle=message.get('ReceiptHandle'))
except Exception as e:
print('Exception in worker > ', e)
sqs.delete_message(QueueUrl=QUEUE_URL, ReceiptHandle=message.get('ReceiptHandle'))
time.sleep(10)
In order to scale up, I am using multi threading to process messages.
if __name__ == '__main__:
for i in range(3):
t = threading.Thread(target=consume_msgs, name='worker-%s' % i)
t.setDaemon(True)
t.start()
while True:
print('Waiting')
time.sleep(5)
The application runs as service. If I need to deploy new release, it has to be restarted. Is there a way have the threads exist gracefully when main process is being terminated? In stead of killing the threads abruptly, they finish with current message first and stop receiving the next messages.
Since your threads keep looping, you cannot just join them, but you need to signal them it's time to break out of the loop too in order to be able to do that. This docs hint might be useful:
Daemon threads are abruptly stopped at shutdown. Their resources (such as open files, database transactions, etc.) may not be released properly. If you want your threads to stop gracefully, make them non-daemonic and use a suitable signalling mechanism such as an Event.
With that, I've put the following example together, which can hopefully help a bit:
from threading import Thread, Event
from time import sleep
def fce(ident, wrap_up_event):
cnt = 0
while True:
print(f"{ident}: {cnt}", wrap_up_event.is_set())
sleep(3)
cnt += 1
if wrap_up_event.is_set():
break
print(f"{ident}: Wrapped up")
if __name__ == '__main__':
wanna_exit = Event()
for i in range(3):
t = Thread(target=fce, args=(i, wanna_exit))
t.start()
sleep(5)
wanna_exit.set()
A single event instance is passed to fce which would just keep running endlessly, but when done with each iteration, before going back to the top check, if the event has been set to True. And before exiting from the script, we set this event to True from the controlling thread. Since the threads are no longer marked as daemon threads, we do not have to explicitly join them.
Depending on how exactly you want to shutdown your script, you will need to handle the incoming signal (SIGTERM perhaps) or KeyboardInterrupt exception for SIGINT. And perform your clean-up before exiting, the mechanics of which remain the same. Apart from not letting python just stop execution right away, you need to let your threads know they should not re-enter the loop and wait for them to be joined.
The SIGINT is a bit simpler, because it's exposed as a python exception and you could do for instance this for the "main" bit:
if __name__ == '__main__':
wanna_exit = Event()
for i in range(3):
t = Thread(target=fce, args=(i, wanna_exit))
t.start()
try:
while True:
sleep(5)
print('Waiting')
except KeyboardInterrupt:
pass
wanna_exit.set()
You can of course send SIGINT to a process with kill and not only from the controlling terminal.
I've got a WebSocket client and I want to process messages in threads that are split to multiple queues based on the content of the data.
I receive messages in specific order, I want to run functions that are processing messages in this order, but only within categories.
What I managed to do is to create a solution that does not respect the messages order:
def start():
ws = websocket.WebSocketApp(
"ws://127.0.0.1:8080/",
on_message=on_message,
)
Thread(target=ws.run_forever).start()
def on_message(ws, data):
Thread(target=process_data, args=(data,).start() # here I want to preserve order
# here should be categorizing and queuing threads
def process_data(data):
# here should wait for running thread in its category to end,
# and then become a running thread and process data
I can't use thread.join() in the on_message because it will block the main thread, I want to preserve messages order in this method. It need to be blocking after categorizing, each category in its own threads queue.
Maybe i should queue messages instead of threads?
Consider this simple approach:
q = Queue()
def worker_thread():
while True:
msg = q.get()
if msg.cat == A:
process A
if msg.cat == B:
process B
Thread(target=worker_thread).start()
while True:
# receive message
if A:
q.put((A, message))
if B:
q.put((B, message))
I need to process frames from a webcam and send a few selected frames to a remote websocket server. The server answers immediately with a confirmation message (much like an echo server).
Frame processing is slow and cpu intensive so I want to do it using a separate thread pool (producer) to use all the available cores. So the client (consumer) just sits idle until the pool has something to send.
My current implementation, see below, works fine only if I add a small sleep inside the producer test loop. If I remove this delay I stop receiving any answer from the server (both the echo server and from my real server). Even the first answer is lost, so I do not think this is a flood protection mechanism.
What am I doing wrong?
import tornado
from tornado.websocket import websocket_connect
from tornado import gen, queues
import time
class TornadoClient(object):
url = None
onMessageReceived = None
onMessageSent = None
ioloop = tornado.ioloop.IOLoop.current()
q = queues.Queue()
def __init__(self, url, onMessageReceived, onMessageSent):
self.url = url
self.onMessageReceived = onMessageReceived
self.onMessageSent = onMessageSent
def enqueueMessage(self, msgData, binary=False):
print("TornadoClient.enqueueMessage")
self.ioloop.add_callback(self.addToQueue, (msgData, binary))
print("TornadoClient.enqueueMessage done")
#gen.coroutine
def addToQueue(self, msgTuple):
yield self.q.put(msgTuple)
#gen.coroutine
def main_loop(self):
connection = None
try:
while True:
while connection is None:
try:
print("Connecting...")
connection = yield websocket_connect(self.url)
print("Connected " + str(connection))
except Exception, e:
print("Exception on connection " + str(e))
connection = None
print("Retry in a few seconds...")
yield gen.Task(self.ioloop.add_timeout, time.time() + 3)
try:
print("Waiting for data to send...")
msgData, binaryVal = yield self.q.get()
print("Writing...")
sendFuture = connection.write_message(msgData, binary=binaryVal)
print("Write scheduled...")
finally:
self.q.task_done()
yield sendFuture
self.onMessageSent("Sent ok")
print("Write done. Reading...")
msg = yield connection.read_message()
print("Got msg.")
self.onMessageReceived(msg)
if msg is None:
print("Connection lost")
connection = None
print("main loop completed")
except Exception, e:
print("ExceptionExceptionException")
print(e)
connection = None
print("Exit main_loop function")
def start(self):
self.ioloop.run_sync(self.main_loop)
print("Main loop completed")
######### TEST METHODS #########
def sendMessages(client):
time.sleep(2) #TEST only: wait for client startup
while True:
client.enqueueMessage("msgData", binary=False)
time.sleep(1) # <--- comment this line to break it
def testPrintMessage(msg):
print("Received: " + str(msg))
def testPrintSentMessage(msg):
print("Sent: " + msg)
if __name__=='__main__':
from threading import Thread
client = TornadoClient("ws://echo.websocket.org", testPrintMessage, testPrintSentMessage)
thread = Thread(target = sendMessages, args = (client, ))
thread.start()
client.start()
My real problem
In my real program I use a "window like" mechanism to protect the consumer (an autobahn.twisted.websocket server): the producer can send up to a maximum number of un-acknowledge messages (the webcam frames), then stops waiting for half of the window to free up.
The consumer sends a "PROCESSED" message back acknowleding one or more messages (just a counter, not by id).
What I see on the consumer log is that the messages are processed and the answer is sent back but these acks vanish somewhere in the network.
I have little experience with asynchio so I wanted to be sure that I'm not missing any yield, annotation or something else.
This is the consumer side log:
2017-05-13 18:59:54+0200 [-] TX Frame to tcp4:192.168.0.5:48964 : fin = True, rsv = 0, opcode = 1, mask = -, length = 21, repeat_length = None, chopsize = None, sync = False, payload = {"type": "PROCESSED"}
2017-05-13 18:59:54+0200 [-] TX Octets to tcp4:192.168.0.5:48964 : sync = False, octets = 81157b2274797065223a202250524f434553534544227d
This is neat code. I believe the reason you need a sleep in your sendMessages thread is because, otherwise, it keeps calling enqueueMessage as fast as possible, millions of times per second. Since enqueueMessage does not wait for the enqueued message to be processed, it keeps calling IOLoop.add_callback as fast as it can, without giving the loop enough opportunity to execute the callbacks.
The loop might make some progress running on the main thread, since you're not actually blocking it. But the sendMessages thread adds callbacks much faster than the loop can handle them. By the time the loop has popped one message from the queue and has begun to process it, millions of new callbacks are added already, which the loop must execute before it can advance to the next stage of message-processing.
Therefore, for your test code, I think it's correct to sleep between calls to enqueueMessage on the thread.
I wrote a program that uses threads to keep a connection alive while the main program loops until it either has an exception or is manually closed. My program runs in 1 hour intervals and the timeout for the connection is 20 minutes, thus I spawn a thread for every connection element that exist inside of my architecture. Thus, if we have two servers to connect to it connects to both these serves and stays connected and loops through each server retrieving data.
the program I wrote works correctly, however I can't seem to find a way to handle when the program it's self throws an exception. This is to say I can't find an appropriate way to dispose of the threads when the main program excepts. When the program excepts it will just hang open because of the thread not excepting as well and it won't close correctly and will have to be closed manually.
Any suggestions on how to handle cleaning up threads on program exit?
This is my thread:
def keep_vc_alive(vcenter,credentials, api):
vm_url = str(vcenter._proxy.binding.url).split('/')[2]
while True:
try:
logging.info('staying connected %s' % str(vm_url))
vcenter.keep_session_alive()
except:
logging.info('unable to call current time of vcenter %s attempting to reconnect.' % str(vm_url))
try:
vcenter = None
connected,api_version,uuid,vcenter = vcenter_open(60, api, * credentials)
except:
logging.critical('unable to call current time of vcenter %s killing application, please have administrator restart the module.' % str(vm_url))
break
time.sleep(60*10)
Then my exception clean up code is as follows, obviously I know.stop() doesn't work, but I honestly have no idea how to do what it is im trying to do.
except Abort: # Exit without clearing the semaphore
logging.exception('ApplicationError')
try:
config_values_vc = metering_config('VSphere',['vcenter-ip','username','password','api-version'])
for k in xrange(0, len(config_values_vc['username'])): # Loop through each vcenter server
vc_thread[config_values_vc['vcenter-ip'][k]].stop()
except:
pass
#disconnect vcenter
try:
for vcenter in list_of_vc_connections:
list_of_vc_connections[vcenter].disconnect()
except:
pass
try: # Close the db is it is open (db is defined)
db.close()
except:
pass
sys.exit(1)
except SystemExit:
raise
except:
logging.exception('ApplicationError')
semaphore('ComputeLoader', False)
logging.critical('Unexpected error: %s' % sys.exc_info()[0])
raise
Instead of sleeping, wait on a threading.Event():
def keep_vc_alive(vcenter,credentials, api, event): # event is a threading.Event()
vm_url = str(vcenter._proxy.binding.url).split('/')[2]
while not event.is_set(): # If the event got set, we exit the thread
try:
logging.info('staying connected %s' % str(vm_url))
vcenter.keep_session_alive()
except:
logging.info('unable to call current time of vcenter %s attempting to reconnect.' % str(vm_url))
try:
vcenter = None
connected,api_version,uuid,vcenter = vcenter_open(60, api, * credentials)
except:
logging.critical('unable to call current time of vcenter %s killing application, please have administrator restart the module.' % str(vm_url))
break
event.wait(timeout=60*10) # Wait until the timeout expires, or the event is set.
Then, in your main thread, set the event in the exception handling code:
except Abort: # Exit without clearing the semaphore
logging.exception('ApplicationError')
event.set() # keep_alive thread will wake up, see that the event is set, and exit
The generally accepted way to stop threads in python is to use the threading.Event object.
The algorithm followed usually is something like the following:
import threading
...
threads = []
#in the main program
stop_event = threading.Event()
#create thread and store thread and stop_event together
thread = threading.Thread(target=keep_vc_alive, args=(stop_event))
threads.append((thread, stop_event))
#execute thread
thread.start()
...
#in thread (i.e. keep_vc_alive)
# check is_set in stop_event
while not stop_event.is_set():
#receive data from server, etc
...
...
#in exception handler
except Abort:
#set the stop_events
for thread, stop_event in threads:
stop_event.set()
#wait for threads to stop
while 1:
#check for any alive threads
all_finished = True
for thread in threads:
if thread.is_alive():
all_finished = False
#keep cpu down
time.sleep(1)