We Keep Coding

How can I ACK A MESSAGE after doing analysis

I am using RabbitMQ to get message from queue.
This messages being processed then sent to different queue in RabbitMQ.
here is how my program works:
I have consuming Thread for message consuming that puts the revised message in local Queue..
Another thread is listening and when a message arrives a nested Thread is created for analysis..
when the analysis is done the message is sent to RabbitMQ.
I AM TRYING TO ACK after this operation is done but since consuming Thread works faster my channel is closed. How can I ACK after finishing my analysis?
Here is my Python code:
import pandas as pd
import pickle
from queue import Queue
from threading import Thread
import time
class MyAnalysisThread (Thread):
def __init__(self, comingQuery,deliverTag):
Thread.__init__(self)
self.comingQuery = comingQuery
self.deliverTag = deliverTag
def run(self):
analysis(self.comingQuery,self.deliverTag)
def sendToRabbit(linkFeatures):
.....send
print(" [x] send to rabbitMQ ")
def analysis(comingQuery,deliverTag):
/////do analysis
sendToRabbit(message)
global ResiveChannel;
# I WANNA ACK HERE after analysis finish and sent
ResiveChannel.basic_ack(delivery_tag = deliverTag,multiple=False)
def analysisCall(messageQueue, consumerClose):
print('//////////////////////////// analysis started')
global sendChannel;
sendChannel.queue_declare(queue='send')
while True:
if(messageQueue.empty()==False):
#get the message
message=messageQueue.get()
comingQuery=message['comingQuery']
deliverTag=message['deliverTag']
messageQueue.task_done()
# each message will create different thread for analysis
analysisThread = MyAnalysisThread (comingQuery, deliverTag)
analysisThread.start()
analysisThread.join()
elif(consumerClose.empty()==False):
# when the consumer is stopped go out the loop
if(consumerClose.get()==True):
print('consumer stopped')
break;
else:
print('sleeping for .....1')
# wait some n sec before next iteration
time.sleep(1)
def consumeFromRabbit(messageQueue, consumerClose):
def callback(ch, method, properties, body):
comingQuery=json.loads(body)
message={'comingQuery':comingQuery,
'deliverTag':method.delivery_tag,
}
messageQueue.put(message)
global ResiveChannel;
ResiveChannel.basic_consume(queue='link_raw', on_message_callback=callback, auto_ack=False)
print(' [*] Waiting for messages. To exit press CTRL+C')
try:
ResiveChannel.start_consuming()
except KeyboardInterrupt:
consumerClose.put(True)
print("consumer stopped")
pass
def main():
print('this is main')
#create shared ques for passing messages :
messageQueue = Queue()
consumerClose=Queue()
consumerThread = Thread(target = consumeFromRabbit, args =(messageQueue, consumerClose))
analysisThreadCall = Thread(target = analysisCall, args =(messageQueue,consumerClose ))
consumerThread.start()
analysisThreadCall.start()
consumerThread.join()
analysisThreadCall.join()
if __name__ == '__main__':
#create connection for sending
Sendconnection = pika.BlockingConnection(pika.ConnectionParameters('host',,'/',credentials))
sendChannel= Sendconnection.channel()
ResiveConnection = pika.BlockingConnection( pika.ConnectionParameters(''host',,'/','/',credentials))
ResiveChannel = ResiveConnection.channel()
try:
main()
except KeyboardInterrupt:
print('Interrupted')
try:
sys.exit(0)
except SystemExit:
os._exit(0)
The error I got after few threads is :
Exception in thread Thread-38:
pika.exceptions.ChannelWrongStateError: Channel is closed.

Timeouts for multiprocessing?

I've searched StackOverflow and although I've found many questions on this, I haven't found an answer that fits for my situation/not a strong python programmer to adapt their answer to fit my need.
I've looked here to no avail:
kill a function after a certain time in windows
Python: kill or terminate subprocess when timeout
signal.alarm replacement in Windows [Python]
I am using multiprocessing to run multiple SAP windows at once to pull reports. The is set up to run on a schedule every 5 minutes. Every once in a while, one of the reports gets stalled due to the GUI interface and never ends. I don't get an error or exception, it just stalls forever. What I would like is to have a timeout function that during this part of the code that is executed in SAP, if it takes longer than 4 minutes, it times out, closes SAP, skips the rest of the code, and waits for next scheduled report time.
I am using Windows Python 2.7
import multiprocessing
from multiprocessing import Manager, Process
import time
import datetime
### OPEN SAP ###
def start_SAP():
print 'opening SAP program'
### REPORTS IN SAP ###
def report_1(q, lock):
while True: # logic to get shared queue
if not q.empty():
lock.acquire()
k = q.get()
time.sleep(1)
lock.release()
break
else:
time.sleep(1)
print 'running report 1'
def report_2(q, lock):
while True: # logic to get shared queue
if not q.empty():
lock.acquire()
k = q.get()
time.sleep(1)
lock.release()
break
else:
time.sleep(1)
print 'running report 2'
def report_3(q, lock):
while True: # logic to get shared queue
if not q.empty():
lock.acquire()
k = q.get()
time.sleep(1)
lock.release()
break
else:
time.sleep(1)
time.sleep(60000) #mimicking the stall for report 3 that takes longer than allotted time
print 'running report 3'
def report_N(q, lock):
while True: # logic to get shared queue
if not q.empty():
lock.acquire()
k = q.get()
time.sleep(1)
lock.release()
break
else:
time.sleep(1)
print 'running report N'
### CLOSES SAP ###
def close_SAP():
print 'closes SAP'
def format_file():
print 'formatting files'
def multi_daily_pull():
lock = multiprocessing.Lock() # creating a lock in multiprocessing
shared_list = range(6) # creating a shared list for all functions to use
q = multiprocessing.Queue() # creating an empty queue in mulitprocessing
for n in shared_list: # putting list into the queue
q.put(n)
print 'Starting process at ', time.strftime('%m/%d/%Y %H:%M:%S')
print 'Starting SAP Pulls at ', time.strftime('%m/%d/%Y %H:%M:%S')
StartSAP = Process(target=start_SAP)
StartSAP.start()
StartSAP.join()
report1= Process(target=report_1, args=(q, lock))
report2= Process(target=report_2, args=(q, lock))
report3= Process(target=report_3, args=(q, lock))
reportN= Process(target=report_N, args=(q, lock))
report1.start()
report2.start()
report3.start()
reportN.start()
report1.join()
report2.join()
report3.join()
reportN.join()
EndSAP = Process(target=close_SAP)
EndSAP.start()
EndSAP.join()
formatfile = Process(target=format_file)
formatfile .start()
formatfile .join()
if __name__ == '__main__':
multi_daily_pull()

One way to do what you want would be to use the optional timeout argument that the Process.join() method accepts. This will make it only block the calling thread at most that length of time.
I also set the daemon attribute of each Process instance so your main thread will be able to terminate even if one of the processes it started is still "running" (or has hung up).
One final point, you don't need a multiprocessing.Lock to control access a multiprocessing.Queue, because they handle that aspect of things automatically, so I removed it. You may still want to have one for some other reason, such as controlling access to stdout so printing to it from the various processes doesn't overlap and mess up what is output to the screen.
import multiprocessing
from multiprocessing import Process
import time
import datetime
def start_SAP():
print 'opening SAP program'
### REPORTS IN SAP ###
def report_1(q):
while True: # logic to get shared queue
if q.empty():
time.sleep(1)
else:
k = q.get()
time.sleep(1)
break
print 'report 1 finished'
def report_2(q):
while True: # logic to get shared queue
if q.empty():
time.sleep(1)
else:
k = q.get()
time.sleep(1)
break
print 'report 2 finished'
def report_3(q):
while True: # logic to get shared queue
if q.empty():
time.sleep(1)
else:
k = q.get()
time.sleep(60000) # Take longer than allotted time
break
print 'report 3 finished'
def report_N(q):
while True: # logic to get shared queue
if q.empty():
time.sleep(1)
else:
k = q.get()
time.sleep(1)
break
print 'report N finished'
def close_SAP():
print 'closing SAP'
def format_file():
print 'formatting files'
def multi_daily_pull():
shared_list = range(6) # creating a shared list for all functions to use
q = multiprocessing.Queue() # creating an empty queue in mulitprocessing
for n in shared_list: # putting list into the queue
q.put(n)
print 'Starting process at ', time.strftime('%m/%d/%Y %H:%M:%S')
print 'Starting SAP Pulls at ', time.strftime('%m/%d/%Y %H:%M:%S')
StartSAP = Process(target=start_SAP)
StartSAP.start()
StartSAP.join()
report1 = Process(target=report_1, args=(q,))
report1.daemon = True
report2 = Process(target=report_2, args=(q,))
report2.daemon = True
report3 = Process(target=report_3, args=(q,))
report3.daemon = True
reportN = Process(target=report_N, args=(q,))
reportN.daemon = True
report1.start()
report2.start()
report3.start()
reportN.start()
report1.join(30)
report2.join(30)
report3.join(30)
reportN.join(30)
EndSAP = Process(target=close_SAP)
EndSAP.start()
EndSAP.join()
formatfile = Process(target=format_file)
formatfile .start()
formatfile .join()
if __name__ == '__main__':
multi_daily_pull()

Python multiprocessing - check status of each processes

I wonder if it is possible to check how long of each processes take.
for example, there are four workers and the job should take no more than 10 seconds, but one of worker take more than 10 seconds.Is there way to raise a alert after 10 seconds and before process finish the job.
My initial thought is using manager, but it seems I have wait till process finished.
Many thanks.

You can check whether process is alive after you tried to join it. Don't forget to set timeout otherwise it'll wait until job is finished.
Here is simple example for you
from multiprocessing import Process
import time
def task():
import time
time.sleep(5)
procs = []
for x in range(2):
proc = Process(target=task)
procs.append(proc)
proc.start()
time.sleep(2)
for proc in procs:
proc.join(timeout=0)
if proc.is_alive():
print "Job is not finished!"

I have found this solution time ago (somewhere here in StackOverflow) and I am very happy with it.
Basically, it uses signal to raise an exception if a process takes more than expected.
All you need to do is to add this class to your code:
import signal
class Timeout:
def __init__(self, seconds=1, error_message='TimeoutError'):
self.seconds = seconds
self.error_message = error_message
def handle_timeout(self, signum, frame):
raise TimeoutError(self.error_message)
def __enter__(self):
signal.signal(signal.SIGALRM, self.handle_timeout)
signal.alarm(self.seconds)
def __exit__(self, type, value, traceback):
signal.alarm(0)
Here is a general example of how it works:
import time
with Timeout(seconds=3, error_message='JobX took too much time'):
try:
time.sleep(10) #your job
except TimeoutError as e:
print(e)
In your case, I would add the with statement to the job that your worker need to perform. Then you catch the Exception and you do what you think is best.
Alternatively, you can periodically check if a process is alive:
timeout = 3 #seconds
start = time.time()
while time.time() - start < timeout:
if any(proces.is_alive() for proces in processes):
time.sleep(1)
else:
print('All processes done')
else:
print("Timeout!")
# do something

Use Pipe and messages
from multiprocessing import Process, Pipe
import numpy as np
caller, worker = Pipe()
val1 = ['der', 'die', 'das']
def worker_function(info):
print (info.recv())
for i in range(10):
print (val1[np.random.choice(3, 1)[0]])
info.send(['job finished'])
info.close()
def request(data):
caller.send(data)
task = Process(target=worker_function, args=(worker,))
if not task.is_alive():
print ("task is requested")
task.start()
if caller.recv() == ['job finished']:
task.join()
print ("finished")
if __name__ == '__main__':
data = {'input': 'here'}
request(data)

Cannot to line up item from one queue to another

I deal with two python queues.
Short description of my issue:
Clients pass through the waiting queue(q1) and they (the clients) are served afterwards. The size of the waiting queue can't be greater than N (10 in my program). If waiting queue becomes full, clients pass to outside queue(q2, size 20). If outside queue becomes full, clients are rejected and not served.
Every client that left a waiting queue allows another client from outside queue to join the waiting queue.
Work with queues should be thread-safe.
Below I implemented approximately what I want. But I'm faced with the problem - enqueuing a client from outside queue (q1) to the waiting queue (q2) during execution serve function. I guess I lost or forgot something important. I think this statement q1.put(client) blocks permanently but don't know why.
import time
import threading
from random import randrange
from Queue import Queue, Full as FullQueue
class Client(object):
def __repr__(self):
return '<{0}: {1}>'.format(self.__class__.__name__, id(self))
def serve(q1, q2):
while True:
if not q2.empty():
client = q2.get()
print '%s leaved outside queue' % client
q1.put(client)
print '%s is in the waiting queue' % client
q2.task_done()
client = q1.get()
print '%s leaved waiting queue for serving' % client
time.sleep(2) # Do something with client
q1.task_done()
def main():
waiting_queue = Queue(10)
outside_queue = Queue(20)
for _ in range(2):
worker = threading.Thread(target=serve, args=(waiting_queue, outside_queue))
worker.setDaemon(True)
worker.start()
delays = [randrange(1, 5) for _ in range(100)]
# Every d seconds 10 clients enter to the waiting queue
for d in delays:
time.sleep(d)
for _ in range(10):
client = Client()
try:
waiting_queue.put_nowait(client)
except FullQueue:
print 'Waiting queue is full. Please line up in outside queue.'
try:
outside_queue.put_nowait(client)
except FullQueue:
print 'Outside queue is full. Please go out.'
waiting_queue.join()
outside_queue.join()
print 'Done'

Finally I found the solution. I check docs more attentive
If full() returns True it doesn’t guarantee that a subsequent call to get() will not block https://docs.python.org/2/library/queue.html#Queue.Queue.full
That's why q1.full() is not reliable in a few threads. I added mutex before inserting item to queues and checking queue is full:
class Client(object):
def __init__(self, ident):
self.ident = ident
def __repr__(self):
return '<{0}: {1}>'.format(self.__class__.__name__, self.ident)
def serve(q1, q2, mutex):
while True:
client = q1.get()
print '%s leaved waiting queue for serving' % client
time.sleep(2) # Do something with client
q1.task_done()
with mutex:
if not q2.empty() and not q1.full():
client = q2.get()
print '%s leaved outside queue' % client
q1.put(client)
print '%s is in the waiting queue' % client
q2.task_done()
def main():
waiting_queue = Queue(10)
outside_queue = Queue(20)
lock = threading.RLock()
for _ in range(2):
worker = threading.Thread(target=serve, args=(waiting_queue, outside_queue, lock))
worker.setDaemon(True)
worker.start()
# Every 1-5 seconds 10 clients enter to the waiting room
i = 1 # Used for unique <int> client's id
while True:
delay = randrange(1, 5)
time.sleep(delay)
for _ in range(10):
client = Client(i)
try:
lock.acquire()
if not waiting_queue.full():
waiting_queue.put(client)
else:
outside_queue.put_nowait(client)
except FullQueue:
# print 'Outside queue is full. Please go out.'
pass
finally:
lock.release()
i += 1
waiting_queue.join()
outside_queue.join()
print 'Done'
Now it works well.

Speed up Redis server inputs

How do speed up this test code in python to Redis on Winxp using python 2.7?
Would multiprocessing be better? The load rate in 6000/s vs publish 100,000/s rates.
I chose 100,000, but could lower in testing. The process takes 15 seconds.
Would changing setting on server help???
import time
from time import strftime
import redis
import threading, Queue
start_time = time.time()
cxn = redis.StrictRedis('127.0.0.1',6379,1)
class WorkerMain(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while 1:
try: # take a job from the queue
row = self.queue.get_nowait()
except Queue.Empty: raise SystemExit
try:
cxn.set(row, "Row")
#print (row, "Row")
except: print 'Setup Error'
if __name__ == '__main__':
connections = 5
sml = range(1,100000)
queue = Queue.Queue()
for row in sml:
queue.put(str(row))
threads = []
for dummy in range(connections):
t = WorkerMain(queue)
t.start()
threads.append(t)
# wait for all threads to finish
for thread in threads:
thread.join()
print
end_time = time.time()
duration = end_time - start_time
print "Duration: %s" % duration
Used the code below for mulitprocessing and "monitored" the data with CLI...not all data went into the server.
from multiprocessing import Pool
import time
import redis
start_time = time.time()
cxn = redis.Redis('127.0.0.1',6379,1)
def rset(var):
cxn.set(var,"value")
if __name__ =='__main__':
sml = range(1,10000)
#for x in sml:print x
pool = Pool(processes=5)
for row in sml:
pool.apply_async(rset, [(row,)])
#print result.get(),
end_time = time.time()
duration = end_time - start_time
print "Duration: %s" % duration
Here is the pipelined code...... I just commented out the threading stuff.
from time import strftime
import redis
import threading, Queue
start_time = time.time()
cxn = redis.StrictRedis('127.0.0.1',6379,0)
pipe = cxn.pipeline(transaction=False)
class WorkerMain(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self):
while 1:
try: # take a job from the queue
row = self.queue.get_nowait()
except Queue.Empty: raise SystemExit
try:
cxn.set(row, "Row")
#print (row, "ROw")
except: print 'Setup Error'
if __name__ == '__main__':
#connections = 5
sml = range(1,100000)
#queue = Queue.Queue()
for row in sml:
#queue.put(str(row))
pipe.set(str(row),"value").execute()# key, value
# threads = []
# for dummy in range(connections):
# t = WorkerMain(queue)
# t.start()
# threads.append(t)
#
# # wait for all threads to finish
# for thread in threads:
# thread.join()
print
end_time = time.time()
duration = end_time - start_time
print "Duration: %s" % duration

Use Pipelines. A Pipeline batches commands so you don't pay for network overheads.
See :
Section on Pipelines over here https://github.com/andymccurdy/redis-py
Pipelining on Redis.io - http://redis.io/topics/pipelining

Using threading for better performance is not a really good idea if you use cpython (the standard python interpreter) because of the gil.
http://wiki.python.org/moin/GlobalInterpreterLock
multiprocessing should work better