I am working on a project that is a gui with a separate thread. When the user clicks a button a function is fired that sends data to the queue and launches the other thread. The other thread then gets the data from the queue and adds new data to it. Where the gui will then get that data and do something. But it gets stuck saying the queue is empty? Why is this and how can I fix it?
def click():
if self.uN.get() and self.pW.get():
self.q.put("LOGIN")
self.q.put(self.uN.get() + "," + self.pW.get())
else:
self.q.put("eFi")
con = Connect(self.q)
con.setDaemon(True)
con.start()
time.sleep(1)
while True:
root.update()
try:
data = self.q.get(False)
except queue.Empty:
pass
else:
print(data + "+")
if data == "Fcon":
tkMessageBox.showerror("ERROR!", "Failed to connect to the server!")
elif data == "nCre":
tkMessageBox.showerror("ERROR!", "A text field is empty!")
elif data == "Gcon":
for item in root.winfo_children():
item.destroy()
self.mScreen()
else:
print("?")
print('!')
break
Here is the other threads code:
class Connect(Thread):
def __init__(self, q):
Thread.__init__(self)
self.s = socket.socket()
self.q = q
def run(self):
while True:
try:
data = self.q.get()
except Exception as e:
pass
else:
if data == "LOGIN":
self.login()
elif data == "eFi":
self.q.put("nCre")
print("??????")
def login(self):
info = self.q.get().split(",")
self.q.put("Gcon")
print("GOD")
The problem was being caused by having two threads access the same queue one thread would grab the information before the proper thread was able to access it. To solve the problem I made two queues one for each thread.
Related
I am using RabbitMQ to get message from queue.
This messages being processed then sent to different queue in RabbitMQ.
here is how my program works:
I have consuming Thread for message consuming that puts the revised message in local Queue..
Another thread is listening and when a message arrives a nested Thread is created for analysis..
when the analysis is done the message is sent to RabbitMQ.
I AM TRYING TO ACK after this operation is done but since consuming Thread works faster my channel is closed. How can I ACK after finishing my analysis?
Here is my Python code:
import pandas as pd
import pickle
from queue import Queue
from threading import Thread
import time
class MyAnalysisThread (Thread):
def __init__(self, comingQuery,deliverTag):
Thread.__init__(self)
self.comingQuery = comingQuery
self.deliverTag = deliverTag
def run(self):
analysis(self.comingQuery,self.deliverTag)
def sendToRabbit(linkFeatures):
.....send
print(" [x] send to rabbitMQ ")
def analysis(comingQuery,deliverTag):
/////do analysis
sendToRabbit(message)
global ResiveChannel;
# I WANNA ACK HERE after analysis finish and sent
ResiveChannel.basic_ack(delivery_tag = deliverTag,multiple=False)
def analysisCall(messageQueue, consumerClose):
print('//////////////////////////// analysis started')
global sendChannel;
sendChannel.queue_declare(queue='send')
while True:
if(messageQueue.empty()==False):
#get the message
message=messageQueue.get()
comingQuery=message['comingQuery']
deliverTag=message['deliverTag']
messageQueue.task_done()
# each message will create different thread for analysis
analysisThread = MyAnalysisThread (comingQuery, deliverTag)
analysisThread.start()
analysisThread.join()
elif(consumerClose.empty()==False):
# when the consumer is stopped go out the loop
if(consumerClose.get()==True):
print('consumer stopped')
break;
else:
print('sleeping for .....1')
# wait some n sec before next iteration
time.sleep(1)
def consumeFromRabbit(messageQueue, consumerClose):
def callback(ch, method, properties, body):
comingQuery=json.loads(body)
message={'comingQuery':comingQuery,
'deliverTag':method.delivery_tag,
}
messageQueue.put(message)
global ResiveChannel;
ResiveChannel.basic_consume(queue='link_raw', on_message_callback=callback, auto_ack=False)
print(' [*] Waiting for messages. To exit press CTRL+C')
try:
ResiveChannel.start_consuming()
except KeyboardInterrupt:
consumerClose.put(True)
print("consumer stopped")
pass
def main():
print('this is main')
#create shared ques for passing messages :
messageQueue = Queue()
consumerClose=Queue()
consumerThread = Thread(target = consumeFromRabbit, args =(messageQueue, consumerClose))
analysisThreadCall = Thread(target = analysisCall, args =(messageQueue,consumerClose ))
consumerThread.start()
analysisThreadCall.start()
consumerThread.join()
analysisThreadCall.join()
if __name__ == '__main__':
#create connection for sending
Sendconnection = pika.BlockingConnection(pika.ConnectionParameters('host',,'/',credentials))
sendChannel= Sendconnection.channel()
ResiveConnection = pika.BlockingConnection( pika.ConnectionParameters(''host',,'/','/',credentials))
ResiveChannel = ResiveConnection.channel()
try:
main()
except KeyboardInterrupt:
print('Interrupted')
try:
sys.exit(0)
except SystemExit:
os._exit(0)
The error I got after few threads is :
Exception in thread Thread-38:
pika.exceptions.ChannelWrongStateError: Channel is closed.
I have recently started working on threads and queues in Python. I have seen the following pattern for multi threading and queuing:
VALID_TARGETS = ['10.192.20.2','10.233.1.3', '10,235,0.3']
queue2 = Queue.Queue(maxsize=10)
for target in VALID_TARGETS:
queue2.put(target)
for i in range(100):
try:
valAuth_obj = ValAuth(queue2, user_id, pwd)
valAuth_obj.daemon = True
valAuth_obj.start()
except Exception, error:
print "ERROR"
break
queue2.join()
Where ValAuth is a class inheriting threading.Threads, whose run function is:
def run(self):
while True:
target = self.queue.get()
self.login(target)
self.queue.task_done()
The program is running as expected. But i have noticed we are calling the class valAuth for every thread i want to run. Then i tried the following, where i pulled out the valAuth() outside the loop.
VALID_TARGETS = ['10.192.20.2','10.233.1.3', '10,235,0.3']
queue2 = Queue.Queue(maxsize=10)
valAuth_obj = ValAuth(queue2, user_id, pwd)
for target in VALID_TARGETS:
queue2.put(target)
for i in range(100):
try:
valAuth_obj.daemon = True
valAuth_obj.start()
except Exception, error:
print "ERROR"
break
queue2.join()
The program still gives me the same result but I can notice there is some difference in terms of speed. Can someone please explain what is the difference in terms of background operations and which one should we stick to?
CODE for valAuth class
class valAuth(threading.Thread):
""" Authenticate all targeted SCM build machines """
def __init__(self, queue, user_id, password):
threading.Thread.__init__(self)
self.queue = queue
self.user = user
self.password = password
def run(self):
while True:
target = self.queue.get()
self.auth(target)
self.queue.task_done()
def auth(self, target):
""" Check all SCM build machines are active with login authentication and
also checks if the account is not AD locked.
"""
print "P1: " + self.user + " target: "+ target
# Company's Authentication Protocol base code ... (cannot show ! SORRY !)
print "P1: " + self.user + " target: "+ target + " Run complete"
Thanks in advance.
So I have written some code which stops a thread when a "STOP" button is pressed. It works, with some caveats. It tends to freeze up resources for a second or two and it stops everything as soon as it is pressed. I would like for the stop button to stop the thread once it is finished what it is doing.
Here is the function I have created for the stop event:
def Stop(self, event):
if myClass.worker != None:
if myClass.browser!=None:
while(myClass.browser.quit()!=None):
myClass.browser.quit()
myClass.worker.stop()
myClass.worker = None
while True:
try:
myfile = open(self.fpath, "r+")
myfile.close()
break
except IOError:
myClass.closeIt(self)
os.unlink(self.fpath)
os.rename(self.temp, self.fpath)
Basically if no thread is spawned (hence it is not None), it quits a browser window and then stops the thread which is scraping information from a webpage on the browser. It then saves the file it is writing to.
The thread that is actually stopped is an instance of the openExcel class I have created:
myClass.worker = openExcel(temp)
Instances of this class call a function called findDomains:
def findDomains(self,searchrow, numrows, sheet, wbook):
for i in range(searchrow, numrows):
domain = sheet.cell_value(i, 2)
if domain == "":
company = sheet.cell_value(i, 1)
self.browser.get('https://www.google.com/')
wait = WebDriverWait(self.browser, 300)
inputGoogle=wait.until(EC.presence_of_element_located((By.NAME, "q")))
inputGoogle.clear()
inputGoogle.send_keys(company)
inputGoogle.submit()
self.browser.refresh()
tries = 0
while tries<1000:
try:
domain=wait.until(EC.presence_of_element_located((By.CSS_SELECTOR, "cite._Rm")))
domain = domain.text
break
except StaleElementReferenceException:
tries+=1
self.browser.refresh()
domainlength=len(domain)
period=0
for charac in range(domainlength):
if domain[charac]==".":
period+=1
if period==1:
if "//" in domain:
domain = domain.split("//",1)[1]
index = domain.find("/")
domain = domain[:index]
elif period > 1:
domain = domain.split(".",1)[1]
index = domain.find("/")
domain = domain[:index]
checkdom=domain.split(".",1)[0]
if (checkdom+".") not in domain:
domain="error"
worksheet.write(i,2,domain)
wbook.save(file)
Instead of stopping everything it is doing in this function, I'd like for the stop feature to wait until the "domain" is saved to the excel sheet. I.E. for it to finish what it is doing, before trying to close a browser and quit.
I'm doing a project involving data collection and logging. I have 2 threads running, a collection thread and a logging thread, both started in main. I'm trying to allow the program to be terminated gracefully when with Ctrl-C.
I'm using a threading.Event to signal to the threads to end their respective loops. It works fine to stop the sim_collectData method, but it doesn't seem to be properly stopping the logData thread. The Collection terminated print statement is never executed, and the program just stalls. (It doesn't end, just sits there).
The second while loop in logData is to make sure everything in the queue is logged. The goal is for Ctrl-C to stop the collection thread immediately, then allow the logging thread to finish emptying the queue, and only then fully terminate the program. (Right now, the data is just being printed out - eventually it's going to be logged to a database).
I don't understand why the second thread never terminates. I'm basing what I've done on this answer: Stopping a thread after a certain amount of time. What am I missing?
def sim_collectData(input_queue, stop_event):
''' this provides some output simulating the serial
data from the data logging hardware.
'''
n = 0
while not stop_event.is_set():
input_queue.put("DATA: <here are some random data> " + str(n))
stop_event.wait(random.randint(0,5))
n += 1
print "Terminating data collection..."
return
def logData(input_queue, stop_event):
n = 0
# we *don't* want to loop based on queue size because the queue could
# theoretically be empty while waiting on some data.
while not stop_event.is_set():
d = input_queue.get()
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
# if the stop event is recieved and the previous loop terminates,
# finish logging the rest of the items in the queue.
print "Collection terminated. Logging remaining data to database..."
while not input_queue.empty():
d = input_queue.get()
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
return
def main():
input_queue = Queue.Queue()
stop_event = threading.Event() # used to signal termination to the threads
print "Starting data collection thread...",
collection_thread = threading.Thread(target=sim_collectData, args=(input_queue, stop_event))
collection_thread.start()
print "Done."
print "Starting logging thread...",
logging_thread = threading.Thread(target=logData, args=(input_queue, stop_event))
logging_thread.start()
print "Done."
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
# stop data collection. Let the logging thread finish logging everything in the queue
stop_event.set()
main()
The problem is that your logger is waiting on d = input_queue.get() and will not check the event. One solution is to skip the event completely and invent a unique message that tells the logger to stop. When you get a signal, send that message to the queue.
import threading
import Queue
import random
import time
def sim_collectData(input_queue, stop_event):
''' this provides some output simulating the serial
data from the data logging hardware.
'''
n = 0
while not stop_event.is_set():
input_queue.put("DATA: <here are some random data> " + str(n))
stop_event.wait(random.randint(0,5))
n += 1
print "Terminating data collection..."
input_queue.put(None)
return
def logData(input_queue):
n = 0
# we *don't* want to loop based on queue size because the queue could
# theoretically be empty while waiting on some data.
while True:
d = input_queue.get()
if d is None:
input_queue.task_done()
return
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
def main():
input_queue = Queue.Queue()
stop_event = threading.Event() # used to signal termination to the threads
print "Starting data collection thread...",
collection_thread = threading.Thread(target=sim_collectData, args=(input_queue, stop_event))
collection_thread.start()
print "Done."
print "Starting logging thread...",
logging_thread = threading.Thread(target=logData, args=(input_queue,))
logging_thread.start()
print "Done."
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
# stop data collection. Let the logging thread finish logging everything in the queue
stop_event.set()
main()
I'm not an expert in threading, but in your logData function the first d=input_queue.get() is blocking, i.e., if the queue is empty it will sit an wait forever until a queue message is received. This is likely why the logData thread never terminates, it's sitting waiting forever for a queue message.
Refer to the [Python docs] to change this to a non-blocking queue read: use .get(False) or .get_nowait() - but either will require some exception handling for cases when the queue is empty.
You are calling a blocking get on your input_queue with no timeout. In either section of logData, if you call input_queue.get() and the queue is empty, it will block indefinitely, preventing the logging_thread from reaching completion.
To fix, you will want to call input_queue.get_nowait() or pass a timeout to input_queue.get().
Here is my suggestion:
def logData(input_queue, stop_event):
n = 0
while not stop_event.is_set():
try:
d = input_queue.get_nowait()
if d.startswith("DATA:"):
print "LOG: " + d
n += 1
except Queue.Empty:
time.sleep(1)
return
You are also signalling the threads to terminate, but not waiting for them to do so. Consider doing this in your main function.
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
stop_event.set()
collection_thread.join()
logging_thread.join()
Based on the answer of tdelaney I created an iterator based approach. The iterator exits when the termination message is encountered. I also added a counter of how many get-calls are currently blocking and a stop-method, which sends just as many termination messages. To prevent a race condition between incrementing and reading the counter, I'm setting a stopping bit there. Furthermore I don't use None as the termination message, because it can not necessarily be compared to other data types when using a PriorityQueue.
There are two restrictions, that I had no need to eliminate. For one the stop-method first waits until the queue is empty before shutting down the threads. The second restriction is, that I did not any code to make the queue reusable after stop. The latter can probably be added quite easily, while the former requires being careful about concurrency and the context in which the code is used.
You have to decide whether you want stop to also wait for all the termination messages to be consumed. I choose to put the necessary join there, but you may just remove it.
So this is the code:
import threading, queue
from functools import total_ordering
#total_ordering
class Final:
def __repr__(self):
return "∞"
def __lt__(self, other):
return False
def __eq__(self, other):
return isinstance(other, Final)
Infty = Final()
class IterQueue(queue.Queue):
def __init__(self):
self.lock = threading.Lock()
self.stopped = False
self.getters = 0
super().__init__()
def __iter__(self):
return self
def get(self):
raise NotImplementedError("This queue may only be used as an iterator.")
def __next__(self):
with self.lock:
if self.stopped:
raise StopIteration
self.getters += 1
data = super().get()
if data == Infty:
self.task_done()
raise StopIteration
with self.lock:
self.getters -= 1
return data
def stop(self):
self.join()
self.stopped = True
with self.lock:
for i in range(self.getters):
self.put(Infty)
self.join()
class IterPriorityQueue(IterQueue, queue.PriorityQueue):
pass
Oh, and I wrote this in python 3.2. So after backporting,
import threading, Queue
from functools import total_ordering
#total_ordering
class Final:
def __repr__(self):
return "Infinity"
def __lt__(self, other):
return False
def __eq__(self, other):
return isinstance(other, Final)
Infty = Final()
class IterQueue(Queue.Queue, object):
def __init__(self):
self.lock = threading.Lock()
self.stopped = False
self.getters = 0
super(IterQueue, self).__init__()
def __iter__(self):
return self
def get(self):
raise NotImplementedError("This queue may only be used as an iterator.")
def next(self):
with self.lock:
if self.stopped:
raise StopIteration
self.getters += 1
data = super(IterQueue, self).get()
if data == Infty:
self.task_done()
raise StopIteration
with self.lock:
self.getters -= 1
return data
def stop(self):
self.join()
self.stopped = True
with self.lock:
for i in range(self.getters):
self.put(Infty)
self.join()
class IterPriorityQueue(IterQueue, Queue.PriorityQueue):
pass
you would use it as
import random
import time
def sim_collectData(input_queue, stop_event):
''' this provides some output simulating the serial
data from the data logging hardware.
'''
n = 0
while not stop_event.is_set():
input_queue.put("DATA: <here are some random data> " + str(n))
stop_event.wait(random.randint(0,5))
n += 1
print "Terminating data collection..."
return
def logData(input_queue):
n = 0
# we *don't* want to loop based on queue size because the queue could
# theoretically be empty while waiting on some data.
for d in input_queue:
if d.startswith("DATA:"):
print d
input_queue.task_done()
n += 1
def main():
input_queue = IterQueue()
stop_event = threading.Event() # used to signal termination to the threads
print "Starting data collection thread...",
collection_thread = threading.Thread(target=sim_collectData, args=(input_queue, stop_event))
collection_thread.start()
print "Done."
print "Starting logging thread...",
logging_thread = threading.Thread(target=logData, args=(input_queue,))
logging_thread.start()
print "Done."
try:
while True:
time.sleep(10)
except (KeyboardInterrupt, SystemExit):
# stop data collection. Let the logging thread finish logging everything in the queue
stop_event.set()
input_queue.stop()
main()
I'm working on server written in python. When the client sends a cmd the server will call a function with unknown running time. So to avoid blocking I used threading. But when looking at the child process it seems that they're not terminating, causing a lot of memory usage.
EDIT : Here is the tree of the directory : http://pastebin.com/WZDxLquC
Following answers I found on stackoverflow I implemented a custom Thread class:
sThreads.py :
import threading
class Thread(threading.Thread):
def __init__(self, aFun, args = ()):
super(Thread, self).__init__(None, aFun, None, args)
self.stopped = threading.Event()
def stop(self):
self.stopped.set()
def isStopped(self):
return self.stopped.isSet()
Then here is the server's loop:
some where in mainServer.py:
def serve_forever(self, aCustomClass, aSize = 1024):
while True:
self.conn, self.addr = self.sock.accept()
msg = self.recvMSG(4096)
if(msg):
self.handShake(msg)
print 'Accepted !'
while True:
msg = self.recvMSG(aSize)
if(msg):
t = sThreads.Thread(self.handle, (aCustomClass,))
t.start()
self.currentThreads.append(t)
if(self.workers > 0):
tt = sThreads.Thread(self.respond)
tt.start()
if(self.workers == 0 and len(self.currentThreads) > 0):
for th in self.currentThreads:
th.stop()
Using a custom Thread class will not solve the issue and it still does not stop the terminated threads!
EDIT : added the handle() and respond() methods :
def handle(self, aClass):
self.workers += 1
self.queue.put(aClass._onRecieve(self.decodeStream()))
def respond(self):
while self.workers > 0:
msgToSend, wantToSend = self.queue.get()
self.workers -= 1
if(wantToSend):
print 'I want to send :', msgToSend
continue #Send is not yet implemented !
It seems that self.queue.get() was causing all the issue ...