We Keep Coding

Tkinter freezes while trying to read stdout from subprocess

I started a Tkinter application but I'm with problems with buffering. I searched the solution but didn't find it.
Correlated links:
Calling python script with subprocess popen and flushing the data
Python C program subprocess hangs at "for line in iter"
Python subprocess standard output to a variable
As an exemple, this app has two buttons: Start and Stop.
When I press the button Start, the spammer.py is called as a subprocess and when I press the button Stop the program must be killed.
# spammer.py
import time
counter = 0
while counter < 40: # This process will last 10 second maximum
print("counter = %d" % counter)
counter += 1
time.sleep(0.25) # 4 messages/second
While the PrintNumbers.py is running, I want the spammer's output be storage in a variable inside the Tkinter to be used in realtime. But once I try to read the buffer with myprocess.stdout.readline, it stucks and it doesn't continue until the subprocess finish and as consequence for exemple I cannot click on the Stop button.
I read that the function is waiting for EOF to continue, and I tried to use tokens as shown here, and the function that should continue when it finds a or a \n, but it did not work.
The Tkinter exemple is bellow. After I click at the Start button, I instantly see the message Started reading stdout, and after 10 seconds it shows a lot of messages, while I wanted to show every message over time.
# PrintNumbers.py
import Tkinter as tk
import subprocess
class App:
def __init__(self, root):
self.root = root
self.myprocess = None
self.logmessages = []
self.createButtons()
self.timedUpdate()
def createButtons(self):
self.ButtonsFrame = tk.Frame(self.root, width=600, height=400)
self.ButtonsFrame.pack()
self.startbutton = tk.Button(self.ButtonsFrame, text="Start",
command=self.__ClickOnStarButton)
self.stopbutton = tk.Button(self.ButtonsFrame, text="Stop",
command=self.__ClickOnStopButton)
self.startbutton.pack()
self.stopbutton.pack()
self.startbutton["state"] = "normal"
self.stopbutton["state"] = "disable"
def __ClickOnStarButton(self):
print("Click on Start Button")
self.startbutton["state"] = "disable"
self.stopbutton["state"] = "normal"
self.startProcess()
def __ClickOnStopButton(self):
print("Click on Stop Button")
self.startbutton["state"] = "normal"
self.stopbutton["state"] = "disable"
self.killProcess()
def startProcess(self):
command = "python spammer.py"
self.myprocess = subprocess.Popen(command, stdout=subprocess.PIPE, bufsize=1)
def killProcess(self):
self.myprocess.terminate()
self.myprocess.wait()
self.myprocess = None
def timedUpdate(self):
if self.myprocess is not None: # There's a process running
self.getLogText() # Will get the info from spammer.py
self.treatOutput() # Do whatever we want with the data
root.after(200, self.timedUpdate) # Every 0.2 seconds we will update
def getLogText(self):
if self.myprocess is None: # There's no process running
return
# The problem is here
print("Started reading stdout")
for line in iter(self.myprocess.stdout.readline, ''):
print(" Inside the loop. line = '%s'" % line)
self.logmessages.append(line)
print("Finished reading stdout")
def treatOutput(self):
# Any function that uses the spammer's output
# it's here just to test
while len(self.logmessages):
line = self.logmessage.pop(0)
line = line.replace("counter = ", "")
mynumber = int(line)
if mynumber % 3:
print(mynumber)
if __name__ == "__main__":
root = tk.Tk()
app = App(root)
root.mainloop()
How can I read the output without getting stuck? I'm still using python 2.7, and I don't know if it's the problem either.

How to stop GUI from freezing when running a loop / function?

I am trying to add threading to a Python 3.63 Tkinter program where a function will run but the GUI will still be responsive, including if the user wants to close the program while the function is running.
In the example below I have tried to run a simple printing to console function on a separate thread to the GUI mainloop so the user could click the X in the top right to close the program while the loop is running if they so wish.
The error I am getting is:
TypeError: start() takes 1 positional argument but 2 were given
try:
import tkinter as tk
import queue as queue
except:
import Tkinter as tk
import Queue as queue
import threading
def center(toplevel,desired_width=None,desired_height=None):
toplevel.update_idletasks()
w, h = toplevel.winfo_screenwidth() - 20, toplevel.winfo_screenheight() - 100
if desired_width and desired_height:
size = (desired_width,desired_height)
else:
size = tuple(int(Q) for Q in toplevel.geometry().split("+")[0].split("x"))
toplevel.geometry("%dx%d+%d+%d" % (size + (w/2 - size[0]/2, h/2 - size[1]/2)))
class ThreadedTask(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self,func):
func()
class app(tk.Tk):
def __init__(self):
tk.Tk.__init__(self)
center(self,desired_width=500,desired_height=400)
self.queue = queue.Queue()
self.run_func_button = tk.Button(self,
text="Run Function",
font=("Calibri",20,"bold"),
command=self.run_func)
self.run_func_button.pack()
def run_func(self):
ThreadedTask(self.queue).start(self.count_to_1500)
def count_to_1500(self):
for i in range(1500):
print (i)
app_start = app()
app_start.mainloop()

See doc threading - start() doesn't use arguments but you use .start(self.count_to_1500) - and this gives your error.
You could use
Thread(target=self.count_to_1500).start()
or
Thread(target=self.count_to_1500, args=(self.queue,)).start()
if you define
def count_to_1500(self, queue):
EDIT: working example with thread which put in quoue and method which get data from queue.
try:
import tkinter as tk
import queue as queue
except:
import Tkinter as tk
import Queue as queue
import threading
import time
def center(toplevel,desired_width=None,desired_height=None):
toplevel.update_idletasks()
w, h = toplevel.winfo_screenwidth() - 20, toplevel.winfo_screenheight() - 100
if desired_width and desired_height:
size = (desired_width,desired_height)
else:
size = tuple(int(Q) for Q in toplevel.geometry().split("+")[0].split("x"))
toplevel.geometry("%dx%d+%d+%d" % (size + (w/2 - size[0]/2, h/2 - size[1]/2)))
class app(tk.Tk):
def __init__(self):
tk.Tk.__init__(self)
center(self,desired_width=500,desired_height=400)
self.queue = queue.Queue()
self.run_func_button = tk.Button(self,
text="Run Function",
font=("Calibri",20,"bold"),
command=self.run_func)
self.run_func_button.pack()
def run_func(self):
threading.Thread(target=self.count_to_1500).start()
threading.Thread(target=self.count_to_1500_with_queue, args=(self.queue,)).start()
self.check_queue()
def count_to_1500(self):
for i in range(10):
print('1:', i)
time.sleep(0.2)
def count_to_1500_with_queue(self, queue):
for i in range(10):
print('put:', i)
queue.put(i)
time.sleep(1)
queue.put('last')
def check_queue(self):
print("check queue")
data = None
if not self.queue.empty():
data = self.queue.get()
print('get:', data)
if data != 'last':
self.after(200, self.check_queue)
app_start = app()
app_start.mainloop()

Thread.start takes no parameters: https://docs.python.org/3/library/threading.html
The correct way to use a Thread is:
# Will call func(*args, **kwargs)
t = threading.Thread(target=func, args=(), kwargs={})
t.start()
t.join()
The join is important. Without it you will have many zombie threads in your app, which will also prevent your app from shutting down cleanly.
Another pattern is to use a daemon thread, which processes a queue. daemon threads are automatically killed when the program exits.
def worker(q):
while True:
try:
f = q.get()
q.task_done()
if f is None: return
f()
except Exception:
import traceback
traceback.print_exc()
q = Queue.Queue()
t = threading.Thread(target=worker, args=(q,))
t.daemon=True
t.start()
# f is a no-arg function to be executed
q.put(f)
# Call at shutdown
q.join()
To run several tasks at the same time, start many threads.
Yet another method, use multiprocessing.pool.ThreadPool
from multiprocessing.pool import ThreadPool
# Create at startup
pool = ThreadPool(8)
# For each treaded task
pool.apply_async(func, args, kwds)
# Call at shutdown
pool.close()
pool.join()
... which works, more or less, as the above.
I recommend reading:
https://docs.python.org/2/library/multiprocessing.html#multiprocessing-programming

Update Tkinter widget from main thread after worker thread completes

I need to update the GUI after a thread completes and call this update_ui function from main thread (like a software interrupt maybe?). How can a worker thread call a function in the main thread?
Sample code:
def thread():
...some long task
update_ui() #But call this in main thread somehow
def main():
start_new_thread(thread)
...other functionality
def update_ui():
Tkinter_widget.update()
I tried to use Queue or any flag accessible to both threads but I have to wait/poll continuously to check if the value has been updated and then call the function - this wait makes the UI unresponsive. e.g.
flag = True
def thread():
...some long task
flag = False
def main():
start_new_thread(thread)
while(flag): sleep(1)
update_ui()
...other functionality

Your code appears to be somewhat hypothetical. Here is some that accomplishes that does what you describe. It creates three labels and initializes their text. It then starts three threads. Each thread updates the tkinter variable associated with the label created in the main thread after a period of time. Now if the main thread really needs to do the updating, queuing does work, but the program must be modified to accomplish that.
import threading
import time
from tkinter import *
import queue
import sys
def createGUI(master, widget_var):
for i in range(3):
Label(master, textvariable=widget_var[i]).grid(row=i, column=0)
widget_var[i].set("Thread " + str(i) + " started")
def sometask(thread_id, delay, queue):
print("Delaying", delay)
time.sleep(delay)
tdict = {'id': thread_id, 'message': 'success'}
# You can put simple strings/ints, whatever in the queue instead
queue.put(tdict)
return
def updateGUI(master, q, widget_var, td):
if not q.empty():
tdict = q.get()
widget_var[tdict['id']].set("Thread " + str(tdict['id']) + " completed with status: " + tdict['message'])
td.append(1)
if len(td) == 3:
print("All threads completed")
master.after(1000, timedExit)
else:
master.after(100, lambda w=master,que=q,v=widget_var, tcount=td: updateGUI(w,que,v,td))
def timedExit():
sys.exit()
root = Tk()
message_q = queue.Queue()
widget_var = []
threads_done = []
for i in range(3):
v = StringVar()
widget_var.append(v)
t = threading.Thread(target=sometask, args=(i, 3 + i * 3, message_q))
t.start()
createGUI(root, widget_var)
updateGUI(root,message_q, widget_var, threads_done)
root.mainloop()

Threaded upload with timeout in Python and PySide

I am looking for a design pattern based on threading.Thread, multiprocessing or Queue to upload a list of items with a timeout. The thread allows the GUI to remain responsive. If the connection hangs, then the timeout should trigger and the program should gracefully exit.
The example below works, but the GUI remains blocked. How could this be improved to allow for uploading of the list, manual canceling of the process, timeout of the upload process plus a non-blocking GUI?
from PySide.QtGui import *
from PySide.QtCore import *
import sys
import time
import threading
class UploadWindow(QDialog):
def __init__(self, parent=None):
super(UploadWindow, self).__init__(parent)
self.uploadBtn = QPushButton('Upload')
mainLayout = QVBoxLayout()
mainLayout.addWidget(self.uploadBtn)
self.uploadBtn.clicked.connect(self.do_upload)
self.progressDialog = QProgressDialog(self)
self.progressDialog.canceled.connect(self.cancelDownload)
self.progressDialog.hide()
self.setLayout(mainLayout)
self.show()
self.raise_()
def do_upload(self):
self.uploadBtn.setEnabled(False)
self.progressDialog.setMaximum(10)
self.progressDialog.show()
self.upload_thread = UploadThread(self)
self.upload_thread.start()
self.upload_thread_stopped = False
#List of items to upload
for i in range(10):
self.upload_thread = UploadThread(i)
self.upload_thread.start()
self.upload_thread.join(5)
self.progressDialog.setValue(i)
if self.upload_thread_stopped:
break
self.progressDialog.hide()
self.uploadBtn.setEnabled(True)
def cancelDownload(self):
self.upload_thread_stopped = True
class UploadThread(threading.Thread):
def __init__(self, i):
super(UploadThread, self).__init__()
self.i = i
self.setDaemon(True)
def run(self):
time.sleep(0.25) #simulate upload time
print self.i
if __name__ == '__main__':
app = QApplication(sys.argv)
w = UploadWindow()
sys.exit(app.exec_())

The GUI is not responsive because you do all the work in do_upload, never going back to the main loop.
Also, you call Thread.join(), that blocks everything until the thread is done (see https://docs.python.org/2/library/threading.html#threading.Thread.join)
You should use PySide.QtCore.QThread to take advantage of signals and slots.
Here is a nice example in C++. I implemented it in Python3.4 with PyQt here, but you should be able to use it with PySide too.
You might also want to look at PySide.QtCore.QProcess, to avoid using threads.

Here, I put some code together that does what I think you want.
For a real project, be sure to keep track of what's uploaded better &/or use something safer than .terminate() to stop the thread on demand.
import sys
from PySide import QtGui, QtCore
import time
class MySigObj(QtCore.QObject):
strSig = QtCore.Signal(str)
tupSig = QtCore.Signal(tuple)
class UploadThread(QtCore.QThread):
def __init__(self, parent=None):
super(UploadThread, self).__init__(parent)
self.endNow = False
self.fileName = None
self.sig = MySigObj()
self.fileNames = []
self.uploaded = []
#QtCore.Slot(str)
def setFileNames(self, t):
self.fileNames = list(t)
def run(self):
while self.fileNames:
print(self.fileNames)
time.sleep(2)
name = self.fileNames.pop(0)
s = 'uploaded file: ' + name + '\n'
print(s)
self.sig.strSig.emit(s)
self.uploaded.append(name)
if len(self.fileNames) == 0:
self.sig.strSig.emit("files transmitted: %s" % str(self.uploaded))
else:
time.sleep(1) #if the thread started but no list, wait 1 sec every cycle thru
#that was this thread should release the Python GIL (Global Interpreter Lock)
class ULoadWin(QtGui.QWidget):
def __init__(self, parent=None):
super(ULoadWin, self).__init__(parent)
self.upThread = UploadThread()
self.sig = MySigObj()
self.sig.tupSig.connect(self.upThread.setFileNames)
self.upThread.sig.strSig.connect(self.txtMsgAppend)
self.sig.tupSig.connect(self.upThread.setFileNames)
self.layout = QtGui.QVBoxLayout()
self.stButton = QtGui.QPushButton("Start")
self.stButton.clicked.connect(self.uploadItems)
self.stpButton = QtGui.QPushButton("Stop")
self.stpButton.clicked.connect(self.killThread)
self.testButton = QtGui.QPushButton("write txt\n not(?) blocked \nbelow")
self.testButton.setMinimumHeight(28)
self.testButton.clicked.connect(self.tstBlking)
self.lbl = QtGui.QTextEdit()
self.lbl.setMinimumHeight(325)
self.lbl.setMinimumWidth(290)
self.layout.addWidget(self.stButton)
self.layout.addWidget(self.stpButton)
self.layout.addWidget(self.testButton)
self.layout.addWidget(self.lbl)
self.setLayout(self.layout)
self.l = ['a', 'list', 'of_files', 'we', 'will_pretend_to_upload', 'st', 'uploading']
self.upThread.start()
def tstBlking(self):
self.lbl.append("txt not(?) blocked")
def uploadItems(self):
t = tuple(self.l)
self.sig.tupSig.emit(t)
self.upThread.start()
def killThread(self):
self.upThread.terminate()
time.sleep(.01)
self.upThread = UploadThread()
#QtCore.Slot(str)
def txtMsgAppend(self, txt):
self.lbl.append(txt + " | ")
if __name__ == '__main__':
app=QtGui.QApplication(sys.argv)
widg=ULoadWin()
widg.show()
sys.exit(app.exec_())

In the end I solved this problem by adapting the approach outlined here, which I find to be an elegant solution. The class I created is shown below:
class UploadThread(threading.Thread):
#input_q and result_q are Queue.Queue objects
def __init__(self, input_q, result_q):
super(UploadThread, self).__init__()
self.input_q = input_q
self.result_q = result_q
self.stoprequest = threading.Event() #threadsafe flag
def run(self):
'''Runs indefinitely until self.join() is called.
As soon as items are placed in the input_q, then the thread will process them until the input_q is emptied.
'''
while not self.stoprequest.isSet(): #stoprequest can be set from the main gui
try:
# Queue.get with timeout to allow checking self.stoprequest
num = self.input_q.get(True, 0.1) #when the queue is empty it waits 100ms before raising the Queue.Empty error
print 'In thread, processing', num
time.sleep(0.5)
self.result_q.put(True) #Indicate to the main thread that an item was successfully processed.
except Queue.Empty as e:
continue
def join(self, timeout=None):
self.stoprequest.set()
super(UploadThread, self).join(timeout)
In the main thread, the upload thread is created and the input_q is loaded with the items to upload. A QTimer is created to regularly check on the progress of the upload by checking what has been placed into the result_q. It also update the progress bar. If no progress has been made within a timeout, this indicates the upload connection has failed.
An advantage of using Queue.Queue objects for communicating between threads, is that multiple threads that share the same input and result queue can be created.

How to communicate with worker thread

I'm using a library which heaviliy uses I/O. For that reason calls to that library can last very long (more than 5 seconds) possible.
Using that directly inside an UI is not a good idea because it will freeze.
For that reason I outsourced the library calls to a thread queue like shown in this example: Python threads: communication and stopping
Nevertheless I'm not very happy with that solution since this has a major drawback:
I cannot really communicate with the UI.
Every lib command returns a return message, which can either be an error message or some computational result.
How would I get this?
Consider a library call do_test(foo):
def do_test(foo):
time.sleep(10)
return random.random() * foo
def ui_btn_click():
threaded_queue.put((do_test, 42))
# Now how to display the result without freezing the UI?
Can someone give me advice how to realize such a pattern?
Edit:
This here is a minimal example:
import os, time, random
import threading, queue
CMD_FOO = 1
CMD_BAR = 2
class ThreadedQueue(threading.Thread):
def __init__(self):
super().__init__()
self.in_queue = queue.Queue()
self.out_queue = queue.Queue()
self.__stoprequest = threading.Event()
def run(self):
while not self.__stoprequest.isSet():
(cmd, arg) = self.in_queue.get(True)
if cmd == CMD_FOO:
ret = self.handle_foo(arg)
elif cmd == CMD_BAR:
ret = self.handle_bar(arg)
else:
print("Unsupported cmd {0}".format(cmd))
self.out_queue.put(ret)
self.in_queue.task_done()
def handle_foo(self, arg):
print("start handle foo")
time.sleep(10)
return random.random() * arg
def handle_bar(self, arg):
print("start handle bar")
time.sleep(2)
return (random.random() * arg, 2 * arg)
if __name__ == "__main__":
print("START")
t = ThreadedQueue()
t.start()
t.in_queue.put((CMD_FOO, 10))
t.in_queue.put((CMD_BAR, 10))
print("Waiting")
while True:
x = t.out_queue.get(True)
t.out_queue.task_done()
print(x)
I personally use PySide but I don't want to depend this library on PySide or any other ui-related library.

I thought a bit about my implementations. THe conclusion is that I start another thread for picking the results of the queue:
class ReceiveThread(threading.Thread):
"""
Processes the output queue and calls a callback for each message
"""
def __init__(self, queue, callback):
super().__init__()
self.__queue = queue
self.__callback = callback
self.__stoprequest = threading.Event()
self.start()
def run(self):
while not self.__stoprequest.isSet():
ret = self.__queue.get(True)
self.__callback(ret)
self.__queue.task_done()
The given callback from an UI or elsewhere is called with every result from the queue.