I am trying to add threading to a Python 3.63 Tkinter program where a function will run but the GUI will still be responsive, including if the user wants to close the program while the function is running.
In the example below I have tried to run a simple printing to console function on a separate thread to the GUI mainloop so the user could click the X in the top right to close the program while the loop is running if they so wish.
The error I am getting is:
TypeError: start() takes 1 positional argument but 2 were given
try:
import tkinter as tk
import queue as queue
except:
import Tkinter as tk
import Queue as queue
import threading
def center(toplevel,desired_width=None,desired_height=None):
toplevel.update_idletasks()
w, h = toplevel.winfo_screenwidth() - 20, toplevel.winfo_screenheight() - 100
if desired_width and desired_height:
size = (desired_width,desired_height)
else:
size = tuple(int(Q) for Q in toplevel.geometry().split("+")[0].split("x"))
toplevel.geometry("%dx%d+%d+%d" % (size + (w/2 - size[0]/2, h/2 - size[1]/2)))
class ThreadedTask(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self,func):
func()
class app(tk.Tk):
def __init__(self):
tk.Tk.__init__(self)
center(self,desired_width=500,desired_height=400)
self.queue = queue.Queue()
self.run_func_button = tk.Button(self,
text="Run Function",
font=("Calibri",20,"bold"),
command=self.run_func)
self.run_func_button.pack()
def run_func(self):
ThreadedTask(self.queue).start(self.count_to_1500)
def count_to_1500(self):
for i in range(1500):
print (i)
app_start = app()
app_start.mainloop()
See doc threading - start() doesn't use arguments but you use .start(self.count_to_1500) - and this gives your error.
You could use
Thread(target=self.count_to_1500).start()
or
Thread(target=self.count_to_1500, args=(self.queue,)).start()
if you define
def count_to_1500(self, queue):
EDIT: working example with thread which put in quoue and method which get data from queue.
try:
import tkinter as tk
import queue as queue
except:
import Tkinter as tk
import Queue as queue
import threading
import time
def center(toplevel,desired_width=None,desired_height=None):
toplevel.update_idletasks()
w, h = toplevel.winfo_screenwidth() - 20, toplevel.winfo_screenheight() - 100
if desired_width and desired_height:
size = (desired_width,desired_height)
else:
size = tuple(int(Q) for Q in toplevel.geometry().split("+")[0].split("x"))
toplevel.geometry("%dx%d+%d+%d" % (size + (w/2 - size[0]/2, h/2 - size[1]/2)))
class app(tk.Tk):
def __init__(self):
tk.Tk.__init__(self)
center(self,desired_width=500,desired_height=400)
self.queue = queue.Queue()
self.run_func_button = tk.Button(self,
text="Run Function",
font=("Calibri",20,"bold"),
command=self.run_func)
self.run_func_button.pack()
def run_func(self):
threading.Thread(target=self.count_to_1500).start()
threading.Thread(target=self.count_to_1500_with_queue, args=(self.queue,)).start()
self.check_queue()
def count_to_1500(self):
for i in range(10):
print('1:', i)
time.sleep(0.2)
def count_to_1500_with_queue(self, queue):
for i in range(10):
print('put:', i)
queue.put(i)
time.sleep(1)
queue.put('last')
def check_queue(self):
print("check queue")
data = None
if not self.queue.empty():
data = self.queue.get()
print('get:', data)
if data != 'last':
self.after(200, self.check_queue)
app_start = app()
app_start.mainloop()
Thread.start takes no parameters: https://docs.python.org/3/library/threading.html
The correct way to use a Thread is:
# Will call func(*args, **kwargs)
t = threading.Thread(target=func, args=(), kwargs={})
t.start()
t.join()
The join is important. Without it you will have many zombie threads in your app, which will also prevent your app from shutting down cleanly.
Another pattern is to use a daemon thread, which processes a queue. daemon threads are automatically killed when the program exits.
def worker(q):
while True:
try:
f = q.get()
q.task_done()
if f is None: return
f()
except Exception:
import traceback
traceback.print_exc()
q = Queue.Queue()
t = threading.Thread(target=worker, args=(q,))
t.daemon=True
t.start()
# f is a no-arg function to be executed
q.put(f)
# Call at shutdown
q.join()
To run several tasks at the same time, start many threads.
Yet another method, use multiprocessing.pool.ThreadPool
from multiprocessing.pool import ThreadPool
# Create at startup
pool = ThreadPool(8)
# For each treaded task
pool.apply_async(func, args, kwds)
# Call at shutdown
pool.close()
pool.join()
... which works, more or less, as the above.
I recommend reading:
https://docs.python.org/2/library/multiprocessing.html#multiprocessing-programming
Related
I have a requirement of creating child processes, receive results using Future and then kill some of them when required.
For this I have subclassed multiprocessing.Process class and return a Future object from the start() method.
The problem is that I am not able to receive the result in the cb() function as it never gets called.
Please help/suggest if this can be done in some other way or something I am missing in my current implementation?
Following is my current approach
from multiprocessing import Process, Queue
from concurrent.futures import _base
import threading
from time import sleep
def foo(x,q):
print('result {}'.format(x*x))
result = x*x
sleep(5)
q.put(result)
class MyProcess(Process):
def __init__(self, target, args):
super().__init__()
self.target = target
self.args = args
self.f = _base.Future()
def run(self):
q = Queue()
worker_thread = threading.Thread(target=self.target, args=(self.args+ (q,)))
worker_thread.start()
r = q.get(block=True)
print('setting result {}'.format(r))
self.f.set_result(result=r)
print('done setting result')
def start(self):
f = _base.Future()
run_thread = threading.Thread(target=self.run)
run_thread.start()
return f
def cb(future):
print('received result in callback {}'.format(future))
def main():
p1 = MyProcess(target=foo, args=(2,))
f = p1.start()
f.add_done_callback(fn=cb)
sleep(10)
if __name__ == '__main__':
main()
print('Main thread dying')
In your start method you create a new Future which you then return. This is a different future then the one you set the result on, this future is just not used at all. Try:
def start(self):
run_thread = threading.Thread(target=self.run)
run_thread.start()
return self.f
However there are more problems with your code. You override the start method of the process, replacing it with execution on a worker thread, therefore actually bypassing multiprocessing. Also you shouldn't import the _base module, that is an implementation detail as seen from the leading underscore. You should import concurrent.futures.Future (it's the same class, but through public API).
This really uses multiprocessing:
from multiprocessing import Process, Queue
from concurrent.futures import Future
import threading
from time import sleep
def foo(x,q):
print('result {}'.format(x*x))
result = x*x
sleep(5)
q.put(result)
class MyProcess(Process):
def __init__(self, target, args):
super().__init__()
self.target = target
self.args = args
self.f = Future()
def run(self):
q = Queue()
worker_thread = threading.Thread(target=self.target, args=(self.args+ (q,)))
worker_thread.start()
r = q.get(block=True)
print('setting result {}'.format(r))
self.f.set_result(result=r)
print('done setting result')
def cb(future):
print('received result in callback {}: {}'.format(future, future.result()))
def main():
p1 = MyProcess(target=foo, args=(2,))
p1.f.add_done_callback(fn=cb)
p1.start()
p1.join()
sleep(10)
if __name__ == '__main__':
main()
print('Main thread dying')
And you're already in a new process now, spawning a worker thread to execute your target function shouldn't really be necessary, you could just execute your target function directly instead. Should the target function raise an Exception you wouldn't know about it, your callback will only be called on success. So if you fix that, then you're left with:
from multiprocessing import Process
from concurrent.futures import Future
import threading
from time import sleep
def foo(x):
print('result {}'.format(x*x))
result = x*x
sleep(5)
return result
class MyProcess(Process):
def __init__(self, target, args):
super().__init__()
self.target = target
self.args = args
self.f = Future()
def run(self):
try:
r = self.target(*self.args)
print('setting result {}'.format(r))
self.f.set_result(result=r)
print('done setting result')
except Exception as ex:
self.f.set_exception(ex)
def cb(future):
print('received result in callback {}: {}'.format(future, future.result()))
def main():
p1 = MyProcess(target=foo, args=(2,))
p1.f.add_done_callback(fn=cb)
p1.start()
p1.join()
sleep(10)
if __name__ == '__main__':
main()
print('Main thread dying')
This is basically what a ProcessPoolExecutor does.
I need to update the GUI after a thread completes and call this update_ui function from main thread (like a software interrupt maybe?). How can a worker thread call a function in the main thread?
Sample code:
def thread():
...some long task
update_ui() #But call this in main thread somehow
def main():
start_new_thread(thread)
...other functionality
def update_ui():
Tkinter_widget.update()
I tried to use Queue or any flag accessible to both threads but I have to wait/poll continuously to check if the value has been updated and then call the function - this wait makes the UI unresponsive. e.g.
flag = True
def thread():
...some long task
flag = False
def main():
start_new_thread(thread)
while(flag): sleep(1)
update_ui()
...other functionality
Your code appears to be somewhat hypothetical. Here is some that accomplishes that does what you describe. It creates three labels and initializes their text. It then starts three threads. Each thread updates the tkinter variable associated with the label created in the main thread after a period of time. Now if the main thread really needs to do the updating, queuing does work, but the program must be modified to accomplish that.
import threading
import time
from tkinter import *
import queue
import sys
def createGUI(master, widget_var):
for i in range(3):
Label(master, textvariable=widget_var[i]).grid(row=i, column=0)
widget_var[i].set("Thread " + str(i) + " started")
def sometask(thread_id, delay, queue):
print("Delaying", delay)
time.sleep(delay)
tdict = {'id': thread_id, 'message': 'success'}
# You can put simple strings/ints, whatever in the queue instead
queue.put(tdict)
return
def updateGUI(master, q, widget_var, td):
if not q.empty():
tdict = q.get()
widget_var[tdict['id']].set("Thread " + str(tdict['id']) + " completed with status: " + tdict['message'])
td.append(1)
if len(td) == 3:
print("All threads completed")
master.after(1000, timedExit)
else:
master.after(100, lambda w=master,que=q,v=widget_var, tcount=td: updateGUI(w,que,v,td))
def timedExit():
sys.exit()
root = Tk()
message_q = queue.Queue()
widget_var = []
threads_done = []
for i in range(3):
v = StringVar()
widget_var.append(v)
t = threading.Thread(target=sometask, args=(i, 3 + i * 3, message_q))
t.start()
createGUI(root, widget_var)
updateGUI(root,message_q, widget_var, threads_done)
root.mainloop()
I have a background Process (using Process from multiprocessing) that is pushing objects to my GUI, however this background process keeps locking up the GUI and the changes being pushed are never being displayed. The objects are being put in to my queue, however the update method in my GUI isn't being called regularly. What can I do make the GUI update more regularly? My GUI is written in Tkinter.
My background process has a infinite loop within it because I always need to keep reading the USB port for more data, so basically my code looks like this:
TracerAccess.py
import usb
from types import *
import sys
from multiprocessing import Process, Queue
import time
__idVendor__ = 0xFFFF
__idProduct__ = 0xFFFF
END_POINT = 0x82
def __printHEXList__(list):
print ' '.join('%02x' % b for b in list)
def checkDeviceConnected():
dev = usb.core.find(idVendor=__idVendor__, idProduct=__idProduct__)
if dev is None:
return False
else:
return True
class LowLevelAccess():
def __init__(self):
self.rawIn = []
self.tracer = usb.core.find(idVendor=__idVendor__, idProduct=__idProduct__)
if self.tracer is None:
raise ValueError("Device not connected")
self.tracer.set_configuration()
def readUSB(self):
"""
This method reads the USB data from the simtracer.
"""
try:
tmp = self.tracer.read(END_POINT, 10000,None, 100000).tolist()
while(self.checkForEmptyData(tmp)):
tmp = self.tracer.read(END_POINT, 10000,None, 100000).tolist()
self.rawIn = tmp
except:
time.sleep(1)
self.readUSB()
def checkForEmptyData(self, raw):
if(len(raw) == 10 or raw[10] is 0x60 or len(raw) == 11):
return True
else:
return False
class DataAbstraction:
def __init__(self, queue):
self.queue = queue
self.lowLevel = LowLevelAccess()
def readInput(self):
while True:
self.lowLevel.readUSB()
raw = self.lowLevel.rawIn
self.queue.put(raw)
ui.py
from Tkinter import *
import time
import TracerAccess as io
from multiprocessing import Process, Queue
from Queue import Empty
from math import ceil
def findNumberOfLines(message):
lines = message.split("\n")
return len(lines)
class Application(Frame):
def addTextToRaw(self, text, changeColour=False, numberOfLines=0):
self.rawText.config(state=NORMAL)
if changeColour is True:
self.rawText.insert(END,text, 'oddLine')
else:
self.rawText.insert(END,text)
self.rawText.config(state=DISABLED)
def updateRaw(self, text):
if(self.numberOfData() % 2 is not 0):
self.addTextToRaw(text, True)
else:
self.addTextToRaw(text)
def startTrace(self):
self.dataAbstraction = io.DataAbstraction(self.queue)
self.splitProc = Process(target=self.dataAbstraction.readInput())
self.stopButton.config(state="normal")
self.startButton.config(state="disabled")
self.splitProc.start()
def pollQueue(self):
try:
data = self.queue.get(0)
self.dataReturned.append(data)
self.updateRaw(str(data).upper())
self.rawText.tag_config("oddLine", background="#F3F6FA")
except Empty:
pass
finally:
try:
if(self.splitProc.is_alive() is False):
self.stopButton.config(state="disabled")
self.startButton.config(state="normal")
except AttributeError:
pass
self.master.after(10, self.pollQueue)
def stopTrace(self):
self.splitProc.join()
self.stopButton.config(state="disabled")
self.startButton.config(state="normal")
def createWidgets(self):
self.startButton = Button(self)
self.startButton["text"] = "Start"
self.startButton["command"] = self.startTrace
self.startButton.grid(row = 0, column=0)
self.stopButton = Button(self)
self.stopButton["text"] = "Stop"
self.stopButton["command"] = self.stopTrace
self.stopButton.config(state="disabled")
self.stopButton.grid(row = 0, column=1)
self.rawText = Text(self, state=DISABLED, width=82)
self.rawText.grid(row=1, columnspan=4)
def __init__(self, master):
Frame.__init__(self, master)
self.queue = Queue()
self.master.after(10, self.pollQueue)
self.pack()
self.dataReturned = []
self.createWidgets()
def numberOfData(self):
return len(self.dataReturned)
Main.py
import ui as ui
if __name__ == "__main__":
root = Tk()
root.columnconfigure(0,weight=1)
app = ui.Application(root)
app.mainloop()
So the background thread never finishes, however when I end the process the UI starts to be displayed before closing. The problem could have appeared because of my design for the TracerAccess.py module as I developed this after moving straight form java and little to no design experience for python.
What multiprocess.Process does, internally, is really a fork(), which effectively duplicated your process. You can perhaps visualize it as:
/ ["background" process] -------------\
[main process] --+ +-- [main process]
\ [main process continued] -----------/
p.join() attempts to "join" the two processes back to one. This effectively means: waiting until the background process is finished. Here's the actual (full) code from the .join() function:
def join(self, timeout=None):
'''
Wait until child process terminates
'''
assert self._parent_pid == os.getpid(), 'can only join a child process'
assert self._popen is not None, 'can only join a started process'
res = self._popen.wait(timeout)
if res is not None:
_current_process._children.discard(self)
Note how self._popen.wait is called.
This is obviously not what you want.
What you probably want, in the context of TKinter, is use the tk event loop, for example like this (Python 3, but the concept also works on Python 2)
from multiprocessing import Process, Queue
import time, tkinter, queue, random, sys
class Test:
def __init__(self, root):
self.root = root
txt = tkinter.Text(root)
txt.pack()
self.q = Queue()
p = Process(target=self.bg)
p.start()
self.checkqueue()
print('__init__ done', end=' ')
def bg(self):
print('Starting bg loop', end=' ')
n = 42
while True:
# Burn some CPU cycles
(int(random.random() * 199999)) ** (int(random.random() * 1999999))
n += 1
self.q.put(n)
print('bg task finished', end=' ')
def checkqueue(self):
try:
print(self.q.get_nowait(), end=' ')
except queue.Empty:
print('Queue empty', end=' ')
sys.stdout.flush()
# Run myself again after 1 second
self.root.after(1000, self.checkqueue)
root = tkinter.Tk()
Test(root)
root.mainloop()
You don't call the .join(), and instead use the .after() method, which schedules a function to run after n microseconds (if you've ever used Javascript, then think setTimeout()) to read the queue.
Depending on the actual content of your bg() function, you may not event need multiprocesing at all, just scheduling a function with .after() may be enough.
Also see:
http://tkinter.unpythonic.net/wiki/UsingTheEventLoop
I'm using a library which heaviliy uses I/O. For that reason calls to that library can last very long (more than 5 seconds) possible.
Using that directly inside an UI is not a good idea because it will freeze.
For that reason I outsourced the library calls to a thread queue like shown in this example: Python threads: communication and stopping
Nevertheless I'm not very happy with that solution since this has a major drawback:
I cannot really communicate with the UI.
Every lib command returns a return message, which can either be an error message or some computational result.
How would I get this?
Consider a library call do_test(foo):
def do_test(foo):
time.sleep(10)
return random.random() * foo
def ui_btn_click():
threaded_queue.put((do_test, 42))
# Now how to display the result without freezing the UI?
Can someone give me advice how to realize such a pattern?
Edit:
This here is a minimal example:
import os, time, random
import threading, queue
CMD_FOO = 1
CMD_BAR = 2
class ThreadedQueue(threading.Thread):
def __init__(self):
super().__init__()
self.in_queue = queue.Queue()
self.out_queue = queue.Queue()
self.__stoprequest = threading.Event()
def run(self):
while not self.__stoprequest.isSet():
(cmd, arg) = self.in_queue.get(True)
if cmd == CMD_FOO:
ret = self.handle_foo(arg)
elif cmd == CMD_BAR:
ret = self.handle_bar(arg)
else:
print("Unsupported cmd {0}".format(cmd))
self.out_queue.put(ret)
self.in_queue.task_done()
def handle_foo(self, arg):
print("start handle foo")
time.sleep(10)
return random.random() * arg
def handle_bar(self, arg):
print("start handle bar")
time.sleep(2)
return (random.random() * arg, 2 * arg)
if __name__ == "__main__":
print("START")
t = ThreadedQueue()
t.start()
t.in_queue.put((CMD_FOO, 10))
t.in_queue.put((CMD_BAR, 10))
print("Waiting")
while True:
x = t.out_queue.get(True)
t.out_queue.task_done()
print(x)
I personally use PySide but I don't want to depend this library on PySide or any other ui-related library.
I thought a bit about my implementations. THe conclusion is that I start another thread for picking the results of the queue:
class ReceiveThread(threading.Thread):
"""
Processes the output queue and calls a callback for each message
"""
def __init__(self, queue, callback):
super().__init__()
self.__queue = queue
self.__callback = callback
self.__stoprequest = threading.Event()
self.start()
def run(self):
while not self.__stoprequest.isSet():
ret = self.__queue.get(True)
self.__callback(ret)
self.__queue.task_done()
The given callback from an UI or elsewhere is called with every result from the queue.
I have been trying to use Multiprocessing module for updating Tkinter GUI but when I run this code, it is giving Pickling error.
# Test Code for Tkinter with threads
import Tkinter
from multiprocessing import Queue
import multiprocessing
import time
# Data Generator which will generate Data
def GenerateData():
global q
for i in range(10):
print "Generating Some Data, Iteration %s" %(i)
time.sleep(2)
q.put("Some Data from iteration %s \n" %(i))
def QueueHandler():
global q, text_wid
while True:
if not q.empty():
str = q.get()
text_wid.insert("end", str)
# Main Tkinter Application
def GUI():
global text_wid
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
text_wid.pack()
tk.mainloop()
if __name__ == '__main__':
# Queue which will be used for storing Data
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
q = multiprocessing .Queue()
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t2 = multiprocessing.Process(target=QueueHandler,args=(q,text_wid))
t1.start()
t2.start()
text_wid.pack()
tk.mainloop()
Error:
PicklingError: Can't pickle <type 'thread.lock'>: it's not found as thread.lock
If I remove the argument text_wid, then no error is reported but text widget is not updated with the data from the queque.
UPDATE :
I modified code so as to call the function to update the GUI whenever there is value in queue, thus preventing Tkinter widgets from being passed to separate process. Now, I am not getting any error but the widget is not updated with the data. However if i use mix of Threading and Multiprocessing module i.e. create a separate thread for handling data from the queue, then it works fine. My question why didn't it worked when i run the handler code in separate process. Am I not passing the data correctly. Below is the modified code:
# Test Code for Tkinter with threads
import Tkinter
import multiprocessing
from multiprocessing import Queue
import time
import threading
# Data Generator which will generate Data
def GenerateData(q):
for i in range(10):
print "Generating Some Data, Iteration %s" %(i)
time.sleep(2)
q.put("Some Data from iteration %s \n" %(i))
def QueueHandler(q):
while True:
if not q.empty():
str = q.get()
update_gui(str)
#text_wid.insert("end", str)
# Main Tkinter Application
def GUI():
global text_wid
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
text_wid.pack()
tk.mainloop()
def update_gui(str):
global text_wid
text_wid.insert("end", str)
if __name__ == '__main__':
# Queue which will be used for storing Data
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
q = multiprocessing.Queue()
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t2 = multiprocessing.Process(target=QueueHandler,args=(q,))
t1.start()
t2.start()
text_wid.pack()
tk.mainloop()
You missed out an important part, you should protect your calls with a __main__ trap:
if __name__ == '__main__':
q = Queue.Queue()
# Create a thread and run GUI & QueueHadnler in it
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t2 = multiprocessing.Process(target=QueueHandler,args=(q,))
....
Note that the Queue is passed as a parameter rather than using a global.
Edit: just spotted another issue, you should be using Queue from the multiprocessing module, not from Queue:
from multiprocessing import Queue
# Test Code for Tkinter with threads
import Tkinter as Tk
import multiprocessing
from Queue import Empty, Full
import time
class GuiApp(object):
def __init__(self,q):
self.root = Tk.Tk()
self.root.geometry('300x100')
self.text_wid = Tk.Text(self.root,height=100,width=100)
self.text_wid.pack(expand=1,fill=Tk.BOTH)
self.root.after(100,self.CheckQueuePoll,q)
def CheckQueuePoll(self,c_queue):
try:
str = c_queue.get(0)
self.text_wid.insert('end',str)
except Empty:
pass
finally:
self.root.after(100, self.CheckQueuePoll, c_queue)
# Data Generator which will generate Data
def GenerateData(q):
for i in range(10):
print "Generating Some Data, Iteration %s" %(i)
time.sleep(2)
q.put("Some Data from iteration %s \n" %(i))
if __name__ == '__main__':
# Queue which will be used for storing Data
q = multiprocessing.Queue()
q.cancel_join_thread() # or else thread that puts data will not term
gui = GuiApp(q)
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t1.start()
gui.root.mainloop()
t1.join()
t2.join()