I have been trying to use Multiprocessing module for updating Tkinter GUI but when I run this code, it is giving Pickling error.
# Test Code for Tkinter with threads
import Tkinter
from multiprocessing import Queue
import multiprocessing
import time
# Data Generator which will generate Data
def GenerateData():
global q
for i in range(10):
print "Generating Some Data, Iteration %s" %(i)
time.sleep(2)
q.put("Some Data from iteration %s \n" %(i))
def QueueHandler():
global q, text_wid
while True:
if not q.empty():
str = q.get()
text_wid.insert("end", str)
# Main Tkinter Application
def GUI():
global text_wid
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
text_wid.pack()
tk.mainloop()
if __name__ == '__main__':
# Queue which will be used for storing Data
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
q = multiprocessing .Queue()
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t2 = multiprocessing.Process(target=QueueHandler,args=(q,text_wid))
t1.start()
t2.start()
text_wid.pack()
tk.mainloop()
Error:
PicklingError: Can't pickle <type 'thread.lock'>: it's not found as thread.lock
If I remove the argument text_wid, then no error is reported but text widget is not updated with the data from the queque.
UPDATE :
I modified code so as to call the function to update the GUI whenever there is value in queue, thus preventing Tkinter widgets from being passed to separate process. Now, I am not getting any error but the widget is not updated with the data. However if i use mix of Threading and Multiprocessing module i.e. create a separate thread for handling data from the queue, then it works fine. My question why didn't it worked when i run the handler code in separate process. Am I not passing the data correctly. Below is the modified code:
# Test Code for Tkinter with threads
import Tkinter
import multiprocessing
from multiprocessing import Queue
import time
import threading
# Data Generator which will generate Data
def GenerateData(q):
for i in range(10):
print "Generating Some Data, Iteration %s" %(i)
time.sleep(2)
q.put("Some Data from iteration %s \n" %(i))
def QueueHandler(q):
while True:
if not q.empty():
str = q.get()
update_gui(str)
#text_wid.insert("end", str)
# Main Tkinter Application
def GUI():
global text_wid
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
text_wid.pack()
tk.mainloop()
def update_gui(str):
global text_wid
text_wid.insert("end", str)
if __name__ == '__main__':
# Queue which will be used for storing Data
tk = Tkinter.Tk()
text_wid = Tkinter.Text(tk)
q = multiprocessing.Queue()
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t2 = multiprocessing.Process(target=QueueHandler,args=(q,))
t1.start()
t2.start()
text_wid.pack()
tk.mainloop()
You missed out an important part, you should protect your calls with a __main__ trap:
if __name__ == '__main__':
q = Queue.Queue()
# Create a thread and run GUI & QueueHadnler in it
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t2 = multiprocessing.Process(target=QueueHandler,args=(q,))
....
Note that the Queue is passed as a parameter rather than using a global.
Edit: just spotted another issue, you should be using Queue from the multiprocessing module, not from Queue:
from multiprocessing import Queue
# Test Code for Tkinter with threads
import Tkinter as Tk
import multiprocessing
from Queue import Empty, Full
import time
class GuiApp(object):
def __init__(self,q):
self.root = Tk.Tk()
self.root.geometry('300x100')
self.text_wid = Tk.Text(self.root,height=100,width=100)
self.text_wid.pack(expand=1,fill=Tk.BOTH)
self.root.after(100,self.CheckQueuePoll,q)
def CheckQueuePoll(self,c_queue):
try:
str = c_queue.get(0)
self.text_wid.insert('end',str)
except Empty:
pass
finally:
self.root.after(100, self.CheckQueuePoll, c_queue)
# Data Generator which will generate Data
def GenerateData(q):
for i in range(10):
print "Generating Some Data, Iteration %s" %(i)
time.sleep(2)
q.put("Some Data from iteration %s \n" %(i))
if __name__ == '__main__':
# Queue which will be used for storing Data
q = multiprocessing.Queue()
q.cancel_join_thread() # or else thread that puts data will not term
gui = GuiApp(q)
t1 = multiprocessing.Process(target=GenerateData,args=(q,))
t1.start()
gui.root.mainloop()
t1.join()
t2.join()
Related
I have a class to start a thread for a while loop. I tried to scheduling the thread class to start within a certain time but it doesn't work:
def test():
if __name__ == "__main__":
main()
schedule.every().day.at("17:25:50").do(test)
The function does not do anything even the time reached "17:25:50"
My full code:
import discord
import random
import time
import asyncio
import schedule
from facebook_scraper import get_posts, _scraper, exceptions
from discord.ext import commands, tasks
import threading
import time
import re
class LEDManager(threading.Thread):
def __init__(self, id_manager):
threading.Thread.__init__(self)
self.id_manager = int(id_manager)
def run(self):
while True:
try:
wanted = "Pecahan setiap negeri (Kumulatif):" # wanted post
for post in get_posts("myhealthkkm", pages=5):
if post.get("post_text") is not None and wanted in post.get("post_text"):
# print("Found", t)
listposts.append(post.get("post_text"))
# append until 3 page finish then go here
time.sleep(1)
print(listposts)
global listView
if listposts != 0:
listView = listposts.copy()
print(listView)
listposts.clear()
except exceptions.TemporarilyBanned:
print("Temporarily banned, sleeping for 10m")
time.sleep(600)
def main():
thread_id = ("0")
led_index = 0
thread_list = list()
for objs in thread_id:
thread = LEDManager(led_index)
thread_list.append(thread)
led_index += 1
for thread in thread_list:
thread.start()
time.sleep(1)
def test():
if __name__ == "__main__":
main()
schedule.every().day.at("17:25:50").do(test)
You forgot to add these lines:
while True:
schedule.run_pending()
time.sleep(1)
You should add them at the end of the file, so the system will keep checking forever, if "the job" needs to be done (if the hour is "17:25:50").
And here is the full documentation to see how to use the schedule module:
https://schedule.readthedocs.io/en/stable/
I am trying to send a dynamic array across multiple processes in python. My first solution was sending the data directly through the Queue/Pipe of multiprocessing class. The problem is that it is limited by the bandwidth of the ethernet connection. Therefore I am trying to use the ctype array and pass just the address of the object. When I try to access the array from the second process (either A.raw or A.value) the process exit without any exception. Does somebody have an idea what is going on? Maybe some problems with lock etc.
from multiprocessing import Process,Queue
from ctypes import c_char,addressof
from time import sleep
import os
class ProcessIn(Process):
def __init__(self,QueueI):
super().__init__(daemon=True)
self.QueueI=QueueI
def run(self):
Array=[]
while True:
N=100000
A=(c_char*N)()
A.value=b'\x01'
Address=addressof(A)
Array.append(A)
print(os.getpid(),'putted',Address)
self.QueueI.put((Address,N))
sleep(2)
class ProcessOut(Process):
def __init__(self,QueueI):
super().__init__(daemon=True)
self.QueueI=QueueI
def run(self):
while True:
print(os.getpid(),'step 1')
Address,N=self.QueueI.get()
print(os.getpid(),'step 2',Address)
A=(c_char*N).from_address(Address)
print(os.getpid(),'step 3')
Value=A.raw #This will fail
print(os.getpid(),'step 4',Value)
sleep(1)
if __name__ == '__main__':
QueueI=Queue()
In=ProcessIn(QueueI)
Out=ProcessOut(QueueI)
print(os.getpid(),'main')
In.start()
Out.start()
input('press key to finish\n')
Ok, I got it - using mmap with tag:
from multiprocessing import Process,Queue
from ctypes import c_char,addressof
import pyarrow as pa
import numpy as np
from time import sleep
import os
import datetime
import mmap
from sys import getsizeof
class ProcessIn(Process):
def __init__(self,QueueI):
super().__init__(daemon=True)
self.QueueI=QueueI
def run(self):
i=0
while True:
N=np.random.randint(10,14)*100000
data = b'abcdefghijklmnopqrstuvwxyz'
Tag='Data_'+str(i)
buf = mmap.mmap(-1, N*len(data),tagname=Tag)
buf[0]=i
NN=N*len(data)
# print(buf[0:10])
print(os.getpid(),'putted',Tag,NN)
if self.QueueI.qsize()==0:
self.QueueI.put((Tag,NN,datetime.datetime.now()))
i+=1
sleep(1)
class ProcessOut(Process):
def __init__(self,QueueI):
super().__init__(daemon=True)
self.QueueI=QueueI
def run(self):
while True:
# print(os.getpid(),'step 1')
Tag,N,start=self.QueueI.get()
buf = mmap.mmap(-1, N,tagname=Tag)
print('got',buf[0:10],Tag)
# data=buf.read()
dt=(datetime.datetime.now()-start).total_seconds()
if dt!=0:
print(os.getpid(),N/dt/1024**2,'MBs',dt*1000,'ms',N/1024**2,'MB',N)
else:
print(os.getpid(),np.nan,'MBs',dt*1000,'ms',N/1024**2,'MB',N)
buf=None
if __name__ == '__main__':
QueueI=Queue()
In=ProcessIn(QueueI)
Out=ProcessOut(QueueI)
print(os.getpid(),'main')
In.start()
Out.start()
input('press key to finish\n')
I am trying to add threading to a Python 3.63 Tkinter program where a function will run but the GUI will still be responsive, including if the user wants to close the program while the function is running.
In the example below I have tried to run a simple printing to console function on a separate thread to the GUI mainloop so the user could click the X in the top right to close the program while the loop is running if they so wish.
The error I am getting is:
TypeError: start() takes 1 positional argument but 2 were given
try:
import tkinter as tk
import queue as queue
except:
import Tkinter as tk
import Queue as queue
import threading
def center(toplevel,desired_width=None,desired_height=None):
toplevel.update_idletasks()
w, h = toplevel.winfo_screenwidth() - 20, toplevel.winfo_screenheight() - 100
if desired_width and desired_height:
size = (desired_width,desired_height)
else:
size = tuple(int(Q) for Q in toplevel.geometry().split("+")[0].split("x"))
toplevel.geometry("%dx%d+%d+%d" % (size + (w/2 - size[0]/2, h/2 - size[1]/2)))
class ThreadedTask(threading.Thread):
def __init__(self,queue):
threading.Thread.__init__(self)
self.queue = queue
def run(self,func):
func()
class app(tk.Tk):
def __init__(self):
tk.Tk.__init__(self)
center(self,desired_width=500,desired_height=400)
self.queue = queue.Queue()
self.run_func_button = tk.Button(self,
text="Run Function",
font=("Calibri",20,"bold"),
command=self.run_func)
self.run_func_button.pack()
def run_func(self):
ThreadedTask(self.queue).start(self.count_to_1500)
def count_to_1500(self):
for i in range(1500):
print (i)
app_start = app()
app_start.mainloop()
See doc threading - start() doesn't use arguments but you use .start(self.count_to_1500) - and this gives your error.
You could use
Thread(target=self.count_to_1500).start()
or
Thread(target=self.count_to_1500, args=(self.queue,)).start()
if you define
def count_to_1500(self, queue):
EDIT: working example with thread which put in quoue and method which get data from queue.
try:
import tkinter as tk
import queue as queue
except:
import Tkinter as tk
import Queue as queue
import threading
import time
def center(toplevel,desired_width=None,desired_height=None):
toplevel.update_idletasks()
w, h = toplevel.winfo_screenwidth() - 20, toplevel.winfo_screenheight() - 100
if desired_width and desired_height:
size = (desired_width,desired_height)
else:
size = tuple(int(Q) for Q in toplevel.geometry().split("+")[0].split("x"))
toplevel.geometry("%dx%d+%d+%d" % (size + (w/2 - size[0]/2, h/2 - size[1]/2)))
class app(tk.Tk):
def __init__(self):
tk.Tk.__init__(self)
center(self,desired_width=500,desired_height=400)
self.queue = queue.Queue()
self.run_func_button = tk.Button(self,
text="Run Function",
font=("Calibri",20,"bold"),
command=self.run_func)
self.run_func_button.pack()
def run_func(self):
threading.Thread(target=self.count_to_1500).start()
threading.Thread(target=self.count_to_1500_with_queue, args=(self.queue,)).start()
self.check_queue()
def count_to_1500(self):
for i in range(10):
print('1:', i)
time.sleep(0.2)
def count_to_1500_with_queue(self, queue):
for i in range(10):
print('put:', i)
queue.put(i)
time.sleep(1)
queue.put('last')
def check_queue(self):
print("check queue")
data = None
if not self.queue.empty():
data = self.queue.get()
print('get:', data)
if data != 'last':
self.after(200, self.check_queue)
app_start = app()
app_start.mainloop()
Thread.start takes no parameters: https://docs.python.org/3/library/threading.html
The correct way to use a Thread is:
# Will call func(*args, **kwargs)
t = threading.Thread(target=func, args=(), kwargs={})
t.start()
t.join()
The join is important. Without it you will have many zombie threads in your app, which will also prevent your app from shutting down cleanly.
Another pattern is to use a daemon thread, which processes a queue. daemon threads are automatically killed when the program exits.
def worker(q):
while True:
try:
f = q.get()
q.task_done()
if f is None: return
f()
except Exception:
import traceback
traceback.print_exc()
q = Queue.Queue()
t = threading.Thread(target=worker, args=(q,))
t.daemon=True
t.start()
# f is a no-arg function to be executed
q.put(f)
# Call at shutdown
q.join()
To run several tasks at the same time, start many threads.
Yet another method, use multiprocessing.pool.ThreadPool
from multiprocessing.pool import ThreadPool
# Create at startup
pool = ThreadPool(8)
# For each treaded task
pool.apply_async(func, args, kwds)
# Call at shutdown
pool.close()
pool.join()
... which works, more or less, as the above.
I recommend reading:
https://docs.python.org/2/library/multiprocessing.html#multiprocessing-programming
I need to update the GUI after a thread completes and call this update_ui function from main thread (like a software interrupt maybe?). How can a worker thread call a function in the main thread?
Sample code:
def thread():
...some long task
update_ui() #But call this in main thread somehow
def main():
start_new_thread(thread)
...other functionality
def update_ui():
Tkinter_widget.update()
I tried to use Queue or any flag accessible to both threads but I have to wait/poll continuously to check if the value has been updated and then call the function - this wait makes the UI unresponsive. e.g.
flag = True
def thread():
...some long task
flag = False
def main():
start_new_thread(thread)
while(flag): sleep(1)
update_ui()
...other functionality
Your code appears to be somewhat hypothetical. Here is some that accomplishes that does what you describe. It creates three labels and initializes their text. It then starts three threads. Each thread updates the tkinter variable associated with the label created in the main thread after a period of time. Now if the main thread really needs to do the updating, queuing does work, but the program must be modified to accomplish that.
import threading
import time
from tkinter import *
import queue
import sys
def createGUI(master, widget_var):
for i in range(3):
Label(master, textvariable=widget_var[i]).grid(row=i, column=0)
widget_var[i].set("Thread " + str(i) + " started")
def sometask(thread_id, delay, queue):
print("Delaying", delay)
time.sleep(delay)
tdict = {'id': thread_id, 'message': 'success'}
# You can put simple strings/ints, whatever in the queue instead
queue.put(tdict)
return
def updateGUI(master, q, widget_var, td):
if not q.empty():
tdict = q.get()
widget_var[tdict['id']].set("Thread " + str(tdict['id']) + " completed with status: " + tdict['message'])
td.append(1)
if len(td) == 3:
print("All threads completed")
master.after(1000, timedExit)
else:
master.after(100, lambda w=master,que=q,v=widget_var, tcount=td: updateGUI(w,que,v,td))
def timedExit():
sys.exit()
root = Tk()
message_q = queue.Queue()
widget_var = []
threads_done = []
for i in range(3):
v = StringVar()
widget_var.append(v)
t = threading.Thread(target=sometask, args=(i, 3 + i * 3, message_q))
t.start()
createGUI(root, widget_var)
updateGUI(root,message_q, widget_var, threads_done)
root.mainloop()
I have a background Process (using Process from multiprocessing) that is pushing objects to my GUI, however this background process keeps locking up the GUI and the changes being pushed are never being displayed. The objects are being put in to my queue, however the update method in my GUI isn't being called regularly. What can I do make the GUI update more regularly? My GUI is written in Tkinter.
My background process has a infinite loop within it because I always need to keep reading the USB port for more data, so basically my code looks like this:
TracerAccess.py
import usb
from types import *
import sys
from multiprocessing import Process, Queue
import time
__idVendor__ = 0xFFFF
__idProduct__ = 0xFFFF
END_POINT = 0x82
def __printHEXList__(list):
print ' '.join('%02x' % b for b in list)
def checkDeviceConnected():
dev = usb.core.find(idVendor=__idVendor__, idProduct=__idProduct__)
if dev is None:
return False
else:
return True
class LowLevelAccess():
def __init__(self):
self.rawIn = []
self.tracer = usb.core.find(idVendor=__idVendor__, idProduct=__idProduct__)
if self.tracer is None:
raise ValueError("Device not connected")
self.tracer.set_configuration()
def readUSB(self):
"""
This method reads the USB data from the simtracer.
"""
try:
tmp = self.tracer.read(END_POINT, 10000,None, 100000).tolist()
while(self.checkForEmptyData(tmp)):
tmp = self.tracer.read(END_POINT, 10000,None, 100000).tolist()
self.rawIn = tmp
except:
time.sleep(1)
self.readUSB()
def checkForEmptyData(self, raw):
if(len(raw) == 10 or raw[10] is 0x60 or len(raw) == 11):
return True
else:
return False
class DataAbstraction:
def __init__(self, queue):
self.queue = queue
self.lowLevel = LowLevelAccess()
def readInput(self):
while True:
self.lowLevel.readUSB()
raw = self.lowLevel.rawIn
self.queue.put(raw)
ui.py
from Tkinter import *
import time
import TracerAccess as io
from multiprocessing import Process, Queue
from Queue import Empty
from math import ceil
def findNumberOfLines(message):
lines = message.split("\n")
return len(lines)
class Application(Frame):
def addTextToRaw(self, text, changeColour=False, numberOfLines=0):
self.rawText.config(state=NORMAL)
if changeColour is True:
self.rawText.insert(END,text, 'oddLine')
else:
self.rawText.insert(END,text)
self.rawText.config(state=DISABLED)
def updateRaw(self, text):
if(self.numberOfData() % 2 is not 0):
self.addTextToRaw(text, True)
else:
self.addTextToRaw(text)
def startTrace(self):
self.dataAbstraction = io.DataAbstraction(self.queue)
self.splitProc = Process(target=self.dataAbstraction.readInput())
self.stopButton.config(state="normal")
self.startButton.config(state="disabled")
self.splitProc.start()
def pollQueue(self):
try:
data = self.queue.get(0)
self.dataReturned.append(data)
self.updateRaw(str(data).upper())
self.rawText.tag_config("oddLine", background="#F3F6FA")
except Empty:
pass
finally:
try:
if(self.splitProc.is_alive() is False):
self.stopButton.config(state="disabled")
self.startButton.config(state="normal")
except AttributeError:
pass
self.master.after(10, self.pollQueue)
def stopTrace(self):
self.splitProc.join()
self.stopButton.config(state="disabled")
self.startButton.config(state="normal")
def createWidgets(self):
self.startButton = Button(self)
self.startButton["text"] = "Start"
self.startButton["command"] = self.startTrace
self.startButton.grid(row = 0, column=0)
self.stopButton = Button(self)
self.stopButton["text"] = "Stop"
self.stopButton["command"] = self.stopTrace
self.stopButton.config(state="disabled")
self.stopButton.grid(row = 0, column=1)
self.rawText = Text(self, state=DISABLED, width=82)
self.rawText.grid(row=1, columnspan=4)
def __init__(self, master):
Frame.__init__(self, master)
self.queue = Queue()
self.master.after(10, self.pollQueue)
self.pack()
self.dataReturned = []
self.createWidgets()
def numberOfData(self):
return len(self.dataReturned)
Main.py
import ui as ui
if __name__ == "__main__":
root = Tk()
root.columnconfigure(0,weight=1)
app = ui.Application(root)
app.mainloop()
So the background thread never finishes, however when I end the process the UI starts to be displayed before closing. The problem could have appeared because of my design for the TracerAccess.py module as I developed this after moving straight form java and little to no design experience for python.
What multiprocess.Process does, internally, is really a fork(), which effectively duplicated your process. You can perhaps visualize it as:
/ ["background" process] -------------\
[main process] --+ +-- [main process]
\ [main process continued] -----------/
p.join() attempts to "join" the two processes back to one. This effectively means: waiting until the background process is finished. Here's the actual (full) code from the .join() function:
def join(self, timeout=None):
'''
Wait until child process terminates
'''
assert self._parent_pid == os.getpid(), 'can only join a child process'
assert self._popen is not None, 'can only join a started process'
res = self._popen.wait(timeout)
if res is not None:
_current_process._children.discard(self)
Note how self._popen.wait is called.
This is obviously not what you want.
What you probably want, in the context of TKinter, is use the tk event loop, for example like this (Python 3, but the concept also works on Python 2)
from multiprocessing import Process, Queue
import time, tkinter, queue, random, sys
class Test:
def __init__(self, root):
self.root = root
txt = tkinter.Text(root)
txt.pack()
self.q = Queue()
p = Process(target=self.bg)
p.start()
self.checkqueue()
print('__init__ done', end=' ')
def bg(self):
print('Starting bg loop', end=' ')
n = 42
while True:
# Burn some CPU cycles
(int(random.random() * 199999)) ** (int(random.random() * 1999999))
n += 1
self.q.put(n)
print('bg task finished', end=' ')
def checkqueue(self):
try:
print(self.q.get_nowait(), end=' ')
except queue.Empty:
print('Queue empty', end=' ')
sys.stdout.flush()
# Run myself again after 1 second
self.root.after(1000, self.checkqueue)
root = tkinter.Tk()
Test(root)
root.mainloop()
You don't call the .join(), and instead use the .after() method, which schedules a function to run after n microseconds (if you've ever used Javascript, then think setTimeout()) to read the queue.
Depending on the actual content of your bg() function, you may not event need multiprocesing at all, just scheduling a function with .after() may be enough.
Also see:
http://tkinter.unpythonic.net/wiki/UsingTheEventLoop