I'm trying to extend this example on multithreading in PyQt5:
Multithreading PyQt applications with QThreadPool (MWE below)
to allow for two different threaded functions, one that uses a callback and the other that does not. In the example above, the progress_callback is hard-coded in the Worker() class __init__, which means that any threaded function must have a signature that accommodates that callback:
def execute_this_fn(self, progress_callback):
which means that if there is a second threaded process, that function's signature would also be required to accommodate the callback in its signature even though the callback is not used.
So instead of hard-coding progress_callback into the __init__ of Worker(), I'd like to pass in the callback when instantiating Worker().
My MWE is below -- I include two commented-out two lines from the original example for reference. When I run it, and press the "DANGER!" button on the GUI, I get:
$ python threadtest.py
Multithreading with maximum 16 threads
progress_callback = <bound PYQT_SIGNAL progress of WorkerSignals object at 0x113bbd160>
n = 0
n = 1
n = 2
n = 3
n = 4
Done.
THREAD COMPLETE!
So the code runs, but the callback function (slot) progress_fn is never called, and I'm not sure why...
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
import time
import traceback, sys
class WorkerSignals(QObject):
''' Defines the signals available from a running worker thread.
Supported signals are:
finished
No data
error
`tuple` (exctype, value, traceback.format_exc() )
result
`object` data returned from processing, anything
progress
`int` indicating % progress
'''
finished = pyqtSignal()
error = pyqtSignal(tuple)
result = pyqtSignal(object)
progress = pyqtSignal(int)
class Worker(QRunnable):
''' Worker thread
Inherits from QRunnable to handler worker thread setup, signals and wrap-up.
:param callback: The function callback to run on this worker thread. Supplied args and
kwargs will be passed through to the runner.
:type callback: function
:param args: Arguments to pass to the callback function
:param kwargs: Keywords to pass to the callback function
'''
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
# Store constructor arguments (re-used for processing)
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
# The old way (hard-code the callback to kwargs)
#self.kwargs['progress_callback'] = self.signals.progress
#pyqtSlot()
def run(self):
'''
Initialise the runner function with passed args, kwargs.
'''
# Retrieve args/kwargs here; and fire processing using them
try:
result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(result) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.counter = 0
self.signals = WorkerSignals()
layout = QVBoxLayout()
self.l = QLabel("Start")
b = QPushButton("DANGER!")
b.pressed.connect(self.oh_no)
layout.addWidget(self.l)
layout.addWidget(b)
w = QWidget()
w.setLayout(layout)
self.setCentralWidget(w)
self.show()
self.threadpool = QThreadPool()
print("Multithreading with maximum %d threads" % self.threadpool.maxThreadCount())
self.timer = QTimer()
self.timer.setInterval(1000)
self.timer.timeout.connect(self.recurring_timer)
self.timer.start()
def progress_fn(self, n):
print("%d%% done" % n) # this does not execute...
def execute_this_fn(self, progress_callback):
print(f'progress_callback = {progress_callback}')
for n in range(0, 5):
time.sleep(1)
print(f'n = {n}')
progress_callback.emit(int(n*100/4))
return "Done."
def print_output(self, s):
print(s)
def thread_complete(self):
print("THREAD COMPLETE!")
def oh_no(self):
# Pass the function to execute
# the old way (callback hardcoded in __init__ of Worker class)
#worker = Worker(self.execute_this_fn)
# the desired new way
worker = Worker(self.execute_this_fn, progress_callback=self.signals.progress)
worker.signals.result.connect(self.print_output)
worker.signals.finished.connect(self.thread_complete)
worker.signals.progress.connect(self.progress_fn)
# Execute
self.threadpool.start(worker)
def recurring_timer(self):
self.counter +=1
self.l.setText("Counter: %d" % self.counter)
app = QApplication([])
window = MainWindow()
app.exec_()
The problem is simple: You have 2 WorkerSignals objects, and in one of them you make the connection and with the other you emit the signal. The solution is to use the same object for the connection and for the emission:
worker = Worker(self.execute_this_fn, progress_callback=self.signals.progress)
worker.signals.result.connect(self.print_output)
worker.signals.finished.connect(self.thread_complete)
self.signals.progress.connect(self.progress_fn)
Although I prefer to create a QObject that implements that logic:
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
import time
import traceback, sys
class WorkerSignals(QObject):
"""Defines the signals available from a running worker thread.
Supported signals are:
finished
No data
error
`tuple` (exctype, value, traceback.format_exc() )
result
`object` data returned from processing, anything
progress
`int` indicating % progress
"""
finished = pyqtSignal()
error = pyqtSignal(tuple)
result = pyqtSignal(object)
progress = pyqtSignal(int)
class Worker(QRunnable):
"""Worker thread
Inherits from QRunnable to handler worker thread setup, signals and wrap-up.
:param callback: The function callback to run on this worker thread. Supplied args and
kwargs will be passed through to the runner.
:type callback: function
:param args: Arguments to pass to the callback function
:param kwargs: Keywords to pass to the callback function
"""
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
# Store constructor arguments (re-used for processing)
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
# The old way (hard-code the callback to kwargs)
# self.kwargs['progress_callback'] = self.signals.progress
#pyqtSlot()
def run(self):
"""
Initialise the runner function with passed args, kwargs.
"""
# Retrieve args/kwargs here; and fire processing using them
try:
result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(result) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
class ProgressCallback(QObject):
progressChanged = pyqtSignal(int)
def __call__(self, value):
self.progressChanged.emit(value)
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.counter = 0
layout = QVBoxLayout()
self.l = QLabel("Start")
b = QPushButton("DANGER!")
b.pressed.connect(self.oh_no)
layout.addWidget(self.l)
layout.addWidget(b)
w = QWidget()
w.setLayout(layout)
self.setCentralWidget(w)
self.show()
self.threadpool = QThreadPool()
print(
"Multithreading with maximum %d threads" % self.threadpool.maxThreadCount()
)
self.timer = QTimer()
self.timer.setInterval(1000)
self.timer.timeout.connect(self.recurring_timer)
self.timer.start()
def progress_fn(self, n):
print("%d%% done" % n) # this does not execute...
def execute_this_fn(self, progress_callback):
print(f"progress_callback = {progress_callback}")
for n in range(0, 5):
time.sleep(1)
print(f"n = {n}")
progress_callback(int(n * 100 / 4))
return "Done."
def print_output(self, s):
print(s)
def thread_complete(self):
print("THREAD COMPLETE!")
def oh_no(self):
callback = ProgressCallback()
worker = Worker(self.execute_this_fn, progress_callback=callback)
worker.signals.result.connect(self.print_output)
worker.signals.finished.connect(self.thread_complete)
callback.progressChanged.connect(self.progress_fn)
# Execute
self.threadpool.start(worker)
def recurring_timer(self):
self.counter += 1
self.l.setText("Counter: %d" % self.counter)
app = QApplication([])
window = MainWindow()
app.exec_()
Related
So basically I created a script called worker which contain 2 classes:
Worker
CreateThread
this script is supposed to handle unexpected exception occurs, while also provide threading solution to running heavy function to prevent GUI freezing.
My problem is that I get unexpected behavior sometimes, for example one time I can run function with the worker class and I will get a good response and everything works fine, and second time things go wrong and I get odd errors and also GUI crashes.
I tried to change the order I manage the thread handling, and I tried addepting the code to the errors but there is a hidden problem that keeps hunting me...
that is my worker code:
# Create a worker class #
class Worker(QObject):
my_started = pyqtSignal()
my_finished = pyqtSignal()
my_error = pyqtSignal(str, str)
my_success = pyqtSignal()
def __init__(self):
super(Worker, self).__init__()
def run(self, func):
try:
self.my_started.emit()
time.sleep(0.1)
x = func()
if x is None:
self.my_success.emit()
self.my_finished.emit()
return
if x[0] == -1:
function_error = x[1]
trace_back = x[2]
print(f'\nFunction error: {function_error}')
print(trace_back)
self.my_error.emit(str(function_error), trace_back,
str(func.__name__))
self.my_finished.emit()
return
except Exception as e:
self.trace_back = str(traceback.format_exc())
print(f'\nFunction error: {e}')
print(self.trace_back)
self.my_error.emit(str(e), self.trace_back)
self.my_finished.emit()
# Creating thread manger class #
class CreateThread(Worker, QObject):
def __init__(self, atp_name):
super(CreateThread, self).__init__()
self.stop_flag = False
self.atp_name = atp_name
self.thread = QThread()
self.worker = Worker()
def start_thread(self, func, **kwargs):
self.kwargs = kwargs
self.func_name = func.__name__
if self.stop_flag:
return
if self.func_name == '<lambda>':
self.func_name = self.kwargs['func_name']
# Thread and Worker init #
self.thread = QThread()
self.worker = Worker()
act = partial(self.worker.run, func)
self.worker.moveToThread(self.thread)
self.thread.started.connect(act)
# Signal to Function connection #
self.worker.my_started.connect(self.started_signal_connection)
self.worker.my_finished.connect(self.finished_signal_connection)
self.worker.my_error.connect(self.error_signal_connection)
self.worker.my_success.connect(self.success_signal_connection)
# Handling the worker and the thread deletion #
self.worker.my_finished.connect(self.thread.quit)
self.worker.my_finished.connect(self.worker.deleteLater)
self.thread.finished.connect(self.thread.deleteLater)
self.thread.start()
# Connect signals from worker #
def started_signal_connection(self):
self.stop_flag = True
print(f"\nFunction {self.func_name} started\n")
def finished_signal_connection(self):
self.stop_flag = False
print("\nWorker finished\n")
def error_signal_connection(self, error_name, trace_back):
self.today = date.today()
self.now = datetime.now()
self.current_date = str(
self.now.strftime("%d/%m/%Y %H:%M:%S")).replace(' ',
'_').replace(
'/', '_').replace(':', '_')
self.header_name = f'{self.atp_name}_{self.func_name}_Date_{self.current_date}'
self.target_folder = 'target_folder'
if os.path.isfile(self.header_name) == False:
file_name = f'{self.target_folder}{self.header_name}.txt'
try:
with open(file_name, 'w', encoding='utf-8') as f:
f.write(f'''ATP: {self.atp_name}\nFunction name: {self.func_name}\nError: {error_name}\nDate: {self.current_date}\n\n{trace_back}''')
except Exception as e:
print(e)
def success_signal_connection(self):
self.stop_flag = False
print("\nFunction success")
Another thing I noticed is that every time I use the worker with function that use Tk - askopenfilename()
I get troubles, here is a traceback example:
line 57, in run
**x = func()**
line 64, in open_main_terminal
**self.path1, self.file_name = upload_item()**
line 62, in upload_item
**filename = askopenfilename()**
line 375, in askopenfilename
**return Open(**options).show()**
line 39, in show
**w = Frame(self.master)**
line 2741, in __init__
**Widget.__init__(self, master, 'frame', cnf, {}, extra)**
line 2296, in __init__ (widgetName, self._w) + extra + self._options(cnf))
**RuntimeError: main thread is not in main loop**
I'm writing a control program that needs multiple configurable, restartable timers which run in their own thread. Originally I implemented this with custom functions and custom event names for all timers. Because this is a really ugly approach, I'm doing a rewrite with a more OO-like approach. Please consider the minimum example below which does everything I need it to do with one exception for which I've not been able to find a solution: I can't access the required class instance attributes (I hope I'm using the right terminology here).
The failing statement is line 74 and commented out. I suspect a problem with the way I pass arguments and kwargs but have run into a brick wall. Can you see where I go wrong and suggest a solution?
#!/usr/bin/python3
# coding: utf-8
"""MRE; needs installation of sine.threads ReStartableThread: pip3 install sine.threads"""
import sys
import time
import threading
from sine.threads import ReStartableThread as Thread
class MyThread(Thread):
"""
drop-in replacement of ReStartableThread with status method added
"""
def __init__(self, name=None, event_name=None, target=None, group=None, args=(), kwargs=None):
Thread.__init__(self)
self.name = name
self.event_name = event_name
self.target = target
self.args = args
self.kwargs = kwargs
super().__init__(name=name, event_name=event_name, target=target, group=group, args=args, kwargs=kwargs)
def status(self, i): # placeholder
print('dummy status #', i)
def threader(f):
""" a threading decorator; put #threaded above the callable that needs threading """
def threading_function(*args, **kwargs):
threading.Thread(name=f.__name__, target=f, args=args, kwargs=kwargs).start()
return threading_function
#threader
def t0():
time.sleep(3)
t1.stop()
t1.join()
class MyTimer:
def __init__(self, name, kwargs):
for key, value in kwargs.items(): # unpack kwargs and convert to variables
setattr(self, key, value)
def __call__(self, stop_event):
i = 5
while not stop_event.is_set() and i > 0:
print('I: ', i)
time.sleep(1)
i -= 1
if stop_event.is_set():
print('timer is CANCELLED')
else:
print('timer is FINISHED')
def main():
global t1
name = 'timer'
t1_kwargs = {'interval': 99, 'message': 'STARTING COUNTDOWN...'}
t1 = MyThread(name=name, target=MyTimer(name, t1_kwargs))
t1.status(1)
# first countdown
t1.start()
"""
The next statement fails with "AttributeError: 'StoppableThread' object has no attribute 'interval'"
where StoppableThread is a member of the sine.threads module.
Requirement: access to attributes (like interval) of instances of MyTimer from main
"""
# print(t1.interval)
t0()
t1.join()
t1.status(2)
# second countdown
t1.start()
t1.status(3)
t1.join()
t1.status(4)
print('END')
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('\b\b\r')
print("PROGRAM TERMINATED AT USER'S REQUEST")
force_exit()
First, you are trying to initialize your base class Thread twice, first with
Thread.__init__(self)
and then with:
super().__init__(name=name, event_name=event_name, target=target, group=group, args=args, kwargs=kwargs)
You should get rid of the first initialization.
Your problem is that you are assigning t1 to your MyThread instance:
t1 = MyThread(name=name, target=MyTimer(name, t1_kwargs))
But interval is an attribute of your MyTimer instance, not your MyThread instance. So t1.interval will fail. You want:
t1 = MyTimer(name, t1_kwargs)
my_thread = MyThread(name, target=t1)
my_thread.status(1)
my_thread.start()
etc.
You just need to keep straight your two different instances:
#!/usr/bin/python3
# coding: utf-8
"""MRE; needs installation of sine.threads ReStartableThread: pip3 install sine.threads"""
import sys
import time
import threading
from sine.threads import ReStartableThread as Thread
class MyThread(Thread):
"""
drop-in replacement of ReStartableThread with status method added
"""
def __init__(self, name=None, event_name=None, target=None, group=None, args=(), kwargs=None):
self.name = name
self.event_name = event_name
self.target = target
self.args = args
self.kwargs = kwargs
super().__init__(name=name, event_name=event_name, target=target, group=group, args=args, kwargs=kwargs)
def status(self, i): # placeholder
print('dummy status #', i)
def threader(f):
""" a threading decorator; put #threaded above the callable that needs threading """
def threading_function(*args, **kwargs):
threading.Thread(name=f.__name__, target=f, args=args, kwargs=kwargs).start()
return threading_function
#threader
def t0():
time.sleep(3)
my_thread.stop()
my_thread.join()
class MyTimer:
def __init__(self, name, kwargs):
for key, value in kwargs.items(): # unpack kwargs and convert to variables
setattr(self, key, value)
def __call__(self, stop_event):
i = 5
while not stop_event.is_set() and i > 0:
print('I: ', i)
time.sleep(1)
i -= 1
if stop_event.is_set():
print('timer is CANCELLED')
else:
print('timer is FINISHED')
def main():
global t1
global my_thread
name = 'timer'
t1_kwargs = {'interval': 99, 'message': 'STARTING COUNTDOWN...'}
t1 = MyTimer(name, t1_kwargs)
my_thread = MyThread(name, target=t1)
my_thread.status(1)
my_thread.start()
print(t1.interval)
t0()
my_thread.join()
my_thread.status(2)
# second countdown
my_thread.start()
my_thread.status(3)
my_thread.join()
my_thread.status(4)
print('END')
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
print('\b\b\r')
print("PROGRAM TERMINATED AT USER'S REQUEST")
force_exit()
Prints:
dummy status # 1
I: 5
99
I: 4
I: 3
timer is CANCELLED
dummy status # 2
I: 5
dummy status # 3
I: 4
I: 3
I: 2
I: 1
timer is FINISHED
dummy status # 4
END
Background :
I have a script that allows me to make spatial queries on a PostgreSQL database via an API coming from a private editor (I can't directly query the database). This API is working with python 3.2. To summarize very quickly, this script is used to download elements of this database in the desired geographical footprint. Depending of the zone, you can obtain between 1 to over 100 elements, each of them having very different sizes (from Ko to Go).
The main window let you to set all options and then start the global process. When launched a console window appears letting you see what’s going on. Once an item has been downloaded a short “report” is displayed on the console. Currently everything is done sequentially one element at a time. As you can imagine, if this element is quite large, the console freezes while waiting for the end of the download process.
Code:
I'm not going to post the full script here, but through a very simple script I will try to show the main problem I'm trying to solve (i.e. avoid locking the user interface / have some sort of real-time output on what's going on).
So, in order to avoid these freezing problems, the use of threads seemed to me to be the best solution. To simulate the download process (see previous chapter) I used the url.request urlretrieve method with multiple urls (pointing to files of different sizes).
import os
import sys
import time
import urllib.request
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class DownloadWorkerSignals(QtCore.QObject):
"""
Defines the signals available from a running download worker thread.
"""
finished = QtCore.pyqtSignal(str)
# ---------------------------------------------------------------------------------
class DownloadWorker(QtCore.QRunnable):
"""
Worker thread
"""
def __init__(self, url, filepath, filename, index):
super(DownloadWorker, self).__init__()
self.url = url
self.file_path = filepath
self.filename = filename
self.index = index
self.signals = DownloadWorkerSignals()
#QtCore.pyqtSlot(str)
def run(self):
t = time.time()
message = 'Thread %d started\n' % self.index
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.file_path,
self.filename))
except IOError as error:
message += str(error) + '\n'
finally:
message += 'Thread %d ended %.2f s\n' % (self.index, time.time() - t)
self.signals.finished.emit(message) # Done
# ---------------------------------------------------------------------------------
class Main(QtGui.QMainWindow):
"""
Main window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Main")
self.setWindowModality(QtCore.Qt.ApplicationModal)
self.centralwidget = QtGui.QWidget(self)
self.setCentralWidget(self.centralwidget)
# Ok / Close
# -------------------------------------------------------------------------
self.buttonBox = QtGui.QDialogButtonBox(self.centralwidget)
self.buttonBox.setStandardButtons(QtGui.QDialogButtonBox.Cancel |
QtGui.QDialogButtonBox.Ok)
self.buttonBox.setGeometry(QtCore.QRect(10, 160, 380, 20))
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.buttonBox,
QtCore.SIGNAL('accepted()'),
self.button_ok_clicked)
self.connect(self.buttonBox,
QtCore.SIGNAL('rejected()'),
self.button_cancel_clicked)
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
self.close()
def button_ok_clicked(self):
# Launch console
console = Console(parent=self)
console.exec_()
# ---------------------------------------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self, parent):
super(self.__class__, self).__init__()
self.parent = parent
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.verticalLayout = QtGui.QVBoxLayout(self)
# Text edit
# -------------------------------------------------------------------------
self.text_edit = QtGui.QPlainTextEdit(self)
self.text_edit.setReadOnly(True)
self.text_edit_cursor = QtGui.QTextCursor(self.text_edit.document())
self.verticalLayout.addWidget(self.text_edit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.verticalLayout.addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.button_cancel_clicked)
# Post initialization
# -------------------------------------------------------------------------
self.threadpool = QtCore.QThreadPool()
self.threadpool.setMaxThreadCount(2)
for index, url in enumerate(urls):
worker = DownloadWorker(url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
worker.signals.finished.connect(self.write_message)
self.threadpool.start(worker)
'''
I have to wait for the end of the thread pool to make a post-processing.
If I use the waitForDone I don't see my console until the all work is done
'''
# self.threadpool.waitForDone()
# self.write_stram('Thread pool finished')
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
if self.threadpool.activeThreadCount() != 0:
pass # How to interrupt the threadpool ?
self.close()
#QtCore.pyqtSlot(str)
def write_message(self, text):
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Main()
window.show()
app.exec_()
Questions:
Everything seems to work as expected but I encounter two difficulties:
At the end of the thread pool process I have to make some
post-processing. If I use the waitForDone method I don't see my
console until the all work is done and it’s not the type of behavior
wanted.
If the Cancel Button in the Console is clicked, I need to interrupt
the threadpool and I don’t know how to manage that.
I had another look at this problem (based largely on this : how-do-i-maintain-a-resposive-gui-using-qthread-with-pyqgis).
So I replaced the previous tandem QThreadPool/QRunnable, by Queue/QThread. The code below gives an overview.
import os
import sys
import time
import urllib.request
import queue
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class WorkerThread(QtCore.QThread):
"""
Worker thread
"""
def __init__(self, parent_thread):
QtCore.QThread.__init__(self, parent_thread)
def run(self):
self.running = True
success = self.do_work()
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def stop(self):
self.running = False
pass
def do_work(self):
return True
def clean_up(self):
pass
# ---------------------------------------------------------------------------------
class LongRunningTask(WorkerThread):
def __init__(self, parent_thread, url, filepath, filename, index):
WorkerThread.__init__(self, parent_thread)
self.url = url
self.filepath = filepath
self.filename = filename
self.index = index
def do_work(self):
t = time.time()
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'), 'Thread %d started\n' % self.index)
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.filepath,
self.filename))
except IOError as error:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d error - ' % self.index + str(error) + '\n')
finally:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d ended %.2f s\n' % (self.index, time.time() - t))
return True
# ---------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.setLayout(QtGui.QVBoxLayout())
# Text edit
# -------------------------------------------------------------------------
self.textEdit = QtGui.QPlainTextEdit(self)
self.textEdit.setReadOnly(True)
self.textEdit_cursor = QtGui.QTextCursor(self.textEdit.document())
self.layout().addWidget(self.textEdit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(False)
self.layout().addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.reject)
# Post-Initialization
# -------------------------------------------------------------------------
self.queue = queue.Queue()
# self.queue = queue.Queue(maxsize=2)
self.run_thread()
# Connect functions
# -----------------------------------------------------------------------------
def cancel_thread(self):
self.workerThread.stop()
def job_finished_from_thread(self, success):
self.workerThread.stop()
self.queue.get()
# Stop the pulsation
if self.queue.empty():
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(True)
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def text_from_thread(self, value):
self.textEdit.insertPlainText(value)
cursor = self.textEdit.textCursor()
self.textEdit.setTextCursor(cursor)
def run_thread(self):
for index, url in enumerate(urls):
self.workerThread = LongRunningTask(parent_thread=self,
url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
self.connect(self.workerThread,
QtCore.SIGNAL('jobFinished(PyQt_PyObject)'),
self.job_finished_from_thread)
self.connect(self.workerThread,
QtCore.SIGNAL('threadText(PyQt_PyObject)'),
self.text_from_thread)
self.queue.put(self.workerThread)
self.workerThread.start()
# If I set the queue to maxsize=2, how to manage it here
'''
while not self.queue.full():
self.queue.put(self.workerThread)
self.workerThread.start()
'''
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Console()
window.show()
app.exec_()
Question:
Unfortunately, I encounter other types of difficulties. In reality, the queue can contain a large amount of threads (over 100). 1. How can I, like the QthreadPool and its setMaxThreadCount method, manage the number of threads running in parallel in order to prevent the system from collapsing completely ?
Below I have an example program. When the button is pressed, it takes a second before it can calculate the value to show. If the user presses the button in rapid succession they end up waiting a long time to see the last answer, which is the only answer they care about. In the code, you can see that the _dataCruncher function needs to know self._count, but self._count does not depend on the output of _dataCruncher.
My question, therefore, is how can I interrupt the normal execution of _dataCruncher on subsequent calls in order to keep the GUI free to do other stuff, and to not waste processing time when it is not needed? I realize that I will likely need to use a thread to run _dataCruncher and some sort of Queue to get the appropriate val to display, but I do not understand how to put this all together.
from PyQt4 import QtGui, QtCore
import sys
import time
import random
import random
class MainWindow(QtGui.QMainWindow):
def __init__(self):
self.app = QtGui.QApplication(sys.argv)
super(MainWindow, self).__init__()
self.count = 0
self.initUI()
def initUI(self):
# Layouts
central = QtGui.QWidget()
layout = QtGui.QVBoxLayout()
self.button = QtGui.QPushButton('Press Me')
self.text = QtGui.QLabel('?')
layout.addWidget(self.button)
layout.addWidget(self.text)
central.setLayout(layout)
self.setCentralWidget(central)
self.button.clicked.connect(self._buttonClicked)
def _dataCruncher(self, val):
time.sleep(1) # takes a long time to process data using val
return val * random.randint(1,10)
def _buttonClicked(self):
self.count += 1
val = self._dataCruncher(self.count)
self.text.setText('Value {}'.format(val))
def startup(self):
self.show()
result = self.app.exec_()
sys.exit(result)
if __name__ == '__main__':
random.seed()
myWindow = MainWindow()
myWindow.startup()
So, finding an answer to this was more complicated than I thought. As #MTset mentions in one of the comments, python does not offer any means by which to cancel the execution of a Thread. So, what I did was create a 'threadHandler' class that, well, handles thread. It keeps track of the last thread that was created and offers a means by which to get the result from the execution of the last thread.
I am posting an modified version of the test code from the original post as well as the threadHandler code in full in case anyone has use for it.
File 1 here
# tester.py, run this file
from PyQt4 import QtGui, QtCore
import random, sys, time
from threadHandler import MyThreadHandler
class MyModel(object):
def dataCruncher(self, val):
delay = random.randint(1,5)
print('{} sleeping for {}'.format(val, delay))
time.sleep(delay) # takes a long time to process data using val
print('{} done sleeping'.format(val))
return val
class MainWindow(QtGui.QMainWindow):
def __init__(self, threadHandler):
self.app = QtGui.QApplication(sys.argv)
super(MainWindow, self).__init__()
self.count = 0
self.initUI()
self.button_clicked_events = Event()
self.threadHandler = threadHandler
def initUI(self):
# Layouts
central = QtGui.QWidget()
layout = QtGui.QVBoxLayout()
self.button = QtGui.QPushButton('Press Me')
self.text = QtGui.QLabel('?')
layout.addWidget(self.button)
layout.addWidget(self.text)
central.setLayout(layout)
self.setCentralWidget(central)
self.button.clicked.connect(self._buttonClicked)
def _buttonClicked(self):
self.count += 1
self.button_clicked_events(self.count)
def setLabel(self, val):
self.text.setText(str(val))
def startup(self):
self.show()
result = self.app.exec_()
return result
class Event(list):
"""Event subscription.
A list of callable objects. Calling an instance of this will cause a
call to each item in the list in ascending order by index.
Example Usage:
>>> def f(x):
... print 'f(%s)' % x
>>> def g(x):
... print 'g(%s)' % x
>>> e = Event()
>>> e()
>>> e.append(f)
>>> e(123)
f(123)
>>> e.remove(f)
>>> e()
>>> e += (f, g)
>>> e(10)
f(10)
g(10)
>>> del e[0]
>>> e(2)
g(2)
"""
def __init__(self):
self.output = {}
def __call__(self, *args, **kwargs):
for f,key in self:
output = f(*args, **kwargs)
self.output[key] = output
return self.output
def __repr__(self):
return "Event({})".format(list.__repr__(self))
if __name__ == '__main__':
def checker(handler, window):
if handler.isLastDone():
val = handler.getLastResult()
window.setLabel(val)
else:
window.setLabel('calculating...')
random.seed()
model = MyModel()
threadHandler = MyThreadHandler()
myWindow = MainWindow(threadHandler)
threadHandler.createTimer(1, checker, threadHandler, myWindow)
def getData(count):
threadHandler.createOneShot(model.dataCruncher, count)
myWindow.button_clicked_events.append((getData, 'dt'))
result = myWindow.startup()
print('ending')
threadHandler.end()
print('ended')
sys.exit(result)
File 2 below
#threadHandler.py, save this file in the same folder as tester.py
import threading, time
class MyThreadHandler(object):
def __init__(self):
self.oneShots = []
self.timers = []
self.oldOneShots = []
self.latest = None
self.cleaning = False
self._startCleaner()
def _startCleaner(self):
print('-'*20+'Starting cleaner'+'-'*20)
self.cleaner = self.createTimer(1, self._cleanupThreads)
def _stopCleaner(self):
print('-'*20+'Stopping cleaner'+'-'*20)
self.cleaner.stop()
def getNumThreads(self):
return len(self.oneShots)
def getNumOldThreads(self):
return len(self.oldOneShots)
def end(self):
for i,timer in enumerate(self.timers):
timer.stop()
self.timers.pop(i)
def createTimer(self, interval, func, *args, **kwargs):
timer = myTimer(interval, func, args, kwargs)
self.timers.append(timer)
return timer
def createOneShot(self, func, *args, **kwargs):
oneshot = myOneShot(func, args, kwargs)
self.oneShots.append(oneshot)
self.latest = oneshot
def isLastDone(self):
if not self.latest is None:
return not self.latest.running()
else:
return None
def getLastResult(self):
if self.latest is None:
raise ValueError('There have not been any oneshots created.')
while self.latest.running():
pass
result = self.latest.getResult()
if len(self.oneShots) > 0:
self.oldOneShots.append(myOneShot(self._cleanAll, (self.oneShots,)))
self.oneShots = []
return result
def _cleanAll(self, toClean):
# loop through toClean and pop up anything that's done. this DOES lock
while len(toClean) > 0:
toClean = self._cleanup(toClean)
def _cleanup(self, toCleanup):
while not self.cleaning:
self.cleaning = True
for i, thread in enumerate(toCleanup):
if not thread.running():
toCleanup.pop(i)
self.cleaning = False
return toCleanup
def _cleanupThreads(self):
# check each of these lists and pop out any threads that are done. This
# does not lock. This function should really only be called by the
# cleaner, which is set up in __init__
self.oneShots = self._cleanup(self.oneShots)
self.timers = self._cleanup(self.timers)
self.oldOneShots = self._cleanup(self.oldOneShots)
class myTimer(object):
def __init__(self, delay, func, args=tuple(), kwargs={}):
self.delay = delay
self.func = func
self.loop = True
self.args = args
self.kwargs = kwargs
self.thread = threading.Thread(target=self.run, daemon=True)
self.thread.start()
self.output = None
def run(self):
while self.loop:
self.output = self.func(*self.args, **self.kwargs)
if self.delay > 0.1:
count = 0
while count <= self.delay:
count += 0.1
time.sleep(0.1)
else:
time.sleep(self.delay)
def stop(self):
self.loop = False
def running(self):
return self.loop
def getResult(self):
return self.output
class myOneShot(object):
def __init__(self, func, args=tuple(), kwargs={}):
self.func = func
self.args = args
self.kwargs = kwargs
self.thread = threading.Thread(target=self.run, daemon=True)
self.thread.start()
self.output = None
def run(self):
self.output = self.func(*self.args, **self.kwargs)
def running(self):
return self.thread.is_alive()
def getResult(self):
return self.output
if __name__ == '__main__':
import random
random.seed()
def longFunc(num):
delay = random.randint(5,8)
if num in (3, 6):
delay = 2
print('-'*30+'func {} has sleep {}'.format(num, delay))
time.sleep(delay)
print('-'*30+'func {} is done'.format(num))
return num
def checker(handler):
if handler.isLastDone():
return handler.getLastResult()
else:
return None
myHandler = MyThreadHandler()
# The 'checker' function simulates something in my program that uses the
# data generated by the 'longFunc'. It waits until there are no more threads
# in the threadHandler, as that would indicate that the user is done
# switching back-and-forth between different values
checkTimer = myHandler.createTimer(1, checker, myHandler)
# create 10 one-shot threads that take a 'long' time. The delay is to keep
# them in order, as this loop is meant to simulate a user switching between
# items using a keyboard or mouse, which I imagine they couldn't do any
# faster than every 1/10th of a second
start = time.time()
for i in range(4):
myHandler.createOneShot(longFunc, i)
time.sleep(0.1)
# wait until there are no more threads executing
last = myHandler.getLastResult()
print('result from last = {}'.format(last))
for i in range(4, 7):
myHandler.createOneShot(longFunc, i)
time.sleep(0.1)
last = myHandler.getLastResult()
print('result from last = {}'.format(last))
while myHandler.getNumOldThreads() >0 or myHandler.getNumThreads() > 0:
pass
myHandler.end()
print('done ending')
You could disable the button after it's pressed until an answer is ready using:
setEnabled(False)
Then reset it just before providing the result.
I'm having problem with QThreads in python.
I would like to start my multi QThread when I push on button Run.
But the compiler outputs following error:
"QThread: Destroyed while thread is still running"
I don't know what is wrong with my code.
Any help would be appreciated.
Here is my code:
# -*- coding: utf-8 -*-
from PySide import QtCore, QtGui
from Ui_MainWindow import Ui_MainWindow
from queue import Queue
import sys, re, random
import time, random, re, urllib.request
from urllib.parse import urljoin
class Worker(QtCore.QThread):
def __init__(self,threadID, name, q, delay):
QtCore.QThread.__init__(self)
self.threadID = threadID
self.name = name
self.q = q
self.delay = delay
self._running = False
def run(self):
self._running = True
print ("start - %s" %self.name)
while self._running:
req = self.request(self.name, self.q, self.delay)
def stop(self, wait=False):
print (self.name)
self._running = False
def request(self, threadName, q1, delay):
while not self.q.empty():
time.sleep(delay)
q = q1.get()
print ("%s: %s %s %s" % (threadName, time.ctime(time.time()), q, delay))
if self.q.empty():
print ("queue empty")
self.stop()
class MainWindow(QtGui.QMainWindow):
def __init__(self, parent=None):
QtGui.QMainWindow.__init__(self, parent)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.backend = Queue()
self.connect(self.ui.actionStart, QtCore.SIGNAL('triggered()'),self.start)
def start(self):
try :
f1 = open('./payload/backend.log')
except FileNotFoundError as e:
return
threadList = ["Thread-1", "Thread-2", "Thread-3", "Thread-4", "Thread-5"]
self.url = "http://test.com"
self.threads = []
threadID = 1
for payload in f1.read().splitlines() :
full_url = urljoin(self.url, payload)
self.backend_dir.put(full_url)
for tName in threadList:
ran_int = random.randint(1, 2)
downloader = Worker(threadID, tName, self.backend, ran_int)
downloader.start()
self.threads.append(downloader)
if __name__ == "__main__":
app = QtGui.QApplication(sys.argv)
window = MainWindow()
window.show()
sys.exit(app.exec_())
log
QThread: Destroyed while thread is still running
QThread: Destroyed while thread is still running
QThread: Destroyed while thread is still running
QThread: Destroyed while thread is still running
You are trying to do something that is a built-in of Qt: a QThreadPool.
I would advice you to read the doc and use it instead.
If you really want to use QThread:
You should not subclass QThread. Instead you should subclass the basic QObject to create your worker and use the moveToThread method:
class WorkerSignals(QObject):
finished = pyqtSignal()
class Worker(QObject):
def __init__():
self.signal = WorkerSignals()
def run(self):
# Your stuff
print('running')
self.signal.finished.emit()
Then somewhere else:
thread = QThread()
worker = Worker(...)
worker.moveToThread(thread)
thread.started.connect(worker.run)
worker.finished.connect(thread.quit)
worker.finished.connect(worker.deleteLater)
thread.finished(thread.deleteLater)
thread.start()
The solution is a rough translation of this one in C++:
https://mayaposch.wordpress.com/2011/11/01/how-to-really-truly-use-qthreads-the-full-explanation/
Hope this helps!
The problem is caused by the way you are using the queue.
All the threads start and begin their tasks normally, up until the queue becomes empty. At that point, the first thread to finish terminates, but the other four threads are left waiting for an item to be returned from the queue, which never comes.
This is because you use get with no arguments, which will block indefinitely until an item becomes available. Instead, you should use get_nowait, and also call stop() unconditionally at the end of request():
from queue import Queue, Empty
...
class Worker(QtCore.QThread):
...
def request(self, threadName, q1, delay):
while not q1.empty():
time.sleep(delay)
try:
q = q1.get_nowait()
except Empty:
break
else:
print ("%s: %s %s %s" % (threadName, time.ctime(time.time()), q, delay))
print ("queue empty")
self.stop()
I believe that you need to call self.threads.append(downloader) before downloader.start() so that the thread doesn't go out of scope and get garbage collected.