I want to acquire a spectrum of a spectrometer every 1s, and below is the code I wrote for this.
However, when the spectrometer is triggered at less than 1 Hz,
the function asking for a spectrum will wait for a trigger and take longer than 1s to process, while the timer has already timed out.
This cause the thread to be very laggy and eventually freeze.
What is a better way to write it so that it calls the function 'acquire_spectrum' 1s after the function finishes itself, instead of just calling it every 1s?
class Spec_Thread(QThread):
def __init__(self):
QThread.__init__(self)
self.signals = Signals()
self.specth = Spectrometer.from_first_available() #connect to the spectrometer
self.threadtimer = QTimer()
self.threadtimer.moveToThread(self)
self.threadtimer.timeout.connect(self.acquire_spectrum)
def acquire_spectrum(self): #acquire the current spectrum from the spectrometer
print('in thread,', device_running)
if device_running == True:
self.specth.open() #open the usb portal
self.wavelengths = self.specth.wavelengths() #acquire wavelengths (will wait for a trigger)
self.intensities = self.specth.intensities() #acquire intensities (will wait for a trigger)
self.specth.close() #close usb portal
self.signals.new_spectrum.emit(self.wavelengths, self.intensities)
else:
print('Device stopped')
return
def run(self):
self.threadtimer.start(10000)
loop = QEventLoop()
loop.exec_()
You have to start a timer after finishing the task execution, and for this you can threading.Timer():
import threading
from PyQt5 import QtCore
import numpy as np
from seabreeze.spectrometers import Spectrometer
class QSpectrometer(QtCore.QObject):
dataChanged = QtCore.pyqtSignal(np.ndarray, np.ndarray)
def __init__(self, parent=None):
super().__init__(parent)
self.specth = Spectrometer.from_first_available()
def start(self, repeat=False):
threading.Thread(target=self._read, args=(repeat,), daemon=True).start()
def _read(self, repeat):
self.specth.open()
wavelengths = self.specth.wavelengths()
intensities = self.specth.intensities()
self.specth.close()
self.dataChanged.emit(wavelengths, intensities)
if repeat:
threading.Timer(1, self._read, (repeat,)).start()
if __name__ == "__main__":
import sys
app = QtCore.QCoreApplication(sys.argv)
spectometer = QSpectrometer()
spectometer.start(True)
spectometer.dataChanged.connect(print)
sys.exit(app.exec_())
Related
Background :
I have a script that allows me to make spatial queries on a PostgreSQL database via an API coming from a private editor (I can't directly query the database). This API is working with python 3.2. To summarize very quickly, this script is used to download elements of this database in the desired geographical footprint. Depending of the zone, you can obtain between 1 to over 100 elements, each of them having very different sizes (from Ko to Go).
The main window let you to set all options and then start the global process. When launched a console window appears letting you see what’s going on. Once an item has been downloaded a short “report” is displayed on the console. Currently everything is done sequentially one element at a time. As you can imagine, if this element is quite large, the console freezes while waiting for the end of the download process.
Code:
I'm not going to post the full script here, but through a very simple script I will try to show the main problem I'm trying to solve (i.e. avoid locking the user interface / have some sort of real-time output on what's going on).
So, in order to avoid these freezing problems, the use of threads seemed to me to be the best solution. To simulate the download process (see previous chapter) I used the url.request urlretrieve method with multiple urls (pointing to files of different sizes).
import os
import sys
import time
import urllib.request
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class DownloadWorkerSignals(QtCore.QObject):
"""
Defines the signals available from a running download worker thread.
"""
finished = QtCore.pyqtSignal(str)
# ---------------------------------------------------------------------------------
class DownloadWorker(QtCore.QRunnable):
"""
Worker thread
"""
def __init__(self, url, filepath, filename, index):
super(DownloadWorker, self).__init__()
self.url = url
self.file_path = filepath
self.filename = filename
self.index = index
self.signals = DownloadWorkerSignals()
#QtCore.pyqtSlot(str)
def run(self):
t = time.time()
message = 'Thread %d started\n' % self.index
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.file_path,
self.filename))
except IOError as error:
message += str(error) + '\n'
finally:
message += 'Thread %d ended %.2f s\n' % (self.index, time.time() - t)
self.signals.finished.emit(message) # Done
# ---------------------------------------------------------------------------------
class Main(QtGui.QMainWindow):
"""
Main window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Main")
self.setWindowModality(QtCore.Qt.ApplicationModal)
self.centralwidget = QtGui.QWidget(self)
self.setCentralWidget(self.centralwidget)
# Ok / Close
# -------------------------------------------------------------------------
self.buttonBox = QtGui.QDialogButtonBox(self.centralwidget)
self.buttonBox.setStandardButtons(QtGui.QDialogButtonBox.Cancel |
QtGui.QDialogButtonBox.Ok)
self.buttonBox.setGeometry(QtCore.QRect(10, 160, 380, 20))
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.buttonBox,
QtCore.SIGNAL('accepted()'),
self.button_ok_clicked)
self.connect(self.buttonBox,
QtCore.SIGNAL('rejected()'),
self.button_cancel_clicked)
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
self.close()
def button_ok_clicked(self):
# Launch console
console = Console(parent=self)
console.exec_()
# ---------------------------------------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self, parent):
super(self.__class__, self).__init__()
self.parent = parent
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.verticalLayout = QtGui.QVBoxLayout(self)
# Text edit
# -------------------------------------------------------------------------
self.text_edit = QtGui.QPlainTextEdit(self)
self.text_edit.setReadOnly(True)
self.text_edit_cursor = QtGui.QTextCursor(self.text_edit.document())
self.verticalLayout.addWidget(self.text_edit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.verticalLayout.addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.button_cancel_clicked)
# Post initialization
# -------------------------------------------------------------------------
self.threadpool = QtCore.QThreadPool()
self.threadpool.setMaxThreadCount(2)
for index, url in enumerate(urls):
worker = DownloadWorker(url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
worker.signals.finished.connect(self.write_message)
self.threadpool.start(worker)
'''
I have to wait for the end of the thread pool to make a post-processing.
If I use the waitForDone I don't see my console until the all work is done
'''
# self.threadpool.waitForDone()
# self.write_stram('Thread pool finished')
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
if self.threadpool.activeThreadCount() != 0:
pass # How to interrupt the threadpool ?
self.close()
#QtCore.pyqtSlot(str)
def write_message(self, text):
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Main()
window.show()
app.exec_()
Questions:
Everything seems to work as expected but I encounter two difficulties:
At the end of the thread pool process I have to make some
post-processing. If I use the waitForDone method I don't see my
console until the all work is done and it’s not the type of behavior
wanted.
If the Cancel Button in the Console is clicked, I need to interrupt
the threadpool and I don’t know how to manage that.
I had another look at this problem (based largely on this : how-do-i-maintain-a-resposive-gui-using-qthread-with-pyqgis).
So I replaced the previous tandem QThreadPool/QRunnable, by Queue/QThread. The code below gives an overview.
import os
import sys
import time
import urllib.request
import queue
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class WorkerThread(QtCore.QThread):
"""
Worker thread
"""
def __init__(self, parent_thread):
QtCore.QThread.__init__(self, parent_thread)
def run(self):
self.running = True
success = self.do_work()
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def stop(self):
self.running = False
pass
def do_work(self):
return True
def clean_up(self):
pass
# ---------------------------------------------------------------------------------
class LongRunningTask(WorkerThread):
def __init__(self, parent_thread, url, filepath, filename, index):
WorkerThread.__init__(self, parent_thread)
self.url = url
self.filepath = filepath
self.filename = filename
self.index = index
def do_work(self):
t = time.time()
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'), 'Thread %d started\n' % self.index)
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.filepath,
self.filename))
except IOError as error:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d error - ' % self.index + str(error) + '\n')
finally:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d ended %.2f s\n' % (self.index, time.time() - t))
return True
# ---------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.setLayout(QtGui.QVBoxLayout())
# Text edit
# -------------------------------------------------------------------------
self.textEdit = QtGui.QPlainTextEdit(self)
self.textEdit.setReadOnly(True)
self.textEdit_cursor = QtGui.QTextCursor(self.textEdit.document())
self.layout().addWidget(self.textEdit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(False)
self.layout().addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.reject)
# Post-Initialization
# -------------------------------------------------------------------------
self.queue = queue.Queue()
# self.queue = queue.Queue(maxsize=2)
self.run_thread()
# Connect functions
# -----------------------------------------------------------------------------
def cancel_thread(self):
self.workerThread.stop()
def job_finished_from_thread(self, success):
self.workerThread.stop()
self.queue.get()
# Stop the pulsation
if self.queue.empty():
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(True)
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def text_from_thread(self, value):
self.textEdit.insertPlainText(value)
cursor = self.textEdit.textCursor()
self.textEdit.setTextCursor(cursor)
def run_thread(self):
for index, url in enumerate(urls):
self.workerThread = LongRunningTask(parent_thread=self,
url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
self.connect(self.workerThread,
QtCore.SIGNAL('jobFinished(PyQt_PyObject)'),
self.job_finished_from_thread)
self.connect(self.workerThread,
QtCore.SIGNAL('threadText(PyQt_PyObject)'),
self.text_from_thread)
self.queue.put(self.workerThread)
self.workerThread.start()
# If I set the queue to maxsize=2, how to manage it here
'''
while not self.queue.full():
self.queue.put(self.workerThread)
self.workerThread.start()
'''
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Console()
window.show()
app.exec_()
Question:
Unfortunately, I encounter other types of difficulties. In reality, the queue can contain a large amount of threads (over 100). 1. How can I, like the QthreadPool and its setMaxThreadCount method, manage the number of threads running in parallel in order to prevent the system from collapsing completely ?
I have been writing a program which runs a remote script on server. So, I need to show the progress with a bar but somehow when I run my code, GUI start to freeze. I have used QThread and SIGNAL but unfortunately couldnt be succedeed.
Here is my code below;
class dumpThread(QThread):
def __init__(self):
QThread.__init__(self)
def __del__(self):
self.wait()
def sendEstablismentCommands(self, connection):
# Commands are sending sequently with proper delay-timers #
connection.sendShell("telnet localhost 21000")
time.sleep(0.5)
connection.sendShell("admin")
time.sleep(0.5)
connection.sendShell("admin")
time.sleep(0.5)
connection.sendShell("cd imdb")
time.sleep(0.5)
connection.sendShell("dump subscriber")
command = input('$ ')
def run(self):
# your logic here
# self.emit(QtCore.SIGNAL('THREAD_VALUE'), maxVal)
self.sendEstablismentCommands(connection)
class progressThread(QThread):
def __init__(self):
QThread.__init__(self)
def __del__(self):
self.wait()
def run(self):
# your logic here
while 1:
maxVal = 100
self.emit(SIGNAL('PROGRESS'), maxVal)
class Main(QtGui.QMainWindow):
def __init__(self):
QtGui.QMainWindow.__init__(self)
self.ui = Ui_MainWindow()
self.ui.setupUi(self)
self.ui.connectButton.clicked.connect(self.connectToSESM)
def connectToSESM(self):
## Function called when pressing connect button, input are being taken from edit boxes. ##
## dumpThread() method has been designed for working thread seperate from GUI. ##
# Connection data are taken from "Edit Boxes"
# username has been set as hardcoded
### Values Should Be Defined As Global ###
username = "ntappadm"
password = self.ui.passwordEdit.text()
ipAddress = self.ui.ipEdit.text()
# Connection has been established through paramiko shell library
global connection
connection = pr.ssh(ipAddress, username, password)
connection.openShell()
pyqtRemoveInputHook() # For remove unnecessary items from console
global get_thread
get_thread = dumpThread() # Run thread - Dump Subscriber
self.progress_thread = progressThread()
self.progress_thread.start()
self.connect(self.progress_thread, SIGNAL('PROGRESS'), self.updateProgressBar)
get_thread.start()
def updateProgressBar(self, maxVal):
for i in range(maxVal):
self.ui.progressBar.setValue(self.ui.progressBar.value() + 1)
time.sleep(1)
maxVal = maxVal - 1
if maxVal == 0:
self.ui.progressBar.setValue(100)
def parseSubscriberList(self):
parsing = reParser()
def done(self):
QtGui.QMessageBox.information(self, "Done!", "Done fetching posts!")
if __name__ == "__main__":
app = QtGui.QApplication(sys.argv)
main = Main()
main.show()
sys.exit(app.exec_())
I am expecting to see updateProgressBar method has called with SIGNAL, so process goes through seperate thread. I coudlnt find where I am missing.
Thanks for any help
There are really two problems. One thing I have noticed is that Python threads are greedy if they are not used for IO operations like reading from a serial port. If you tell a thread to run a calculation or something that is not IO related a thread will take up all of the processing and doesn't like to let the main thread / event loop run. The second problem is that signals are slow ... very slow. I've noticed that if you emit a signal from a thread and do it very fast it can drastically slow down a program.
So at the heart of the issue, the thread is taking up all of the time and you are emitting a signal very very fast which will cause slow downs.
For clarity and ease of use I would use the new style signal and slots.
http://pyqt.sourceforge.net/Docs/PyQt4/new_style_signals_slots.html
class progressThread(QThread):
progress_update = QtCore.Signal(int) # or pyqtSignal(int)
def __init__(self):
QThread.__init__(self)
def __del__(self):
self.wait()
def run(self):
# your logic here
while 1:
maxVal = 100
self.progress_update.emit(maxVal) # self.emit(SIGNAL('PROGRESS'), maxVal)
# Tell the thread to sleep for 1 second and let other things run
time.sleep(1)
To connect to the new style signal
...
self.progress_thread.start()
self.process_thread.progress_update.connect(self.updateProgressBar) # self.connect(self.progress_thread, SIGNAL('PROGRESS'), self.updateProgressBar)
...
EDIT
Sorry there is another problem. When a signal calls a function you cannot stay in that function forever. That function is not running in a separate thread it is running on the main event loop and the main event loop waits to run until you exit that function.
Update progress sleeps for 1 second and keeps looping. The hanging is coming from staying in this function.
def updateProgressBar(self, maxVal):
for i in range(maxVal):
self.ui.progressBar.setValue(self.ui.progressBar.value() + 1)
time.sleep(1)
maxVal = maxVal - 1
if maxVal == 0:
self.ui.progressBar.setValue(100)
It would be better to write the progress bar like
class progressThread(QThread):
progress_update = QtCore.Signal(int) # or pyqtSignal(int)
def __init__(self):
QThread.__init__(self)
def __del__(self):
self.wait()
def run(self):
# your logic here
while 1:
maxVal = 1 # NOTE THIS CHANGED to 1 since updateProgressBar was updating the value by 1 every time
self.progress_update.emit(maxVal) # self.emit(SIGNAL('PROGRESS'), maxVal)
# Tell the thread to sleep for 1 second and let other things run
time.sleep(1)
def updateProgressBar(self, maxVal):
self.ui.progressBar.setValue(self.ui.progressBar.value() + maxVal)
if maxVal == 0:
self.ui.progressBar.setValue(100)
I am looking for a design pattern based on threading.Thread, multiprocessing or Queue to upload a list of items with a timeout. The thread allows the GUI to remain responsive. If the connection hangs, then the timeout should trigger and the program should gracefully exit.
The example below works, but the GUI remains blocked. How could this be improved to allow for uploading of the list, manual canceling of the process, timeout of the upload process plus a non-blocking GUI?
from PySide.QtGui import *
from PySide.QtCore import *
import sys
import time
import threading
class UploadWindow(QDialog):
def __init__(self, parent=None):
super(UploadWindow, self).__init__(parent)
self.uploadBtn = QPushButton('Upload')
mainLayout = QVBoxLayout()
mainLayout.addWidget(self.uploadBtn)
self.uploadBtn.clicked.connect(self.do_upload)
self.progressDialog = QProgressDialog(self)
self.progressDialog.canceled.connect(self.cancelDownload)
self.progressDialog.hide()
self.setLayout(mainLayout)
self.show()
self.raise_()
def do_upload(self):
self.uploadBtn.setEnabled(False)
self.progressDialog.setMaximum(10)
self.progressDialog.show()
self.upload_thread = UploadThread(self)
self.upload_thread.start()
self.upload_thread_stopped = False
#List of items to upload
for i in range(10):
self.upload_thread = UploadThread(i)
self.upload_thread.start()
self.upload_thread.join(5)
self.progressDialog.setValue(i)
if self.upload_thread_stopped:
break
self.progressDialog.hide()
self.uploadBtn.setEnabled(True)
def cancelDownload(self):
self.upload_thread_stopped = True
class UploadThread(threading.Thread):
def __init__(self, i):
super(UploadThread, self).__init__()
self.i = i
self.setDaemon(True)
def run(self):
time.sleep(0.25) #simulate upload time
print self.i
if __name__ == '__main__':
app = QApplication(sys.argv)
w = UploadWindow()
sys.exit(app.exec_())
The GUI is not responsive because you do all the work in do_upload, never going back to the main loop.
Also, you call Thread.join(), that blocks everything until the thread is done (see https://docs.python.org/2/library/threading.html#threading.Thread.join)
You should use PySide.QtCore.QThread to take advantage of signals and slots.
Here is a nice example in C++. I implemented it in Python3.4 with PyQt here, but you should be able to use it with PySide too.
You might also want to look at PySide.QtCore.QProcess, to avoid using threads.
Here, I put some code together that does what I think you want.
For a real project, be sure to keep track of what's uploaded better &/or use something safer than .terminate() to stop the thread on demand.
import sys
from PySide import QtGui, QtCore
import time
class MySigObj(QtCore.QObject):
strSig = QtCore.Signal(str)
tupSig = QtCore.Signal(tuple)
class UploadThread(QtCore.QThread):
def __init__(self, parent=None):
super(UploadThread, self).__init__(parent)
self.endNow = False
self.fileName = None
self.sig = MySigObj()
self.fileNames = []
self.uploaded = []
#QtCore.Slot(str)
def setFileNames(self, t):
self.fileNames = list(t)
def run(self):
while self.fileNames:
print(self.fileNames)
time.sleep(2)
name = self.fileNames.pop(0)
s = 'uploaded file: ' + name + '\n'
print(s)
self.sig.strSig.emit(s)
self.uploaded.append(name)
if len(self.fileNames) == 0:
self.sig.strSig.emit("files transmitted: %s" % str(self.uploaded))
else:
time.sleep(1) #if the thread started but no list, wait 1 sec every cycle thru
#that was this thread should release the Python GIL (Global Interpreter Lock)
class ULoadWin(QtGui.QWidget):
def __init__(self, parent=None):
super(ULoadWin, self).__init__(parent)
self.upThread = UploadThread()
self.sig = MySigObj()
self.sig.tupSig.connect(self.upThread.setFileNames)
self.upThread.sig.strSig.connect(self.txtMsgAppend)
self.sig.tupSig.connect(self.upThread.setFileNames)
self.layout = QtGui.QVBoxLayout()
self.stButton = QtGui.QPushButton("Start")
self.stButton.clicked.connect(self.uploadItems)
self.stpButton = QtGui.QPushButton("Stop")
self.stpButton.clicked.connect(self.killThread)
self.testButton = QtGui.QPushButton("write txt\n not(?) blocked \nbelow")
self.testButton.setMinimumHeight(28)
self.testButton.clicked.connect(self.tstBlking)
self.lbl = QtGui.QTextEdit()
self.lbl.setMinimumHeight(325)
self.lbl.setMinimumWidth(290)
self.layout.addWidget(self.stButton)
self.layout.addWidget(self.stpButton)
self.layout.addWidget(self.testButton)
self.layout.addWidget(self.lbl)
self.setLayout(self.layout)
self.l = ['a', 'list', 'of_files', 'we', 'will_pretend_to_upload', 'st', 'uploading']
self.upThread.start()
def tstBlking(self):
self.lbl.append("txt not(?) blocked")
def uploadItems(self):
t = tuple(self.l)
self.sig.tupSig.emit(t)
self.upThread.start()
def killThread(self):
self.upThread.terminate()
time.sleep(.01)
self.upThread = UploadThread()
#QtCore.Slot(str)
def txtMsgAppend(self, txt):
self.lbl.append(txt + " | ")
if __name__ == '__main__':
app=QtGui.QApplication(sys.argv)
widg=ULoadWin()
widg.show()
sys.exit(app.exec_())
In the end I solved this problem by adapting the approach outlined here, which I find to be an elegant solution. The class I created is shown below:
class UploadThread(threading.Thread):
#input_q and result_q are Queue.Queue objects
def __init__(self, input_q, result_q):
super(UploadThread, self).__init__()
self.input_q = input_q
self.result_q = result_q
self.stoprequest = threading.Event() #threadsafe flag
def run(self):
'''Runs indefinitely until self.join() is called.
As soon as items are placed in the input_q, then the thread will process them until the input_q is emptied.
'''
while not self.stoprequest.isSet(): #stoprequest can be set from the main gui
try:
# Queue.get with timeout to allow checking self.stoprequest
num = self.input_q.get(True, 0.1) #when the queue is empty it waits 100ms before raising the Queue.Empty error
print 'In thread, processing', num
time.sleep(0.5)
self.result_q.put(True) #Indicate to the main thread that an item was successfully processed.
except Queue.Empty as e:
continue
def join(self, timeout=None):
self.stoprequest.set()
super(UploadThread, self).join(timeout)
In the main thread, the upload thread is created and the input_q is loaded with the items to upload. A QTimer is created to regularly check on the progress of the upload by checking what has been placed into the result_q. It also update the progress bar. If no progress has been made within a timeout, this indicates the upload connection has failed.
An advantage of using Queue.Queue objects for communicating between threads, is that multiple threads that share the same input and result queue can be created.
While using:
def myFunction(arg):
for i in range(10000):
pass
from multiprocessing import Pool
pool = Pool(processes=3)
pool.map_async( myFunction, ['first','second','third'] )
I want the user to be able to pause an execution of the multiprocessing's Pool at any given time after the Pool was started. Then I would like the user to be able to unpause (to continue) with the rest of the items in a Pool. How to achieve it?
EDIT:
Here is the working implementation of suggestions posted by Blckknght. Thanks Blckknght!
import multiprocessing
from PyQt4 import QtGui, QtCore
def setup(event):
global unpaused
unpaused = event
def myFunction( arg=None):
unpaused.wait()
print "Task started...", arg
for i in range(15000000):
pass
print '...task completed.', arg
class MyApp(object):
def __init__(self):
super(MyApp, self).__init__()
app = QtGui.QApplication(sys.argv)
self.mainWidget = QtGui.QWidget()
self.mainLayout = QtGui.QVBoxLayout()
self.mainWidget.setLayout(self.mainLayout)
self.groupbox = QtGui.QGroupBox()
self.layout = QtGui.QVBoxLayout()
self.groupbox.setLayout(self.layout)
self.pauseButton = QtGui.QPushButton('Pause')
self.pauseButton.clicked.connect(self.pauseButtonClicked)
self.layout.addWidget(self.pauseButton)
self.okButton = QtGui.QPushButton('Start Pool')
self.okButton.clicked.connect(self.startPool)
self.layout.addWidget(self.okButton)
self.layout.addWidget(self.pauseButton)
self.mainLayout.addWidget(self.groupbox)
self.mainWidget.show()
sys.exit(app.exec_())
def startPool(self):
self.event = multiprocessing.Event()
self.pool=multiprocessing.Pool(1, setup, (self.event,))
self.result=self.pool.map_async(myFunction, [1,2,3,4,5,6,7,8,9,10])
self.event.set()
# self.result.wait()
def pauseJob(self):
self.event.clear()
def continueJob(self):
self.event.set()
def pauseButtonClicked(self):
if self.pauseButton.text()=='Pause':
print '\n\t\t ...pausing job...','\n'
self.pauseButton.setText('Resume')
self.pauseJob()
else:
print '\n\t\t ...resuming job...','\n'
self.pauseButton.setText('Pause')
self.continueJob()
if __name__ == '__main__':
MyApp()
It sounds like you want to use a multiprocessing.Event to control the running of your worker function. You can create one, then pass it to an initializer of the pool, then wait on it in myFunction.
Here's an example that runs workers that print their argument every second. The workers can be paused by clearing the event, and restarted by setting it again.
from time import sleep
import multiprocessing
def setup(event):
global unpaused
unpaused = event
def myFunction(arg):
for i in range(10):
unpaused.wait()
print(arg)
sleep(1)
if __name__ == "__main__":
event = multiprocessing.Event() # initially unset, so workers will be paused at first
pool = multiprocessing.Pool(3, setup, (event,))
result = pool.map_async(myFunction, ["foo", "bar", "baz"])
event.set() # unpause workers
sleep(5)
event.clear() # pause after five seconds
sleep(5)
event.set() # unpause again after five more seconds
result.wait() # wait for the rest of the work to be completed
The worker processes should print "foo", "bar" and "baz" ten times each, with a one second delay between each repetition. The workers will be paused after the first five seconds though, and restarted after a five more seconds. There are probably various ways to improve this code, depending on what your actual use case is, but hopefully it is enough to get you headed in the right direction.
I'm using a library which heaviliy uses I/O. For that reason calls to that library can last very long (more than 5 seconds) possible.
Using that directly inside an UI is not a good idea because it will freeze.
For that reason I outsourced the library calls to a thread queue like shown in this example: Python threads: communication and stopping
Nevertheless I'm not very happy with that solution since this has a major drawback:
I cannot really communicate with the UI.
Every lib command returns a return message, which can either be an error message or some computational result.
How would I get this?
Consider a library call do_test(foo):
def do_test(foo):
time.sleep(10)
return random.random() * foo
def ui_btn_click():
threaded_queue.put((do_test, 42))
# Now how to display the result without freezing the UI?
Can someone give me advice how to realize such a pattern?
Edit:
This here is a minimal example:
import os, time, random
import threading, queue
CMD_FOO = 1
CMD_BAR = 2
class ThreadedQueue(threading.Thread):
def __init__(self):
super().__init__()
self.in_queue = queue.Queue()
self.out_queue = queue.Queue()
self.__stoprequest = threading.Event()
def run(self):
while not self.__stoprequest.isSet():
(cmd, arg) = self.in_queue.get(True)
if cmd == CMD_FOO:
ret = self.handle_foo(arg)
elif cmd == CMD_BAR:
ret = self.handle_bar(arg)
else:
print("Unsupported cmd {0}".format(cmd))
self.out_queue.put(ret)
self.in_queue.task_done()
def handle_foo(self, arg):
print("start handle foo")
time.sleep(10)
return random.random() * arg
def handle_bar(self, arg):
print("start handle bar")
time.sleep(2)
return (random.random() * arg, 2 * arg)
if __name__ == "__main__":
print("START")
t = ThreadedQueue()
t.start()
t.in_queue.put((CMD_FOO, 10))
t.in_queue.put((CMD_BAR, 10))
print("Waiting")
while True:
x = t.out_queue.get(True)
t.out_queue.task_done()
print(x)
I personally use PySide but I don't want to depend this library on PySide or any other ui-related library.
I thought a bit about my implementations. THe conclusion is that I start another thread for picking the results of the queue:
class ReceiveThread(threading.Thread):
"""
Processes the output queue and calls a callback for each message
"""
def __init__(self, queue, callback):
super().__init__()
self.__queue = queue
self.__callback = callback
self.__stoprequest = threading.Event()
self.start()
def run(self):
while not self.__stoprequest.isSet():
ret = self.__queue.get(True)
self.__callback(ret)
self.__queue.task_done()
The given callback from an UI or elsewhere is called with every result from the queue.