I'm writing a small application in Python 3.6 and PyQt 5.13.1 to compare file checksums for directories across multiple remote servers. I'm trying to split out the main logic of my application into its own thread, and from that thread use a threadpool to create several more threads to SSH into the remote servers and get information back.
Everything is mostly working, except when the signal I have made for the SSH threads is called, one of the positional arguments ends up missing and I get the following error:
TypeError: ssh_result() missing 1 required positional argument: 'message'
Here is a stripped down version of what I currently have, but is exhibiting the same issue:
import paramiko
from PyQt5.QtWidgets import *
from PyQt5.QtCore import QThread, QThreadPool, QRunnable, pyqtSignal, pyqtSlot, QObject
class SSHWorkerSignals(QObject):
result = pyqtSignal('PyQt_PyObject', 'PyQt_PyObject')
class SSHWorker(QRunnable):
def __init__(self):
super(SSHWorker, self).__init__()
self.ssh_signals = SSHWorkerSignals()
self.ssh_server = 'test.server.corp'
#pyqtSlot()
def run(self):
try:
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy)
ssh_client.connect(self.ssh_server, port=22, username='test', password='test')
stdin, stdout, stderr = ssh_client.exec_command('uname -a')
std_out = str(stdout.read())
self.ssh_signals.result.emit(self.ssh_server, std_out)
except Exception as e:
self.ssh_signals.result.emit(self.ssh_server, str(e))
finally:
ssh_client.close()
class CompareWorker(QObject):
done = pyqtSignal()
def __init__(self):
super(CompareWorker, self).__init__()
self.ssh_threadpool = QThreadPool()
self.ssh_threadpool.setMaxThreadCount(10)
#pyqtSlot()
def execute(self):
ssh_worker = SSHWorker()
ssh_worker.ssh_signals.result.connect(MainWindow.ssh_result)
self.ssh_threadpool.start(ssh_worker)
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.button_compare = QPushButton('Compare')
self.button_compare.clicked.connect(self.compare)
self.setCentralWidget(self.button_compare)
def compare(self):
compare_thread = QThread(self)
self.compare_worker = CompareWorker()
compare_thread.started.connect(self.compare_worker.execute)
self.compare_worker.done.connect(compare_thread.quit)
self.compare_worker.done.connect(self.compare_worker.deleteLater)
self.compare_worker.moveToThread(compare_thread)
compare_thread.start()
def ssh_result(self, server, message):
print('server: ', server, ', result: ', message)
if __name__ == '__main__':
app = QApplication([])
window = MainWindow()
window.show()
app.exec_()
If I change the ssh_result() function so that message is an optional argument (def ssh_result(self, server, message=''):), I can see that the first positional argument is disappearing, leaving what should be the second argument to be the first instead, like so:
server: [Errno 11001] getaddrinfo failed , result:
Can anyone help me figure out why this is happening?
The main problem is that MainWindow.ssh_result is not a method that belongs to some instance, so the first "self" parameter does not exist. I also don't see the need for the creation of CompareWorker.
Considering the above, the solution is:
# ...
class SSHWorkerSignals(QObject):
result = pyqtSignal(
"PyQt_PyObject", "PyQt_PyObject"
) # or pyqtSignal(object, object)
class SSHWorker(QRunnable):
# ...
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.button_compare = QPushButton("Compare")
self.button_compare.clicked.connect(self.compare)
self.setCentralWidget(self.button_compare)
self.ssh_threadpool = QThreadPool()
self.ssh_threadpool.setMaxThreadCount(10)
def compare(self):
ssh_worker = SSHWorker()
ssh_worker.ssh_signals.result.connect(self.ssh_result)
self.ssh_threadpool.start(ssh_worker)
def ssh_result(self, server, message):
print("server: ", server, ", result: ", message)
# ...
Related
I'm trying to extend this example on multithreading in PyQt5:
Multithreading PyQt applications with QThreadPool (MWE below)
to allow for two different threaded functions, one that uses a callback and the other that does not. In the example above, the progress_callback is hard-coded in the Worker() class __init__, which means that any threaded function must have a signature that accommodates that callback:
def execute_this_fn(self, progress_callback):
which means that if there is a second threaded process, that function's signature would also be required to accommodate the callback in its signature even though the callback is not used.
So instead of hard-coding progress_callback into the __init__ of Worker(), I'd like to pass in the callback when instantiating Worker().
My MWE is below -- I include two commented-out two lines from the original example for reference. When I run it, and press the "DANGER!" button on the GUI, I get:
$ python threadtest.py
Multithreading with maximum 16 threads
progress_callback = <bound PYQT_SIGNAL progress of WorkerSignals object at 0x113bbd160>
n = 0
n = 1
n = 2
n = 3
n = 4
Done.
THREAD COMPLETE!
So the code runs, but the callback function (slot) progress_fn is never called, and I'm not sure why...
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
import time
import traceback, sys
class WorkerSignals(QObject):
''' Defines the signals available from a running worker thread.
Supported signals are:
finished
No data
error
`tuple` (exctype, value, traceback.format_exc() )
result
`object` data returned from processing, anything
progress
`int` indicating % progress
'''
finished = pyqtSignal()
error = pyqtSignal(tuple)
result = pyqtSignal(object)
progress = pyqtSignal(int)
class Worker(QRunnable):
''' Worker thread
Inherits from QRunnable to handler worker thread setup, signals and wrap-up.
:param callback: The function callback to run on this worker thread. Supplied args and
kwargs will be passed through to the runner.
:type callback: function
:param args: Arguments to pass to the callback function
:param kwargs: Keywords to pass to the callback function
'''
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
# Store constructor arguments (re-used for processing)
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
# The old way (hard-code the callback to kwargs)
#self.kwargs['progress_callback'] = self.signals.progress
#pyqtSlot()
def run(self):
'''
Initialise the runner function with passed args, kwargs.
'''
# Retrieve args/kwargs here; and fire processing using them
try:
result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(result) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.counter = 0
self.signals = WorkerSignals()
layout = QVBoxLayout()
self.l = QLabel("Start")
b = QPushButton("DANGER!")
b.pressed.connect(self.oh_no)
layout.addWidget(self.l)
layout.addWidget(b)
w = QWidget()
w.setLayout(layout)
self.setCentralWidget(w)
self.show()
self.threadpool = QThreadPool()
print("Multithreading with maximum %d threads" % self.threadpool.maxThreadCount())
self.timer = QTimer()
self.timer.setInterval(1000)
self.timer.timeout.connect(self.recurring_timer)
self.timer.start()
def progress_fn(self, n):
print("%d%% done" % n) # this does not execute...
def execute_this_fn(self, progress_callback):
print(f'progress_callback = {progress_callback}')
for n in range(0, 5):
time.sleep(1)
print(f'n = {n}')
progress_callback.emit(int(n*100/4))
return "Done."
def print_output(self, s):
print(s)
def thread_complete(self):
print("THREAD COMPLETE!")
def oh_no(self):
# Pass the function to execute
# the old way (callback hardcoded in __init__ of Worker class)
#worker = Worker(self.execute_this_fn)
# the desired new way
worker = Worker(self.execute_this_fn, progress_callback=self.signals.progress)
worker.signals.result.connect(self.print_output)
worker.signals.finished.connect(self.thread_complete)
worker.signals.progress.connect(self.progress_fn)
# Execute
self.threadpool.start(worker)
def recurring_timer(self):
self.counter +=1
self.l.setText("Counter: %d" % self.counter)
app = QApplication([])
window = MainWindow()
app.exec_()
The problem is simple: You have 2 WorkerSignals objects, and in one of them you make the connection and with the other you emit the signal. The solution is to use the same object for the connection and for the emission:
worker = Worker(self.execute_this_fn, progress_callback=self.signals.progress)
worker.signals.result.connect(self.print_output)
worker.signals.finished.connect(self.thread_complete)
self.signals.progress.connect(self.progress_fn)
Although I prefer to create a QObject that implements that logic:
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
import time
import traceback, sys
class WorkerSignals(QObject):
"""Defines the signals available from a running worker thread.
Supported signals are:
finished
No data
error
`tuple` (exctype, value, traceback.format_exc() )
result
`object` data returned from processing, anything
progress
`int` indicating % progress
"""
finished = pyqtSignal()
error = pyqtSignal(tuple)
result = pyqtSignal(object)
progress = pyqtSignal(int)
class Worker(QRunnable):
"""Worker thread
Inherits from QRunnable to handler worker thread setup, signals and wrap-up.
:param callback: The function callback to run on this worker thread. Supplied args and
kwargs will be passed through to the runner.
:type callback: function
:param args: Arguments to pass to the callback function
:param kwargs: Keywords to pass to the callback function
"""
def __init__(self, fn, *args, **kwargs):
super(Worker, self).__init__()
# Store constructor arguments (re-used for processing)
self.fn = fn
self.args = args
self.kwargs = kwargs
self.signals = WorkerSignals()
# The old way (hard-code the callback to kwargs)
# self.kwargs['progress_callback'] = self.signals.progress
#pyqtSlot()
def run(self):
"""
Initialise the runner function with passed args, kwargs.
"""
# Retrieve args/kwargs here; and fire processing using them
try:
result = self.fn(*self.args, **self.kwargs)
except:
traceback.print_exc()
exctype, value = sys.exc_info()[:2]
self.signals.error.emit((exctype, value, traceback.format_exc()))
else:
self.signals.result.emit(result) # Return the result of the processing
finally:
self.signals.finished.emit() # Done
class ProgressCallback(QObject):
progressChanged = pyqtSignal(int)
def __call__(self, value):
self.progressChanged.emit(value)
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.counter = 0
layout = QVBoxLayout()
self.l = QLabel("Start")
b = QPushButton("DANGER!")
b.pressed.connect(self.oh_no)
layout.addWidget(self.l)
layout.addWidget(b)
w = QWidget()
w.setLayout(layout)
self.setCentralWidget(w)
self.show()
self.threadpool = QThreadPool()
print(
"Multithreading with maximum %d threads" % self.threadpool.maxThreadCount()
)
self.timer = QTimer()
self.timer.setInterval(1000)
self.timer.timeout.connect(self.recurring_timer)
self.timer.start()
def progress_fn(self, n):
print("%d%% done" % n) # this does not execute...
def execute_this_fn(self, progress_callback):
print(f"progress_callback = {progress_callback}")
for n in range(0, 5):
time.sleep(1)
print(f"n = {n}")
progress_callback(int(n * 100 / 4))
return "Done."
def print_output(self, s):
print(s)
def thread_complete(self):
print("THREAD COMPLETE!")
def oh_no(self):
callback = ProgressCallback()
worker = Worker(self.execute_this_fn, progress_callback=callback)
worker.signals.result.connect(self.print_output)
worker.signals.finished.connect(self.thread_complete)
callback.progressChanged.connect(self.progress_fn)
# Execute
self.threadpool.start(worker)
def recurring_timer(self):
self.counter += 1
self.l.setText("Counter: %d" % self.counter)
app = QApplication([])
window = MainWindow()
app.exec_()
I have a websocket server running in python, and for every new connection a new thread will be created and the requests will be served.
In the main-thread [Gui-thread],i am initialing QApplication([]). the use case is, when i process the request i wanted to wait and get text response from the user through QInputDialog.
when ever i run it, there is a event-loop running but is not showing the gui. because all the gui elements can be displayed from Gui-thread itself.
I have tried various approaches using QSignals/slots and Pypubsub but unable to achieve what is required. please do suggest some idea to get the use-case done. a pseudo-code is well appreciated.
Below mentioned code are some examples i tried. i am using thread in below examples because, as i mentioned each request from a connection is executed with the thread assigned to the connection. and the text from QInputDialog is required by the thread.
thanks in advance.
below is the websockets server code which serves the request calling server_extentions function, i have to show QInputDialog everytime i get a incoming request.
import websockets
import asyncio
from PyQt5.QtWidgets import QInputDialog, QApplication
app = QApplication([])
async def server_extentions(websocket, path):
try:
while(True):
request = await websocket.recv()
# this is where i need to show input dialog.
text, ok = QInputDialog.getText(None, "Incoming message", request)
if ok:
response = text
else:
response = "NO REPLY"
await websocket.send(response)
except websockets.ConnectionClosed as exp:
print("connection closed.")
start_server = websockets.serve(server_extentions, '127.0.0.1', 5588)
loop = asyncio.get_event_loop()
try:
loop.run_until_complete(start_server)
loop.run_forever()
finally:
loop.run_until_complete(loop.shutdown_asyncgens())
loop.close()
----edit-----
Below is some general idea, i tried using pypubsub.
import threading
import pubsub.pub
from PyQt5.QtWidgets import QInputDialog, QApplication
class MainThread:
def __init__(self):
self.app = QApplication([])
pubsub.pub.subscribe(self.pub_callback, "lala")
def pub_callback(self):
print("this is Main thread's pub callback.")
QInputDialog.getText(None, "main-thread", "lala call back : ")
def start_thread(self):
self.th = threading.Thread(target=self.thread_proc)
self.th.start()
def thread_proc(self):
pubsub.pub.sendMessage("lala")
m = MainThread()
m.start_thread()
-----edit 2 -------
below is something i tried with QSignal. [check the comment in the code, How to call a function with Mainthread].
import threading
from PyQt5.QtWidgets import QInputDialog, QApplication
from PyQt5.QtCore import pyqtSignal, QObject, QThread
class TextDialog(QObject):
sig = pyqtSignal(str)
def __init__(self):
QObject.__init__(self)
def get_text(self):
print("class Thread2, showing QInputDialog.")
text, ok = QInputDialog.getText(None, "Lala", "give me some text : ")
if ok:
self.sig.emit(text)
return
self.sig.emit("NO TEXT")
return
class Thread1:
def thread_proc(self):
td = TextDialog()
td.sig.connect(self.get_text_callback)
td.moveToThread(m.main_thread)
# here i dont understand how to invoke MainThread's show_dialog with main thread. [GUI Thread]
#m.show_dialog(td)
def get_text_callback(self, txt):
print("this is get_text_callback, input : " + str(txt))
class MainThread:
def __init__(self):
self.app = QApplication([])
self.main_thread = QThread.currentThread()
def main_proc(self):
th1 = Thread1()
th = threading.Thread(target=th1.thread_proc)
th.start()
def show_dialog(self, text_dialog: TextDialog):
print("got a call to MainThread's show_dialog.")
text_dialog.get_text()
m = MainThread()
m.main_proc()
exit()
For this type of applications it is better to implement the worker-thread approach. This approach the main idea is to implement QObjects, move them to a new thread and invoke the slots asynchronously (through QEvents, pyqtSignals, QTimer.singleShot(...), QMetaObject::invokeMethod(...), etc) so that the tasks are executed in the thread that Live the QObject.
import threading
from functools import partial
from PyQt5 import QtCore, QtWidgets
class TextDialog(QtCore.QObject):
sig = QtCore.pyqtSignal(str)
#QtCore.pyqtSlot()
def get_text(self):
print("class Thread2, showing QInputDialog.")
text, ok = QtWidgets.QInputDialog.getText(
None, "Lala", "give me some text : "
)
if ok:
self.sig.emit(text)
return
self.sig.emit("NO TEXT")
return
class Worker1(QtCore.QObject):
#QtCore.pyqtSlot(QtCore.QObject)
def thread_proc(self, manager):
print(
"current: {}- main: {}".format(
threading.current_thread(), threading.main_thread()
)
)
manager.td.sig.connect(self.get_text_callback)
QtCore.QTimer.singleShot(0, manager.show_dialog)
#QtCore.pyqtSlot(str)
def get_text_callback(self, txt):
print(
"current: {}- main: {}".format(
threading.current_thread(), threading.main_thread()
)
)
print("this is get_text_callback, input : %s" % (txt,))
class Manager(QtCore.QObject):
def __init__(self, parent=None):
super().__init__(parent)
self.td = TextDialog()
#QtCore.pyqtSlot()
def show_dialog(self):
print("got a call to MainThread's show_dialog.")
self.td.get_text()
class Application:
def __init__(self):
print(
"current: {}- main: {}".format(
threading.current_thread(), threading.main_thread()
)
)
self.app = QtWidgets.QApplication([])
# By default if after opening a window all the windows are closed
# the application will be terminated, in this case after opening
# and closing the QInputDialog the application will be closed avoiding
# that it will be noticed that get_text_callback is called,
# to avoid the above it is deactivated behavior.
self.app.setQuitOnLastWindowClosed(False)
self.manager = Manager()
def main_proc(self):
#
self.thread = QtCore.QThread()
self.thread.start()
self.worker = Worker1()
# move the worker so that it lives in the thread that handles the QThread
self.worker.moveToThread(self.thread)
# calling function asynchronously
# will cause the function to run on the worker's thread
QtCore.QTimer.singleShot(
0, partial(self.worker.thread_proc, self.manager)
)
def run(self):
return self.app.exec_()
if __name__ == "__main__":
import sys
m = Application()
m.main_proc()
ret = m.run()
sys.exit(ret)
Background :
I have a script that allows me to make spatial queries on a PostgreSQL database via an API coming from a private editor (I can't directly query the database). This API is working with python 3.2. To summarize very quickly, this script is used to download elements of this database in the desired geographical footprint. Depending of the zone, you can obtain between 1 to over 100 elements, each of them having very different sizes (from Ko to Go).
The main window let you to set all options and then start the global process. When launched a console window appears letting you see what’s going on. Once an item has been downloaded a short “report” is displayed on the console. Currently everything is done sequentially one element at a time. As you can imagine, if this element is quite large, the console freezes while waiting for the end of the download process.
Code:
I'm not going to post the full script here, but through a very simple script I will try to show the main problem I'm trying to solve (i.e. avoid locking the user interface / have some sort of real-time output on what's going on).
So, in order to avoid these freezing problems, the use of threads seemed to me to be the best solution. To simulate the download process (see previous chapter) I used the url.request urlretrieve method with multiple urls (pointing to files of different sizes).
import os
import sys
import time
import urllib.request
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class DownloadWorkerSignals(QtCore.QObject):
"""
Defines the signals available from a running download worker thread.
"""
finished = QtCore.pyqtSignal(str)
# ---------------------------------------------------------------------------------
class DownloadWorker(QtCore.QRunnable):
"""
Worker thread
"""
def __init__(self, url, filepath, filename, index):
super(DownloadWorker, self).__init__()
self.url = url
self.file_path = filepath
self.filename = filename
self.index = index
self.signals = DownloadWorkerSignals()
#QtCore.pyqtSlot(str)
def run(self):
t = time.time()
message = 'Thread %d started\n' % self.index
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.file_path,
self.filename))
except IOError as error:
message += str(error) + '\n'
finally:
message += 'Thread %d ended %.2f s\n' % (self.index, time.time() - t)
self.signals.finished.emit(message) # Done
# ---------------------------------------------------------------------------------
class Main(QtGui.QMainWindow):
"""
Main window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Main")
self.setWindowModality(QtCore.Qt.ApplicationModal)
self.centralwidget = QtGui.QWidget(self)
self.setCentralWidget(self.centralwidget)
# Ok / Close
# -------------------------------------------------------------------------
self.buttonBox = QtGui.QDialogButtonBox(self.centralwidget)
self.buttonBox.setStandardButtons(QtGui.QDialogButtonBox.Cancel |
QtGui.QDialogButtonBox.Ok)
self.buttonBox.setGeometry(QtCore.QRect(10, 160, 380, 20))
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.buttonBox,
QtCore.SIGNAL('accepted()'),
self.button_ok_clicked)
self.connect(self.buttonBox,
QtCore.SIGNAL('rejected()'),
self.button_cancel_clicked)
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
self.close()
def button_ok_clicked(self):
# Launch console
console = Console(parent=self)
console.exec_()
# ---------------------------------------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self, parent):
super(self.__class__, self).__init__()
self.parent = parent
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.verticalLayout = QtGui.QVBoxLayout(self)
# Text edit
# -------------------------------------------------------------------------
self.text_edit = QtGui.QPlainTextEdit(self)
self.text_edit.setReadOnly(True)
self.text_edit_cursor = QtGui.QTextCursor(self.text_edit.document())
self.verticalLayout.addWidget(self.text_edit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.verticalLayout.addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.button_cancel_clicked)
# Post initialization
# -------------------------------------------------------------------------
self.threadpool = QtCore.QThreadPool()
self.threadpool.setMaxThreadCount(2)
for index, url in enumerate(urls):
worker = DownloadWorker(url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
worker.signals.finished.connect(self.write_message)
self.threadpool.start(worker)
'''
I have to wait for the end of the thread pool to make a post-processing.
If I use the waitForDone I don't see my console until the all work is done
'''
# self.threadpool.waitForDone()
# self.write_stram('Thread pool finished')
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
if self.threadpool.activeThreadCount() != 0:
pass # How to interrupt the threadpool ?
self.close()
#QtCore.pyqtSlot(str)
def write_message(self, text):
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Main()
window.show()
app.exec_()
Questions:
Everything seems to work as expected but I encounter two difficulties:
At the end of the thread pool process I have to make some
post-processing. If I use the waitForDone method I don't see my
console until the all work is done and it’s not the type of behavior
wanted.
If the Cancel Button in the Console is clicked, I need to interrupt
the threadpool and I don’t know how to manage that.
I had another look at this problem (based largely on this : how-do-i-maintain-a-resposive-gui-using-qthread-with-pyqgis).
So I replaced the previous tandem QThreadPool/QRunnable, by Queue/QThread. The code below gives an overview.
import os
import sys
import time
import urllib.request
import queue
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class WorkerThread(QtCore.QThread):
"""
Worker thread
"""
def __init__(self, parent_thread):
QtCore.QThread.__init__(self, parent_thread)
def run(self):
self.running = True
success = self.do_work()
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def stop(self):
self.running = False
pass
def do_work(self):
return True
def clean_up(self):
pass
# ---------------------------------------------------------------------------------
class LongRunningTask(WorkerThread):
def __init__(self, parent_thread, url, filepath, filename, index):
WorkerThread.__init__(self, parent_thread)
self.url = url
self.filepath = filepath
self.filename = filename
self.index = index
def do_work(self):
t = time.time()
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'), 'Thread %d started\n' % self.index)
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.filepath,
self.filename))
except IOError as error:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d error - ' % self.index + str(error) + '\n')
finally:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d ended %.2f s\n' % (self.index, time.time() - t))
return True
# ---------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.setLayout(QtGui.QVBoxLayout())
# Text edit
# -------------------------------------------------------------------------
self.textEdit = QtGui.QPlainTextEdit(self)
self.textEdit.setReadOnly(True)
self.textEdit_cursor = QtGui.QTextCursor(self.textEdit.document())
self.layout().addWidget(self.textEdit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(False)
self.layout().addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.reject)
# Post-Initialization
# -------------------------------------------------------------------------
self.queue = queue.Queue()
# self.queue = queue.Queue(maxsize=2)
self.run_thread()
# Connect functions
# -----------------------------------------------------------------------------
def cancel_thread(self):
self.workerThread.stop()
def job_finished_from_thread(self, success):
self.workerThread.stop()
self.queue.get()
# Stop the pulsation
if self.queue.empty():
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(True)
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def text_from_thread(self, value):
self.textEdit.insertPlainText(value)
cursor = self.textEdit.textCursor()
self.textEdit.setTextCursor(cursor)
def run_thread(self):
for index, url in enumerate(urls):
self.workerThread = LongRunningTask(parent_thread=self,
url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
self.connect(self.workerThread,
QtCore.SIGNAL('jobFinished(PyQt_PyObject)'),
self.job_finished_from_thread)
self.connect(self.workerThread,
QtCore.SIGNAL('threadText(PyQt_PyObject)'),
self.text_from_thread)
self.queue.put(self.workerThread)
self.workerThread.start()
# If I set the queue to maxsize=2, how to manage it here
'''
while not self.queue.full():
self.queue.put(self.workerThread)
self.workerThread.start()
'''
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Console()
window.show()
app.exec_()
Question:
Unfortunately, I encounter other types of difficulties. In reality, the queue can contain a large amount of threads (over 100). 1. How can I, like the QthreadPool and its setMaxThreadCount method, manage the number of threads running in parallel in order to prevent the system from collapsing completely ?
I'm getting stuck on what I think is a basic multiprocess and threading issue. I've got a multiprocess set up, and within this a thread. However, when I set up the thread class within the init function, I get the following error:
"TypeError: can't pickle thread.lock objects".
However, this does not happen if the thread is set up outside of the init function. Does anyone know why this is happening? Note I'm using Windows.
Some code is below to illustrate the issue. As typed below, it runs fine. However if print_hello() is called from within the DoStuff init def, then the error occurs, if it's called within the multi-process run() def then it's fine.
Can anyone point me in the right direction so it runs fine when called from init? thanks!
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("starting DoStuff")
# This works fine if the line below is uncommented and __init__ self.print_hello() is commented...
self.dostuff.print_hello()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
# If the following is uncommented, the error occurs...
# Note it also occurs if the lines in start_thead are pasted here...
# self.print_hello()
def print_hello(self):
print "hello"
self.start_thread()
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("Starting MyThread")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
# mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
It looks like there is no simple answer, and it appears to be a restriction of Windows (Win 7, python 3.6 in my case); on Windows it looks like you need to start the process before you can start the worker thread inside the owned object.
There appears to be no such restriction on Unix (CentOS 7, python 2.7.5).
As an experiment I modified your code as follows; this version checks the OS and starts either the process first, or the thread first:
import multiprocessing
import threading
import time
import os
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff(self)
def run(self):
print("MyProcess.run()")
print("MyProcess.ident = " + repr(self.ident))
if os.name == 'nt':
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, owner, **kwargs):
super(DoStuff, self).__init__(**kwargs)
self.owner = owner
if os.name != 'nt':
self.start_thread()
def start_thread(self):
print("DoStuff.start_thread()")
self.my_thread_instance = MyThread(self)
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self, owner):
super(MyThread, self).__init__()
self.owner = owner
def run(self):
print("MyThread.run()")
print("MyThread.ident = " + repr(self.ident))
print("MyThread.owner.owner.ident = " + repr(self.owner.owner.ident))
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
... and got the following on Windows, where the process starts first:
MyProcess.run()
MyProcess.ident = 14700
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 14220
MyThread.owner.owner.ident = 14700
... and the following on Linux, where the thread is started first:
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 140316342347520
MyThread.owner.owner.ident = None
MyProcess.run()
MyProcess.ident = 4358
If it were my code I'd be tempted to always start the process first, then create the thread within that process; the following version works fine for me across both platforms:
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("MyProcess.run()")
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("MyThread.run()")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
Trying to convert a server written in C++ into Python. The server was written to be Asynchronous/Non Blocking. What works in C++ doesn't seem to want to work for me in Python
I am using PyQT4. I read Python you have to create the event loop or something along those lines any ideas are greatly appreciated
I should mention what seems to not work is that the incomingConnection function in Class Server is never called.
*cheers
import sys
from PyQt4.QtCore import *
from PyQt4.QtNetwork import *
class Client(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
QThreadPool.globalInstance().setMaxThreadCount(15)
def SetSocket(self, Descriptor):
self.socket = QTcpSocket(self)
self.connect(self.socket, SIGNAL("connected()"), SLOT(self.connected()))
self.connect(self.socket, SIGNAL("disconnected()"), SLOT(self.disconnected()))
self.connect(self.socket, SIGNAL("readyRead()"), SLOT(self.readyRead()))
self.socket.setSocketDescriptor(Descriptor)
print "Client Connected from IP %s" % self.socket.peerAddress().toString()
def connected(self):
print "Client Connected Event"
def disconnected(self):
print "Client Disconnected"
def readyRead(self):
msg = self.socket.readAll()
print msg
class Server(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
def incomingConnection(self, handle):
print "incoming"
self.client = Client(self)
self.client.SetSocket(handle)
def StartServer(self):
self.server = QTcpServer()
if self.server.listen(QHostAddress("0.0.0.0"), 8888):
print "Server is awake"
else:
print "Server couldn't wake up"
def main():
app = QCoreApplication(sys.argv)
Server().StartServer()
sys.exit(app.exec_())
if __name__ == '__main__':
main()
incomingConnection is not called because QTcpServer's base implementation of the function is called. as incomingConnection is a vitual function, you just have to assign your's to QTcpServer's incomingConnection attribute, like this:
class Server(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
def incomingConnection(self, handle):
print "incoming"
self.client = Client(self)
self.client.SetSocket(handle)
def StartServer(self):
self.server = QTcpServer()
self.server.incomingConnection = self.incomingConnection
if self.server.listen(QHostAddress("0.0.0.0"), 8888):
print "Server is awake"
else:
print "Server couldn't wake up"
you can check out PySide's documentation, as it's much more pythonic than PyQt's, currently hosted only here:
http://srinikom.github.com/pyside-docs/