I'm trying to run a few slow processes but, I need to keep updated the QDialog to show the progress (maybe I put a progress bar too).
So I decide to use QThread, but on the first try, it doesn't work as I expected.
In my example code:
1- I'm using a simple ping to my default gateway
2- I'm pinging to my dns resolver
As you can see on imagem below, the information is showed according the thread is finalizing, but it is a mess to me.
Is possible to respect the threads order to show the informations?
Thanks.
Follow my example code:
# -*- coding: utf8 -*-
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from ping3 import ping, verbose_ping
import socket
import dns.resolver
class ExternalTests(QThread):
data_collected = pyqtSignal(object)
def __init__(self, title, arg=None):
QThread.__init__(self)
self.title = title
self.arg = arg
def run(self):
resp = ping(self.arg)
self.data_collected.emit('%s: %s' % (self.title, resp))
class MainMenu(QMenu):
def __init__(self, parent=None):
QMenu.__init__(self)
self.setStyleSheet("background-color: #3a80cd; color: rgb(255,255,255); selection-color: white; selection-background-color: #489de4;")
# Diagnostics
self.check_internet = QAction("Diagnosys")
self.check_internet.setIcon(QIcon(QPixmap("..\\img\\lupa.png")))
self.check_internet.triggered.connect(self.diagnosticNetwork)
self.addAction(self.check_internet)
self.addSeparator()
# To quit the app
self.quit = QAction("Quit")
self.quit.triggered.connect(app.quit)
self.addAction(self.quit)
def diagnosticNetwork(self):
self.check_internet_dialog = QDialog()
self.check_internet_dialog.setWindowTitle("Check external connections")
self.check_internet_dialog.setWindowModality(Qt.ApplicationModal)
self.check_internet_dialog.setGeometry(150, 100, 700, 500)
# text box
self.textbox = QTextBrowser(self.check_internet_dialog)
self.textbox.move(20, 20)
self.textbox.resize(660,400)
self.textbox.setFont(QFont("Courier New", 12))
self.textbox.setStyleSheet("background-color: black;")
#button copy
btn_copy = QPushButton("Copy", self.check_internet_dialog)
btn_copy.setIcon(QIcon(QPixmap("..\\img\\copy.png")))
btn_copy.move(520,450)
btn_copy.clicked.connect(self.dialogClickCopy)
#button close
btn_copy = QPushButton("Close", self.check_internet_dialog)
btn_copy.setIcon(QIcon(QPixmap("..\\img\\close.png")))
btn_copy.move(605,450)
btn_copy.clicked.connect(self.dialogClickClose)
# tests
self.textbox.setTextColor(QColor("white"))
self.textbox.append("Diagnosys")
self.textbox.append("--------------------------------------------------")
self.textbox.setTextColor(QColor("cyan"))
self.threads = []
#QCoreApplication.processEvents()
''' ping default gateway '''
ping_default_gw = ExternalTests("default gatewat is reacheble", "192.168.0.1")
ping_default_gw.data_collected.connect(self.onDataReady)
self.threads.append(ping_default_gw)
ping_default_gw.start()
''' ping dns resolver '''
ping_dns_resolvers = dns.resolver.Resolver().nameservers
for dns_resolver in ping_dns_resolvers:
ping_dns_resolver = ExternalTests("dns resolver is reacheble %s" % dns_resolver, dns_resolver)
ping_dns_resolver.data_collected.connect(self.onDataReady)
self.threads.append(ping_dns_resolver)
ping_dns_resolver.start()
self.check_internet_dialog.exec_()
def onDataReady(self, data):
print(data)
if data:
self.textbox.append(data)
else:
self.textbox.append("error")
def dialogClickCopy(self):
pass
def dialogClickClose(self):
self.check_internet_dialog.close()
class SystemTrayIcon(QSystemTrayIcon):
def __init__(self, menu, parent=None):
QSystemTrayIcon.__init__(self)
self.setIcon(QIcon("..\\img\\icon.png"))
self.setVisible(True)
self.setContextMenu(menu)
if __name__ == "__main__":
import sys
app = QApplication([])
app.setQuitOnLastWindowClosed(False)
app.setApplicationName('pkimonitor')
app.setApplicationVersion('0.1')
app.setWindowIcon(QIcon("..\\img\\icon.png"))
menu = MainMenu()
widget = QWidget()
trayIcon = SystemTrayIcon(menu, widget)
trayIcon.show()
sys.exit(app.exec_())
I tried to create a scheme to organize by "run position" and it works. Follow my code.
In 'diagnosticNetwork':
self.data_list = []
self.data_hold = []
In 'onDataReady':
if len(self.data_list) > 0:
if self.data_list[-1][0] + 1 == data[0]:
self.data_list.append(data)
if data[2]:
self.textbox.append("%s: %s" % (data[1], 'OK'))
else:
self.textbox.append("%s: %s" % (data[1], 'NOK'))
elif self.data_list[-1][0] < data[0]:
self.data_hold.append(data)
else:
self.data_list.append(data)
if data[2]:
self.textbox.append("%s: %s" % (data[1], 'OK'))
else:
self.textbox.append("%s: %s" % (data[1], 'NOK'))
if len(self.data_hold) > 0:
hold_sorted = self.data_hold[:]
hold_sorted.sort()
for line_hold in hold_sorted:
if self.data_list[-1][0] + 1 == line_hold[0]:
if line_hold[2]:
self.textbox.append("%s: %s" % (line_hold[1], 'OK'))
else:
self.textbox.append("%s: %s" % (line_hold[1], 'NOK'))
self.data_list.append(line_hold)
del self.data_hold[0]
I worked with two lists, data_list and data_hold. The results that I received from the threads, I filled out the main list according to the order that I pre-established, if the result that entered was not the next one in the sequence, it goes to data_hold, then scan this list to fill the rest of my textbox.
Thank you for all help!
Related
I have made a Desktop Application using Python and used PyQt5 and Pytube which could download video from youtube. When download is in Progress, I want to show user an animation. In Fact I did it, but when the file is getting downloaded the PyQt window seems like freezing and everything just gets paused until the download is complete. So, Does anyone know why is this happening? How do I fix it?
Here's the code snippet:
def download_created(self, qual): # Used in 'selection' method
selected_stream = yt.streams.get_by_resolution(qual)
self.progress_func()
try:
self.download_btn.setCurrentIndex(-1)
selected_stream.download(self.askLocation() + "/")
except:
pass
# This gets the quality that the user chooses
def selection(self):
global quality
quality = self.download_btn.currentText()
try:
self.download_created(quality) # Calls a method called 'download'
except:
self.start_anime()
# Fetching the details about the Link from Youtube
def download_youtube(self):
global check
if check != self.get_input():
check = self.get_input()
self.download_btn.clear()
enter_url = self.get_input()
try:
global yt
yt = pytube.YouTube(
enter_url,
on_progress_callback = on_progress,
on_complete_callback = self.complete_func)
self.start_anime()
except:
self.input_error()
VIDEO_TITLE = (yt.title)
global VIDEO_ID
VIDEO_ID = (yt.video_id)
videos = yt.streams.filter(mime_type="video/mp4", progressive="True")
# Display all the available qualities
for i in videos:
self.download_btn.addItem(i.resolution)
self.download_btn.currentIndexChanged.connect(self.selection)
You have to execute the time consuming tasks in another thread, for example in your case the task of getting the streams and downloading.
import sys
import threading
from functools import cached_property
from PyQt5 import QtCore, QtWidgets
import pytube
class QPyTube(QtCore.QObject):
initialized = QtCore.pyqtSignal(bool, str)
download_started = QtCore.pyqtSignal()
download_progress_changed = QtCore.pyqtSignal(int)
download_finished = QtCore.pyqtSignal()
def __init__(self, url):
super().__init__()
self._url = url
self._yt = None
self._mutex = threading.Lock()
threading.Thread(target=self._init, daemon=True).start()
#property
def url(self):
return self._url
#cached_property
def resolutions(self):
return list()
def _init(self):
with self._mutex:
self.resolutions.clear()
try:
self._yt = pytube.YouTube(
self.url,
on_progress_callback=self._on_progress,
on_complete_callback=self._on_complete,
)
streams = self._yt.streams.filter(mime_type="video/mp4", progressive="True")
except Exception as e:
self.initialized.emit(False, str(e))
return
with self._mutex:
self.resolutions = [stream.resolution for stream in streams]
self.initialized.emit(True, "")
def download(self, resolution, directory):
threading.Thread(
target=self._download, args=(resolution, directory), daemon=True
).start()
def _download(self, resolution, directory):
stream = self._yt.streams.get_by_resolution(resolution)
self.download_started.emit()
stream.download(directory)
def _on_progress(self, stream, chunk, bytes_remaining):
self.download_progress_changed.emit(
100 * (stream.filesize - bytes_remaining) // stream.filesize
)
def _on_complete(self, stream, filepath):
self.download_finished.emit()
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, parent=None):
super().__init__(parent)
self.le_url = QtWidgets.QLineEdit("http://youtube.com/watch?v=2lAe1cqCOXo")
self.lbl_error = QtWidgets.QLabel()
self.btn_search = QtWidgets.QPushButton("Search")
self.cmb_resolutions = QtWidgets.QComboBox()
self.le_directory = QtWidgets.QLineEdit("")
self.btn_download = QtWidgets.QPushButton("Download")
self.pgb_download = QtWidgets.QProgressBar()
central_widget = QtWidgets.QWidget()
self.setCentralWidget(central_widget)
lay = QtWidgets.QGridLayout(central_widget)
lay.addWidget(self.le_url, 0, 0)
lay.addWidget(self.btn_search, 0, 1)
lay.addWidget(self.cmb_resolutions, 1, 0)
lay.addWidget(self.le_directory, 1, 1)
lay.addWidget(self.btn_download, 1, 2)
lay.addWidget(self.pgb_download, 2, 0, 1, 3)
self.btn_download.setEnabled(False)
self._qpytube = None
self.btn_search.clicked.connect(self.handle_search_clicked)
self.btn_download.clicked.connect(self.handle_download_clicked)
def handle_search_clicked(self):
self.cmb_resolutions.clear()
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.lbl_error.clear()
self._qpytube = QPyTube(self.le_url.text())
self._qpytube.initialized.connect(self.handle_initialized)
self._qpytube.download_progress_changed.connect(self.pgb_download.setValue)
self._qpytube.download_started.connect(self.handle_download_started)
self._qpytube.download_finished.connect(self.handle_download_finished)
#QtCore.pyqtSlot(bool, str)
def handle_initialized(self, status, error=""):
if status:
self.cmb_resolutions.addItems(self._qpytube.resolutions)
self.btn_download.setEnabled(True)
else:
self.lbl_error.setText(error)
self.btn_search.setEnabled(True)
def handle_download_clicked(self):
self._qpytube.download(
self.cmb_resolutions.currentText(), self.le_directory.text()
)
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.le_directory.setEnabled(False)
def handle_download_started(self):
self.lbl_error.clear()
print("started")
def handle_download_finished(self):
self.pgb_download.setValue(100)
self.btn_search.setEnabled(True)
self.btn_download.setEnabled(True)
self.le_directory.setEnabled(True)
print("finished")
def main(args):
app = QtWidgets.QApplication(args)
w = MainWindow()
w.show()
app.exec_()
if __name__ == "__main__":
main(sys.argv)
I have created a desktop application by PYQT5 and python 3.7 to download a video by clicking the download button and save it locally in the PC.
The code will fetch the video link from (lineEdit.text()) which is labeled "URL" and save it in the local directory in (lineEdit_2.text()) which is labeled "SAVE AS". If the download stops for any reason, it will be resumed again by press the start download button. In addition, the ProgressBar start from 1% until 100% along with downloading the video. Everything working smoothly.
The question is, once the video stops in the middle for any reason then it resumes the downloading again but the ProgressBar should start from where it stopped but it is not. For example, if it stops in 50% then should be resumed from 50% and continue. However, it starts from 0% (from the beginning).
```def curl_progress(self,total, existing, totalfrac,fracmb):
global frac,tsize,size,save_location
try:
frac= float(existing)/float(total)
self.progressBar.setValue(totalfrac)
QApplication.processEvents()
except (ZeroDivisionError, RuntimeError, TypeError, NameError):
frac = 0
self.textBrowser.append("Downloaded %d/%d %d%%" % (existing, total, totalfrac))
if frac ==1.0:
self.textBrowser.append("")
size = os.path.getsize(save_location)
tsize= (size /1024 /1024)
QMessageBox.information(self,"Download Completed", "The Download is Finished and the size is %03.2f MB" %(tsize,))
self.textBrowser.append('Size of file is %03.2f MB' %(tsize,))
self.progressBar.setValue(0)
self.lineEdit.setText('')
self.lineEdit_2.setText('')
QMessageBox.close(self)
else:
self.textBrowser.append("Downloaded %d/%d %d%%" % (existing, total, totalfrac))
def curl_limit_rate(self,rate_limit):
global tsize,size,save_location
url= self.lineEdit.text()
save_location = self.lineEdit_2.text()
if len(url) == 0 and len(save_location) == 0:
QMessageBox.information(self, "Error", "Please put the links")
return
if len(url) > 0 and len(save_location) == 0:
QMessageBox.information(self, "Error", "Please put the location")
return
if len(url) == 0 and len(save_location) > 0:
QMessageBox.information(self, "Error", "Please put the link")
return
if len(url) > 0 and len(save_location) > 0:
c = pycurl.Curl()
c.setopt(pycurl.CAINFO, certifi.where())
c.setopt(c.URL,url)
c.setopt(c.MAX_RECV_SPEED_LARGE, rate_limit)
if os.path.exists(save_location):
file_id = open(save_location, "ab")
c.setopt(c.RESUME_FROM, os.path.getsize(save_location))
else:
file_id = open(save_location, "wb")
c.setopt(c.WRITEDATA, file_id)
c.setopt(c.NOPROGRESS, 0)
c.setopt(c.PROGRESSFUNCTION, self.curl_progress)
c.perform()
c.close()
else:
QMessageBox.information(self, "Error", "Unknown error!")```
The picture
Many thanks in advance,
Before pointing out the solution, I must point out that you should not run pycurl in the main thread since it is blocking, instead you must execute it in another thread and send the information to the main thread so that it can be shown.
Going to the point, the idea is that when you calculate the percentage it is using the following formula:
progress = 100 * (bytes_downloaded + size_of_resume_file) / (total_bytes + size_of_resume_file)
Considering the above the solution is:
import os
import certifi
import pycurl
from PyQt5 import QtCore, QtWidgets
class Downloader(QtCore.QObject):
started = QtCore.pyqtSignal()
finished = QtCore.pyqtSignal()
progressChanged = QtCore.pyqtSignal(int)
error = QtCore.pyqtSignal(int, str)
bytesChanged = QtCore.pyqtSignal(int, int)
#QtCore.pyqtSlot(str, str)
def download(self, url, save_location):
pass
class PycURLDownloader(Downloader):
def __init__(self, parent=None):
super().__init__(parent)
self._resume_size = 0
self._c = pycurl.Curl()
self._flag_stop = 0
def download(self, url, save_location):
self._flag_stop = 0
exist_path = os.path.exists(save_location)
self.started.emit()
with open(save_location, "ab" if exist_path else "wb") as file_id:
self._c.setopt(pycurl.CAINFO, certifi.where())
self._c.setopt(pycurl.URL, url)
self._c.setopt(pycurl.MAX_RECV_SPEED_LARGE, 1024)
if exist_path:
self._c.setopt(pycurl.RESUME_FROM, os.path.getsize(save_location))
self._resume_size = os.path.getsize(save_location)
else:
self._resume_size = 0
self._c.setopt(pycurl.WRITEDATA, file_id)
self._c.setopt(pycurl.NOPROGRESS, 0)
self._c.setopt(pycurl.PROGRESSFUNCTION, self._progress_callaback)
try:
self._c.perform()
except pycurl.error as e:
self.error.emit(*e.args)
else:
self.finished.emit()
self._c.close()
#QtCore.pyqtSlot()
def stop(self):
self._flag_stop = 1
def _progress_callaback(self, total, existing, totalfrac, fracmb):
frac = 0
if existing > 0 and total > 0:
frac = int(
100 * (existing + self._resume_size) / (total + self._resume_size)
)
self.bytesChanged.emit(existing, total)
self.progressChanged.emit(frac)
if QtCore.QThread.currentThread().isInterruptionRequested():
return 1
class Widget(QtWidgets.QWidget):
def __init__(self, parent=None):
super().__init__(parent)
self.url_lineedit = QtWidgets.QLineEdit()
self.save_location_lineedit = QtWidgets.QLineEdit()
browse_button = QtWidgets.QPushButton(self.tr("Browse"))
self.download_progressbar = QtWidgets.QProgressBar(minimum=0, maximum=100)
self.download_log_browser = QtWidgets.QTextBrowser()
self.start_download_button = QtWidgets.QPushButton(self.tr("Start Download"))
widget = QtWidgets.QWidget()
widget.setContentsMargins(0, 0, 0, 0)
hlay = QtWidgets.QHBoxLayout(widget)
hlay.addWidget(self.save_location_lineedit)
hlay.addWidget(browse_button)
hlay.setContentsMargins(0, 0, 0, 0)
flay = QtWidgets.QFormLayout()
flay.addRow("URL", self.url_lineedit)
flay.addRow("Save as", widget)
flay.addRow("", self.download_progressbar)
flay.addRow("", QtWidgets.QLabel(self.tr("Packets output in Bytes")))
flay.addRow("", self.download_log_browser)
hlay2 = QtWidgets.QHBoxLayout()
hlay2.addStretch()
hlay2.addWidget(self.start_download_button)
hlay2.addStretch()
vlay = QtWidgets.QVBoxLayout(self)
vlay.addLayout(flay)
vlay.addLayout(hlay2)
self.start_download_button.clicked.connect(self.start_download)
browse_button.clicked.connect(self.select_save_location)
self._thread = QtCore.QThread(self)
self._thread.start()
self._downloader = PycURLDownloader()
self._downloader.moveToThread(self._thread)
self._downloader.progressChanged.connect(self.download_progressbar.setValue)
self._downloader.bytesChanged.connect(self.on_bytesChanged)
self._downloader.started.connect(self.on_started)
self._downloader.finished.connect(self.on_finished)
self.url_lineedit.setText("http://techslides.com/demos/sample-videos/small.mp4")
#QtCore.pyqtSlot()
def start_download(self):
url = self.url_lineedit.text()
save_location = self.save_location_lineedit.text()
if not url:
QtWidgets.QMessageBox.information(self, "Error", "Please put the links")
return
elif not save_location:
QtWidgets.QMessageBox.information(self, "Error", "Please put the location")
return
wrapper = partial(self._downloader.download, url, save_location)
QtCore.QTimer.singleShot(0, wrapper)
#QtCore.pyqtSlot()
def select_save_location(self):
filename, _ = QtWidgets.QFileDialog.getSaveFileName(self, "Select")
if filename:
self.save_location_lineedit.setText(filename)
#QtCore.pyqtSlot(int, str)
def on_error(self, t, msg):
QtWidgets.QMessageBox.information(self, "Error", msg)
#QtCore.pyqtSlot(int, int)
def on_bytesChanged(self, existing, total):
self.download_log_browser.append(
"Downloaded %d/%d %d%%" % (existing, total, 100 * existing / total)
)
#QtCore.pyqtSlot()
def on_started(self):
self.start_download_button.setEnabled(False)
#QtCore.pyqtSlot()
def on_finished(self):
self.start_download_button.setEnabled(True)
def closeEvent(self, event):
self._thread.requestInterruption()
self._thread.quit()
self._thread.wait()
super().closeEvent(event)
if __name__ == "__main__":
from functools import partial
import sys
app = QtWidgets.QApplication(sys.argv)
w = Widget()
w.resize(640, 480)
w.show()
ret = app.exec_()
sys.exit(ret)
I'm new at phyton programming and developing gui interface has gtk framework and serial port. It has a treeview whose liststore model. I could the insert new rows easily.
I'm using the serialport recive callback in different thread from main gui thread avoid to not miss any data. After the received a new data, it should update the treeview. However, since the serialport is in different thread, I don't know how to update the list. Please help me to do this.
the gui class:
class MainGUI():
def __init__(self):
self.builder = Gtk.Builder()
self.builder.add_from_file("main.glade")
self.builder.connect_signals(MainGUI)
self.window = self.builder.get_object("window1")
self.mycombobox = self.builder.get_object('comboboxtext1')
self.toggle = self.builder.get_object('togglebutton1')
self.table = self.builder.get_object('treeview2')
self.list = self.builder.get_object('liststore1')
self.scroll_window = self.builder.get_object('scrolledwindow1')
def show(self):
print("App main thread number", format(threading.get_ident()))
self.window.show()
Gtk.main()
#staticmethod
def connect_toggled(_self):
if main.toggle.get_active():
main.toggle.set_label("Disconnect")
serial_port.connect(main.mycombobox.get_active_text())
t3 = threading.Thread(target=serial_port.read_from_port)
t3.start()
serial_port.disconnect()
def row_inserted_event(self, path, iter):
"""The actual scrolling method"""
adj = main.scroll_window.get_vadjustment()
adj.set_value(adj.get_upper() - adj.get_page_size())
def update_table(self):
# for i in range(256):
# main.list.append(['aaa', 'ddds', i])
# if len(main.list) > 50:
# main.list.remove(main.list.get_iter(0))
main.list.append(['aaa', 'ddds', 0])
if len(main.list) > 50:
main.list.remove(main.list.get_iter(0))
print(len(main.list))
if __name__ == "__main__":
serial_port = SerialPort()
ports = SerialPort().list_ports()
main = MainGUI()
for port in ports:
main.mycombobox.append_text(port)
main.mycombobox.set_active(0)
main.toggle.set_label("Connect")
main.update_table()
main.show()
the serial port class:
class SerialPort:
def __init__(self):
self.ser = serial.Serial()
self.baud_rate = 115200
def write(self, data):
self.ser.write(bytes(data))
print(data)
def connect(self, port):
print("serial port thread number = %d" % (threading.get_ident()))
print("connected the port = %s" % (port))
self.ser.port = port
self.ser.baudrate = self.baud_rate
self.ser.timeout = 0
if self.ser.isOpen():
print("already connected this port = %s" % (port))
else:
self.ser.open()
def disconnect(self):
if self.ser.isOpen():
self.ser.close()
print("disconnected port")
def read_from_port(self):
while True:
if self.ser.isOpen():
reading = self.ser.readline()
if len(reading) > 0:
self.received_callback(reading)
time.sleep(0.1)
def received_callback(self, data):
print(data)
def list_ports(self):
if sys.platform.startswith('win'):
ports = ['COM%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
# this excludes your current terminal "/dev/tty"
# ports = glob.glob('/dev/tty[A-Za-z]*')
ports = ['/dev/pts/%s' % (i + 1) for i in range(256)]
elif sys.platform.startswith('darwin'):
ports = glob.glob('/dev/tty.*')
else:
raise EnvironmentError('Unsupported platform')
result = []
for port in ports:
try:
s = serial.Serial(port)
s.close()
result.append(port)
except (OSError, serial.SerialException):
pass
return result
I believe that your problem is more related to threading + GUI, than GTK.
As far as I know, when you modify the liststore that is the model for the treeview, the latter should be updated instantly. So, there should be no problem there.
A fundamental principle when working with threads and a GUI, is that you should only update the GUI from within its own thread (main loop). So what you need to do, is have your worker thread (serial port connection thread) send the update to the main GUI thread and let it update the treeview. The update can be scheduled with the GLib.idle_add function to let GTK do it when most convenient.
Now, to communicate between threads, you could use the queue module.
I don't quite understand your code. So I'll write a simple example (using gtk3 PyGObject, since you didn't specify).
import threading
import queue
import gi
gi.require_version('Gtk', '3.0')
gi.require_version('GLib', '2.0')
from gi.repository import Gtk, GLib
def do_work(com_queue):
# do some work
com_queue.put("update for your treeview")
# continue
class MainGUI(object):
def __init__(self):
self.com_queue = queue.Queue()
self.worker_thread = None
self.liststore = None
# more gui initialization...
def launch_worker_thread(self):
self.worker_thread = threading.Thread(target=do_work, args=(self.com_queue,))
self.worker_thread.start()
Glib.timeout_add(1000, self.check_queue) # run check_queue every 1 second
def check_queue(self):
if self.worker_thread.is_alive():
try:
update = self.com_queue.get()
GLib.idle_add(self.update_treeview, (update,)) # send tuple
except queue.Empty:
pass
return True # to keep timeout running
else:
return False # to end timeout
def update_treeview(self, update):
self.liststore.append(update) # here update the treeview model with tuple
if __name__ == "__main__":
gui = MainGUI()
Gtk.main()
I hope this helps.
Background :
I have a script that allows me to make spatial queries on a PostgreSQL database via an API coming from a private editor (I can't directly query the database). This API is working with python 3.2. To summarize very quickly, this script is used to download elements of this database in the desired geographical footprint. Depending of the zone, you can obtain between 1 to over 100 elements, each of them having very different sizes (from Ko to Go).
The main window let you to set all options and then start the global process. When launched a console window appears letting you see what’s going on. Once an item has been downloaded a short “report” is displayed on the console. Currently everything is done sequentially one element at a time. As you can imagine, if this element is quite large, the console freezes while waiting for the end of the download process.
Code:
I'm not going to post the full script here, but through a very simple script I will try to show the main problem I'm trying to solve (i.e. avoid locking the user interface / have some sort of real-time output on what's going on).
So, in order to avoid these freezing problems, the use of threads seemed to me to be the best solution. To simulate the download process (see previous chapter) I used the url.request urlretrieve method with multiple urls (pointing to files of different sizes).
import os
import sys
import time
import urllib.request
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class DownloadWorkerSignals(QtCore.QObject):
"""
Defines the signals available from a running download worker thread.
"""
finished = QtCore.pyqtSignal(str)
# ---------------------------------------------------------------------------------
class DownloadWorker(QtCore.QRunnable):
"""
Worker thread
"""
def __init__(self, url, filepath, filename, index):
super(DownloadWorker, self).__init__()
self.url = url
self.file_path = filepath
self.filename = filename
self.index = index
self.signals = DownloadWorkerSignals()
#QtCore.pyqtSlot(str)
def run(self):
t = time.time()
message = 'Thread %d started\n' % self.index
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.file_path,
self.filename))
except IOError as error:
message += str(error) + '\n'
finally:
message += 'Thread %d ended %.2f s\n' % (self.index, time.time() - t)
self.signals.finished.emit(message) # Done
# ---------------------------------------------------------------------------------
class Main(QtGui.QMainWindow):
"""
Main window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Main")
self.setWindowModality(QtCore.Qt.ApplicationModal)
self.centralwidget = QtGui.QWidget(self)
self.setCentralWidget(self.centralwidget)
# Ok / Close
# -------------------------------------------------------------------------
self.buttonBox = QtGui.QDialogButtonBox(self.centralwidget)
self.buttonBox.setStandardButtons(QtGui.QDialogButtonBox.Cancel |
QtGui.QDialogButtonBox.Ok)
self.buttonBox.setGeometry(QtCore.QRect(10, 160, 380, 20))
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.buttonBox,
QtCore.SIGNAL('accepted()'),
self.button_ok_clicked)
self.connect(self.buttonBox,
QtCore.SIGNAL('rejected()'),
self.button_cancel_clicked)
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
self.close()
def button_ok_clicked(self):
# Launch console
console = Console(parent=self)
console.exec_()
# ---------------------------------------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self, parent):
super(self.__class__, self).__init__()
self.parent = parent
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.verticalLayout = QtGui.QVBoxLayout(self)
# Text edit
# -------------------------------------------------------------------------
self.text_edit = QtGui.QPlainTextEdit(self)
self.text_edit.setReadOnly(True)
self.text_edit_cursor = QtGui.QTextCursor(self.text_edit.document())
self.verticalLayout.addWidget(self.text_edit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.verticalLayout.addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.button_cancel_clicked)
# Post initialization
# -------------------------------------------------------------------------
self.threadpool = QtCore.QThreadPool()
self.threadpool.setMaxThreadCount(2)
for index, url in enumerate(urls):
worker = DownloadWorker(url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
worker.signals.finished.connect(self.write_message)
self.threadpool.start(worker)
'''
I have to wait for the end of the thread pool to make a post-processing.
If I use the waitForDone I don't see my console until the all work is done
'''
# self.threadpool.waitForDone()
# self.write_stram('Thread pool finished')
# Connect functions
# -----------------------------------------------------------------------------
def button_cancel_clicked(self):
if self.threadpool.activeThreadCount() != 0:
pass # How to interrupt the threadpool ?
self.close()
#QtCore.pyqtSlot(str)
def write_message(self, text):
self.text_edit.insertPlainText(text)
cursor = self.text_edit.textCursor()
self.text_edit.setTextCursor(cursor)
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Main()
window.show()
app.exec_()
Questions:
Everything seems to work as expected but I encounter two difficulties:
At the end of the thread pool process I have to make some
post-processing. If I use the waitForDone method I don't see my
console until the all work is done and it’s not the type of behavior
wanted.
If the Cancel Button in the Console is clicked, I need to interrupt
the threadpool and I don’t know how to manage that.
I had another look at this problem (based largely on this : how-do-i-maintain-a-resposive-gui-using-qthread-with-pyqgis).
So I replaced the previous tandem QThreadPool/QRunnable, by Queue/QThread. The code below gives an overview.
import os
import sys
import time
import urllib.request
import queue
from PyQt4 import QtCore, QtGui
url_1m = 'http://ipv4.sbg.proof.ovh.net/files/1Mio.dat'
url_10m = 'http://ipv4.sbg.proof.ovh.net/files/10Mio.dat'
url_100m = 'http://ipv4.sbg.proof.ovh.net/files/100Mio.dat'
url_1g = 'http://ipv4.sbg.proof.ovh.net/files/1Gio.dat'
url_10g = 'http://ipv4.sbg.proof.ovh.net/files/10Gio.dat'
urls = (url_1m, url_10m, url_100m, url_1g, url_10g)
# ---------------------------------------------------------------------------------
class WorkerThread(QtCore.QThread):
"""
Worker thread
"""
def __init__(self, parent_thread):
QtCore.QThread.__init__(self, parent_thread)
def run(self):
self.running = True
success = self.do_work()
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def stop(self):
self.running = False
pass
def do_work(self):
return True
def clean_up(self):
pass
# ---------------------------------------------------------------------------------
class LongRunningTask(WorkerThread):
def __init__(self, parent_thread, url, filepath, filename, index):
WorkerThread.__init__(self, parent_thread)
self.url = url
self.filepath = filepath
self.filename = filename
self.index = index
def do_work(self):
t = time.time()
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'), 'Thread %d started\n' % self.index)
try:
# The urlretrieve method will copy a network object to a local file
urllib.request.urlretrieve(url=self.url,
filename=os.path.join(self.filepath,
self.filename))
except IOError as error:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d error - ' % self.index + str(error) + '\n')
finally:
self.emit(QtCore.SIGNAL('threadText(PyQt_PyObject)'),
'Thread %d ended %.2f s\n' % (self.index, time.time() - t))
return True
# ---------------------------------------------------------------------------------
class Console(QtGui.QDialog):
"""
Console window
"""
def __init__(self):
super(self.__class__, self).__init__()
self.resize(400, 200)
self.setWindowTitle("Console")
self.setModal(True)
self.setLayout(QtGui.QVBoxLayout())
# Text edit
# -------------------------------------------------------------------------
self.textEdit = QtGui.QPlainTextEdit(self)
self.textEdit.setReadOnly(True)
self.textEdit_cursor = QtGui.QTextCursor(self.textEdit.document())
self.layout().addWidget(self.textEdit)
# Ok / Close
# -------------------------------------------------------------------------
self.button_box = QtGui.QDialogButtonBox(self)
self.button_box.setStandardButtons(QtGui.QDialogButtonBox.Close)
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(False)
self.layout().addWidget(self.button_box)
# Connect definition
# -------------------------------------------------------------------------
self.connect(self.button_box.button(QtGui.QDialogButtonBox.Close),
QtCore.SIGNAL('clicked()'),
self.reject)
# Post-Initialization
# -------------------------------------------------------------------------
self.queue = queue.Queue()
# self.queue = queue.Queue(maxsize=2)
self.run_thread()
# Connect functions
# -----------------------------------------------------------------------------
def cancel_thread(self):
self.workerThread.stop()
def job_finished_from_thread(self, success):
self.workerThread.stop()
self.queue.get()
# Stop the pulsation
if self.queue.empty():
self.button_box.button(QtGui.QDialogButtonBox.Close).setEnabled(True)
self.emit(QtCore.SIGNAL('jobFinished(PyQt_PyObject)'), success)
def text_from_thread(self, value):
self.textEdit.insertPlainText(value)
cursor = self.textEdit.textCursor()
self.textEdit.setTextCursor(cursor)
def run_thread(self):
for index, url in enumerate(urls):
self.workerThread = LongRunningTask(parent_thread=self,
url=url,
filepath='C:\\Users\\philippe\\Downloads',
filename='url_%d.txt' % index,
index=index)
self.connect(self.workerThread,
QtCore.SIGNAL('jobFinished(PyQt_PyObject)'),
self.job_finished_from_thread)
self.connect(self.workerThread,
QtCore.SIGNAL('threadText(PyQt_PyObject)'),
self.text_from_thread)
self.queue.put(self.workerThread)
self.workerThread.start()
# If I set the queue to maxsize=2, how to manage it here
'''
while not self.queue.full():
self.queue.put(self.workerThread)
self.workerThread.start()
'''
# ---------------------------------------------------------------------------------
if __name__ == '__main__':
app = QtGui.QApplication(sys.argv)
window = Console()
window.show()
app.exec_()
Question:
Unfortunately, I encounter other types of difficulties. In reality, the queue can contain a large amount of threads (over 100). 1. How can I, like the QthreadPool and its setMaxThreadCount method, manage the number of threads running in parallel in order to prevent the system from collapsing completely ?
I updated somethings like calling the on_event method from the thread class and now i have this code :
# -*- coding: utf-8 -*-
from PyQt4 import QtCore, QtGui
import sys, pyodbc, serial
import os
import time
import thread
from PyQt4.QtCore import pyqtSignal
from PyQt4.QtCore import QObject, pyqtSignal
#Variables
Code_Zone = "d"
class MainWindow(QtGui.QWidget):
def __init__(self, main):
super(MainWindow, self).__init__()
self.main = main
self.grid = QtGui.QGridLayout(self)
self.welcome = QtGui.QLabel("Bienvenue sur www.developpez.net", self)
self.grid.addWidget(self.welcome, 2, 2, 1, 5)
class AccessWindow(QtGui.QWidget):
def __init__(self):
super(AccessWindow, self).__init__()
self.setMinimumSize(150, 50)
self.grid = QtGui.QGridLayout(self)
self.label = QtGui.QLabel(self)
self.grid.addWidget(self.label, 1, 1, 1, 1)
class Main(object):
global EPC_Code
def __init__(self):
self.accueil = MainWindow(self)
self.accueil.show()
self.access = AccessWindow()
def on_event(self, data):
def refresh():
self.toggle_widget(False)
#c = AThread()
c.run()
#self.wait()
# vérification des données
EPC_Code = data
cnxn = """DRIVER={SQL Server};SERVER=Aziz-PC\SQLEXPRESS;PORT=1433;UID=poleindus;PWD=poleindus;DATABASE=indusdb"""
db_connection = pyodbc.connect(cnxn)
db_cursor = db_connection.cursor()
print ('Connected TO DB & READY')
sql_command = "EXECUTE [Get_Access_RFID] #Code_RFID = '"+EPC_Code+"', #Code_Zone = '"+Code_Zone+"'"
db_cursor.execute("EXECUTE [Get_Access_RFID] #Code_RFID = '"+EPC_Code+"', #Code_Zone = '"+Code_Zone+"'")
rows = db_cursor.fetchone()
result= str(rows[0])
print ("result = " + str(result))
if result == "True":
# si OK
self.access.label.setText('ACCESS GRANTED')
else:
# si pas OK
self.access.label.setText('ACCESS DENIED')
self.toggle_widget(True)
QtCore.QTimer.singleShot(2000, refresh)
def toggle_widget(self, b):
self.accueil.setVisible(not b)
self.access.setVisible(b)
class AThread(QtCore.QThread):
rfid_event = pyqtSignal(str, name='rfid_event')
def run(self):
while 1:
ser = serial.Serial(port='COM6', baudrate=115200)
a = ser.read(19).encode('hex')
ser.close()
if len(a) == 38:
EPC_Code = a[14:]
print ('EPC is : ' + EPC_Code)
self.rfid_event.emit(EPC_Code)
if __name__=='__main__':
#cnxn = """DRIVER={SQL Server};SERVER=Aziz-PC\SQLEXPRESS;PORT=1433;UID=poleindus;PWD=poleindus;DATABASE=indusdb"""
#db_connection = pyodbc.connect(cnxn)
#db_cursor = db_connection.cursor()
#print ('Connected TO DB & READY')
app = QtGui.QApplication(sys.argv)
main = Main()
thread = AThread()
thread.rfid_event.connect(Main().on_event)
thread.start()
#main = Main()
sys.exit(app.exec_())
I have already added this line at the top of the code and now, i have another error: The first argument to execute must be a string or unicode query.when i checked the value of the sql command, i find it equals to: PyQt4.QtCore.QString(u"EXECUTE [Get_Access_RFID] #Code_RFID = 'e20010712904019411709bdd', #Code_Zone = 'd'")
You create a new Main in AThread. That means you now have two of them (and worse, you create one every time you process an RFID event).
Also, you're not allowed to call methods from Main from inside a thread. Imagine what would happen if the thread would call them again before the first call would return: Data would be corrupted.
Instead, you must declare a signal in AThread and call emit() to send it. Your Main from the last block of code can then connect to this signal.
class AThread(QtCore.QThread):
rfid_event = pyqtSignal(str, name='rfid_event')
def run(self):
while 1:
ser = serial.Serial(port='COM6', baudrate=115200)
a = ser.read(19).encode('hex')
ser.close()
if len(a) == 38:
EPC_Code = a[14:]
print ('EPC is : ' + EPC_Code)
self.rfid_event.emit(EPC_Code)
and later in the __main__ code:
main = Main()
thread = AThread()
thread.rfid_event.coonect(Main.on_event)
thread.start()
#main = Main()
sys.exit(app.exec_())
Also note that thread.finished.connect(app.exit) won't work: The thread contains an endless loop and never finishes. Instead, you should terminate the thread when the main window is closed.
I also fixed access to the serial port; you must close it every time.