We Keep Coding

how to update a progress bar in a pyqt5 program and the pytube library [duplicate]

I have made a Desktop Application using Python and used PyQt5 and Pytube which could download video from youtube. When download is in Progress, I want to show user an animation. In Fact I did it, but when the file is getting downloaded the PyQt window seems like freezing and everything just gets paused until the download is complete. So, Does anyone know why is this happening? How do I fix it?
Here's the code snippet:
def download_created(self, qual): # Used in 'selection' method
selected_stream = yt.streams.get_by_resolution(qual)
self.progress_func()
try:
self.download_btn.setCurrentIndex(-1)
selected_stream.download(self.askLocation() + "/")
except:
pass
# This gets the quality that the user chooses
def selection(self):
global quality
quality = self.download_btn.currentText()
try:
self.download_created(quality) # Calls a method called 'download'
except:
self.start_anime()
# Fetching the details about the Link from Youtube
def download_youtube(self):
global check
if check != self.get_input():
check = self.get_input()
self.download_btn.clear()
enter_url = self.get_input()
try:
global yt
yt = pytube.YouTube(
enter_url,
on_progress_callback = on_progress,
on_complete_callback = self.complete_func)
self.start_anime()
except:
self.input_error()
VIDEO_TITLE = (yt.title)
global VIDEO_ID
VIDEO_ID = (yt.video_id)
videos = yt.streams.filter(mime_type="video/mp4", progressive="True")
# Display all the available qualities
for i in videos:
self.download_btn.addItem(i.resolution)
self.download_btn.currentIndexChanged.connect(self.selection)

You have to execute the time consuming tasks in another thread, for example in your case the task of getting the streams and downloading.
import sys
import threading
from functools import cached_property
from PyQt5 import QtCore, QtWidgets
import pytube
class QPyTube(QtCore.QObject):
initialized = QtCore.pyqtSignal(bool, str)
download_started = QtCore.pyqtSignal()
download_progress_changed = QtCore.pyqtSignal(int)
download_finished = QtCore.pyqtSignal()
def __init__(self, url):
super().__init__()
self._url = url
self._yt = None
self._mutex = threading.Lock()
threading.Thread(target=self._init, daemon=True).start()
#property
def url(self):
return self._url
#cached_property
def resolutions(self):
return list()
def _init(self):
with self._mutex:
self.resolutions.clear()
try:
self._yt = pytube.YouTube(
self.url,
on_progress_callback=self._on_progress,
on_complete_callback=self._on_complete,
)
streams = self._yt.streams.filter(mime_type="video/mp4", progressive="True")
except Exception as e:
self.initialized.emit(False, str(e))
return
with self._mutex:
self.resolutions = [stream.resolution for stream in streams]
self.initialized.emit(True, "")
def download(self, resolution, directory):
threading.Thread(
target=self._download, args=(resolution, directory), daemon=True
).start()
def _download(self, resolution, directory):
stream = self._yt.streams.get_by_resolution(resolution)
self.download_started.emit()
stream.download(directory)
def _on_progress(self, stream, chunk, bytes_remaining):
self.download_progress_changed.emit(
100 * (stream.filesize - bytes_remaining) // stream.filesize
)
def _on_complete(self, stream, filepath):
self.download_finished.emit()
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, parent=None):
super().__init__(parent)
self.le_url = QtWidgets.QLineEdit("http://youtube.com/watch?v=2lAe1cqCOXo")
self.lbl_error = QtWidgets.QLabel()
self.btn_search = QtWidgets.QPushButton("Search")
self.cmb_resolutions = QtWidgets.QComboBox()
self.le_directory = QtWidgets.QLineEdit("")
self.btn_download = QtWidgets.QPushButton("Download")
self.pgb_download = QtWidgets.QProgressBar()
central_widget = QtWidgets.QWidget()
self.setCentralWidget(central_widget)
lay = QtWidgets.QGridLayout(central_widget)
lay.addWidget(self.le_url, 0, 0)
lay.addWidget(self.btn_search, 0, 1)
lay.addWidget(self.cmb_resolutions, 1, 0)
lay.addWidget(self.le_directory, 1, 1)
lay.addWidget(self.btn_download, 1, 2)
lay.addWidget(self.pgb_download, 2, 0, 1, 3)
self.btn_download.setEnabled(False)
self._qpytube = None
self.btn_search.clicked.connect(self.handle_search_clicked)
self.btn_download.clicked.connect(self.handle_download_clicked)
def handle_search_clicked(self):
self.cmb_resolutions.clear()
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.lbl_error.clear()
self._qpytube = QPyTube(self.le_url.text())
self._qpytube.initialized.connect(self.handle_initialized)
self._qpytube.download_progress_changed.connect(self.pgb_download.setValue)
self._qpytube.download_started.connect(self.handle_download_started)
self._qpytube.download_finished.connect(self.handle_download_finished)
#QtCore.pyqtSlot(bool, str)
def handle_initialized(self, status, error=""):
if status:
self.cmb_resolutions.addItems(self._qpytube.resolutions)
self.btn_download.setEnabled(True)
else:
self.lbl_error.setText(error)
self.btn_search.setEnabled(True)
def handle_download_clicked(self):
self._qpytube.download(
self.cmb_resolutions.currentText(), self.le_directory.text()
)
self.btn_search.setEnabled(False)
self.btn_download.setEnabled(False)
self.le_directory.setEnabled(False)
def handle_download_started(self):
self.lbl_error.clear()
print("started")
def handle_download_finished(self):
self.pgb_download.setValue(100)
self.btn_search.setEnabled(True)
self.btn_download.setEnabled(True)
self.le_directory.setEnabled(True)
print("finished")
def main(args):
app = QtWidgets.QApplication(args)
w = MainWindow()
w.show()
app.exec_()
if __name__ == "__main__":
main(sys.argv)

PyQT QThread follow the thread execution order (wait)

I'm trying to run a few slow processes but, I need to keep updated the QDialog to show the progress (maybe I put a progress bar too).
So I decide to use QThread, but on the first try, it doesn't work as I expected.
In my example code:
1- I'm using a simple ping to my default gateway
2- I'm pinging to my dns resolver
As you can see on imagem below, the information is showed according the thread is finalizing, but it is a mess to me.
Is possible to respect the threads order to show the informations?
Thanks.
Follow my example code:
# -*- coding: utf8 -*-
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
from ping3 import ping, verbose_ping
import socket
import dns.resolver
class ExternalTests(QThread):
data_collected = pyqtSignal(object)
def __init__(self, title, arg=None):
QThread.__init__(self)
self.title = title
self.arg = arg
def run(self):
resp = ping(self.arg)
self.data_collected.emit('%s: %s' % (self.title, resp))
class MainMenu(QMenu):
def __init__(self, parent=None):
QMenu.__init__(self)
self.setStyleSheet("background-color: #3a80cd; color: rgb(255,255,255); selection-color: white; selection-background-color: #489de4;")
# Diagnostics
self.check_internet = QAction("Diagnosys")
self.check_internet.setIcon(QIcon(QPixmap("..\\img\\lupa.png")))
self.check_internet.triggered.connect(self.diagnosticNetwork)
self.addAction(self.check_internet)
self.addSeparator()
# To quit the app
self.quit = QAction("Quit")
self.quit.triggered.connect(app.quit)
self.addAction(self.quit)
def diagnosticNetwork(self):
self.check_internet_dialog = QDialog()
self.check_internet_dialog.setWindowTitle("Check external connections")
self.check_internet_dialog.setWindowModality(Qt.ApplicationModal)
self.check_internet_dialog.setGeometry(150, 100, 700, 500)
# text box
self.textbox = QTextBrowser(self.check_internet_dialog)
self.textbox.move(20, 20)
self.textbox.resize(660,400)
self.textbox.setFont(QFont("Courier New", 12))
self.textbox.setStyleSheet("background-color: black;")
#button copy
btn_copy = QPushButton("Copy", self.check_internet_dialog)
btn_copy.setIcon(QIcon(QPixmap("..\\img\\copy.png")))
btn_copy.move(520,450)
btn_copy.clicked.connect(self.dialogClickCopy)
#button close
btn_copy = QPushButton("Close", self.check_internet_dialog)
btn_copy.setIcon(QIcon(QPixmap("..\\img\\close.png")))
btn_copy.move(605,450)
btn_copy.clicked.connect(self.dialogClickClose)
# tests
self.textbox.setTextColor(QColor("white"))
self.textbox.append("Diagnosys")
self.textbox.append("--------------------------------------------------")
self.textbox.setTextColor(QColor("cyan"))
self.threads = []
#QCoreApplication.processEvents()
''' ping default gateway '''
ping_default_gw = ExternalTests("default gatewat is reacheble", "192.168.0.1")
ping_default_gw.data_collected.connect(self.onDataReady)
self.threads.append(ping_default_gw)
ping_default_gw.start()
''' ping dns resolver '''
ping_dns_resolvers = dns.resolver.Resolver().nameservers
for dns_resolver in ping_dns_resolvers:
ping_dns_resolver = ExternalTests("dns resolver is reacheble %s" % dns_resolver, dns_resolver)
ping_dns_resolver.data_collected.connect(self.onDataReady)
self.threads.append(ping_dns_resolver)
ping_dns_resolver.start()
self.check_internet_dialog.exec_()
def onDataReady(self, data):
print(data)
if data:
self.textbox.append(data)
else:
self.textbox.append("error")
def dialogClickCopy(self):
pass
def dialogClickClose(self):
self.check_internet_dialog.close()
class SystemTrayIcon(QSystemTrayIcon):
def __init__(self, menu, parent=None):
QSystemTrayIcon.__init__(self)
self.setIcon(QIcon("..\\img\\icon.png"))
self.setVisible(True)
self.setContextMenu(menu)
if __name__ == "__main__":
import sys
app = QApplication([])
app.setQuitOnLastWindowClosed(False)
app.setApplicationName('pkimonitor')
app.setApplicationVersion('0.1')
app.setWindowIcon(QIcon("..\\img\\icon.png"))
menu = MainMenu()
widget = QWidget()
trayIcon = SystemTrayIcon(menu, widget)
trayIcon.show()
sys.exit(app.exec_())

I tried to create a scheme to organize by "run position" and it works. Follow my code.
In 'diagnosticNetwork':
self.data_list = []
self.data_hold = []
In 'onDataReady':
if len(self.data_list) > 0:
if self.data_list[-1][0] + 1 == data[0]:
self.data_list.append(data)
if data[2]:
self.textbox.append("%s: %s" % (data[1], 'OK'))
else:
self.textbox.append("%s: %s" % (data[1], 'NOK'))
elif self.data_list[-1][0] < data[0]:
self.data_hold.append(data)
else:
self.data_list.append(data)
if data[2]:
self.textbox.append("%s: %s" % (data[1], 'OK'))
else:
self.textbox.append("%s: %s" % (data[1], 'NOK'))
if len(self.data_hold) > 0:
hold_sorted = self.data_hold[:]
hold_sorted.sort()
for line_hold in hold_sorted:
if self.data_list[-1][0] + 1 == line_hold[0]:
if line_hold[2]:
self.textbox.append("%s: %s" % (line_hold[1], 'OK'))
else:
self.textbox.append("%s: %s" % (line_hold[1], 'NOK'))
self.data_list.append(line_hold)
del self.data_hold[0]
I worked with two lists, data_list and data_hold. The results that I received from the threads, I filled out the main list according to the order that I pre-established, if the result that entered was not the next one in the sequence, it goes to data_hold, then scan this list to fill the rest of my textbox.
Thank you for all help!

Am I using Qthread correctly?

I have a job to create a stm mcu downloader using uart communication.
I used to python and pyqt5.
Results were bad because of the program GUI freezing when starting the download. (download progress is well)
So I changed the code to the way I use Qthread.
But nothing change.
I have 2 Questions.
Am I use Qthread correctly?
How can I fix GUI freezing when start download?
Attach the code below
from PyQt5.QtWidgets import QMainWindow, QApplication, QFileDialog, QTextBrowser
from PyQt5 import uic
from PyQt5.QtCore import *
import sys, serial, time, struct, os
form_class = uic.loadUiType('USART_Downloader.ui')[0]
class Worker(QObject):
ports=['COM%s'%(i+1)for i in range(256)]
check_connect = 0
serial = serial.Serial()
time_instance = 0.00008
finished = pyqtSignal(list)
finished2 = pyqtSignal(str)
finished3 = pyqtSignal(str)
finished4 = pyqtSignal(int)
#pyqtSlot()
def Find_Com(self):
result = []
for i in self.ports:
try:
temp = serial.Serial(i)
temp.close()
result.append(i)
except serial.SerialException:
pass
self.finished.emit(result)
#pyqtSlot(str)
def Apply_Com(self, COM_Port):
self.check_connect=0
self.serial.close()
try :
self.serial = serial.Serial(COM_Port, 115200, timeout = 1)
self.check_connect = 1
self.finished2.emit('Successfully Port Connection')
except serial.SerialException:
self.finished2.emit('Failure Port Connection')
if self.check_connect :
self.serial.write(b'\x7F')
time.sleep(self.time_instance)
res = self.serial.read()
if res == b'\x79' :
self.finished2.emit('Successfully Enter Boot Mode')
else:
self.finished2.emit('Failure Enter Boot Mode')
#pyqtSlot(str)
def Download_Flash(self, fileName):
if self.check_connect:
self.serial.write(b'\x44')
time.sleep(self.time_instance)
self.serial.write(b'\xBB')
time.sleep(self.time_instance)
res = self.serial.read()
if res == b'\x79' :
self.finished3.emit('Erasing')
else:
self.finished3.emit('Failure Enter Erase Flash')
return
self.serial.write(b'\xFF')
time.sleep(self.time_instance)
self.serial.write(b'\xFF')
time.sleep(self.time_instance)
self.serial.write(b'\x00')
time.sleep(self.time_instance)
res = self.serial.read()
if res == b'\x79' :
self.finished3.emit('Successfully Erase Flash')
else:
self.finished3.emit('Failure Enter Erase Flash')
return
else :
self.finished3.emit('Need USART Connection')
Bit_Num = 256
f = open(fileName, 'rb')
download_binary = f.read()
download_binary_len = len(download_binary)
f.close()
start_address = b'\x00\x00\x00\x08'
total_data = []
while(True):
if(download_binary_len > Bit_Num):
total_data.append(Bit_Num)
download_binary_len -= Bit_Num
elif(download_binary_len > 0):
total_data.append(download_binary_len)
download_binary_len = 0
else:
break
self.finished3.emit('Downloading')
for i, k in enumerate(total_data):
self.serial.write(b'\x31')
time.sleep(self.time_instance)
self.serial.write(b'\xCE')
time.sleep(self.time_instance)
res = self.serial.read()
if res == b'\x79' :
pass
else:
self.finished3.emit('Failure Download1')
return
self.serial.write(start_address[3:4])
time.sleep(self.time_instance)
self.serial.write(start_address[2:3])
time.sleep(self.time_instance)
self.serial.write(start_address[1:2])
time.sleep(self.time_instance)
self.serial.write(start_address[0:1])
time.sleep(self.time_instance)
temp2 = struct.unpack('4B', start_address)
check_sum = temp2[0] ^ temp2[1] ^ temp2[2] ^ temp2[3]
check_sum = struct.pack('B', check_sum)
self.serial.write(check_sum)
time.sleep(self.time_instance)
res = self.serial.read()
if res == b'\x79' :
pass
else:
self.finished3.emit('Failure Download2')
return
check_sum = (k-1)
self.serial.write(struct.pack('B', check_sum))
time.sleep(self.time_instance)
for j in range(k):
self.serial.write(download_binary[(i*Bit_Num)+j:(i*Bit_Num)+j+1])
time.sleep(self.time_instance)
for j in download_binary[(i*Bit_Num):(i*Bit_Num)+j+1] :
check_sum = check_sum ^ j
check_sum = struct.pack('B', check_sum)
self.serial.write(check_sum)
time.sleep(self.time_instance)
res = self.serial.read()
if res == b'\x79' :
pass
else:
self.finished3.emit('Failure Download3')
return
temp3 = struct.unpack('i', start_address)[0]
temp3 = temp3 + Bit_Num
start_address = struct.pack('i', temp3)
self.finished4.emit(i/(len(total_data)-1)*100)
self.finished3.emit('Success Download')
class MyWindowClass(QMainWindow, form_class):
def __init__(self, parent=None):
QMainWindow.__init__(self, parent)
self.setupUi(self)
self.fileName = ''
self.worker_thread = QThread()
self.worker = Worker()
self.worker.moveToThread(self.worker_thread)
self.worker_thread.start()
self.worker.finished.connect(self.Refresh_Com)
self.worker.finished2.connect(self.Print_textBrowser)
self.worker.finished3.connect(self.Print_textBrowser_3)
self.worker.finished4.connect(self.Set_ProgressBar)
self.pushButton.clicked.connect(self.Find_Binary)
self.pushButton_2.clicked.connect(lambda:self.worker.Download_Flash(self.fileName))
self.pushButton_3.clicked.connect(lambda:self.worker.Apply_Com(self.comboBox.currentText()))
self.pushButton_4.clicked.connect(self.worker.Find_Com)
self.textBrowser.setAcceptRichText(True)
self.textBrowser.setOpenExternalLinks(True)
self.textBrowser_3.setAcceptRichText(True)
self.textBrowser_3.setOpenExternalLinks(True)
self.progressBar.reset()
self.worker.Find_Com()
#pyqtSlot(list)
def Refresh_Com(self, result):
self.comboBox.clear()
for i in result:
self.comboBox.addItem(i)
self.textBrowser.append('COM Port Refresh Done')
#pyqtSlot(str)
def Print_textBrowser(self, text):
self.textBrowser.append(text)
#pyqtSlot(str)
def Print_textBrowser_3(self, text):
self.textBrowser_3.append(text)
#pyqtSlot(int)
def Set_ProgressBar(self, percent):
self.progressBar.setValue(percent)
def Find_Binary(self):
options = QFileDialog.Options()
self.fileName, _ = QFileDialog.getOpenFileName(self, "QFileDialog.getOpenFileName()", "", "Binary Files (*.bin)", options=options)
self.lineEdit.setText(self.fileName)
if __name__ == "__main__":
app = QApplication(sys.argv)
myWindow = MyWindowClass()
myWindow.show()
app.exec_()

You're using time.sleep() in your code which blocks your GUI. I would recommend you to use QTimer.singleShot() instead. according from this source, you can use time.sleep equivalent like this QtTest.QTest.qWait(msecs)

Have PyQt5 Labels dynamically update while sending http requests

So I've been working on a async python scraper with http requests based on modules. So for that I've been using asks and importlib, and I'd like to make a simple little GUI that updates with the status codes of requests. And I did.
Everything is great, but I seem to have an issue as the requests are sent sucessfully, the GUI shows up, but it only shows up once all the requests are sent, and not dynamically while the requests are being sent
I've been playing around with tutorials and Qtimer, but in all the tutorials and help threads I've seen such as:
https://www.riverbankcomputing.com/pipermail/pyqt/2013-July/033053.html
PyQt5: Updating Label?
I have tried to implement the code to my situation, but the only thing I've manage to do is for the GUI to show up at the same time the requests are sent, but it stays frozen (not responding) until all the requests are finished
import trio
from asks import Session
import importlib
from PyQt5.QtWidgets import QLabel, QMainWindow, QApplication, QWidget, QVBoxLayout
from PyQt5 import QtCore
import qdarkstyle
app = QApplication(sys.argv)
app.setStyleSheet(qdarkstyle.load_stylesheet_pyqt5())
module = importlib.import_module("get_module")
good = 0
bad = 0
total = 0
class Menu(QMainWindow):
def __init__(self):
global good, bad, total
super().__init__()
self.setWindowTitle("Status Codes")
self.central_widget = QWidget()
self.setCentralWidget(self.central_widget)
lay = QVBoxLayout(self.central_widget)
self.resize(500, 350)
ok = QLabel("200: <font color='green'>0</font>")
ok.setAlignment(QtCore.Qt.AlignHCenter)
bad = QLabel("400: <font color='yellow'>0</font>")
bad.setAlignment(QtCore.Qt.AlignHCenter)
total = QLabel("Total: <font color='#00FF00'>0</font>")
total.setAlignment(QtCore.Qt.AlignHCenter)
r_total, r_good, r_bad = self.check()
QtCore.QTimer.singleShot(1000, lambda: self.updateLabels(r_total, r_good, r_bad))
lay.addWidget(ok)
lay.addWidget(bad)
lay.addWidget(total)
self.show()
def check(self):
async def worker1(s):
global ok
global bad
global total
if module.method.lower() == "get":
r = await s.get(module.request(), params=module.settings())
elif module.method.lower() == "post":
r = await s.post(module.request(), data=module.settings())
if any(x in r.status_code for x in module.error):
print("BAD -- " + module.request())
r_total += 1
r_invalid += 1
else:
print("GOOD -- " + module.request())
r_total += 1
r_valid += 1
print(r.text)
async def worker2(s):
global ok
global bad
global total
if module.method.lower() == "get":
r = await s.get(module.request(), params=module.settings())
elif module.method.lower() == "post":
r = await s.post(module.request(), data=module.settings())
if any(x in r.status_code for x in module.error):
print("BAD -- " + module.request())
r_total += 1
r_invalid += 1
else:
print("GOOD -- " + module.request())
r_total += 1
r_valid += 1
print(r.text)
async def example():
s = Session(connections=module.connections)
for i in range(10):
async with trio.open_nursery() as nursery:
nursery.start_soon(worker1, s)
nursery.start_soon(worker2, s)
trio.run(example)
print("Total:", r_total)
print("Total good:", r_valid)
print("Total bad:", r_invalid)
return r_total, r_valid, r_invalid
def updateLabels(self, r_total, r_card, r_invalid):
good.setText("200: <font color='green'>%s</font>" % (r_valid))
bad.setText("400: <font color='#00FF00'>%s</font>" % (r_invalid))
total.setText("Total: <font color='#F40D30'>%s</font>" % (r.total))
if __name__ == '__main__':
ex = Menu()
sys.exit(app.exec_())
Now what I would like it to do is that the GUI shows up and that dynamically (or every 1 second) the 200, 400 and total labels show how many requests have been made, and how many returned 200 and 400.
However instead it'll show them (it does show the total, total 200 and total 400) but only once all the requests are finished and not dynamically on the side

Asynchronous tasks block the thread of the GUI, so those tasks must be executed in another thread and send it through signals to the GUI, also that allows us to separate the business logic and the GUI having a cleaner code:
get_module.py
# coding: utf8
#======================= IMPORT AREA =======================
#here's where you import any module needed for the website
from random import randint
#===========================================================
#====================== SETTINGS AREA ======================
#here's where you declare the settings of the website such
#as method, error key, success key, custom settings, etc...
name = "GET Test Config"
method = 'GET' #Method is either GET, POST or CUSTOM
error = ['400', '401', '402', '403', '404', '405', '406', '407', '408', '409', '410', '411', '412', '413', '414', '415', '416', '417', '418', '421', '422', '423', '424', '425', '426', '428', '429', '431', '451','500', '501', '502', '503', '504', '505', '506', '507', '508', '510', '511']
connections = 5
#===========================================================
#====================== DEFINITON AREA =====================
#here's where the definitions are made.
#There's 2 defs:
#def request(): Which returns the url (with or without modifications)
#def settings(): returns the data to send in the GET request
#====== SETTINGS AREA ======
def request():
url = "https://httpbin.org/anything"
return url
def settings():
data = {'example':'example'}
return (data)
#===========================================================
main.py
import importlib
import multio
import qdarkstyle
import trio
from PyQt5 import QtCore, QtWidgets
from asks import Session
module = importlib.import_module("get_module")
class TaskWorker(QtCore.QObject):
totalChanged = QtCore.pyqtSignal(int)
validChanged = QtCore.pyqtSignal(int)
invalidChanged = QtCore.pyqtSignal(int)
def __init__(self, parent=None):
super(TaskWorker, self).__init__(parent)
self._total = 0
self._valid = 0
self._invalid = 0
#QtCore.pyqtProperty(int, notify=totalChanged)
def total(self):
return self._total
#total.setter
def total(self, value):
if self._total == value:
return
self._total = value
self.totalChanged.emit(self._total)
#QtCore.pyqtProperty(int, notify=validChanged)
def valid(self):
return self._valid
#valid.setter
def valid(self, value):
if self._valid == value:
return
self._valid = value
self.validChanged.emit(self._valid)
#QtCore.pyqtProperty(int, notify=invalidChanged)
def invalid(self):
return self._invalid
#invalid.setter
def invalid(self, value):
if self._invalid == value:
return
self._invalid = value
self.invalidChanged.emit(self._invalid)
#QtCore.pyqtSlot()
def check(self):
async def worker1(s):
if module.method.lower() == "get":
r = await s.get(module.request(), params=module.settings())
elif module.method.lower() == "post":
r = await s.post(module.request(), data=module.settings())
# if any(x in r.status_code for x in module.error):
if str(r.status_code) in module.error:
print("BAD -- " + module.request())
self.total += 1
self.invalid += 1
else:
print("GOOD -- " + module.request())
self.total += 1
self.valid += 1
print(r.text)
async def worker2(s):
if module.method.lower() == "get":
r = await s.get(module.request(), params=module.settings())
elif module.method.lower() == "post":
r = await s.post(module.request(), data=module.settings())
# if any(x in r.status_code for x in module.error):
if str(r.status_code) in module.error:
print("BAD -- " + module.request())
self.total += 1
self.invalid += 1
else:
print("GOOD -- " + module.request())
self.total += 1
self.valid += 1
print(r.text)
async def example():
s = Session(connections=module.connections)
for i in range(40):
async with trio.open_nursery() as nursery:
nursery.start_soon(worker1, s)
nursery.start_soon(worker2, s)
multio.init("trio")
trio.run(example)
class Menu(QtWidgets.QMainWindow):
def __init__(self):
super().__init__()
self.setWindowTitle("Status Codes")
self.central_widget = QtWidgets.QWidget()
self.setCentralWidget(self.central_widget)
lay = QtWidgets.QVBoxLayout(self.central_widget)
self.resize(500, 350)
self.ok = QtWidgets.QLabel(
"200: <font color='green'>0</font>",
alignment=QtCore.Qt.AlignHCenter,
)
self.bad = QtWidgets.QLabel(
"400: <font color='yellow'>0</font>",
alignment=QtCore.Qt.AlignHCenter,
)
self.total = QtWidgets.QLabel(
"Total: <font color='#00FF00'>0</font>",
alignment=QtCore.Qt.AlignHCenter,
)
lay.addWidget(self.ok)
lay.addWidget(self.bad)
lay.addWidget(self.total)
thread = QtCore.QThread(self)
thread.start()
self.worker = TaskWorker()
self.worker.moveToThread(thread)
self.worker.totalChanged.connect(self.updateTotal)
self.worker.validChanged.connect(self.updateValid)
self.worker.invalidChanged.connect(self.updateInvalid)
QtCore.QTimer.singleShot(0, self.worker.check)
#QtCore.pyqtSlot(int)
def updateTotal(self, total):
self.total.setText("Total: <font color='#F40D30'>%s</font>" % (total))
#QtCore.pyqtSlot(int)
def updateValid(self, valid):
self.ok.setText("200: <font color='green'>%s</font>" % (valid))
#QtCore.pyqtSlot(int)
def updateInvalid(self, invalid):
self.bad.setText("400: <font color='#00FF00'>%s</font>" % (invalid))
if __name__ == "__main__":
import sys
app = QtWidgets.QApplication(sys.argv)
app.setStyleSheet(qdarkstyle.load_stylesheet_pyqt5())
ex = Menu()
ex.show()
sys.exit(app.exec_())

Web Scraping Multiple Links with PyQt / QtWebkit

I'm trying to scrape a large website of government records which requires a "snowball" method, i.e., starting at the main search page and then following each link that the scraper finds to the next page.
I've been able to load the main page using PyQt this SiteScraper tutorial.
import sys
from PySide.QtGui import *
from PySide.QtCore import *
from PySide.QtWebKit import *
from BeautifulSoup import BeautifulSoup
class Render(QWebPage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
self.app.exec_()
def _loadFinished(self, result):
self.frame = self.mainFrame()
self.app.quit()
def main():
baseUrl = 'http://www.thesite.gov'
url = 'http://www.thesite.gov/search'
r = Render(url)
html = r.frame.toHtml()
# use BeautifulSoup to cycle through each regulation
soup = BeautifulSoup(html)
regs = soup.find('div',{'class':'x-grid3-body'}).findAll('a')
# cycle through list and call up each page separately
for reg in regs:
link = baseUrl + reg['href']
link = str(link)
# use Qt to load each regulation page
r = Render(link)
html = r.frame.toHtml() # get actual rendered web page
The problem is I get this error when I try to render a new webpage:
RuntimeError: A QApplication instance already exists.
I get it that the function is trying to call another QApplication instance. But how do I navigate to a new page with the same instance?
class Render(QWebPage):
def __init__(self, app, url):
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
app.exec_()
def _loadFinished(self, result):
self.frame = self.mainFrame()
def main():
app = QApplication(sys.argv)
baseUrl = 'http://www.thesite.gov'
url = 'http://www.thesite.gov/search'
r = Render(app, url)
html = r.frame.toHtml()

I had the same problem (needing to load multiple pages with QWebPage) but I couldn't get any of these answers to work for me. Here's what did work, the key is to use a QEventLoop and connect loadFinished to loop.quit:
from PySide import QtCore, QtGui, QtWebKit
import sys
def loadPage(url):
page = QtWebKit.QWebPage()
loop = QtCore.QEventLoop() # Create event loop
page.mainFrame().loadFinished.connect(loop.quit) # Connect loadFinished to loop quit
page.mainFrame().load(url)
loop.exec_() # Run event loop, it will end on loadFinished
return page.mainFrame().toHtml()
app = QtGui.QApplication(sys.argv)
urls = ['https://google.com', 'http://reddit.com', 'http://wikipedia.org']
for url in urls:
print '-----------------------------------------------------'
print 'Loading ' + url
html = loadPage(url)
print html
app.exit()
Posting a simplified example here compared to OP's to demonstrate the essential problem and solution.

You're crazy man! QT has a much better DOM than beautifulsoup.
Replace:
soup = BeautifulSoup(html)
With
page = QWebPage()
page.settings().setAttribute(QWebSettings.AutoLoadImages, False)
page.settings().setAttribute(QWebSettings.PluginsEnabled, False)
page.mainFrame().setHtml(html)
dom = page.mainFrame().documentElement()
Then you can simply scrape data like so:
li = dom.findFirst("body div#content div#special ul > li")
if not li.isNull():
class = li.attribute("class")
text = li.toPlainText()
Finally you should use QWebView instead of QWebPage. You can set it up to act like a server which can be controlled with a socket. This is what I do:
class QTimerWithPause(QTimer):
def __init__(self, parent = None):
super(QTimerWithPause, self).__init__ (parent)
self.startTime = 0
self.interval = 0
return
def start(self, interval):
from time import time
self.interval = interval
self.startTime = time()
super(QTimerWithPause, self).start(interval)
return
def pause(self):
from time import time
if self.isActive ():
self.stop()
elapsedTime = self.startTime - time()
self.startTime -= elapsedTime
# time() returns float secs, interval is int msec
self.interval -= int(elapsedTime*1000)+1
return
def resume(self):
if not self.isActive():
self.start(self.interval)
return
class CrawlerWebServer(QWebView):
TIMEOUT = 60
STUPID = r"(bing|yahoo|google)"
def __init__(self, host="0.0.0.0", port=50007, parent=None, enableImages=True, enablePlugins=True):
# Constructor
super(CrawlerWebServer, self).__init__(parent)
self.command = None
self.isLoading = True
self.isConnected = False
self.url = QUrl("http://mast3rpee.tk/")
self.timeout = QTimerWithPause(self)
self.socket = QTcpServer(self)
# 1: Settings
self.settings().enablePersistentStorage()
self.settings().setAttribute(QWebSettings.AutoLoadImages, enableImages)
self.settings().setAttribute(QWebSettings.PluginsEnabled, enablePlugins)
self.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, True)
# 2: Server
if args.verbosity > 0: print "Starting server..."
self.socket.setProxy(QNetworkProxy(QNetworkProxy.NoProxy))
self.socket.listen(QHostAddress(host), int(port))
self.connect(self.socket, SIGNAL("newConnection()"), self._connect)
if args.verbosity > 1:
print " Waiting for connection(" + host + ":" + str(port) + ")..."
# 3: Default page
self._load(10*1000, self._loadFinished)
return
def __del__(self):
try:
self.conn.close()
self.socket.close()
except:
pass
return
def _sendAuth(self):
self.conn.write("Welcome to WebCrawler server (http://mast3rpee.tk)\r\n\rLicenced under GPL\r\n\r\n")
def _connect(self):
self.disconnect(self.socket, SIGNAL("newConnection()"), self._connect)
self.conn = self.socket.nextPendingConnection()
self.conn.nextBlockSize = 0
self.connect(self.conn, SIGNAL("readyRead()"), self.io)
self.connect(self.conn, SIGNAL("disconnected()"), self.close)
self.connect(self.conn, SIGNAL("error()"), self.close)
self._sendAuth()
if args.verbosity > 1:
print " Connection by:", self.conn.peerAddress().toString()
self.isConnected = True
if self.isLoading == False:
self.conn.write("\r\nEnter command:")
return
def io(self):
if self.isLoading: return None
if args.verbosity > 0:
print "Reading command..."
data = self.conn.read(1024).strip(" \r\n\t")
if not data: return None
elif self.command is not None:
r = self.command(data)
self.command = None
return r
return self._getCommand(data)
def _getCommand(self, d):
from re import search
d = unicode(d, errors="ignore")
if search(r"(help|HELP)", d) is not None:
self.conn.write("URL | JS | WAIT | QUIT\r\n\r\nEnter Command:")
elif search(r"(url|URL)", d) is not None:
self.command = self._print
self.conn.write("Enter address:")
elif search(r"(js|JS|javascript|JAVASCRIPT)", d) is not None:
self.command = self._js
self.conn.write("Enter javascript to execte:")
elif search(r"(wait|WAIT)", d) is not None:
self.loadFinished.connect(self._loadFinishedPrint)
self.loadFinished.connect(self._loadFinished)
elif search(r"(quit|QUIT|exit|EXIT)", d) is not None:
self.close()
else:
self.conn.write("Invalid command!\r\n\r\nEnter Command:")
return
def _print(self, d):
u = d[:250]
self.out(u)
return True
def _js(self, d):
try:
self.page().mainFrame().evaluateJavaScript(d)
except:
pass
self.conn.write("Enter Javascript:")
return True
def _stop(self):
from time import sleep
if self.isLoading == False: return
if args.verbosity > 0:
print " Stopping..."
self.timeout.stop()
self.stop()
def _load(self, timeout, after):
# Loads a page into frame / sets up timeout
self.timeout.timeout.connect(self._stop)
self.timeout.start(timeout)
self.loadFinished.connect(after)
self.load(self.url)
return
def _loadDone(self, disconnect = None):
from re import search
from time import sleep
self.timeout.timeout.disconnect(self._stop)
self.timeout.stop()
if disconnect is not None:
self.loadFinished.disconnect(disconnect)
# Stick a while on the page
if search(CrawlerWebServer.STUPID, self.url.toString(QUrl.RemovePath)) is not None:
sleep(5)
else:
sleep(1)
return
def _loadError(self):
from time import sleep, time
if not self.timeout.isActive(): return True
if args.verbosity > 0: print " Error retrying..."
# 1: Pause timeout
self.timeout.pause()
# 2: Check for internet connection
while self.page().networkAccessManager().networkAccessible() == QNetworkAccessManager.NotAccessible: sleep(1)
# 3: Wait then try again
sleep(2)
self.reload()
self.timeout.resume()
return False
def go(self, url, after = None):
# Go to a specific address
global args
if after is None:
after = self._loadFinished
if args.verbosity > 0:
print "Loading url..."
self.url = QUrl(url)
self.isLoading = True
if args.verbosity > 1:
print " ", self.url.toString()
self._load(CrawlerWebServer.TIMEOUT * 1000, after)
return
def out(self, url):
# Print html of a a specific url
self.go(url, self._loadFinishedPrint)
return
def createWindow(self, windowType):
# Load links in the same web-view.
return self
def _loadFinished(self, ok):
# Default LoadFinished
from time import sleep
from re import search
if self.isLoading == False: return
if ok == False:
if not self._loadError(): return
self._loadDone(self._loadFinished)
if args.verbosity > 1:
print " Done"
if self.isConnected == True:
self.conn.write("\r\nEnter command:")
self.isLoading = False
return
def _loadFinishedPrint(self, ok):
# Print the evaluated HTML to stdout
if self.isLoading == False: return
if ok == False:
if not self._loadError(): return
self._loadDone(self._loadFinishedPrint)
if args.verbosity > 1:
print " Done"
h = unicode( self.page().mainFrame().toHtml(), errors="ignore" )
if args.verbosity > 2:
print "------------------\n" + h + "\n--------------------"
self.conn.write(h)
self.conn.write("\r\nEnter command:")
self.isLoading = False
return
def contextMenuEvent(self, event):
# Context Menu
menu = self.page().createStandardContextMenu()
menu.addSeparator()
action = menu.addAction('ReLoad')
#action.triggered.connect
def refresh():
self.load(self.url)
menu.exec_(QCursor.pos())
class CrawlerWebClient(object):
def __init__(self, host, port):
import socket
global args
# CONNECT TO SERVER
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((host, port))
o = self.read()
if args.verbosity > 2:
print "\n------------------------------\n" + o + "\n------------------------------\n"
return
def __del__(self):
try: self.socket.close()
except: pass
def read(self):
from re import search
r = ""
while True:
out = self.socket.recv(64*1024).strip("\r\n")
if out.startswith(r"Enter"):
break
if out.endswith(r"Enter command:"):
r += out[:-14]
break
r += out
return r
def command(self, command):
global args
if args.verbosity > 2:
print " Command: [" + command + "]\n------------------------------"
self.socket.sendall(unicode(command))
r = self.read()
if args.verbosity > 2:
print r, "\n------------------------------\n"
return r

OK then. If you really need JavaScript. (Can you get the answer from JSON at all? That would probably be easier still with simplejson or json.) The answer is don't make more than one QApplication. You're not allowed to. Make main make a QApplication and then use the QWebPage without bothering to call QApplication.exec_(). If that doesn't work, run it all in another QThread.

I am not familiar with PyQt, but as an option, you could write your script without using a class. That way, you can more easily re-use that application instance.
Hope it helps.