We Keep Coding

Python Program Loses Track of Time

[EDIT]
I've added an incremental counter to the data, to see if I'm missing some data, I don't miss data. These are the times between packets in nanoseconds:
650255.8786692759 (Agrees with the oscilloscope)
500667.7866927593 (wrong)
491091.92954990215 (wrong)
484505.240704501 (wrong)
488062.3091976517 (wrong)
487157.1350293542 (wrong)
485264.33268101764 (wrong)
I have a Python Program that reads the serial port. And draws it on a Chart. Using Qt.
The data comes in every 300 microseconds (Confirmed with an Oscilloscope).
Problem is after the first run through of the worker Thread (traad()), it says the time is 100 microseconds between reads. (which is clearly wrong)
I just can't figure out what's wrong!
import serial
from PyQt5 import QtWidgets
from pyqtgraph.Qt import QtCore
import pyqtgraph as pg
import sys
from scipy.fft import fft, fftfreq
import numpy as np
import threading
import time
import re
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.lock = threading.Lock()
self.Voltage = []
self.tid = []
self.xdata = []
self.ydata = []
self.dataReady = False
self.dataRead = True
self.samples = 512
self.x = threading.Thread(target=self.traad, args=(self.samples,))
self.graphWidget = pg.PlotWidget()
self.setCentralWidget(self.graphWidget)
self.x.start()
self.timer = QtCore.QTimer()
self.timer.timeout.connect(self.updater)
self.timer.start(100)
def traad(self, s):
ser = serial.Serial('/dev/ttyUSB0', baudrate=576000, stopbits=2, bytesize=8)
while True:
while self.dataRead:
self.dataRead = False
self.lock.acquire()
self.xdata = []
self.ydata = []
self.Voltage = []
self.tid = []
counter = 0
self.lock.release()
while counter < s:
if ser.read(1) == b'?':
data = ser.readline().strip()
stid = time.time_ns()
data = re.search(b"\\d+\\.\\d+", data)
if data:
self.lock.acquire()
self.Voltage.append(data.group(0))
self.tid.append(stid)
self.lock.release()
print(data.group(0))
counter += 1
tot = self.get_samplerate(self.tid)
xf, yf = self.CalcFft(self.Voltage, tot)
self.lock.acquire()
self.xdata = xf
self.ydata = yf
self.dataReady = True
self.lock.release()
def CalcFft(self, volt, deltatid):
N = len(volt)
T = deltatid / 1000000000
yf = fft(volt)
xf = fftfreq(N, T)[:N // 2]
return xf, yf
def get_samplerate(self, tider):
total = []
delta = 0
test = 0
for x in tider:
total.append(x-test)
test = x
total.pop(0)
for e in total:
delta += e
l = len(total)
d = delta / l
print(d)
return d
def updater(self):
if self.dataReady:
self.graphWidget.plot(self.xdata, 2.0 / self.samples * np.abs(self.ydata[0:self.samples // 2]))
self.lock.acquire()
self.dataReady = False
self.dataRead = True
self.lock.release()
def main():
app = QtWidgets.QApplication(sys.argv)
m = MainWindow()
m.show()
sys.exit(app.exec_())
if __name__ == '__main__':
main()

Why python pickle does not work dos to unix?

Few days ago I coded a mini game to learn socket and threading. When I run my game server and client on windows it just works perfectly but when I moved my server file to my test server it gives me this pickle error:
Exception in thread Thread-2:
Traceback (most recent call last):
File "/usr/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/usr/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "server.py", line 46, in handle_client
obj = pickle.loads(conn.recv(obj_length))
_pickle.UnpicklingError: invalid load key, ' '.
What could be the problem?
Whole game files: ---------------------------------------------------------------------------------
Codes that might help:
server.py:
import socket
import threading
import pickle
import time
import random
import ast
#-------------------------------------------------------------------------
class Server():
def __init__(self):
self.HEADER = 2048
self.PORT = 6000
self.SERVER = "ip"
self.ADDR = (self.SERVER, self.PORT)
self.FORMAT = 'utf-8'
self.DISCONNECT_MESSAGE = "!DISCONNECT"
self.ROLES = ["Mafya", "Mafya", "Köylü", "Doktor","Gözcü"]
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind(self.ADDR)
#---------------------------------------------------------------------------------------------
self.names = list()
self.addresses = list()
self.lobby_dict = dict()
self.game_dict = dict()
self.ready_list = list()
self.alive_list = list()
self.vote = list()
self.kill_vote = list()
self.who_voted = list()
self.ready_for_day = list()
self.protected = None
def handle_client(self, conn, addr):
try:
if addr[0] not in self.addresses:
print(f"[NEW CONNECTION] {addr[0]} connected.")
self.addresses.append(addr[0])
connected = True
while connected:
obj_length = conn.recv(self.HEADER).decode(self.FORMAT)
if obj_length:
obj_length = int(obj_length)
obj = pickle.loads(conn.recv(obj_length))
if obj == self.DISCONNECT_MESSAGE:
connected = False
print(f"[DISCONNECTED] {addr[0]} disconnected.")
elif "?ONLINE" in obj:
lobby_id = obj.split(":")[1]
conn.send(pickle.dumps(self.lobby_dict[lobby_id]["Players"]))
elif "!NEWLOBBY" in obj:
splitted_obj = obj.split(":")
lobby_id = splitted_obj[1]
admin = splitted_obj[2]
capacity = splitted_obj[3]
self.lobby_dict[lobby_id] = {"Players":[admin],"Capacity":capacity}
elif "!JOINLOBBY" in obj:
splitted_obj = obj.split(":")
lobby_id = splitted_obj[1]
name = splitted_obj[2]
if lobby_id in self.lobby_dict.keys():
self.lobby_dict[lobby_id]["Players"].append(name)
conn.send(pickle.dumps(f"True:{self.lobby_dict[lobby_id]['Capacity']}"))
else:
conn.send(pickle.dumps("False"))
elif "?ALIVE" in obj:
conn.send(pickle.dumps(self.alive_list))
#-----------------------------------------------------------------------------------------------
#Game commands:
elif "!NAME" in obj:
name = obj.split(":")[1]
self.names.append(name)
elif "!ALIVE" in obj:
conn.send(pickle.dumps(self.names))
elif "!READY" in obj:
ready_player = obj.split(":")[1]
self.ready_list.append(ready_player)
elif "!SHUFFLE" in obj:
if len(self.ready_list) == len(self.names):
temp = self.ROLES
if len(self.names) > len(self.ROLES):
for i in range(0,len(self.names) - len(self.ROLES)):
temp.append("Köylü")
random.shuffle(temp)
for i in range(len(self.names)):
self.game_dict[self.names[i]] = temp[i]
conn.send(pickle.dumps(f"True/{self.game_dict}"))
with open("shuffled_roles.txt", "w", encoding="utf-8") as file:
file.write(str(self.game_dict))
print(f"[SHUFFLED LIST] {self.game_dict}")
else:
conn.send(pickle.dumps("False"))
elif "!ROLES" in obj:
if len(self.ready_list) == len(self.names):
with open("shuffled_roles.txt", "r", encoding="utf-8") as file:
line = file.readline()
self.game_dict = ast.literal_eval(line)
conn.send(pickle.dumps(f"True/{self.game_dict}"))
else:
conn.send(pickle.dumps("False"))
elif "!VOTE" in obj:
voted_player = obj.split(":")[1]
who = obj.split(":")[2] + ": " + voted_player
self.who_voted.append(who)
self.vote.append(voted_player)
elif "!VRESULTS" in obj:
conn.send(pickle.dumps(self.vote))
conn.send(pickle.dumps(self.who_voted))
elif "!VCLEAN" in obj:
self.vote = []
self.who_voted = []
elif "!PROTECTED" in obj:
protected = obj.split(":")[1]
self.protected = obj
elif "!NIGHT_KILL" in obj:
kill = obj.split(":")[1]
self.kill_vote.append(kill)
elif "!NKRESULTS" in obj:
nk_results = self.kill_vote
nk_protected = self.protected
if len(nk_results) == 1:
if nk_results[0] != nk_protected:
conn.send(pickle.dumps(nk_results[0]))
elif len(nk_results) == 2:
if nk_results[0] == nk_results[1]:
if nk_results[0] != nk_protected and nk_results[0] != "None":
conn.send(pickle.dumps(nk_results[0]))
elif nk_results[0] == "None" and nk_results[1] != "None":
conn.send(pickle.dumps(nk_results[1]))
elif nk_results[1] == "None" and nk_results[0] != "None":
conn.send(pickle.dumps(nk_results[0]))
else:
conn.send(pickle.dumps(None))
else:
conn.send(pickle.dumps(None))
elif "!NKCLEAN" in obj:
self.protected = "None"
self.kill_vote = []
elif "!RFORDAY" in obj:
rplayer = obj.split(":")[1]
self.ready_for_day.append(rplayer)
elif "!RFDLIST" in obj:
conn.send(pickle.dumps(self.ready_for_day))
elif "!RFDCLEAN" in obj:
self.ready_for_day = list()
else:
print(f"[{addr}] {obj}") #İsimler Buradan -> addr
except ConnectionResetError:
print(f"[CONNECTION] {addr} Connection reset exception has been handled.")
finally:
conn.close()
def start(self):
print("[STARTING] Server is starting...")
self.server.listen()
print("[LISTENING] Server is listening on {}".format(self.SERVER))
while True:
conn, addr = self.server.accept()
thread = threading.Thread(target=self.handle_client, args=(conn, addr))
thread.start()
if __name__ == "__main__":
server = Server()
server.start()
app.py:
import sys
import os
import time
import random
import socket
import threading
import pickle
class Config():
def __init__(self):
self.HEADER = 2048
self.PORT = 6000
self.SERVER = "ip" #socket.gethostbyname(socket.gethostname())
self.ADDR = (self.SERVER, self.PORT)
self.FORMAT = 'utf-8'
self.DISCONNECT_MESSAGE = "!DISCONNECT"
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.menuText = """
░██╗░░░░░░░██╗██╗░██████╗███████╗  ████████╗░█████╗░░██╗░░░░░░░██╗███╗░░██╗
░██║░░██╗░░██║██║██╔════╝██╔════╝  ╚══██╔══╝██╔══██╗░██║░░██╗░░██║████╗░██║
░╚██╗████╗██╔╝██║╚█████╗░█████╗░░  ░░░██║░░░██║░░██║░╚██╗████╗██╔╝██╔██╗██║
░░████╔═████║░██║░╚═══██╗██╔══╝░░  ░░░██║░░░██║░░██║░░████╔═████║░██║╚████║
░░╚██╔╝░╚██╔╝░██║██████╔╝███████╗  ░░░██║░░░╚█████╔╝░░╚██╔╝░╚██╔╝░██║░╚███║
░░░╚═╝░░░╚═╝░░╚═╝╚═════╝░╚══════╝  ░░░╚═╝░░░░╚════╝░░░░╚═╝░░░╚═╝░░╚═╝░░╚══╝
"""
class Client(Config):
def __init__(self):
super().__init__()
self.client.connect(self.ADDR)
self.admin = None
def send(self, obj):
obj = pickle.dumps(obj)
obj_length = len(obj)
send_length = str(obj_length).encode(self.FORMAT)
send_length += b' ' * (self.HEADER - len(send_length))
self.client.send(send_length)
self.client.send(obj)
def messenger(self):
while True:
msg = input("Mesaj: ")
self.send(msg)
class Menu(Client):
def __init__(self):
super().__init__()
class MainMenu(Menu):
def __init__(self, player_name):
super().__init__()
self.player_name = player_name
def printMainMenu(self, game_is_on = False):
print("\n" * 7 + self.menuText)
time.sleep(2)
os.system("cls")
print(self.menuText)
print("""
1. Yeni Oyun Oluştur
2. Oyuna Katıl
""")
class Lobby(Menu):
def __init__(self):
super().__init__()
def lobbyMenu(self):
name_send = f"!NAME:{self.player_name}"
self.send(name_send)
os.system("cls")
print(self.menuText)
print(f"Lobby ID: {self.id}")
self.send(f"?ONLINE:{self.id}")
self.online_list = pickle.loads(self.client.recv(2048))
temp_list = self.online_list
sys.stdout.write("Aktif Oyuncular:| ")
for i in self.online_list:
sys.stdout.write(f"{i} | ")
sys.stdout.flush()
while not self.game_started:
time.sleep(1)
self.send("?ONLINE:" + str(self.id))
self.online_list = pickle.loads(self.client.recv(2048))
if temp_list != self.online_list:
sys.stdout.write("\rAktif Oyuncular:| ")
for i in self.online_list:
sys.stdout.write(f"{i} | ")
sys.stdout.flush()
temp_list = self.online_list
if len(self.online_list) == self.capacity:
self.game_started = True
class CreateLobby(Lobby):
def __init__(self, capacity, player_name, admin):
super().__init__()
self.player_name = player_name
self.admin = admin
self.id = random.randint(100000,999999)
self.capacity = int(capacity)
self.game_started = False
self.send(f"!NEWLOBBY:{self.id}:{self.player_name}:{self.capacity}")
self.lobbyMenu()
class JoinLobby(Lobby):
def __init__(self, id, player_name):
super().__init__()
self.id = id
self.player_name = player_name
self.game_started = False
self.lobby_joiner()
def lobby_joiner(self):
self.send(f"!JOINLOBBY:{self.id}:{self.player_name}")
bool_obj = pickle.loads(self.client.recv(2048))
while bool_obj == "False":
print("Bu ID'ye ait lobby bulunmamaktadır.")
self.id = input(" Lobby ID: ")
self.send(f"!JOINLOBBY:{self.id}:{self.player_name}")
bool_obj = pickle.loads(self.client.recv(2048))
self.capacity = int(bool_obj.split(":")[1])
self.lobbyMenu()
#-------------------------------------------------------------------------------------------------------------

Your code includes
obj = pickle.loads(conn.recv(obj_length))
The problem is that TCP is a streaming protocol and the entire obj_length of data may not have been received when the call is made. When you run client and server on the same machine, you don't have the real network with real segmentation and delays, so you don't see the problem.
The solution is your own receiver that knows to keep asking for data until it sees it all
def recvall(conn, count):
recvlist = []
recvcount = 0
while recvcount < count:
buf = conn.recv(count-recvcount)
if not buf:
# replace with your error handling here
raise OSError("Connection terminated")
recvlist.append(buf)
recvcount += len(buf)
return b"".join(recvlist)
Replace your original line with
obj = pickle.loads(recvall(conn, obj_length))
and it should work

ProgressBar resume from where it stopped

I have created a desktop application by PYQT5 and python 3.7 to download a video by clicking the download button and save it locally in the PC.
The code will fetch the video link from (lineEdit.text()) which is labeled "URL" and save it in the local directory in (lineEdit_2.text()) which is labeled "SAVE AS". If the download stops for any reason, it will be resumed again by press the start download button. In addition, the ProgressBar start from 1% until 100% along with downloading the video. Everything working smoothly.
The question is, once the video stops in the middle for any reason then it resumes the downloading again but the ProgressBar should start from where it stopped but it is not. For example, if it stops in 50% then should be resumed from 50% and continue. However, it starts from 0% (from the beginning).
```def curl_progress(self,total, existing, totalfrac,fracmb):
global frac,tsize,size,save_location
try:
frac= float(existing)/float(total)
self.progressBar.setValue(totalfrac)
QApplication.processEvents()
except (ZeroDivisionError, RuntimeError, TypeError, NameError):
frac = 0
self.textBrowser.append("Downloaded %d/%d %d%%" % (existing, total, totalfrac))
if frac ==1.0:
self.textBrowser.append("")
size = os.path.getsize(save_location)
tsize= (size /1024 /1024)
QMessageBox.information(self,"Download Completed", "The Download is Finished and the size is %03.2f MB" %(tsize,))
self.textBrowser.append('Size of file is %03.2f MB' %(tsize,))
self.progressBar.setValue(0)
self.lineEdit.setText('')
self.lineEdit_2.setText('')
QMessageBox.close(self)
else:
self.textBrowser.append("Downloaded %d/%d %d%%" % (existing, total, totalfrac))
def curl_limit_rate(self,rate_limit):
global tsize,size,save_location
url= self.lineEdit.text()
save_location = self.lineEdit_2.text()
if len(url) == 0 and len(save_location) == 0:
QMessageBox.information(self, "Error", "Please put the links")
return
if len(url) > 0 and len(save_location) == 0:
QMessageBox.information(self, "Error", "Please put the location")
return
if len(url) == 0 and len(save_location) > 0:
QMessageBox.information(self, "Error", "Please put the link")
return
if len(url) > 0 and len(save_location) > 0:
c = pycurl.Curl()
c.setopt(pycurl.CAINFO, certifi.where())
c.setopt(c.URL,url)
c.setopt(c.MAX_RECV_SPEED_LARGE, rate_limit)
if os.path.exists(save_location):
file_id = open(save_location, "ab")
c.setopt(c.RESUME_FROM, os.path.getsize(save_location))
else:
file_id = open(save_location, "wb")
c.setopt(c.WRITEDATA, file_id)
c.setopt(c.NOPROGRESS, 0)
c.setopt(c.PROGRESSFUNCTION, self.curl_progress)
c.perform()
c.close()
else:
QMessageBox.information(self, "Error", "Unknown error!")```
The picture
Many thanks in advance,

Before pointing out the solution, I must point out that you should not run pycurl in the main thread since it is blocking, instead you must execute it in another thread and send the information to the main thread so that it can be shown.
Going to the point, the idea is that when you calculate the percentage it is using the following formula:
progress = 100 * (bytes_downloaded + size_of_resume_file) / (total_bytes + size_of_resume_file)
Considering the above the solution is:
import os
import certifi
import pycurl
from PyQt5 import QtCore, QtWidgets
class Downloader(QtCore.QObject):
started = QtCore.pyqtSignal()
finished = QtCore.pyqtSignal()
progressChanged = QtCore.pyqtSignal(int)
error = QtCore.pyqtSignal(int, str)
bytesChanged = QtCore.pyqtSignal(int, int)
#QtCore.pyqtSlot(str, str)
def download(self, url, save_location):
pass
class PycURLDownloader(Downloader):
def __init__(self, parent=None):
super().__init__(parent)
self._resume_size = 0
self._c = pycurl.Curl()
self._flag_stop = 0
def download(self, url, save_location):
self._flag_stop = 0
exist_path = os.path.exists(save_location)
self.started.emit()
with open(save_location, "ab" if exist_path else "wb") as file_id:
self._c.setopt(pycurl.CAINFO, certifi.where())
self._c.setopt(pycurl.URL, url)
self._c.setopt(pycurl.MAX_RECV_SPEED_LARGE, 1024)
if exist_path:
self._c.setopt(pycurl.RESUME_FROM, os.path.getsize(save_location))
self._resume_size = os.path.getsize(save_location)
else:
self._resume_size = 0
self._c.setopt(pycurl.WRITEDATA, file_id)
self._c.setopt(pycurl.NOPROGRESS, 0)
self._c.setopt(pycurl.PROGRESSFUNCTION, self._progress_callaback)
try:
self._c.perform()
except pycurl.error as e:
self.error.emit(*e.args)
else:
self.finished.emit()
self._c.close()
#QtCore.pyqtSlot()
def stop(self):
self._flag_stop = 1
def _progress_callaback(self, total, existing, totalfrac, fracmb):
frac = 0
if existing > 0 and total > 0:
frac = int(
100 * (existing + self._resume_size) / (total + self._resume_size)
)
self.bytesChanged.emit(existing, total)
self.progressChanged.emit(frac)
if QtCore.QThread.currentThread().isInterruptionRequested():
return 1
class Widget(QtWidgets.QWidget):
def __init__(self, parent=None):
super().__init__(parent)
self.url_lineedit = QtWidgets.QLineEdit()
self.save_location_lineedit = QtWidgets.QLineEdit()
browse_button = QtWidgets.QPushButton(self.tr("Browse"))
self.download_progressbar = QtWidgets.QProgressBar(minimum=0, maximum=100)
self.download_log_browser = QtWidgets.QTextBrowser()
self.start_download_button = QtWidgets.QPushButton(self.tr("Start Download"))
widget = QtWidgets.QWidget()
widget.setContentsMargins(0, 0, 0, 0)
hlay = QtWidgets.QHBoxLayout(widget)
hlay.addWidget(self.save_location_lineedit)
hlay.addWidget(browse_button)
hlay.setContentsMargins(0, 0, 0, 0)
flay = QtWidgets.QFormLayout()
flay.addRow("URL", self.url_lineedit)
flay.addRow("Save as", widget)
flay.addRow("", self.download_progressbar)
flay.addRow("", QtWidgets.QLabel(self.tr("Packets output in Bytes")))
flay.addRow("", self.download_log_browser)
hlay2 = QtWidgets.QHBoxLayout()
hlay2.addStretch()
hlay2.addWidget(self.start_download_button)
hlay2.addStretch()
vlay = QtWidgets.QVBoxLayout(self)
vlay.addLayout(flay)
vlay.addLayout(hlay2)
self.start_download_button.clicked.connect(self.start_download)
browse_button.clicked.connect(self.select_save_location)
self._thread = QtCore.QThread(self)
self._thread.start()
self._downloader = PycURLDownloader()
self._downloader.moveToThread(self._thread)
self._downloader.progressChanged.connect(self.download_progressbar.setValue)
self._downloader.bytesChanged.connect(self.on_bytesChanged)
self._downloader.started.connect(self.on_started)
self._downloader.finished.connect(self.on_finished)
self.url_lineedit.setText("http://techslides.com/demos/sample-videos/small.mp4")
#QtCore.pyqtSlot()
def start_download(self):
url = self.url_lineedit.text()
save_location = self.save_location_lineedit.text()
if not url:
QtWidgets.QMessageBox.information(self, "Error", "Please put the links")
return
elif not save_location:
QtWidgets.QMessageBox.information(self, "Error", "Please put the location")
return
wrapper = partial(self._downloader.download, url, save_location)
QtCore.QTimer.singleShot(0, wrapper)
#QtCore.pyqtSlot()
def select_save_location(self):
filename, _ = QtWidgets.QFileDialog.getSaveFileName(self, "Select")
if filename:
self.save_location_lineedit.setText(filename)
#QtCore.pyqtSlot(int, str)
def on_error(self, t, msg):
QtWidgets.QMessageBox.information(self, "Error", msg)
#QtCore.pyqtSlot(int, int)
def on_bytesChanged(self, existing, total):
self.download_log_browser.append(
"Downloaded %d/%d %d%%" % (existing, total, 100 * existing / total)
)
#QtCore.pyqtSlot()
def on_started(self):
self.start_download_button.setEnabled(False)
#QtCore.pyqtSlot()
def on_finished(self):
self.start_download_button.setEnabled(True)
def closeEvent(self, event):
self._thread.requestInterruption()
self._thread.quit()
self._thread.wait()
super().closeEvent(event)
if __name__ == "__main__":
from functools import partial
import sys
app = QtWidgets.QApplication(sys.argv)
w = Widget()
w.resize(640, 480)
w.show()
ret = app.exec_()
sys.exit(ret)

Update client widget from server with rpyc callback

I want to edit all client QTextBrowser logs when any client sends some message to server.
my intended procedure is this:
[client]enter -> [client]chatUI.handleEnter -> (RFC)[server]exposed_send -> [server]broadcast -> (RFC)[clients]update.emit() -> [clients]listen -> log changed
When I run this code, other clients logs are not changed and only the client that give input to server has an updated log.
How I can solve this to update all clients properly?
chat_server.py
import rpyc
import random
import string
from threading import RLock
users = dict()
callbacks = dict()
user_num = 0
lock = RLock()
buf = dict()
class chatService(rpyc.Service):
def on_connect(self):
global user_num
with lock:
user_num = user_num+1
print ("connect user: %d" % user_num)
def on_disconnect(self):
global user_num
with lock:
user_num = user_num-1
print ("disconnect user: %d" % user_num)
def exposed_accept(self, idt, callback):
with lock:
global users
global callbacks
if not isinstance(idt, str) or len(idt) != 6:
return False
elif idt in users:
return -1
else:
pw = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(32))
users[idt] = pw
callbacks[idt] = rpyc.async(callback)
return pw
def exposed_send(self, target, msg, idt, pw):
print ('here')
name = self.identify(idt, pw)
if name == False:
print ('here2')
return False
else:
print ('here3')
global callbacks
if target == None:
self.broadcast("[%s] %s" % (name, msg))
elif msg.target in callbacks:
self.send("[%s] %s" %(name, msg), target)
else:
return False
def exposed_order(self, msg, idt, pw):
pass
def identify(self, idt, pw):
global users
if users[idt] == pw:
return idt
else:
return False
def broadcast(self, msg):
with lock:
print("bloadcast calls")
global callbacks
global buf
for user, callback in callbacks.items():
if user not in buf or buf[user] == None:
buf[user] = (msg,)
else:
buf[user] = buf[user] + (msg,)
callback()
def send(self, msg, target):
global callbacks
global buf
if user not in buf or buf[user] == None:
buf[target] = (msg,)
else:
buf[target] = buf[target] + (msg,)
callbacks[target]()
def exposed_get_buf(self, user):
global buf
temp = buf[user]
buf[user] = None
return temp
if __name__ == '__main__':
from rpyc.utils.server import ThreadedServer
t = ThreadedServer(chatService, port = 3743)
t.start()
chat_client.py
from chatUI import *
import rpyc
import random
import string
if __name__ == '__main__':
service = rpyc.connect('floating.tk', 3743)
app, chat = UIReady(service)
while True:
idt = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(6))
pw = service.root.accept(idt, chat.update.update.emit)
if pw != False and pw != -1:
break
chat.idt = idt
chat.pw = pw
sys.exit(app.exec_())
chatUI.py
import sys
from PyQt5.QtWidgets import *
from PyQt5.QtCore import *
class Updater(QObject):
update = pyqtSignal()
class Chat(QWidget):
log = None
enter = None
def __init__(self, service) :
super().__init__()
self.service = service
self.idt = None
self.pw = None
self.initUI()
self.update = Updater()
self.update.update.connect(self.listen)
def initUI(self):
logLabel = QLabel('chat log')
enterLabel = QLabel('enter')
self.log = QTextBrowser()
self.enter = QLineEdit()
self.enter.returnPressed.connect(self.handleEnter)
layout = QGridLayout()
layout.addWidget(logLabel, 0, 0)
layout.addWidget(self.log, 0, 1, 5, 1)
layout.addWidget(enterLabel, 6, 0)
layout.addWidget(self.enter, 6, 1)
self.setLayout(layout)
self.setWindowTitle('chat')
self.resize(600, 600)
self.show()
def handleEnter(self):
msg = self.enter.text()
self.enter.setText("")
self.service.root.send(None, msg, self.idt, self.pw)
print('get enter')
def listen(self):
msg = self.service.root.get_buf(self.idt)
for m in msg:
self.log.append(m)
def UIReady(service):
app = QApplication(sys.argv)
chat = Chat(service)
return app, chat

Web Scraping Multiple Links with PyQt / QtWebkit

I'm trying to scrape a large website of government records which requires a "snowball" method, i.e., starting at the main search page and then following each link that the scraper finds to the next page.
I've been able to load the main page using PyQt this SiteScraper tutorial.
import sys
from PySide.QtGui import *
from PySide.QtCore import *
from PySide.QtWebKit import *
from BeautifulSoup import BeautifulSoup
class Render(QWebPage):
def __init__(self, url):
self.app = QApplication(sys.argv)
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
self.app.exec_()
def _loadFinished(self, result):
self.frame = self.mainFrame()
self.app.quit()
def main():
baseUrl = 'http://www.thesite.gov'
url = 'http://www.thesite.gov/search'
r = Render(url)
html = r.frame.toHtml()
# use BeautifulSoup to cycle through each regulation
soup = BeautifulSoup(html)
regs = soup.find('div',{'class':'x-grid3-body'}).findAll('a')
# cycle through list and call up each page separately
for reg in regs:
link = baseUrl + reg['href']
link = str(link)
# use Qt to load each regulation page
r = Render(link)
html = r.frame.toHtml() # get actual rendered web page
The problem is I get this error when I try to render a new webpage:
RuntimeError: A QApplication instance already exists.
I get it that the function is trying to call another QApplication instance. But how do I navigate to a new page with the same instance?
class Render(QWebPage):
def __init__(self, app, url):
QWebPage.__init__(self)
self.loadFinished.connect(self._loadFinished)
self.mainFrame().load(QUrl(url))
app.exec_()
def _loadFinished(self, result):
self.frame = self.mainFrame()
def main():
app = QApplication(sys.argv)
baseUrl = 'http://www.thesite.gov'
url = 'http://www.thesite.gov/search'
r = Render(app, url)
html = r.frame.toHtml()

I had the same problem (needing to load multiple pages with QWebPage) but I couldn't get any of these answers to work for me. Here's what did work, the key is to use a QEventLoop and connect loadFinished to loop.quit:
from PySide import QtCore, QtGui, QtWebKit
import sys
def loadPage(url):
page = QtWebKit.QWebPage()
loop = QtCore.QEventLoop() # Create event loop
page.mainFrame().loadFinished.connect(loop.quit) # Connect loadFinished to loop quit
page.mainFrame().load(url)
loop.exec_() # Run event loop, it will end on loadFinished
return page.mainFrame().toHtml()
app = QtGui.QApplication(sys.argv)
urls = ['https://google.com', 'http://reddit.com', 'http://wikipedia.org']
for url in urls:
print '-----------------------------------------------------'
print 'Loading ' + url
html = loadPage(url)
print html
app.exit()
Posting a simplified example here compared to OP's to demonstrate the essential problem and solution.

You're crazy man! QT has a much better DOM than beautifulsoup.
Replace:
soup = BeautifulSoup(html)
With
page = QWebPage()
page.settings().setAttribute(QWebSettings.AutoLoadImages, False)
page.settings().setAttribute(QWebSettings.PluginsEnabled, False)
page.mainFrame().setHtml(html)
dom = page.mainFrame().documentElement()
Then you can simply scrape data like so:
li = dom.findFirst("body div#content div#special ul > li")
if not li.isNull():
class = li.attribute("class")
text = li.toPlainText()
Finally you should use QWebView instead of QWebPage. You can set it up to act like a server which can be controlled with a socket. This is what I do:
class QTimerWithPause(QTimer):
def __init__(self, parent = None):
super(QTimerWithPause, self).__init__ (parent)
self.startTime = 0
self.interval = 0
return
def start(self, interval):
from time import time
self.interval = interval
self.startTime = time()
super(QTimerWithPause, self).start(interval)
return
def pause(self):
from time import time
if self.isActive ():
self.stop()
elapsedTime = self.startTime - time()
self.startTime -= elapsedTime
# time() returns float secs, interval is int msec
self.interval -= int(elapsedTime*1000)+1
return
def resume(self):
if not self.isActive():
self.start(self.interval)
return
class CrawlerWebServer(QWebView):
TIMEOUT = 60
STUPID = r"(bing|yahoo|google)"
def __init__(self, host="0.0.0.0", port=50007, parent=None, enableImages=True, enablePlugins=True):
# Constructor
super(CrawlerWebServer, self).__init__(parent)
self.command = None
self.isLoading = True
self.isConnected = False
self.url = QUrl("http://mast3rpee.tk/")
self.timeout = QTimerWithPause(self)
self.socket = QTcpServer(self)
# 1: Settings
self.settings().enablePersistentStorage()
self.settings().setAttribute(QWebSettings.AutoLoadImages, enableImages)
self.settings().setAttribute(QWebSettings.PluginsEnabled, enablePlugins)
self.settings().setAttribute(QWebSettings.DeveloperExtrasEnabled, True)
# 2: Server
if args.verbosity > 0: print "Starting server..."
self.socket.setProxy(QNetworkProxy(QNetworkProxy.NoProxy))
self.socket.listen(QHostAddress(host), int(port))
self.connect(self.socket, SIGNAL("newConnection()"), self._connect)
if args.verbosity > 1:
print " Waiting for connection(" + host + ":" + str(port) + ")..."
# 3: Default page
self._load(10*1000, self._loadFinished)
return
def __del__(self):
try:
self.conn.close()
self.socket.close()
except:
pass
return
def _sendAuth(self):
self.conn.write("Welcome to WebCrawler server (http://mast3rpee.tk)\r\n\rLicenced under GPL\r\n\r\n")
def _connect(self):
self.disconnect(self.socket, SIGNAL("newConnection()"), self._connect)
self.conn = self.socket.nextPendingConnection()
self.conn.nextBlockSize = 0
self.connect(self.conn, SIGNAL("readyRead()"), self.io)
self.connect(self.conn, SIGNAL("disconnected()"), self.close)
self.connect(self.conn, SIGNAL("error()"), self.close)
self._sendAuth()
if args.verbosity > 1:
print " Connection by:", self.conn.peerAddress().toString()
self.isConnected = True
if self.isLoading == False:
self.conn.write("\r\nEnter command:")
return
def io(self):
if self.isLoading: return None
if args.verbosity > 0:
print "Reading command..."
data = self.conn.read(1024).strip(" \r\n\t")
if not data: return None
elif self.command is not None:
r = self.command(data)
self.command = None
return r
return self._getCommand(data)
def _getCommand(self, d):
from re import search
d = unicode(d, errors="ignore")
if search(r"(help|HELP)", d) is not None:
self.conn.write("URL | JS | WAIT | QUIT\r\n\r\nEnter Command:")
elif search(r"(url|URL)", d) is not None:
self.command = self._print
self.conn.write("Enter address:")
elif search(r"(js|JS|javascript|JAVASCRIPT)", d) is not None:
self.command = self._js
self.conn.write("Enter javascript to execte:")
elif search(r"(wait|WAIT)", d) is not None:
self.loadFinished.connect(self._loadFinishedPrint)
self.loadFinished.connect(self._loadFinished)
elif search(r"(quit|QUIT|exit|EXIT)", d) is not None:
self.close()
else:
self.conn.write("Invalid command!\r\n\r\nEnter Command:")
return
def _print(self, d):
u = d[:250]
self.out(u)
return True
def _js(self, d):
try:
self.page().mainFrame().evaluateJavaScript(d)
except:
pass
self.conn.write("Enter Javascript:")
return True
def _stop(self):
from time import sleep
if self.isLoading == False: return
if args.verbosity > 0:
print " Stopping..."
self.timeout.stop()
self.stop()
def _load(self, timeout, after):
# Loads a page into frame / sets up timeout
self.timeout.timeout.connect(self._stop)
self.timeout.start(timeout)
self.loadFinished.connect(after)
self.load(self.url)
return
def _loadDone(self, disconnect = None):
from re import search
from time import sleep
self.timeout.timeout.disconnect(self._stop)
self.timeout.stop()
if disconnect is not None:
self.loadFinished.disconnect(disconnect)
# Stick a while on the page
if search(CrawlerWebServer.STUPID, self.url.toString(QUrl.RemovePath)) is not None:
sleep(5)
else:
sleep(1)
return
def _loadError(self):
from time import sleep, time
if not self.timeout.isActive(): return True
if args.verbosity > 0: print " Error retrying..."
# 1: Pause timeout
self.timeout.pause()
# 2: Check for internet connection
while self.page().networkAccessManager().networkAccessible() == QNetworkAccessManager.NotAccessible: sleep(1)
# 3: Wait then try again
sleep(2)
self.reload()
self.timeout.resume()
return False
def go(self, url, after = None):
# Go to a specific address
global args
if after is None:
after = self._loadFinished
if args.verbosity > 0:
print "Loading url..."
self.url = QUrl(url)
self.isLoading = True
if args.verbosity > 1:
print " ", self.url.toString()
self._load(CrawlerWebServer.TIMEOUT * 1000, after)
return
def out(self, url):
# Print html of a a specific url
self.go(url, self._loadFinishedPrint)
return
def createWindow(self, windowType):
# Load links in the same web-view.
return self
def _loadFinished(self, ok):
# Default LoadFinished
from time import sleep
from re import search
if self.isLoading == False: return
if ok == False:
if not self._loadError(): return
self._loadDone(self._loadFinished)
if args.verbosity > 1:
print " Done"
if self.isConnected == True:
self.conn.write("\r\nEnter command:")
self.isLoading = False
return
def _loadFinishedPrint(self, ok):
# Print the evaluated HTML to stdout
if self.isLoading == False: return
if ok == False:
if not self._loadError(): return
self._loadDone(self._loadFinishedPrint)
if args.verbosity > 1:
print " Done"
h = unicode( self.page().mainFrame().toHtml(), errors="ignore" )
if args.verbosity > 2:
print "------------------\n" + h + "\n--------------------"
self.conn.write(h)
self.conn.write("\r\nEnter command:")
self.isLoading = False
return
def contextMenuEvent(self, event):
# Context Menu
menu = self.page().createStandardContextMenu()
menu.addSeparator()
action = menu.addAction('ReLoad')
#action.triggered.connect
def refresh():
self.load(self.url)
menu.exec_(QCursor.pos())
class CrawlerWebClient(object):
def __init__(self, host, port):
import socket
global args
# CONNECT TO SERVER
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((host, port))
o = self.read()
if args.verbosity > 2:
print "\n------------------------------\n" + o + "\n------------------------------\n"
return
def __del__(self):
try: self.socket.close()
except: pass
def read(self):
from re import search
r = ""
while True:
out = self.socket.recv(64*1024).strip("\r\n")
if out.startswith(r"Enter"):
break
if out.endswith(r"Enter command:"):
r += out[:-14]
break
r += out
return r
def command(self, command):
global args
if args.verbosity > 2:
print " Command: [" + command + "]\n------------------------------"
self.socket.sendall(unicode(command))
r = self.read()
if args.verbosity > 2:
print r, "\n------------------------------\n"
return r

OK then. If you really need JavaScript. (Can you get the answer from JSON at all? That would probably be easier still with simplejson or json.) The answer is don't make more than one QApplication. You're not allowed to. Make main make a QApplication and then use the QWebPage without bothering to call QApplication.exec_(). If that doesn't work, run it all in another QThread.

I am not familiar with PyQt, but as an option, you could write your script without using a class. That way, you can more easily re-use that application instance.
Hope it helps.