We Keep Coding

PyQt Signal positional argument disappearing

I'm writing a small application in Python 3.6 and PyQt 5.13.1 to compare file checksums for directories across multiple remote servers. I'm trying to split out the main logic of my application into its own thread, and from that thread use a threadpool to create several more threads to SSH into the remote servers and get information back.
Everything is mostly working, except when the signal I have made for the SSH threads is called, one of the positional arguments ends up missing and I get the following error:
TypeError: ssh_result() missing 1 required positional argument: 'message'
Here is a stripped down version of what I currently have, but is exhibiting the same issue:
import paramiko
from PyQt5.QtWidgets import *
from PyQt5.QtCore import QThread, QThreadPool, QRunnable, pyqtSignal, pyqtSlot, QObject
class SSHWorkerSignals(QObject):
result = pyqtSignal('PyQt_PyObject', 'PyQt_PyObject')
class SSHWorker(QRunnable):
def __init__(self):
super(SSHWorker, self).__init__()
self.ssh_signals = SSHWorkerSignals()
self.ssh_server = 'test.server.corp'
#pyqtSlot()
def run(self):
try:
ssh_client = paramiko.SSHClient()
ssh_client.load_system_host_keys()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy)
ssh_client.connect(self.ssh_server, port=22, username='test', password='test')
stdin, stdout, stderr = ssh_client.exec_command('uname -a')
std_out = str(stdout.read())
self.ssh_signals.result.emit(self.ssh_server, std_out)
except Exception as e:
self.ssh_signals.result.emit(self.ssh_server, str(e))
finally:
ssh_client.close()
class CompareWorker(QObject):
done = pyqtSignal()
def __init__(self):
super(CompareWorker, self).__init__()
self.ssh_threadpool = QThreadPool()
self.ssh_threadpool.setMaxThreadCount(10)
#pyqtSlot()
def execute(self):
ssh_worker = SSHWorker()
ssh_worker.ssh_signals.result.connect(MainWindow.ssh_result)
self.ssh_threadpool.start(ssh_worker)
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.button_compare = QPushButton('Compare')
self.button_compare.clicked.connect(self.compare)
self.setCentralWidget(self.button_compare)
def compare(self):
compare_thread = QThread(self)
self.compare_worker = CompareWorker()
compare_thread.started.connect(self.compare_worker.execute)
self.compare_worker.done.connect(compare_thread.quit)
self.compare_worker.done.connect(self.compare_worker.deleteLater)
self.compare_worker.moveToThread(compare_thread)
compare_thread.start()
def ssh_result(self, server, message):
print('server: ', server, ', result: ', message)
if __name__ == '__main__':
app = QApplication([])
window = MainWindow()
window.show()
app.exec_()
If I change the ssh_result() function so that message is an optional argument (def ssh_result(self, server, message=''):), I can see that the first positional argument is disappearing, leaving what should be the second argument to be the first instead, like so:
server: [Errno 11001] getaddrinfo failed , result:
Can anyone help me figure out why this is happening?

The main problem is that MainWindow.ssh_result is not a method that belongs to some instance, so the first "self" parameter does not exist. I also don't see the need for the creation of CompareWorker.
Considering the above, the solution is:
# ...
class SSHWorkerSignals(QObject):
result = pyqtSignal(
"PyQt_PyObject", "PyQt_PyObject"
) # or pyqtSignal(object, object)
class SSHWorker(QRunnable):
# ...
class MainWindow(QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
self.button_compare = QPushButton("Compare")
self.button_compare.clicked.connect(self.compare)
self.setCentralWidget(self.button_compare)
self.ssh_threadpool = QThreadPool()
self.ssh_threadpool.setMaxThreadCount(10)
def compare(self):
ssh_worker = SSHWorker()
ssh_worker.ssh_signals.result.connect(self.ssh_result)
self.ssh_threadpool.start(ssh_worker)
def ssh_result(self, server, message):
print("server: ", server, ", result: ", message)
# ...

Python - "can't pickle thread.lock" error when creating a thread under a multiprocess in Windows

I'm getting stuck on what I think is a basic multiprocess and threading issue. I've got a multiprocess set up, and within this a thread. However, when I set up the thread class within the init function, I get the following error:
"TypeError: can't pickle thread.lock objects".
However, this does not happen if the thread is set up outside of the init function. Does anyone know why this is happening? Note I'm using Windows.
Some code is below to illustrate the issue. As typed below, it runs fine. However if print_hello() is called from within the DoStuff init def, then the error occurs, if it's called within the multi-process run() def then it's fine.
Can anyone point me in the right direction so it runs fine when called from init? thanks!
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("starting DoStuff")
# This works fine if the line below is uncommented and __init__ self.print_hello() is commented...
self.dostuff.print_hello()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
# If the following is uncommented, the error occurs...
# Note it also occurs if the lines in start_thead are pasted here...
# self.print_hello()
def print_hello(self):
print "hello"
self.start_thread()
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("Starting MyThread")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
# mp_target.daemon = True
mp_target.start()
time.sleep(0.1)

It looks like there is no simple answer, and it appears to be a restriction of Windows (Win 7, python 3.6 in my case); on Windows it looks like you need to start the process before you can start the worker thread inside the owned object.
There appears to be no such restriction on Unix (CentOS 7, python 2.7.5).
As an experiment I modified your code as follows; this version checks the OS and starts either the process first, or the thread first:
import multiprocessing
import threading
import time
import os
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff(self)
def run(self):
print("MyProcess.run()")
print("MyProcess.ident = " + repr(self.ident))
if os.name == 'nt':
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, owner, **kwargs):
super(DoStuff, self).__init__(**kwargs)
self.owner = owner
if os.name != 'nt':
self.start_thread()
def start_thread(self):
print("DoStuff.start_thread()")
self.my_thread_instance = MyThread(self)
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self, owner):
super(MyThread, self).__init__()
self.owner = owner
def run(self):
print("MyThread.run()")
print("MyThread.ident = " + repr(self.ident))
print("MyThread.owner.owner.ident = " + repr(self.owner.owner.ident))
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)
... and got the following on Windows, where the process starts first:
MyProcess.run()
MyProcess.ident = 14700
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 14220
MyThread.owner.owner.ident = 14700
... and the following on Linux, where the thread is started first:
DoStuff.start_thread()
MyThread.run()
MyThread.ident = 140316342347520
MyThread.owner.owner.ident = None
MyProcess.run()
MyProcess.ident = 4358
If it were my code I'd be tempted to always start the process first, then create the thread within that process; the following version works fine for me across both platforms:
import multiprocessing
import threading
import time
class MyProcess(multiprocessing.Process):
def __init__(self, **kwargs):
super(MyProcess, self).__init__(**kwargs)
self.dostuff = DoStuff()
def run(self):
print("MyProcess.run()")
self.dostuff.start_thread()
class DoStuff(object):
def __init__(self, **kwargs):
super(DoStuff, self).__init__(**kwargs)
def start_thread(self):
self.my_thread_instance = MyThread()
self.my_thread_instance.start()
time.sleep(0.1)
class MyThread(threading.Thread):
def __init__(self):
super(MyThread, self).__init__()
def run(self):
print("MyThread.run()")
if __name__ == '__main__':
mp_target = MyProcess() # Also pass the pipe to transfer data
mp_target.daemon = True
mp_target.start()
time.sleep(0.1)

Callback function does not see correct values in instance

I'm having a strange phenomena in Python with callback functions and handlers.
I use ZMQ to handle my communication and use a stream for the socket. I have the base class:
import multiprocessing
import zmq
from concurrent.futures import ThreadPoolExecutor
from zmq.eventloop import ioloop, zmqstream
from zmq.utils import jsonapi as json
# Types of messages
TYPE_A = 'type_a'
TYPE_B = 'type_b'
class ZmqProcess(multiprocessing.Process):
def __init__(self):
super(ZmqProcess, self).__init__()
self.context = None
self.loop = None
self.handle_stream = None
def setup(self):
self.context = zmq.Context()
self.loop = ioloop.IOLoop.instance()
def send(self, msg_type, msg, host, port):
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://%s:%s' % (host, port))
sock.send_json([msg_type, msg])
def stream(self, sock_type, addr):
sock = self.context.socket(sock_type)
if isinstance(addr, str):
addr = addr.split(':')
host, port = addr if len(addr) == 2 else (addr[0], None)
if port:
sock.bind('tcp://%s:%s' % (host, port))
else:
port = sock.bind_to_random_port('tcp://%s' % host)
stream = zmqstream.ZMQStream(sock, self.loop)
return stream, int(port)
class MessageHandler(object):
def __init__(self, json_load=-1):
self._json_load = json_load
self.pool = ThreadPoolExecutor(max_workers=10)
def __call__(self, msg):
i = self._json_load
msg_type, data = json.loads(msg[i])
msg[i] = data
if msg_type.startswith('_'):
raise AttributeError('%s starts with an "_"' % msg_type)
getattr(self, msg_type)(*msg)
And I have a class that inherits from it:
import zmq
import zmq_base
class ZmqServerMeta(zmq_base.ZmqProcess):
def __init__(self, bind_addr, handlers):
super(ZmqServerMeta, self).__init__()
self.bind_addr = bind_addr
self.handlers = handlers
def setup(self):
super(ZmqServerMeta, self).setup()
self.handle_stream, _ = self.stream(zmq.PAIR, self.bind_addr)
self.handle_stream.on_recv(StreamHandler(self.handle_stream, self.stop,
self.handlers))
def run(self):
self.setup()
self.loop.start()
def stop(self):
self.loop.stop()
class StreamHandler(zmq_base.MessageHandler):
def __init__(self, handle_stream, stop, handlers):
super(StreamHandler, self).__init__()
self._handle_stream = handle_stream
self._stop = stop
self._handlers = handlers
def type_a(self, data):
if zmq_base.TYPE_A in self._handlers:
if self._handlers[zmq_base.TYPE_A]:
for handle in self._handlers[zmq_base.TYPE_A]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def type_b(self, data):
if zmq_base.TYPE_B in self._handlers:
if self._handlers[zmq_base.TYPE_B]:
for handle in self._handlers[zmq_base.TYPE_B]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def endit(self):
self._stop()
Additionally, I have a class that I want to use as storage. And here is where the trouble starts:
import threading
import zmq_server_meta as server
import zmq_base as base
class Storage:
def __init__(self):
self.list = []
self.list_lock = threading.RLock()
self.zmq_server = None
self.host = '127.0.0.1'
self.port = 5432
self.bind_addr = (self.host, self.port)
def setup(self):
handlers = {base.TYPE_A: [self. remove]}
self.zmq_server = server.ZmqServerMeta(handlers=handlers, bind_addr=self.bind_addr)
self.zmq_server.start()
def add(self, data):
with self.list_lock:
try:
self.list.append(data)
except:
print "Didn't work"
def remove(self, msg):
with self.list_lock:
try:
self.list.remove(msg)
except:
print "Didn't work"
The idea is that that class stores some global information that it receives.
It is all started in a file to test:
import sys
import time
import storage
import zmq_base as base
import zmq_server_meta as server
def printMsg(msg):
print msg
store = storage.Storage()
store.setup()
handlers = {base.TYPE_B: [printMsg]}
client = server.ZmqServerMeta(handlers=handlers, bind_addr=('127.0.0.1', 5431))
client.start()
message = "Test"
store.add(message)
client.send(base.TYPE_A, message, '127.0.0.1', 5432)
I simplified it to reduce clutter. Instead of just adding it, it is usually send and then a response comes back. The response, the client sending, should be processed by the correct callback, remove(), and it should remove something out of the list. The problem that occurs is, that the remove() function sees an empty list, although there should be an element in the list. If I check from the testing file, I can see the element after it was added, and if I call remove() from there, I see a non-empty list and can remove it. My question is, why does the callback sees an empty list and how can I make sure it does see the correct elements in the list?
Kind regards
Patrick

I believe the problem lays in the fact that the ZmqProcess class inherits from multiprocessing.Process. Multiprocessing does not allow to share objects among different processes, except by using a shared memory map using Value or Array ( as can be seen in the documentation: https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes )
If you want to use your custom object, you can use a Server process / proxy object, which can be found in on the same page of the documentation.
So you can, for instance, define a manager in the init function of the Storage class like: self.manager = Manager() Afterwards you put self.list = self.manager.list(). This should do the trick.

Python PyQt4: Using QTcpServer to handle data from client, but I can't receive the data

I am confused that why the QTcpSocket instance could send text to client before connected in the official example. While a connection came in, the result is:
New connection came
0
New connection ran into thread: 944
My code is at the bottom of the content, it base on the official example[threadedfortuneserver.py].
# -*- coding: utf-8 -*-
__author__ = 'erliang'
# TODO:消息处理以及UI响应
from PyQt4.QtCore import QThread, pyqtSignal
from PyQt4.QtGui import QMessageBox, QApplication
from PyQt4.QtNetwork import QTcpSocket, QTcpServer, QHostAddress
from ui import ServerWindow
class Socket(QTcpSocket):
def __init__(self, parent=None):
super(Socket, self).__init__(parent)
self.connected.connect(self.on_connected)
self.disconnected.connect(self.on_disconnected)
self.readyRead.connect(self.on_ready_read)
self.error.connect(self.on_error)
def on_connected(self):
print "Socket connected!"
def on_disconnected(self):
print "Socket disconnected!"
def on_ready_read(self):
print "Received data:", self.readAll()
def on_error(self):
print "Error occurred:", self.errorString()
class ChatThread(QThread):
error = pyqtSignal(QTcpSocket.SocketError)
def __init__(self, socketDescriptor, parent):
super(ChatThread, self).__init__(parent)
self.socketDescriptor = socketDescriptor
self.received_data = None
def run(self):
# 在此处新建线程处理客户端消息
self.tcp = Socket()
print self.tcp.state()
print "New connection ran into thread:", self.socketDescriptor
if not self.tcp.setSocketDescriptor(self.socketDescriptor):
self.error.emit(self.tcp.error())
return
# self.tcp.disconnectFromHost()
# self.tcp.waitForDisconnected()
class ChatServer(QTcpServer):
def __init__(self, parent=None):
super(ChatServer, self).__init__(parent)
self.newConnection.connect(self.on_new_connection)
def incomingConnection(self, socketDescriptor):
# print "New connection came!", socketDescriptor
self.thread = ChatThread(socketDescriptor, self)
self.thread.finished.connect(self.thread.deleteLater)
self.thread.start()
def on_new_connection(self):
print "New connection came"
class Server(ServerWindow):
def __init__(self):
super(Server, self).__init__()
self.StartServerButton.clicked.connect(self.start_server)
self.StopServerButton.clicked.connect(self.stop_server)
self.chat_server = ChatServer()
# self.ClientLogTable.setRowCount(2)
# self.ClientLogTable.setItem(1, 1, QTableWidgetItem('123'))
def start_server(self):
# GUi与Socket操作不能在同一线程, 每次打开服务器便新建一个线程
if not self.chat_server.isListening():
addr = QHostAddress()
addr.setAddress('127.0.0.1')
if not self.chat_server.listen(addr, 8888):
QMessageBox.critical(self, u"服务器", u"无法初始化服务器: %s." % self.server.errorString())
self.close()
return
self.InfoLabel.setText(u'服务器IP:{0} 端口：{1}'.format(self.chat_server.serverAddress().toString(),
self.chat_server.serverPort()))
def stop_server(self):
if self.chat_server.isListening():
self.chat_server.close()
self.InfoLabel.setText(u'服务器尚未启动！')
if __name__ == '__main__':
import sys
app = QApplication(sys.argv)
client = Server()
client.show()
sys.exit(app.exec_())

Python, Call a class function within another class

Can you anyone please help me call the broadcast function from class BroadcastServerFactory in class test, as per attached code
I have tried so many methods of call a function from another class, but no solution
import sys
from twisted.internet import reactor
from twisted.python import log
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, \
listenWS
class test():
//call broadcast function from here
class BroadcastServerProtocol(WebSocketServerProtocol):
def onOpen(self):
self.factory.register(self)
def onMessage(self, msg, binary):
if not binary:
self.factory.broadcast("'%s' from %s" % (msg, self.peerstr))
def connectionLost(self, reason):
WebSocketServerProtocol.connectionLost(self, reason)
self.factory.unregister(self)
class BroadcastServerFactory(WebSocketServerFactory):
"""
Simple broadcast server broadcasting any message it receives to all
currently connected clients.
"""
def __init__(self, url, debug = False, debugCodePaths = False):
WebSocketServerFactory.__init__(self, url, debug = debug, debugCodePaths = debugCodePaths)
self.clients = []
self.tickcount = 0
self.tick()
def tick(self):
self.tickcount += 1
self.broadcast("'tick %d' from server" % self.tickcount)
reactor.callLater(1, self.tick)
def register(self, client):
if not client in self.clients:
print "registered client " + client.peerstr
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print "unregistered client " + client.peerstr
self.clients.remove(client)
def broadcast(self, msg):
print "broadcasting message '%s' .." % msg
for c in self.clients:
c.sendMessage(msg)
print "message sent to " + c.peerstr
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
log.startLogging(sys.stdout)
debug = True
else:
debug = False
ServerFactory = BroadcastServerFactory
#ServerFactory = BroadcastPreparedServerFactory
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
factory.protocol = BroadcastServerProtocol
factory.setProtocolOptions(allowHixie76 = True)
listenWS(factory)
webdir = File(".")
web = Site(webdir)
reactor.listenTCP(8080, web)
reactor.run()

class test():
def __init__(self, factory):
factory.broadcast("I don't know what I'm doing!")
Meanwhile, in main...
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
test(factory)
This will do what you want, but it seems you're missing some core concepts about classes and instances. For the test class to call anything on another class, there needs to be an instance of it first (bar the case of static methods).