We Keep Coding

Python HTTP server endless loop on POST response while returing multiprocessing queue

Heyo 👋
I have basic HTTPServer and pcsc_listener process that waits for user card scan. Both are run in using multiprocessing.
I have difficulty passing scanned card id from first pcsc_listener to HTTPServer as POST response - it just endlessly loops on first request, won't close and on 2nd request it just breaks.
My main function:
# Main routine
def main():
with Manager() as manager:
queue = manager.Queue()
p1 = Process(target=pcsc_listener, args=(queue,)).start()
p2 = Process(target=http_server, args=(queue,)).start()
while True:
msg = queue.get()
if msg:
if msg[0] == "active_card":
print("Card scanned " + msg[1])
Since card scanning pcsc_listener simply adds list to queue via queue.put([...]), it is safe to assume it works, so I won't add its code here.
My http server:
def http_server(queue):
def error_handler(request, client_address):
raise
class http_handler(BaseHTTPRequestHandler):
def do_GET(self):
'''
We are totally ignoring GET requests, but we going to leave it open for them.
Let's just return 501
'''
self.error_content_type('application/json')
self.send_error(501)
self.end_headers()
def do_POST(self):
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length).decode("utf-8") \
if content_length > 0 else ""
post_data = json.loads(post_data)
pass_data = {}
while True:
msg = queue.get()
if msg[0] == "active_card":
print(msg[1])
break
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(json.dumps(post_data).encode("utf-8"))
raise(RuntimeWarning(post_data))
def log_message(self, format, *args):
return
Why is it not returning Queue and endlesly loops HTTP POST and doesn't return response? 🤔
What am I doing wrong. I basically need to share variable between these processes, so I'm okay with other suggestions.

TCP client not able to write to socket but receive works fine

I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?

You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.

How to put queue in a Python 2 BaseHTTPRequestHandler?

With Python 2.7, I have extended the BaseHTTPServer.BaseHTTPRequestHandler to support a do_POST method. I would like to give the request handler a queue, so that it can put the posted data on a queue to be processed by another thread.
Here is a stripped down version of my class:
import BaseHTTPServer
import json
class PostHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.end_headers()
length = int(self.headers['Content-Length'])
self.post_data = self.rfile.read(length)
try:
if self.headers['Content-Type'] == 'application/json':
self.post_data = json.loads(self.post_data)
self.log_message(json.dumps(self.post_data))
### WANT TO PUT self.post_data ON A QUEUE HERE ###
except KeyError as error:
self.log_message('No Content-Type header')
except ValueError as error:
self.log_message("%s" % error)
Since then handler gets created by the BaseHTTPServer, I don't think I can alter the init method to pass in a queue.
I'd like my main() to look something like this:
def main():
import logging
import Queue
import signal
import threading
import traceback
try:
# set stoprequest to accept Ctrl+c
stoprequest = threading.Event()
signal.signal(signal.SIGINT, lambda signal, frame: stoprequest.set())
args = _get_main_args()
### HERE IS THE QUEUE, HOW TO I GIVE A REFERENCE TO THE HANDLER??? ###
data_queue = Queue.Queue()
handler = PostHTTPRequestHandler
server = BaseHTTPServer.HTTPServer((args.address, args.port), handler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
while not stoprequest.is_set():
try:
data = data_queue.get(False)
### I WANT TO PROCESS THE DATA HERE ###
except Queue.Empty:
pass
server.shutdown()
#logging.debug("Exiting with return code 0")
return 0
except Exception:
sys.stderr.write("%s" % traceback.format_exc())
return 1

I now see that the init for the BaseRequestHandler looks like this:
class BaseRequestHandler:
def __init__(self, request, client_address, server):
self.request = request
self.client_address = client_address
self.server = server
self.setup()
try:
self.handle()
finally:
self.finish()
So I will extend the BaseHTTPServer to contain a queue, then it will be available to the handler.
class QueuingHTTPServer(BaseHTTPServer.HTTPServer):
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.data_queue = Queue.Queue()
So now the handler looks like this:
class PostHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.end_headers()
length = int(self.headers['Content-Length'])
self.post_data = self.rfile.read(length)
try:
if self.headers['Content-Type'] == 'application/json':
self.post_data = json.loads(self.post_data)
self.log_message(json.dumps(self.post_data))
try:
self.server.data_queue.put(self.post_data)
except Queue.Full:
pass
except KeyError as error:
self.log_message('No Content-Type header')
except ValueError as error:
self.log_message("%s" % error)

I modified the code submitted by Josh to use a shared Queue reference instead of creating one during instantiation.
class QueuingHTTPServer(BaseHTTPServer.HTTPServer):
def __init__(self, server_address, RequestHandlerClass, data_queue, bind_and_activate=True):
BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.data_queue = queue

PyQT4 Asynchronous QTcpServer (How to create event loop)

Trying to convert a server written in C++ into Python. The server was written to be Asynchronous/Non Blocking. What works in C++ doesn't seem to want to work for me in Python
I am using PyQT4. I read Python you have to create the event loop or something along those lines any ideas are greatly appreciated
I should mention what seems to not work is that the incomingConnection function in Class Server is never called.
*cheers
import sys
from PyQt4.QtCore import *
from PyQt4.QtNetwork import *
class Client(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
QThreadPool.globalInstance().setMaxThreadCount(15)
def SetSocket(self, Descriptor):
self.socket = QTcpSocket(self)
self.connect(self.socket, SIGNAL("connected()"), SLOT(self.connected()))
self.connect(self.socket, SIGNAL("disconnected()"), SLOT(self.disconnected()))
self.connect(self.socket, SIGNAL("readyRead()"), SLOT(self.readyRead()))
self.socket.setSocketDescriptor(Descriptor)
print "Client Connected from IP %s" % self.socket.peerAddress().toString()
def connected(self):
print "Client Connected Event"
def disconnected(self):
print "Client Disconnected"
def readyRead(self):
msg = self.socket.readAll()
print msg
class Server(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
def incomingConnection(self, handle):
print "incoming"
self.client = Client(self)
self.client.SetSocket(handle)
def StartServer(self):
self.server = QTcpServer()
if self.server.listen(QHostAddress("0.0.0.0"), 8888):
print "Server is awake"
else:
print "Server couldn't wake up"
def main():
app = QCoreApplication(sys.argv)
Server().StartServer()
sys.exit(app.exec_())
if __name__ == '__main__':
main()

incomingConnection is not called because QTcpServer's base implementation of the function is called. as incomingConnection is a vitual function, you just have to assign your's to QTcpServer's incomingConnection attribute, like this:
class Server(QObject):
def __init__(self, parent=None):
QObject.__init__(self)
def incomingConnection(self, handle):
print "incoming"
self.client = Client(self)
self.client.SetSocket(handle)
def StartServer(self):
self.server = QTcpServer()
self.server.incomingConnection = self.incomingConnection
if self.server.listen(QHostAddress("0.0.0.0"), 8888):
print "Server is awake"
else:
print "Server couldn't wake up"
you can check out PySide's documentation, as it's much more pythonic than PyQt's, currently hosted only here:
http://srinikom.github.com/pyside-docs/

Confirmation box on file transfer over sockets using PyQT

I have a confirmation box which gets called when sending a file over the network. However, the function for the Signal is in the MainWindow class and uses variables from the Worker thread. But it doesn't work because the variables in the worker thread are not in the right scope.
Any idea how I can pass the msg variable to the saveFile function in the MainWindow class from the signal?
Edit: What I needed to do was pass an argument to my signal but I figured it out. Sorry for the confusion. I was unsure what needed to be done.
Here's a working example:
import socket
import select
import sys, os
from PyQt4.QtCore import *
from PyQt4.QtGui import *
CMD_FILE = 1
CLIENT_PORT = 9001
CLIENT_HOST = '127.0.0.1'
class MainWindow(QWidget):
def __init__(self, parent=None):
#super(QWidget, self).__init__(parent)
super(MainWindow, self).__init__(parent)
self.resize(100, 50)
self.thread = Worker()
self.thread.start()
self.file_button = QPushButton('Send a File')
self.connect(self.file_button, SIGNAL("released()"), self.sendFile)
self.connect_button = QPushButton('Connect To Server')
self.connect(self.connect_button, SIGNAL("released()"), self.connectToServer)
self.connect(self.thread, SIGNAL('triggered(PyQt_PyObject)'), self.saveFile)
self.layout = QFormLayout()
self.layout.addRow(self.file_button, self.connect_button)
self.setLayout(self.layout)
def connectToServer(self):
global s
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((str(CLIENT_HOST).strip(), CLIENT_PORT))
print "Connected to ", str(CLIENT_HOST)
except:
print "unable to connect"
def clientAll(self, cmd, msg):
#check if connnected send a message
s.sendall(cmd + msg)
def sendFile(self):
filename=QFileDialog.getOpenFileName(self, 'Open File', '.')
f = open(filename, 'rb')
myFile = f.read()
self.clientAll(chr(CMD_FILE), myFile)
f.close()
print filename, "Sent\n"
def saveFile(self, msg):
reply = QMessageBox.question(self, 'Message',
"Are you sure you wish to download this file?", QMessageBox.Yes |
QMessageBox.No, QMessageBox.No)
if reply == QMessageBox.Yes:
filename = QFileDialog.getSaveFileName(self, 'Save File', '.')
f = open(filename, 'wb')
f.write(msg)
f.close()
print filename, "Recieved"
else:
pass
class Worker(QThread):
def __init__(self):
QThread.__init__(self)
self.exiting = False
def __del__(self):
self.exiting = True
self.wait()
def run(self):
source_ip = ''
#socket.gethostbyname(socket.gethostname())
PORT = 9001
### Initialize socket
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((source_ip, PORT))
#
server_socket.listen(5)
read_list = [server_socket]
### Start receive loop
while True:
readable, writable, errored = select.select(read_list, [], [])
for s in readable:
if s is server_socket:
conn, addr = s.accept()
read_list.append(conn)
else:
msg = conn.recv(12024)
if msg:
cmd, msg = ord(msg[0]),msg[1:]
if cmd == CMD_FILE:
if not msg: break
self.emit(SIGNAL('triggered(PyQt_PyObject)'), msg)
else:
s.close()
read_list.remove(s)
if __name__ == '__main__':
app = QApplication(sys.argv)
win = MainWindow()
win.setWindowTitle('CATS!!! <3')
win.show()
sys.exit(app.exec_())

I think I don't understand exactly your situation.
However, did you try to define a new (custom) signal?
If you define a new signal with pyqtSignal,
you can emit the signal with parameters.
Documentation on signal and slots (new style).