With Python 2.7, I have extended the BaseHTTPServer.BaseHTTPRequestHandler to support a do_POST method. I would like to give the request handler a queue, so that it can put the posted data on a queue to be processed by another thread.
Here is a stripped down version of my class:
import BaseHTTPServer
import json
class PostHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.end_headers()
length = int(self.headers['Content-Length'])
self.post_data = self.rfile.read(length)
try:
if self.headers['Content-Type'] == 'application/json':
self.post_data = json.loads(self.post_data)
self.log_message(json.dumps(self.post_data))
### WANT TO PUT self.post_data ON A QUEUE HERE ###
except KeyError as error:
self.log_message('No Content-Type header')
except ValueError as error:
self.log_message("%s" % error)
Since then handler gets created by the BaseHTTPServer, I don't think I can alter the init method to pass in a queue.
I'd like my main() to look something like this:
def main():
import logging
import Queue
import signal
import threading
import traceback
try:
# set stoprequest to accept Ctrl+c
stoprequest = threading.Event()
signal.signal(signal.SIGINT, lambda signal, frame: stoprequest.set())
args = _get_main_args()
### HERE IS THE QUEUE, HOW TO I GIVE A REFERENCE TO THE HANDLER??? ###
data_queue = Queue.Queue()
handler = PostHTTPRequestHandler
server = BaseHTTPServer.HTTPServer((args.address, args.port), handler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
while not stoprequest.is_set():
try:
data = data_queue.get(False)
### I WANT TO PROCESS THE DATA HERE ###
except Queue.Empty:
pass
server.shutdown()
#logging.debug("Exiting with return code 0")
return 0
except Exception:
sys.stderr.write("%s" % traceback.format_exc())
return 1
I now see that the init for the BaseRequestHandler looks like this:
class BaseRequestHandler:
def __init__(self, request, client_address, server):
self.request = request
self.client_address = client_address
self.server = server
self.setup()
try:
self.handle()
finally:
self.finish()
So I will extend the BaseHTTPServer to contain a queue, then it will be available to the handler.
class QueuingHTTPServer(BaseHTTPServer.HTTPServer):
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.data_queue = Queue.Queue()
So now the handler looks like this:
class PostHTTPRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
def do_POST(self):
self.send_response(200)
self.end_headers()
length = int(self.headers['Content-Length'])
self.post_data = self.rfile.read(length)
try:
if self.headers['Content-Type'] == 'application/json':
self.post_data = json.loads(self.post_data)
self.log_message(json.dumps(self.post_data))
try:
self.server.data_queue.put(self.post_data)
except Queue.Full:
pass
except KeyError as error:
self.log_message('No Content-Type header')
except ValueError as error:
self.log_message("%s" % error)
I modified the code submitted by Josh to use a shared Queue reference instead of creating one during instantiation.
class QueuingHTTPServer(BaseHTTPServer.HTTPServer):
def __init__(self, server_address, RequestHandlerClass, data_queue, bind_and_activate=True):
BaseHTTPServer.HTTPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.data_queue = queue
Related
I have separately developed programs for a pyqt5 app and a client for communicate with a server.
GUI:
from PyQt5 import QtCore, QtWidgets, QtGui
from screen_stack import Ui_MainWindow_screenstack
from screen1 import Ui_Form_screen1
from screen2 import Ui_Form_screen2
from screen3 import Ui_Form_screen3
from client_cli_oop_with_exception_printing import mainProg
import socket
import threading #maybe this won't useful
import sys
import os
class MainApp(QtWidgets.QMainWindow):
def __init__(self):
super(MainApp, self).__init__() #if init was provided with paren=None, then inside this init parent argument can be passed
self.screenstack = Ui_MainWindow_screenstack()
self.screenstack.setupUi(self)
self.screen1 = scr1()
self.screen2 = scr2()
self.screen3 = scr3()
self.screenstack.stackedWidget.addWidget(self.screen1)
self.screenstack.stackedWidget.addWidget(self.screen2)
self.screenstack.stackedWidget.addWidget(self.screen3)
self.screenstack.stackedWidget.setCurrentWidget(self.screen1)
# self.screen1.s1.connect_tcp.clicked.connect(
# lambda: self.screenstack.stackedWidget.setCurrentWidget(self.screen2)
# )
self.screen1.s1.connect_tcp.clicked.connect(self.init_connect_server)
self.screen2.s2.register_name.clicked.connect(
lambda: self.screenstack.stackedWidget.setCurrentWidget(self.screen3)
)
self.sock = None
self.errScr1 = QtWidgets.QMessageBox()
self.errScr1.setWindowTitle('Error')
def init_connect_server(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.sock.connect((self.screen1.s1.tcp_link.text(), int(self.screen1.s1.tcp_port.text())))
self.screenstack.stackedWidget.setCurrentWidget(self.screen2)
except Exception as e:
self.errScr1.setText(str(e))
self.errScr1.exec_()
def start_client_threads(self):
#all the communication threads should start here..
pass
class scr1(QtWidgets.QWidget):
def __init__(self):
super(scr1,self).__init__()
self.s1=Ui_Form_screen1()
self.s1.setupUi(self)
class scr2(QtWidgets.QWidget):
def __init__(self):
super(scr2,self).__init__()
self.s2=Ui_Form_screen2()
self.s2.setupUi(self)
class scr3(QtWidgets.QWidget):
def __init__(self):
super(scr3,self).__init__()
self.s3=Ui_Form_screen3()
self.s3.setupUi(self)
if __name__ == '__main__':
app = QtWidgets.QApplication(sys.argv)
sw = MainApp()
sw.show()
sys.exit(app.exec_())
Client:
import socket
import threading
import time
import os
class mainProg:
def __init__(self, host, port):
self.is_connection_lost = threading.Event()
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.client.connect((host, port))
except Exception as e:
print(e)
print('Server is not valid...quitting...')
time.sleep(5)
os._exit(0)
self.start_threads()
def recieve(self):
while True:
try:
message = self.client.recv(1024).decode('utf-8')
if message == 'Nickname':
self.client.send(nickname.encode('utf-8'))
elif message == '': #when server gone, we will recieve a lot of empty strings continuously.. we can make the clients shut the python if server gone, but client waiting server to init again will be cooler
self.is_connection_lost.set()
self.client.close()
print('Connection Lost: waiting for the connection,')
while True:
print('waiting...')
time.sleep(3)
if not self.is_connection_lost.is_set():
print('Connection restored!!!')
break
else:
print(message)
#print(type(message))
except Exception as e:
print(e)
print('An error occured!, Not yet connected!')
#client.close()
#is_connection_lost = True
break
def send_messages(self):
while True:
try:
message = f"{nickname}: {input('')}"
self.client.send(message.encode('utf-8'))
except Exception:
pass
def reconnect_server(self):
time.sleep(3)
while True:
if self.is_connection_lost.is_set():
print('Trying to connect!!!!')
time.sleep(3)
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.client.connect((srv,port))
except Exception as e:
print(f'Reconnect failed due to {e}')
continue
if self.client.recv(1024).decode('utf-8') == '':
self.client.close()
pass
else:
self.client.send(nickname.encode('utf-8'))
self.is_connection_lost.clear() #no problem here, it connects to server when server is up again...
def start_threads(self):
t1 = threading.Thread(target=self.recieve)
t2 = threading.Thread(target=self.send_messages)
t3 = threading.Thread(target=self.reconnect_server)
t1.start()
t2.start()
t3.start()
if __name__ == '__main__':
nickname = input('Choose a nickname: ')
srv = '0.tcp.in.ngrok.io'
port = 14112
mainProg(srv, port)
I know that using qthreads I can rewrite the code for client in GUI and make a functional app. But is there a way to connect both classes, so that I can use self.labelMessage.setText(message) instead of print(message) in client. In another way I am asking is there a way to inherit from multiple classes (MainApp and mainProg) so that I can easily connect both codes?
I'm having a strange phenomena in Python with callback functions and handlers.
I use ZMQ to handle my communication and use a stream for the socket. I have the base class:
import multiprocessing
import zmq
from concurrent.futures import ThreadPoolExecutor
from zmq.eventloop import ioloop, zmqstream
from zmq.utils import jsonapi as json
# Types of messages
TYPE_A = 'type_a'
TYPE_B = 'type_b'
class ZmqProcess(multiprocessing.Process):
def __init__(self):
super(ZmqProcess, self).__init__()
self.context = None
self.loop = None
self.handle_stream = None
def setup(self):
self.context = zmq.Context()
self.loop = ioloop.IOLoop.instance()
def send(self, msg_type, msg, host, port):
sock = zmq.Context().socket(zmq.PAIR)
sock.connect('tcp://%s:%s' % (host, port))
sock.send_json([msg_type, msg])
def stream(self, sock_type, addr):
sock = self.context.socket(sock_type)
if isinstance(addr, str):
addr = addr.split(':')
host, port = addr if len(addr) == 2 else (addr[0], None)
if port:
sock.bind('tcp://%s:%s' % (host, port))
else:
port = sock.bind_to_random_port('tcp://%s' % host)
stream = zmqstream.ZMQStream(sock, self.loop)
return stream, int(port)
class MessageHandler(object):
def __init__(self, json_load=-1):
self._json_load = json_load
self.pool = ThreadPoolExecutor(max_workers=10)
def __call__(self, msg):
i = self._json_load
msg_type, data = json.loads(msg[i])
msg[i] = data
if msg_type.startswith('_'):
raise AttributeError('%s starts with an "_"' % msg_type)
getattr(self, msg_type)(*msg)
And I have a class that inherits from it:
import zmq
import zmq_base
class ZmqServerMeta(zmq_base.ZmqProcess):
def __init__(self, bind_addr, handlers):
super(ZmqServerMeta, self).__init__()
self.bind_addr = bind_addr
self.handlers = handlers
def setup(self):
super(ZmqServerMeta, self).setup()
self.handle_stream, _ = self.stream(zmq.PAIR, self.bind_addr)
self.handle_stream.on_recv(StreamHandler(self.handle_stream, self.stop,
self.handlers))
def run(self):
self.setup()
self.loop.start()
def stop(self):
self.loop.stop()
class StreamHandler(zmq_base.MessageHandler):
def __init__(self, handle_stream, stop, handlers):
super(StreamHandler, self).__init__()
self._handle_stream = handle_stream
self._stop = stop
self._handlers = handlers
def type_a(self, data):
if zmq_base.TYPE_A in self._handlers:
if self._handlers[zmq_base.TYPE_A]:
for handle in self._handlers[zmq_base.TYPE_A]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def type_b(self, data):
if zmq_base.TYPE_B in self._handlers:
if self._handlers[zmq_base.TYPE_B]:
for handle in self._handlers[zmq_base.TYPE_B]:
self.pool.submit(handle, data)
else:
pass
else:
pass
def endit(self):
self._stop()
Additionally, I have a class that I want to use as storage. And here is where the trouble starts:
import threading
import zmq_server_meta as server
import zmq_base as base
class Storage:
def __init__(self):
self.list = []
self.list_lock = threading.RLock()
self.zmq_server = None
self.host = '127.0.0.1'
self.port = 5432
self.bind_addr = (self.host, self.port)
def setup(self):
handlers = {base.TYPE_A: [self. remove]}
self.zmq_server = server.ZmqServerMeta(handlers=handlers, bind_addr=self.bind_addr)
self.zmq_server.start()
def add(self, data):
with self.list_lock:
try:
self.list.append(data)
except:
print "Didn't work"
def remove(self, msg):
with self.list_lock:
try:
self.list.remove(msg)
except:
print "Didn't work"
The idea is that that class stores some global information that it receives.
It is all started in a file to test:
import sys
import time
import storage
import zmq_base as base
import zmq_server_meta as server
def printMsg(msg):
print msg
store = storage.Storage()
store.setup()
handlers = {base.TYPE_B: [printMsg]}
client = server.ZmqServerMeta(handlers=handlers, bind_addr=('127.0.0.1', 5431))
client.start()
message = "Test"
store.add(message)
client.send(base.TYPE_A, message, '127.0.0.1', 5432)
I simplified it to reduce clutter. Instead of just adding it, it is usually send and then a response comes back. The response, the client sending, should be processed by the correct callback, remove(), and it should remove something out of the list. The problem that occurs is, that the remove() function sees an empty list, although there should be an element in the list. If I check from the testing file, I can see the element after it was added, and if I call remove() from there, I see a non-empty list and can remove it. My question is, why does the callback sees an empty list and how can I make sure it does see the correct elements in the list?
Kind regards
Patrick
I believe the problem lays in the fact that the ZmqProcess class inherits from multiprocessing.Process. Multiprocessing does not allow to share objects among different processes, except by using a shared memory map using Value or Array ( as can be seen in the documentation: https://docs.python.org/3/library/multiprocessing.html#sharing-state-between-processes )
If you want to use your custom object, you can use a Server process / proxy object, which can be found in on the same page of the documentation.
So you can, for instance, define a manager in the init function of the Storage class like: self.manager = Manager() Afterwards you put self.list = self.manager.list(). This should do the trick.
I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?
You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.
I am running an application which cannot sit and wait the successful/unsuccessful connection to a Python Manager. The client application should try to send some info to the supposedly running server, and in case it fails, another measure is taken. The problem is that whenever the server is down the connection takes a lot of time to return the control to the client application, and it cannot waste time waiting for it because there is other stuff to do.
I came up with a scheme where an intermediary object is in charge of the connection but it only works once. Let's say that for the first time, when there is still no connection to the server, this intermediary object handles the connecting part without blocking the client application. If, for some reason, the server goes down and comes back again, I can't get it to work anymore.
Suppose I have the following server:
# server.py
from multiprocessing import Queue, managers
from multiprocessing.queues import Empty
import select
import threading
class RServer(object):
def __init__(self, items_buffer):
self.items_buffer = items_buffer
def receive_items(self):
while True:
(_, [], []) = select.select([self.items_buffer._reader], [], [])
while True:
try:
item = self.items_buffer.get(block=False)
# do something with item
print('item received')
except Empty:
break
class SharedObjectsManager(managers.BaseManager):
pass
if __name__ == '__main__':
items_buffer = Queue()
remote_server = RServer(items_buffer)
remote_server_th = threading.Thread(target=remote_server.receive_items)
remote_server_th.start()
SharedObjectsManager.register('items_buffer', callable=lambda: items_buffer)
shared_objects_manager = SharedObjectsManager(address=('localhost', 5001),
authkey=str.encode('my_server'),
serializer='xmlrpclib')
s = shared_objects_manager.get_server()
s.serve_forever()
And here is the intermediary object to handle the connection:
# bridge.py
from multiprocessing.managers import BaseManager
import threading
import socket
class ConnectionManager():
def __init__(self):
self.remote_manager = BaseManager(address=('localhost', 5001),
authkey=b'my_server',
serializer='xmlrpclib')
self.remote_manager.register('items_buffer')
self.items_buffer = None
self.items_buffer_lock = threading.Lock()
self.connecting = False
self.connecting_lock = threading.Lock()
self.connection_started_condition = threading.Condition()
def transmit_item(self, item):
try:
with self.items_buffer_lock:
self.items_buffer.put(item)
except (AttributeError, EOFError, IOError):
with self.connection_started_condition:
with self.connecting_lock:
if not self.connecting:
self.connecting = True
connect_th = threading.Thread(target=self.connect_to_server,
name='Client Connect')
connect_th.start()
self.connection_started_condition.notify()
raise ConnectionError('Connection Error')
def connect_to_server(self):
with self.connection_started_condition:
self.connection_started_condition.wait()
try:
self.remote_manager.connect()
except socket.error:
pass
else:
try:
with self.items_buffer_lock:
self.items_buffer = self.remote_manager.items_buffer()
except (AssertionError, socket.error):
pass
with self.connecting_lock:
self.connecting = False
class ConnectionError(Exception):
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
And finally the client application:
# client.py
import time
from bridge import ConnectionManager, ConnectionError
remote_buffer = ConnectionManager()
while True:
try:
remote_buffer.transmit_item({'rubish': None})
print('item sent')
except ConnectionError:
# do something else
print('item not sent')
# do other stuff
print('doing other stuff')
time.sleep(15)
I am for sure doing something wrong with the thread but I can't figure it out. Any idea?
Can you anyone please help me call the broadcast function from class BroadcastServerFactory in class test, as per attached code
I have tried so many methods of call a function from another class, but no solution
import sys
from twisted.internet import reactor
from twisted.python import log
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, \
listenWS
class test():
//call broadcast function from here
class BroadcastServerProtocol(WebSocketServerProtocol):
def onOpen(self):
self.factory.register(self)
def onMessage(self, msg, binary):
if not binary:
self.factory.broadcast("'%s' from %s" % (msg, self.peerstr))
def connectionLost(self, reason):
WebSocketServerProtocol.connectionLost(self, reason)
self.factory.unregister(self)
class BroadcastServerFactory(WebSocketServerFactory):
"""
Simple broadcast server broadcasting any message it receives to all
currently connected clients.
"""
def __init__(self, url, debug = False, debugCodePaths = False):
WebSocketServerFactory.__init__(self, url, debug = debug, debugCodePaths = debugCodePaths)
self.clients = []
self.tickcount = 0
self.tick()
def tick(self):
self.tickcount += 1
self.broadcast("'tick %d' from server" % self.tickcount)
reactor.callLater(1, self.tick)
def register(self, client):
if not client in self.clients:
print "registered client " + client.peerstr
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print "unregistered client " + client.peerstr
self.clients.remove(client)
def broadcast(self, msg):
print "broadcasting message '%s' .." % msg
for c in self.clients:
c.sendMessage(msg)
print "message sent to " + c.peerstr
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
log.startLogging(sys.stdout)
debug = True
else:
debug = False
ServerFactory = BroadcastServerFactory
#ServerFactory = BroadcastPreparedServerFactory
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
factory.protocol = BroadcastServerProtocol
factory.setProtocolOptions(allowHixie76 = True)
listenWS(factory)
webdir = File(".")
web = Site(webdir)
reactor.listenTCP(8080, web)
reactor.run()
class test():
def __init__(self, factory):
factory.broadcast("I don't know what I'm doing!")
Meanwhile, in main...
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
test(factory)
This will do what you want, but it seems you're missing some core concepts about classes and instances. For the test class to call anything on another class, there needs to be an instance of it first (bar the case of static methods).