I would like to pass a Queue object to a base ThreadedHTTPServer implementation. My existing code works just fine, but I would like a safe way to send calls to and from my HTTP requests. Normally this would probably be handled by a web framework, but this is a HW limited environment.
My main confusion comes on how to pass the Queue (or any) object to allow access to other modules in my environment.
The base code template I have currently running:
import base64,threading,urlparse,urllib2,os,re,cgi,sys,time
import Queue
class DemoHttpHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server,qu):
BaseHTTPRequestHandler.__init__(self, request, client_address, server)
def do_GET(self):
...
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
def main():
listen_interface = "localhost"
listen_port = 2323
server = startLocalServer.ThreadedHTTPServer((listen_interface, listen_port), startLocalServer.DemoHttpHandler)
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
print 'started httpserver thread...'
Your code does not run but I modified it so that it will run:
import base64,threading,urlparse,urllib2,os,re,cgi,sys,time
import Queue
class DemoHttpHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server):
BaseHTTPRequestHandler.__init__(self, request, client_address, server)
self.qu = server.qu # save the queue here.
def do_GET(self):
...
self.qu # access the queue self.server.qu
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
def main():
listen_interface = "localhost"
listen_port = 2323
qu = Queue.Queue()
server = startLocalServer.ThreadedHTTPServer((listen_interface, listen_port), startLocalServer.DemoHttpHandler)
server.qu = qu # store the queue in the server
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True
server_thread.start()
print 'started httpserver thread...'
Related
I am writing a custom SSL proxy with Twisted. I keep running in to an issue that happens every so often and I cant figure out what the problem is.
When I try to connect the client transport to the server's transport through the registerProducer function exactly as twisted.protocols.portforward functions I keep getting this error.
File "/opt/Memory/Mobile/Proxy/forwarder.py", line 40, in connectionMade
self.peer.transport.registerProducer(self.transport, True)
File "/usr/lib/python3.9/site-packages/twisted/protocols/tls.py", line 602, in registerProducer
self.transport.registerProducer(producer, True)
File "/usr/lib/python3.9/site-packages/twisted/internet/_newtls.py", line 233, in registerProducer
FileDescriptor.registerProducer(self, producer, streaming)
File "/usr/lib/python3.9/site-packages/twisted/internet/abstract.py", line 104, in registerProducer
raise RuntimeError(
builtins.RuntimeError: Cannot register producer <twisted.protocols.tls._ProducerMembrane object at 0x7fb5799f0910>, because producer <twisted.protocols.tls._ProducerMembrane object at 0x7fb579b474c0> was never unregistered.
Here are my Inherited Classes from twisted?
from twisted.internet import reactor
from twisted.internet import ssl
from twisted.protocols import portforward
from twisted.internet import protocol
from twisted.python import log
import sys
##SSLProxy base class that will be inherited
class SSLProxy(protocol.Protocol):
noisy = True
peer = None
def setPeer(self, peer):
#log.msg("SSLProxy.setPeer")
self.peer = peer
def connectionLost(self, reason):
#log.msg("SSLProxy.connectionLost")
if self.peer is not None:
self.peer.transport.loseConnection()
self.peer = None
elif self.noisy:
log.msg("Unable to connect to peer: {}".format(reason))
def dataReceived(self, data):
#log.msg("SSLProxy.dataReceived")
if self.peer is not None:
self.peer.transport.write(data)
##Foward data from Proxy to => Remote Server
class SSLProxyClient(SSLProxy):
def connectionMade(self):
#log.msg("SSLProxyClient.connectionMade")
self.peer.setPeer(self)
self.transport.registerProducer(self.peer.transport, True)
self.peer.transport.registerProducer(self.transport, True)
# We're connected, everybody can read to their hearts content.
self.peer.transport.resumeProducing()
class SSLProxyClientFactory(protocol.ClientFactory):
protocol = SSLProxyClient
def setServer(self, server):
#log.msg("SSLProxyClientFactory.setServer")
self.server = server
def buildProtocol(self, *args, **kw):
#log.msg("SSLProxyClientFactory.buildProtocol")
prot = protocol.ClientFactory.buildProtocol(self, *args, **kw)
prot.setPeer(self.server)
return prot
def clientConnectionFailed(self, connector, reason):
#log.msg("SSLProxyClientFactory.clientConnectionFailed")
self.server.transport.loseConnection()
class SSLProxyServer(SSLProxy):
clientProtocolFactory = SSLProxyClientFactory
reactor = None
def connectionMade(self):
log.msg("SSLProxyServer.connectionMade")
#Get Current SSL Context
ssl_context = self.transport._tlsConnection.get_context()
#Hack to get SNI to do two functions in diffrent classes
ssl_context._finishSNI = self.SNICallback
def SNICallback(self, connection):
#log.msg("SSLProxyServer.SNICallback: {}".format(connection))
#print(connection.get_context().new_host)
self.transport.pauseProducing()
#self.transport.transport.pauseProducing()
#print(dir())
self.dst_host, self.dst_port = connection.get_context().new_host
#Setup Clients
self.client = self.clientProtocolFactory()
self.client.setServer(self)
#Start stuff
log.msg('Redirecting to {}:{}'.format(self.dst_host, self.dst_port))
if self.reactor is None:
self.reactor = reactor
log.msg("Making Connection to Dest Server: {}:{}".format(self.dst_host, self.dst_port))
self.reactor.connectSSL(self.dst_host, self.dst_port, self.client, ssl.ClientContextFactory())
#self.transport.resumeProducing()
#Client -> Proxy
def dataReceived(self, data):
log.msg("SSLProxyServer.dataReceived: {}".format(data))
#Call Inherited Function
super().dataReceived(data)
class SSLProxyFactory(protocol.Factory):
"""Factory for port forwarder."""
protocol = SSLProxyServer
def __init__(self):
super().__init__()
#log.msg("SSLProxyFactory.__init__")
def sslToSSL(localport, remotehost, remoteport, serverContextFactory):
log.msg("SSL on localhost:{} forwarding to SSL {}:{}".format(localport, remotehost, remoteport))
return reactor.listenSSL(localport, SSLProxyFactory(), serverContextFactory)
Any guidance would be appreciated.
I found that if you just unregister the current Producer you can register the new one.
##Foward data from Proxy to => Remote Server
class SSLProxyClient(SSLProxy):
def connectionMade(self):
#log.msg("SSLProxyClient.connectionMade")
self.peer.setPeer(self)
self.transport.registerProducer(self.peer.transport, True)
self.peer.transport.unregisterProducer()
self.peer.transport.registerProducer(self.transport, True)
# We're connected, everybody can read to their hearts content.
self.peer.transport.resumeProducing()
I have a simple multithreading server, But it creates a new thread for each socket, I don't want to create a lot of threads. My idea is to receive the messages in other way: when the user send a message, it will add the message to a queue of messages and with a threadpool the server will handle these requests.
The simple multithreaded server:
import socket
import threading
class ThreadedServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind((self.host, self.port))
def listen(self):
self.sock.listen(5)
while True:
client, address = self.sock.accept()
client.settimeout(60)
threading.Thread(target = self.listenToClient,args = (client,address)).start()
def listenToClient(self, client, address):
size = 1024
while True:
try:
data = client.recv(size)
if data:
# Set the response to echo back the recieved data
response = data
client.send(response)
else:
raise error('Client disconnected')
except:
client.close()
return False
if __name__ == "__main__":
port_num = input("Port? ")
ThreadedServer('',port_num).listen()
How can I implement my idea or is there better way to do it?
The question seems to be pretty old but i also stumble upon the same issue while working on the socket server, so here is the below code which you can use to make threaded socket server which doesnt spawn new threads on arrival.
Just to give gist ThreadingMixIn classes is overided with threaded pool.
class ThreadPoolMixIn(socketserver.ThreadingMixIn):
'''
use a thread pool instead of a new thread on every request
'''
# numThreads = 50
allow_reuse_address = True # seems to fix socket.error on server restart
def serve_forever(self):
'''
Handle one request at a time until doomsday.
'''
print('[X] Server is Running with No of Threads :- {}'.format(self.numThreads))
# set up the threadpool
self.requests = Queue(self.numThreads)
for x in range(self.numThreads):
t = threading.Thread(target = self.process_request_thread)
t.setDaemon(1)
t.start()
# server main loop
while True:
self.handle_request()
self.server_close()
def process_request_thread(self):
'''
obtain request from queue instead of directly from server socket
'''
while True:
socketserver.ThreadingMixIn.process_request_thread(self, *self.requests.get())
def handle_request(self):
'''
simply collect requests and put them on the queue for the workers.
'''
try:
request, client_address = self.get_request()
except socket.error:
return
if self.verify_request(request, client_address):
self.requests.put((request, client_address))
And then it is called in ThreadedTCPRequest Handler and override the numThreads parameter :
class ThreadedTCPServer(ThreadPoolMixIn, socketserver.TCPServer):
#Extend base class and overide the thread paramter to control the number of threads.
def __init__(self, no_of_threads, server_address, ThreadedTCPRequestHandler):
self.numThreads = no_of_threads
super().__init__(server_address, ThreadedTCPRequestHandler)
Ultimately creating the server which serves forever :
def create_multi_threaded_socket(CONFIG, HandlerClass = ThreadedTCPRequestHandler,
ServerClass = ThreadedTCPServer,
protocol="HTTP/1.0"):
server_address = ('', CONFIG.port)
HandlerClass.protocol_version = protocol
# httpd = ServerClass(server_address, HandlerClass)
server = ThreadedTCPServer(CONFIG.no_of_threads, server_address, ThreadedTCPRequestHandler)
sa = server.socket.getsockname()
print("Serving HTTP on {} port : {}".format(sa[0], sa[1]))
server.serve_forever()
I got the sample code from :
http://code.activestate.com/recipes/574454-thread-pool-mixin-class-for-use-with-socketservert/
Modified bit according to my need.
Hope this helps :) .
I have the following two Python classes:
import socketserver
class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self, server_address):
socketserver.TCPServer.__init__(self, server_address, MyTcpHandler)
self.allow_reuse_address = True
self.serve_forever()
class MyTcpHandler(socketserver.BaseRequestHandler):
data = ""
def handle(self):
self.data = self.request.recv(BUFF_SIZE).strip()
if self.data == b"shutdown":
self.request.close()
import threading
threading.Thread(target=SERVER.shutdown).start()
Thus, when the client sends "shutdown", the server itself should shutdown. As a workaround I set the global variable SERVER to the MyServer object, then I call SERVER.shutdown in another thread, as shown above.
But using a global variable is ugly as hell. So how can I communicate directly from the request handler with the socket server instead?
The server is available as self.server according to the Python docs here https://docs.python.org/2/library/socketserver.html#SocketServer.BaseRequestHandler.handle
I want to write a single threaded program that hosts a webserver using Tornado and also receive messages on a ZMQ socket (using PyZMQ Tornado event loop: http://learning-0mq-with-pyzmq.readthedocs.org/en/latest/pyzmq/multisocket/tornadoeventloop.html), but I'm not sure how to structure it. Should I be using
from zmq.eventloop import ioloop
or
from tornado.ioloop import IOLoop
or both?
Before all Tornado imports you need import zmq.eventloop.ioloop and call zmq.eventloop.ioloop.install function. Then you may import Tornado ioloop and use it.
See:
http://zeromq.github.io/pyzmq/eventloop.html
Here is an example with Tornado HTTP server with zeroMQ PUB SUB sockets.
#!/usr/bin/env python
import json
import tornado
import tornado.web
import zmq
from tornado import httpserver
from zmq.eventloop import ioloop
from zmq.eventloop.zmqstream import ZMQStream
ioloop.install()
tornado.ioloop = ioloop
import sys
def ping_remote():
"""callback to keep the connection with remote server alive while we wait
Network routers between raspberry pie and cloud server will close the socket
if there is no data exchanged for long time.
"""
pub_inst.send_json_data(msg="Ping", req_id="##")
sys.stdout.write('.')
sys.stdout.flush()
pending_requests = {}
class ZMQSub(object):
def __init__(self, callback):
self.callback = callback
context = zmq.Context()
socket = context.socket(zmq.SUB)
# socket.connect('tcp://127.0.0.1:5559')
socket.bind('tcp://*:8081')
self.stream = ZMQStream(socket)
self.stream.on_recv(self.callback)
socket.setsockopt(zmq.SUBSCRIBE, "")
def shutdown_zmq_sub(self):
self.stream.close()
class ZMQPub(object):
def __init__(self):
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind('tcp://*:8082')
self.publish_stream = ZMQStream(socket)
def send_json_data(self, msg, req_id):
topic = str(req_id)
self.publish_stream.send_multipart([topic, msg])
def shutdown_zmq_sub(self):
self.publish_stream.close()
def SensorCb(msg):
# decode message from raspberry pie and the channel ID.
key, msg = (i for i in msg)
if not key == "##":
msg = json.loads(msg)
if key in pending_requests.keys():
req_inst = pending_requests[key]
req_inst.write(msg)
req_inst.finish()
del pending_requests[key]
else:
print "no such request"
print pending_requests
else:
print "received ping"
class Handler(tornado.web.RequestHandler):
def __init__(self, *args, **kwargs):
super(Handler, self).__init__(*args, **kwargs)
# get the unique req id
self.req_id = str(self.application.req_id) + "#"
self.application.req_id += 1
# set headers
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers", "x-requested-with")
self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS, PUT')
#tornado.web.asynchronous
def get(self):
print self.request
if self.req_id not in pending_requests.keys():
pending_requests[self.req_id] = self
else:
print "WTF"
pub_inst.send_json_data(msg=json.dumps({"op": "ServiceCall"}), req_id=self.req_id)
if __name__ == "__main__":
pub_inst = ZMQPub()
sub_inst = ZMQSub(callback=SensorCb)
application = tornado.web.Application(
[(r'/get_sensor_data', Handler), (r'/(.*)')])
application.req_id = 0
server = httpserver.HTTPServer(application, )
port = 8080
server.listen(port)
print "Sensor server ready on port: ", port
ping = ioloop.PeriodicCallback(ping_remote, 3000)
ping.start()
tornado.ioloop.IOLoop.instance().start()
I'm trying to create an multi-threaded web server using BaseHttpServer and ThreadingMixIn (as seen on various examples). Pseudo code would be something like:
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
pass
def do_POST(self):
pass
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
if __name__ == '__main__':
server = ThreadedHTTPServer(('localhost', 9999), Handler)
print 'Starting server, use <Ctrl-C> to stop'
server.serve_forever()
This works as expected, but my problem is that not every request gets a thread, but threading is done per URL. I've tested it like this: I have an URL bound to execute the following method:
import time
import datetime
def request_with_pause(self):
print datetime.datetime.now().strftime("%H:%M:%S.%f"), 'REQUEST RECEIVED'
time.sleep(10)
print datetime.datetime.now().strftime("%H:%M:%S.%f"), 'SENT RESPONSE'
It works fine, except when I call the url twice with a 5 second pause (click the URL, wait 5 seconds and click it another time) - both "responses" arrive after 10 seconds (response of first click).
In Python 2.7:
from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
from threading import Thread
class ThreadedHTTPServer(HTTPServer):
def process_request(self, request, client_address):
thread = Thread(target=self.__new_request, args=(self.RequestHandlerClass, request, client_address, self))
thread.start()
def __new_request(self, handlerClass, request, address, server):
handlerClass(request, address, server)
self.shutdown_request(request)
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write("hello world")
server = ThreadedHTTPServer(('', 80), Handler)
#server.serve_forever()
You can find the main source code of the HTTPServer class in SocketServer.py which you can find in the Lib folder in the Python directory. (HTTPServer is inherited from TCPServer, TCPServer is inherited from BaseServer.)
The important line is 315:
def process_request(self, request, client_address):
self.finish_request(request, client_address)
self.shutdown_request(request)
def finish_request(self, request, client_address):
self.RequestHandlerClass(request, client_address, self)
In this point the server create new request object with your Handler class. The BaseRequestHandler constructor automatically call the self.setup(), self.handle() and the self.finish() method.
So what I did was to override the process_request method to move this stuff in a new thread.