Heyo 👋
I have basic HTTPServer and pcsc_listener process that waits for user card scan. Both are run in using multiprocessing.
I have difficulty passing scanned card id from first pcsc_listener to HTTPServer as POST response - it just endlessly loops on first request, won't close and on 2nd request it just breaks.
My main function:
# Main routine
def main():
with Manager() as manager:
queue = manager.Queue()
p1 = Process(target=pcsc_listener, args=(queue,)).start()
p2 = Process(target=http_server, args=(queue,)).start()
while True:
msg = queue.get()
if msg:
if msg[0] == "active_card":
print("Card scanned " + msg[1])
Since card scanning pcsc_listener simply adds list to queue via queue.put([...]), it is safe to assume it works, so I won't add its code here.
My http server:
def http_server(queue):
def error_handler(request, client_address):
raise
class http_handler(BaseHTTPRequestHandler):
def do_GET(self):
'''
We are totally ignoring GET requests, but we going to leave it open for them.
Let's just return 501
'''
self.error_content_type('application/json')
self.send_error(501)
self.end_headers()
def do_POST(self):
content_length = int(self.headers['Content-Length'])
post_data = self.rfile.read(content_length).decode("utf-8") \
if content_length > 0 else ""
post_data = json.loads(post_data)
pass_data = {}
while True:
msg = queue.get()
if msg[0] == "active_card":
print(msg[1])
break
self.send_response(200)
self.send_header('Content-type', 'application/json')
self.end_headers()
self.wfile.write(json.dumps(post_data).encode("utf-8"))
raise(RuntimeWarning(post_data))
def log_message(self, format, *args):
return
Why is it not returning Queue and endlesly loops HTTP POST and doesn't return response? 🤔
What am I doing wrong. I basically need to share variable between these processes, so I'm okay with other suggestions.
Related
So i have a code for my tcp server. It gets a string, echoes it and closes the connection. What i want to do is that server will get more than one request and respond to more than one client and closes connection only if there is a mistake or client disconnected. How can i do this?
import threading
import socket
def run_server(port):
serv_sock = create_serv_sock(port)
cid = 0
while True:
client_sock = accept_client_conn(serv_sock, cid)
t = threading.Thread(target=serve_client,
args=(client_sock, cid))
t.start()
cid += 1
def serve_client(client_sock, cid):
request = read_request(client_sock)
if request is None:
print(f'Client #{cid} unexpectedly disconnected')
else:
response = handle_request(request)
write_response(client_sock, response, cid)
def create_serv_sock(serv_port):
serv_sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM,proto=0)
serv_sock.bind(('localhost', serv_port))
serv_sock.listen()
return serv_sock
def accept_client_conn(serv_sock, cid):
client_sock, client_addr = serv_sock.accept()
print(f'Client #{cid} connected '
f'{client_addr[0]}:{client_addr[1]}')
return client_sock
def read_request(client_sock, delimiter=b'!'):
request = bytearray()
while True:
try:
chunk = client_sock.recv(4)
if not chunk:
return None
request += chunk
if delimiter in request:
return request
except ConnectionResetError:
return None
except:
raise
def handle_request(request):
return request[::-1]
def write_response(client_sock, response, cid):
client_sock.sendall(response)
client_sock.close()
print(f'Client #{cid} has been served')
if __name__ == '__main__':
run_server(port = 9090)
i tried to make 2 independent funtions with threading.Thread like this
def response(message,client_sock):
client_sock.sendall(f'you said: {message}')
def handle(client_sock):
while True:
try:
request = client_sock.recv(1024)
response(request,client_sock)
except:
client_sock.close()
print ("client has disconected")
break
def receive():
while True:
client_sock, client_addr = server_sock.accept()
print(f"Connected with {client_addr[0]}:{client_addr[1]}")
thread = threading.Thread(target = handle, args=(client_sock))
thread.start()
But I always get a lot of thread errors
1 - Replace this function, I removed its While loop.
def read_request(client_sock, delimiter=b'!'):
request = bytearray()
try:
chunk = client_sock.recv(4)
if not chunk:
return None
request += chunk
if delimiter in request:
return request
except ConnectionResetError:
return None
except:
raise
2 - Replace this function, I moved the while loop here, which will keep running and checking for new messages and processing them
def serve_client(client_sock, cid):
while True:
request = read_request(client_sock)
if request is None:
print(f'Client #{cid} unexpectedly disconnected')
else:
response = handle_request(request)
write_response(client_sock, response, cid)
3 - Replace this function, I removed the client_sock.close() as it would close the socket once you received one message, now the connection between client and server should stay on.
def write_response(client_sock, response, cid):
client_sock.sendall(response)
print(f'Client #{cid} has been served')
although I know Python syntaxis and wrote some scripts for data processing and analysis (spectra and images) which do the job, I've never really worked with networking or streaming and guess I must admit that my programming skills are pretty low. Perhaps, I'm trying to handle more than my current skills allow, but that's probably a common situation for development.
Anyways, I'm working on (yet another) gui-client to control Raspberry Pi camera - both for fun and for the sake of learning. To cut a long story short, I would like to run a streaming http server from this gui. I went for a ready solution and followed this recipe
http://picamera.readthedocs.io/en/latest/recipes2.html#web-streaming
import io
import picamera
import logging
import socketserver
from threading import Condition
from http import server
PAGE="""\
<html>
description of webpage
</html>
"""
class StreamingOutput(object):
def __init__(self):
self.frame = None
self.buffer = io.BytesIO()
self.condition = Condition()
def write(self, buf):
if buf.startswith(b'\xff\xd8'):
# New frame, copy the existing buffer's content and notify all
# clients it's available
self.buffer.truncate()
with self.condition:
self.frame = self.buffer.getvalue()
self.condition.notify_all()
self.buffer.seek(0)
return self.buffer.write(buf)
class StreamingHandler(server.BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/':
self.send_response(301)
self.send_header('Location', '/index.html')
self.end_headers()
elif self.path == '/index.html':
content = PAGE.encode('utf-8')
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.send_header('Content-Length', len(content))
self.end_headers()
self.wfile.write(content)
elif self.path == '/stream.mjpg':
self.send_response(200)
self.send_header('Age', 0)
self.send_header('Cache-Control', 'no-cache, private')
self.send_header('Pragma', 'no-cache')
self.send_header('Content-Type', 'multipart/x-mixed-replace; boundary=FRAME')
self.end_headers()
try:
while True:
with output.condition:
output.condition.wait()
frame = output.frame
self.wfile.write(b'--FRAME\r\n')
self.send_header('Content-Type', 'image/jpeg')
self.send_header('Content-Length', len(frame))
self.end_headers()
self.wfile.write(frame)
self.wfile.write(b'\r\n')
except Exception as e:
logging.warning(
'Removed streaming client %s: %s',
self.client_address, str(e))
else:
self.send_error(404)
self.end_headers()
class StreamingServer(socketserver.ThreadingMixIn, server.HTTPServer):
allow_reuse_address = True
daemon_threads = True
with picamera.PiCamera(resolution='640x480', framerate=24) as camera:
output = StreamingOutput()
camera.start_recording(output, format='mjpeg')
try:
address = ('', 8000)
server = StreamingServer(address, StreamingHandler)
server.serve_forever()
finally:
camera.stop_recording()
OK, so this code works fine if it is run as a standalone application. But if I instead try to have it run as a function, i.e. if I want to do smth like this after the classes construction
def main():
with picamera.PiCamera(resolution='640x480', framerate=24) as camera:
output = StreamingOutput()
camera.start_recording(output, format='mjpeg')
try:
address = ('', 8000)
server = StreamingServer(address, StreamingHandler)
server.serve_forever()
except(KeyboardInterrupt):
camera.stop_recording()
if __name__ == '__main__':
main()
then it won't stream, although the output and server objects are created. I am really puzzled, could anyone, please, answer why? - I would not be surprised if the answer turns out to be simple and the question is stupid, and therefore would be grateful if someone could recommend some tutorials or simplistic reading on writing a server/client for streaming/receiving the data.
Another thing is that I'd like to be able to kill this server upon request - for this, I guess the good solution is to use threading module and have the gui and the server running in separate threads?
Many thanks
N
You're right, the first answer is pretty straightforward.
The problem is that the variable output which you try to read in StreamingHandler is not in scope when your code that defines it is inside a function called main.
output = 5
def test():
print(output)
# the following statement runs fine, output is in scope because it
# was defined in the top-level scope
test()
def test_2():
print(output_2)
def main():
output_2 = 6
test_2()
# error! test_2 doesn't know the value of output_2 because the
# output_2 variable was declared within main()
main()
So, you need to work out a way to pass the output variable to the server. My approach is to declare output as a class variable in StreamingHandler, and add output as an argument when instantiating a new StreamingServer as follows:
import io
import picamera
import logging
import socketserver
from threading import Condition
from http import server
PAGE="""\
<html>
description of webpage
</html>
"""
class StreamingOutput(object):
def __init__(self):
self.frame = None
self.buffer = io.BytesIO()
self.condition = Condition()
def write(self, buf):
if buf.startswith(b'\xff\xd8'):
# New frame, copy the existing buffer's content and notify all
# clients it's available
self.buffer.truncate()
with self.condition:
self.frame = self.buffer.getvalue()
self.condition.notify_all()
self.buffer.seek(0)
return self.buffer.write(buf)
class StreamingHandler(server.BaseHTTPRequestHandler):
output = None
def do_GET(self):
if self.path == '/':
self.send_response(301)
self.send_header('Location', '/index.html')
self.end_headers()
elif self.path == '/index.html':
content = PAGE.encode('utf-8')
self.send_response(200)
self.send_header('Content-Type', 'text/html')
self.send_header('Content-Length', len(content))
self.end_headers()
self.wfile.write(content)
elif self.path == '/stream.mjpg':
self.send_response(200)
self.send_header('Age', 0)
self.send_header('Cache-Control', 'no-cache, private')
self.send_header('Pragma', 'no-cache')
self.send_header('Content-Type', 'multipart/x-mixed-replace; boundary=FRAME')
self.end_headers()
try:
while True:
with StreamingHandler.output.condition:
StreamingHandler.output.condition.wait()
frame = StreamingHandler.output.frame
self.wfile.write(b'--FRAME\r\n')
self.send_header('Content-Type', 'image/jpeg')
self.send_header('Content-Length', len(frame))
self.end_headers()
self.wfile.write(frame)
self.wfile.write(b'\r\n')
except Exception as e:
logging.warning(
'Removed streaming client %s: %s',
self.client_address, str(e))
else:
self.send_error(404)
self.end_headers()
class StreamingServer(socketserver.ThreadingMixIn, server.HTTPServer):
allow_reuse_address = True
daemon_threads = True
def __init__(self, address, handler, output):
handler.output = output
super().__init__(address, handler)
def main():
with picamera.PiCamera(resolution='640x480', framerate=24) as camera:
output = StreamingOutput()
camera.start_recording(output, format='mjpeg')
try:
address = ('', 8000)
server = StreamingServer(address, StreamingHandler, output)
server.serve_forever()
except(KeyboardInterrupt):
camera.stop_recording()
if __name__ == '__main__':
main()
I leave your question about killing the server to my learned colleages for now.
i have a list of url_handler and i would want to make asyncronous httprequest using tornado. When all response structure is arrived i need to use it for other targets.
Here a simple example of my code:
(...)
self.number = 0
self.counter = 0
self.data = {}
(...)
#tornado.web.asynchronous
def post(self):
list_url = [url_service1, url_service2]
self.number = len(list_url)
http_client = AsyncHTTPClient()
for service in list_url:
request = tornado.httpclient.HTTPRequest(url=service, method='POST', headers={'content-type': 'application/json'}, body=json.dumps({..params..}))
http_client.fetch(request, callback=self.handle_response)
# Loop for is finished. Use self.data for example in other funcions...
# if i print(self.data) i have empty dict...
# do_something(self.data)
def handle_response(self,response):
if response.error:
print("Error")
else:
self.counter = self.counter + 1
print("Response {} / {} from {}".format(self.counter, self.number, response.effective_url))
self.data[response.effective_url] = json_decode(response.body)
# number is 2
if self.counter == self.number:
print("Finish response")
def do_something(data):
# code with data parameter
I hope my problem is well explained
Since you know AsyncHTTPClient is asynchronous, that means, the requests will run in background.
So, when the for loop is finished, that does not mean all the requests are also finished - they are running in the background even when the loop finishes.
That is why self.data is empty, because the requests aren't completed yet.
How to fix this
As you know the handle_response callback is called after every request is completed. You can call do_something function from this callback when all the requests are completed. Like this:
def handle_response(...):
...
if self.counter == self.number:
self.do_something(self.data)
print("Finish response")
I am trying to build a coap server, in which I can add a new resource without the need to stop the server, recode it and restart .my server is suppossed to host two types of resources, "sensors(Sens-Me)" and "Actuators(Act-Me)" . I want that if I press the A key, a new instance of actuator should be added to the server, likewise If i Press S for Sensor .Below is my code :
from coapthon.resources.resource import Resource
from coapthon.server.coap import CoAP
class Sensor(Resource):
def __init__(self,name="Sensor",coap_server=None):
super(Sensor,self).__init__(name,coap_server,visible=True,observable=True,allow_children=True)
self.payload = "This is a new sensor"
self.resource_type = "rt1"
self.content_type = "application/json"
self.interface_type = "if1"
self.var = 0
def render_GET(self,request):
self.payload = "new sensor value ::{}".format(str(int(self.var+1)))
self.var +=1
return self
class Actuator(Resource):
def __init__(self,name="Actuator",coap_server=None):
super(Actuator,self).__init__(name,coap_server,visible=True,observable=True)
self.payload="This is an actuator"
self.resource_type="rt1"
def render_GET(self,request):
return self
class CoAPServer(CoAP):
def __init__(self, host, port, multicast=False):
CoAP.__init__(self,(host,port),multicast)
self.add_resource('sens-Me/',Sensor())
self.add_resource('act-Me/',Actuator())
print "CoAP server started on {}:{}".format(str(host),str(port))
print self.root.dump()
def main():
ip = "0.0.0.0"
port = 5683
multicast=False
server = CoAPServer(ip,port,multicast)
try:
server.listen(10)
print "executed after listen"
except KeyboardInterrupt:
server.close()
if __name__=="__main__":
main()
I am not sure what exactly do you want to do.
Is it just to replace a resource on the same route or add a new one?
Replace a resource
It is not possible according to the current coapthon version source:
https://github.com/Tanganelli/CoAPthon/blob/b6983fbf48399bc5687656be55ac5b9cce4f4718/coapthon/server/coap.py#L279
try:
res = self.root[actual_path]
except KeyError:
res = None
if res is None:
if len(paths) != i:
return False
resource.path = actual_path
self.root[actual_path] = resource
Alternatively, you can solve it in scope of request.
Say, have a registry of handlers which are used by resources and can be changed on a user input event. Well, you'll not be able to add new routes.
If you absolutely need that feature, you may request it from a developer or contribute to that project.
Add a new resource
I have extended your snippet a little bit.
I have a little experience in Python so I an not sure I've made everything properly, but it works.
There is a separate thread polling the user input and adding the same resource. Add the needed code there.
from coapthon.resources.resource import Resource
from coapthon.server.coap import CoAP
from threading import Thread
import sys
class Sensor(Resource):
def __init__(self,name="Sensor",coap_server=None):
super(Sensor,self).__init__(name,coap_server,visible=True,observable=True,allow_children=True)
self.payload = "This is a new sensor"
self.resource_type = "rt1"
self.content_type = "application/json"
self.interface_type = "if1"
self.var = 0
def render_GET(self,request):
self.payload = "new sensor value ::{}".format(str(int(self.var+1)))
self.var +=1
return self
class Actuator(Resource):
def __init__(self,name="Actuator",coap_server=None):
super(Actuator,self).__init__(name,coap_server,visible=True,observable=True)
self.payload="This is an actuator"
self.resource_type="rt1"
def render_GET(self,request):
return self
class CoAPServer(CoAP):
def __init__(self, host, port, multicast=False):
CoAP.__init__(self,(host,port),multicast)
self.add_resource('sens-Me/',Sensor())
self.add_resource('act-Me/',Actuator())
print "CoAP server started on {}:{}".format(str(host),str(port))
print self.root.dump()
def pollUserInput(server):
while 1:
user_input = raw_input("Some input please: ")
print user_input
server.add_resource('sens-Me2/', Sensor())
def main():
ip = "0.0.0.0"
port = 5683
multicast=False
server = CoAPServer(ip,port,multicast)
thread = Thread(target = pollUserInput, args=(server,))
thread.setDaemon(True)
thread.start()
try:
server.listen(10)
print "executed after listen"
except KeyboardInterrupt:
print server.root.dump()
server.close()
sys.exit()
if __name__=="__main__":
main()
I'm trying to understand my code's behavior.
I'm using zeromq to create a server that sends a "ping" and waits for "pong" responses.
What i'm seeing is that when I send a ping, only one client receives it.
when I run this code and send "ping" for the first time i receive:
pong: A
and when i run it again, i get
pong: B
why is that? I want to send one "ping" and receive two pongs.
here's the code:
from threading import Thread
import zmq
class zmqdealer(object):
def __init__(self, port):
context = zmq.Context()
self.sock = context.socket(zmq.DEALER)
#self.sock.setsockopt(zmq.RCVTIMEO, 1000)
self.sock.bind("tcp://*:%s" % port)
thread = Thread(target=lambda: self.poll())
thread.daemon = True
thread.start()
def poll(self):
while True:
reply = self.sock.recv()
if reply != "":
print(reply)
def ping(self):
self.sock.send_multipart(['', 'ping'])
class zmqrep(object):
def __init__(self, ident,host, port):
context = zmq.Context()
self.sock = context.socket(zmq.REP)
self.sock.connect("tcp://%s:%s" % (host, port))
self.ident = ident
thread = Thread(target=lambda: self.pong())
thread.daemon = True
thread.start()
def pong(self):
while True:
request = self.sock.recv()
if request == "ping":
msg = "pong: %s" % self.ident
self.sock.send(msg)
if __name__ == "__main__":
port = 11112
host = "localhost"
server = zmqdealer(port)
client1 = zmqrep('A',host,port)
client2 = zmqrep('B',host,port)
answer = raw_input('press <ENTER> to exit or type \'ping\' to get a pong\n')
while True:
if answer == "":
break
if answer == "ping":
server.ping()
answer = raw_input()
EDIT
I found a way to make this work. I really hope there is another way because i genuinely hate this one! so it looks like dealer sends to the clients in a round robin fashion. so to make my ping work i had to send it to all the clients. how? i subscribed to the monitor socket and added every connected client to a list. every time i ping, i ping to every client. look:
import threading
import zmq
from zmq.utils import monitor
def threadify(func, daemon=True):
thread = threading.Thread(target=func)
thread.daemon = daemon
thread.start()
class zmqdealer(object):
def __init__(self, port):
context = zmq.Context()
self.sock = context.socket(zmq.DEALER)
self.monitor_sock = self.sock.get_monitor_socket()
self.sock.bind("tcp://*:%s" % port)
self.connected_clients = {}
threadify(func=self.poll)
threadify(func=self.monitor)
def poll(self):
while True:
reply = self.sock.recv()
if reply != "":
print reply
def add_client(self, event):
endpoint = event['endpoint']
value = event['value']
if endpoint in self.connected_clients:
self.connected_clients[endpoint].append(value)
else:
self.connected_clients[endpoint] = [value]
def remove_client(self, event):
endpoint = event['endpoint']
value = event['value']
if endpoint in self.connected_clients \
and value in self.connected_clients[endpoint]:
self.connected_clients[endpoint].remove(value)
def monitor(self):
options = {zmq.EVENT_ACCEPTED: lambda e: self.add_client(e),
zmq.EVENT_DISCONNECTED: lambda e: self.remove_client(e)}
while True:
event = monitor.recv_monitor_message(self.monitor_sock)
event_type = event['event']
if event_type in options:
options[event_type](event)
event['event'] = event_types[event_type]
print event
def ping(self):
connected_clients_amount = sum([len(clients) for clients in self.connected_clients.values()])
for i in xrange(connected_clients_amount):
self.sock.send_multipart(['', 'ping'])
if connected_clients_amount <= 0:
print "there are no connected clients!"
class zmqrep(object):
def __init__(self, ident, host, port):
context = zmq.Context()
self.sock = context.socket(zmq.REP)
self.sock.connect("tcp://%s:%s" % (host, port))
self.identity = ident
self.stopped = threading.Event()
threadify(self.pong)
def pong(self):
while not self.stopped.isSet():
request = self.sock.recv()
if request == "ping":
msg = "pong: %s" % self.identity
self.sock.send(msg)
self.sock.close()
def stop(self):
self.stopped.set()
if __name__ == "__main__":
port = 11112
host = "localhost"
num = 5
server = zmqdealer(port)
clients = [zmqrep(i.__str__(), host, port) for i in xrange(num)]
answer = raw_input('press <ENTER> to exit or type \'ping\' to get a pong\n')
while True:
if answer == "":
break
if answer == "ping":
server.ping()
if answer == "kill":
if len(clients) > 0:
die = clients[0]
clients.remove(die)
die.stop()
else:
print "there are no connected clients!\n"
answer = raw_input()
Router/Dealer sockets are best used for distributing tasks. Say you have 10 tasks and 2 workers, you do not care who does what. Dealer/Router will distribute in a round robin fashion.
Maybe Pub/Sub or Push/Pull sockets would fit your usecase better? They are both broadcast sockets.
Here's an example of Push/Pull used in a similar fashion as what you're doing.
You often end up doing pairs of sockets, one to transmit and one other to receive results. You could for example do a PUSH with a ping message + random identifier, and ask clients to answer on PUB/SUB where you subscribe to this random identifier. This way you can match requests and responses.