I want to write a single threaded program that hosts a webserver using Tornado and also receive messages on a ZMQ socket (using PyZMQ Tornado event loop: http://learning-0mq-with-pyzmq.readthedocs.org/en/latest/pyzmq/multisocket/tornadoeventloop.html), but I'm not sure how to structure it. Should I be using
from zmq.eventloop import ioloop
or
from tornado.ioloop import IOLoop
or both?
Before all Tornado imports you need import zmq.eventloop.ioloop and call zmq.eventloop.ioloop.install function. Then you may import Tornado ioloop and use it.
See:
http://zeromq.github.io/pyzmq/eventloop.html
Here is an example with Tornado HTTP server with zeroMQ PUB SUB sockets.
#!/usr/bin/env python
import json
import tornado
import tornado.web
import zmq
from tornado import httpserver
from zmq.eventloop import ioloop
from zmq.eventloop.zmqstream import ZMQStream
ioloop.install()
tornado.ioloop = ioloop
import sys
def ping_remote():
"""callback to keep the connection with remote server alive while we wait
Network routers between raspberry pie and cloud server will close the socket
if there is no data exchanged for long time.
"""
pub_inst.send_json_data(msg="Ping", req_id="##")
sys.stdout.write('.')
sys.stdout.flush()
pending_requests = {}
class ZMQSub(object):
def __init__(self, callback):
self.callback = callback
context = zmq.Context()
socket = context.socket(zmq.SUB)
# socket.connect('tcp://127.0.0.1:5559')
socket.bind('tcp://*:8081')
self.stream = ZMQStream(socket)
self.stream.on_recv(self.callback)
socket.setsockopt(zmq.SUBSCRIBE, "")
def shutdown_zmq_sub(self):
self.stream.close()
class ZMQPub(object):
def __init__(self):
context = zmq.Context()
socket = context.socket(zmq.PUB)
socket.bind('tcp://*:8082')
self.publish_stream = ZMQStream(socket)
def send_json_data(self, msg, req_id):
topic = str(req_id)
self.publish_stream.send_multipart([topic, msg])
def shutdown_zmq_sub(self):
self.publish_stream.close()
def SensorCb(msg):
# decode message from raspberry pie and the channel ID.
key, msg = (i for i in msg)
if not key == "##":
msg = json.loads(msg)
if key in pending_requests.keys():
req_inst = pending_requests[key]
req_inst.write(msg)
req_inst.finish()
del pending_requests[key]
else:
print "no such request"
print pending_requests
else:
print "received ping"
class Handler(tornado.web.RequestHandler):
def __init__(self, *args, **kwargs):
super(Handler, self).__init__(*args, **kwargs)
# get the unique req id
self.req_id = str(self.application.req_id) + "#"
self.application.req_id += 1
# set headers
self.set_header("Access-Control-Allow-Origin", "*")
self.set_header("Access-Control-Allow-Headers", "x-requested-with")
self.set_header('Access-Control-Allow-Methods', 'POST, GET, OPTIONS, PUT')
#tornado.web.asynchronous
def get(self):
print self.request
if self.req_id not in pending_requests.keys():
pending_requests[self.req_id] = self
else:
print "WTF"
pub_inst.send_json_data(msg=json.dumps({"op": "ServiceCall"}), req_id=self.req_id)
if __name__ == "__main__":
pub_inst = ZMQPub()
sub_inst = ZMQSub(callback=SensorCb)
application = tornado.web.Application(
[(r'/get_sensor_data', Handler), (r'/(.*)')])
application.req_id = 0
server = httpserver.HTTPServer(application, )
port = 8080
server.listen(port)
print "Sensor server ready on port: ", port
ping = ioloop.PeriodicCallback(ping_remote, 3000)
ping.start()
tornado.ioloop.IOLoop.instance().start()
Related
I am working on Rabbitmq and Server sent events for this I use tornado-eventsource (https://github.com/guilhermef/tornado-eventsource/tree/master/tornado_eventsource) from this code I tried to make my connection of EventSource. When my server receives messages it suooise to send it to client but it gives this error message and messages are not watched by server to send
But I guess due to a change in versions as I use python3 I get an error.
File "C:\Users\Admin\AppData\Local\Programs\Python\Python37\lib\site-packages\tornado_eventsource\handler.py", line 71, in write_message
if isinstance(msg, str) or isinstance(msg, unicode):
NameError: name 'unicode' is not defined
Please guide me regards this.
Server side code here:
import _thread
import asyncio
import time
import os
import tornado.ioloop
import tornado.web
from DateTime import DateTime
import pika
from tornado.web import Application, RequestHandler
from threading import Thread
import logging
import six
#from tornadose.handlers
import tornado_eventsource.handler
from tornado_eventsource.handler import EventSourceHandler
logging.basicConfig(level=logging.INFO)
# web socket clients connected.
clients = []
connection = pika.BlockingConnection()
logging.info('Connected:localhost')
channel = connection.channel()
print("Connection Time: ", time.time()*1000)
print("Connection Time: ", DateTime())
def get_connection():
credentials = pika.PlainCredentials('guest', 'guest')
conn = pika.BlockingConnection(pika.ConnectionParameters(host="localhost", port=5672,
virtual_host="/",
credentials=credentials))
return conn
def callback(ch, method, properties, body):
print(" [x] Received %r", fetch_DateTime(), ':', body)
time.sleep(body.count(b'.'))
#print(fetch_DateTime())
for itm in clients:
itm.write_message(body)
def start_consumers():
asyncio.set_event_loop(asyncio.new_event_loop())
channel = get_connection().channel()
channel.queue_declare(queue="my_queue")
channel.basic_consume(
queue="my_queue",
on_message_callback=callback,
auto_ack=True)
channel.start_consuming()
def fetch_DateTime():
time_milliseconds = time.time() * 1000
return(time_milliseconds)
def disconnect_to_rabbitmq():
channel.stop_consuming()
connection.close()
logging.info('Disconnected from Rabbitmq')
class EventHandler(tornado_eventsource.handler.EventSourceHandler):
def open(self):
logging.info('Server-Sent Events opened')
clients.append(self)
def on_close(self):
logging.info('Server-Sent Events closed')
clients.remove(self)
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.render("SSE_client.html")
def make_app():
return tornado.web.Application([
(r'/', EventHandler,),
(r"/", MainHandler),
])
class WebServer(tornado.web.Application):
def __init__(self):
handlers = [ (r'/', EventHandler,),
(r"/", MainHandler), ]
settings = {'debug': True}
super().__init__(handlers, **settings)
def run(self, port=8888):
self.listen(port)
tornado.ioloop.IOLoop.instance().start()
class TestHandler(tornado.web.RequestHandler):
def get(self):
self.write("test success")
ws = WebServer()
def start_server():
asyncio.set_event_loop(asyncio.new_event_loop())
ws.run()
if __name__ == "__main__":
logging.info('Starting thread Tornado')
threadC = Thread(target=start_consumers)
threadC.start()
from threading import Thread
t = Thread(target=start_s
erver, args=())
t.daemon = True
t.start()
t.join()
try:
input("Server ready. Press enter to stop\n")
except SyntaxError:
pass
try:
logging.info('Disconnecting from RabbitMQ..')
disconnect_to_rabbitmq()
except Exception:
pass
stopTornado();
logging.info('See you...')
Support to Python 3.6+ was added on the version 2.0.0rc2
I am currently trying to create a small demo where I connect a web socket between my computer and my localhost ws://localhost:8080/ws. I want the web socket to monitor a file on my computer for changes. If there are changes, send a message. The connection and output is being monitored using Advanced Rest Client.
Is there a specific method I can use on the class to constantly check the contents of this file?
EDIT
I have implemented an observer using watchdog that detects any event for files in a specified directory. However, my message is not being sent in the sendFSEvent method and I also realized that my client isn't being registered when I connect to the web socket.
Here is my code in server.py
import sys
import os
from watchdog.observers import Observer
from twisted.web.static import File
from twisted.python import log
from twisted.web.server import Site
from twisted.internet import reactor, defer
from autobahn.twisted.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, listenWS
from MessangerEventHandler import MessangerEventHandler
class WsProtocol(WebSocketServerProtocol):
def connectionMade(self):
print("Connection made")
WebSocketServerProtocol.connectionMade(self)
def onOpen(self):
WebSocketServerProtocol.onOpen(self)
print("WebSocket connection open")
def onMessage(self, payload, isBinary):
print("Message was: {}".format(payload))
self.sendMessage("message received")
def sendFSEvent(self, json):
WebSocketProtocol.sendMessage(self, json)
print('Sent FS event')
def onClose(self, wasClean, code, reason):
print("Connection closed: {}".format(reason))
WebSocketServerProtocol.onClose(self, wasClean, code, reason)
class WsServerFactory(WebSocketServerFactory):
protocol = WsProtocol
def __init__(self, url='ws://localhost', port=8080):
addr = url + ':' + str(port)
print("Listening on: {}".format(addr))
WebSocketServerFactory.__init__(self, addr)
self.clients = []
def register(self, client):
if not client in self.clients:
print("Registered client: {}".format(client))
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print("Unregistered client: {}".format(client))
self.clients.remove(client)
self._printConnected()
def _printConnected(self):
print("Connected clients:[")
def notify_clients(self, message):
print("Broadcasting: {}".format(message))
for c in self.clients:
c.sendFSEvent(message)
print("\nSent messages")
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: python server_defer.py <dirs>")
sys.exit(1)
log.startLogging(sys.stdout)
ffactory = WsServerFactory("ws://localhost", 8080)
ffactory.protocol = WsProtocol
listenWS(ffactory)
observers = []
for arg in sys.argv[1:]:
dir_path = os.path.abspath(arg)
if not os.path.exists(dir_path):
print('{} does not exist.'.format(dir_path))
sys.exit(1)
if not os.path.isdir(dir_path):
print('{} is not a directory.'.format(dir_path))
sys.exit(1)
# Check for and handle events
event_handler = MessangerEventHandler(ffactory, reactor, os.getcwd())
observer = Observer()
observer.schedule(event_handler, path=dir_path, recursive=True)
observer.start()
observers.append(observer)
try:
reactor.run()
except KeyboardInterrupt:
for obs in observers:
obs.stop()
reactor.stop()
print("\nGoodbye")
sys.exit(1)
Any help would be greatly appreciated.
Thank you,
Brian
Most enterprise distros come with inotify which is really well suited for monitoring files and directories. The basic idea is to capture a list of connected web socket clients as they connect. Then create a callback that will execute when a change occurs on the files you're monitoring. Within this callback, you can iterate the clients and send them a message like 'file: "blah/blah.txt" has changed'. It's a little wonky, but the code snippet should clear things up for you.
from functools import partial
from twisted.internet import inotify
from twisted.python import filepath
# the rest of your imports ...
class SomeServerProtocol(WebSocketServerProtocol):
def onConnect(self, request):
self.factory.append(self) # <== append this client to the list in the factory
def notification_callback(ignored, filepath, mask, ws_clients):
"""
function that will execute when files are modified
"""
payload = "event on {0}".format(filepath)
for client in ws_clients:
client.sendMessage(
payload.encode('utf8'), # <== don't forget to encode the str to bytes before sending!
isBinary = False)
if __name__ == '__main__':
root = File(".")
factory = WebSocketServerFactory(u"ws://127.0.01:8080")
factory.protocol = SomeServerProtocol
factory.clients = [] # <== create a container for the clients that connect
# inotify stuff
notify = partial(notification_callback, ws_clients=factory.clients) # <== use functools.partial to pass extra params
notifier = inotify.INotify()
notifier.startReading()
notifier.watch(filepath.FilePath("/some/directory"), callbacks=[notify])
# the rest of your code ...
Is it possible, if multiple socket clients are connected to a tornado websocket server, to send a message to a specific one?
I don't know if there is a possibility of getting a client's id and then send a message to that id.
My client code is:
from tornado.ioloop import IOLoop, PeriodicCallback
from tornado import gen
from tornado.websocket import websocket_connect
class Client(object):
def __init__(self, url, timeout):
self.url = url
self.timeout = timeout
self.ioloop = IOLoop.instance()
self.ws = None
self.connect()
PeriodicCallback(self.keep_alive, 20000, io_loop=self.ioloop).start()
self.ioloop.start()
#gen.coroutine
def connect(self):
print "trying to connect"
try:
self.ws = yield websocket_connect(self.url)
except Exception, e:
print "connection error"
else:
print "connected"
self.run()
#gen.coroutine
def run(self):
while True:
msg = yield self.ws.read_message()
print msg
if msg is None:
print "connection closed"
self.ws = None
break
def keep_alive(self):
if self.ws is None:
self.connect()
else:
self.ws.write_message("keep alive")
if __name__ == "__main__":
client = Client("ws://xxx", 5 )
When your client connects to the websocket server will be invoked method 'open' on WebSocketHandler, in this method you can keep socket in the Appliaction.
Like this:
from tornado import web
from tornado.web import url
from tornado.websocket import WebSocketHandler
class Application(web.Application):
def __init__(self):
handlers = [
url(r"/websocket/server/(?P<some_id>[0-9]+)/", WebSocketServer),
]
web.Application.__init__(self, handlers)
self.sockets = {}
class WebSocketServer(WebSocketHandler):
def open(self, some_id):
self.application.sockets[some_id] = self
def on_message(self, message):
self.write_message(u"You said: " + message)
def on_close(self):
print("WebSocket closed")
You also may use message for connection, in this message you have to tell socket id and save socket into the Application.
I am attempting to create a program that allows many clients to connect to 1 server simultaneously. These connections should be interactive on the server side, meaning that I can send requests from the server to the client, after the client has connected.
The following asyncore example code simply replies back with an echo, I need instead of an echo a way to interactively access each session. Somehow background each connection until I decided to interact with it. If I have 100 sessions I would like to chose a particular one or choose all of them or a subset of them to send a command to. Also I am not 100% sure that the asyncore lib is the way to go here, any help is appreciated.
import asyncore
import socket
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
data = self.recv(8192)
if data:
self.send(data)
class EchoServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
handler = EchoHandler(sock)
server = EchoServer('localhost', 8080)
asyncore.loop()
Here's a Twisted server:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import serverFromString
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
class HubConnection(LineReceiver, object):
def __init__(self, hub):
self.name = b'unknown'
self.hub = hub
def connectionMade(self):
self.hub.append(self)
def lineReceived(self, line):
words = line.split(" ", 1)
if words[0] == b'identify':
self.name = words[1]
else:
for connection in self.hub:
connection.sendLine("<{}> {}".format(
self.name, line
).encode("utf-8"))
def connectionLost(self, reason):
self.hub.remove(self)
def main(reactor, listen="tcp:4321"):
hub = []
endpoint = serverFromString(reactor, listen)
endpoint.listen(Factory.forProtocol(lambda: HubConnection(hub)))
return Deferred()
react(main, sys.argv[1:])
and command-line client:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import clientFromString
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.protocol import Factory
from twisted.internet.stdio import StandardIO
from twisted.protocols.basic import LineReceiver
from twisted.internet.fdesc import setBlocking
class HubClient(LineReceiver):
def __init__(self, name, output):
self.name = name
self.output = output
def lineReceived(self, line):
self.output.transport.write(line + b"\n")
def connectionMade(self):
self.sendLine("identify {}".format(self.name).encode("utf-8"))
def say(self, words):
self.sendLine("say {}".format(words).encode("utf-8"))
class TerminalInput(LineReceiver, object):
delimiter = "\n"
hubClient = None
def lineReceived(self, line):
if self.hubClient is None:
self.output.transport.write("Connecting, please wait...\n")
else:
self.hubClient.sendLine(line)
#inlineCallbacks
def main(reactor, name, connect="tcp:localhost:4321"):
endpoint = clientFromString(reactor, connect)
terminalInput = TerminalInput()
StandardIO(terminalInput)
setBlocking(0)
hubClient = yield endpoint.connect(
Factory.forProtocol(lambda: HubClient(name, terminalInput))
)
terminalInput.transport.write("Connecting...\n")
terminalInput.hubClient = hubClient
terminalInput.transport.write("Connected.\n")
yield Deferred()
react(main, sys.argv[1:])
which implement a basic chat server. Hopefully the code is fairly self-explanatory; you can run it to test with python hub_server.py in one terminal, python hub_client.py alice in a second and python hub_client.py bob in a third; then type into alice and bob's sessions and you can see what it does.
Review Requirements
You want
remote calls in client/server manner
probably using TCP communication
using sessions in the call
It is not very clear, how you really want to use sessions, so I will consider, that session is just one of call parameters, which has some meaning on server as well client side and will skip implementing it.
zmq as easy and reliable remote messaging platform
ZeroMQ is lightweight messaging platform, which does not require complex server infrastructure. It can handle many messaging patterns, following example showing request/reply pattern using multipart messages.
There are many alternatives, you can use simple messages encoded to some format like JSON and skip using multipart messages.
server.py
import zmq
class ZmqServer(object):
def __init__(self, url="tcp://*:5555"):
context = zmq.Context()
self.sock = context.socket(zmq.REP)
self.sock.bind(url)
self.go_on = False
def echo(self, message, priority=None):
priority = priority or "not urgent"
msg = "Echo your {priority} message: '{message}'"
return msg.format(**locals())
def run(self):
self.go_on = True
while self.go_on:
args = self.sock.recv_multipart()
if 1 <= len(args) <= 2:
code = "200"
resp = self.echo(*args)
else:
code = "401"
resp = "Bad request, 1-2 arguments expected."
self.sock.send_multipart([code, resp])
def stop(self):
self.go_on = False
if __name__ == "__main__":
ZmqServer().run()
client.py
import zmq
import time
class ZmqClient(object):
def __init__(self, url="tcp://localhost:5555"):
context = zmq.Context()
self.socket = context.socket(zmq.REQ)
self.socket.connect(url)
def call_echo(self, message, priority=None):
args = [message]
if priority:
args.append(priority)
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "200"
return resp
def too_long_call(self, message, priority, extrapriority):
args = [message, priority, extrapriority]
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "401"
return resp
def test_server(self):
print "------------------"
rqmsg = "Hi There"
print "rqmsg", rqmsg
print "response", self.call_echo(rqmsg)
print "------------------"
time.sleep(2)
rqmsg = ["Hi There", "very URGENT"]
print "rqmsg", rqmsg
print "response", self.call_echo(*rqmsg)
print "------------------"
time.sleep(2)
time.sleep(2)
rqmsg = []
print "too_short_call"
print "response", self.too_long_call("STOP", "VERY URGENT", "TOO URGENT")
print "------------------"
if __name__ == "__main__":
ZmqClient().test_server()
Play with the toy
Start the server:
$ python server.py
Now it runs and awaits requests.
Now start the client:
$ python client.py
------------------
rqmsg Hi There
response Echo your not urgent message: 'Hi There'
------------------
rqmsg ['Hi There', 'very URGENT']
response Echo your very URGENT message: 'Hi There'
------------------
too_short_call
response Bad request, 1-2 arguments expected.
------------------
Now experiment a bit:
start client first, then server
stop the server during processing, restart later on
start multiple clients
All these scenarios shall be handled by zmq without adding extra lines of Python code.
Conclusions
ZeroMQ provides very convenient remote messaging solution, try counting lines of messaging related code and compare with any other solution, providing the same level of stability.
Sessions (which were part of OP) can be considered just extra parameter of the call. As we saw, multiple parameters are not a problem.
Maintaining sessions, different backends can be used, they could live in memory (for single server instance), in database, or in memcache or Redis. This answer does not elaborate further on sessions, as it is not much clear, what use is expected.
I want to run both a websocket and a flash policy file server on port 80 using Tornado. The reason for not wanting to run a server on the default port 843 is that it's often closed in corporate networks. Is it possible to do this and if so, how should I do this?
I tried the following structure, which does not seem to work: the websocket connection works, but the policy file request is not routed to the TCPHandler.
#!/usr/bin/python
import tornado.httpserver
import tornado.ioloop
import tornado.web
import tornado.websocket
import tornado.gen
from tornado.options import define, options
from tornado.tcpserver import TCPServer
define("port", default=80, help="run on the given port", type=int)
class FlashPolicyServer(TCPServer):
def handle_stream(self, stream, address):
self._stream = stream
self._read_line()
def _read_line(self):
self._stream.read_until('\n', self._handle_read)
def _handle_read(self, data):
policyFile = ""
self._stream.write(policyFile)
self._read_line()
class WebSocketHandler(tornado.websocket.WebSocketHandler):
def open(self):
pass
def on_message(self, message):
pass
def on_close(self):
pass
def main():
tornado.options.parse_command_line()
mainLoop = tornado.ioloop.IOLoop.instance()
app = tornado.web.Application(
handlers=[
(r"/websocket", WebSocketHandler),
(r"/", FlashPolicyServer)
], main_loop=mainLoop
)
httpServer = tornado.httpserver.HTTPServer(app)
httpServer.listen(options.port)
mainLoop.start()
if __name__ == "__main__":
main()
Any ideas? If this is not possible, would another idea be to serve the policy file via port 443?
There are two problems with your approach:
TCPServer is used like this:
server = TCPServer()
server.listen(6666)
IOLoop.instance().start()
That would do the trick, after you remove the (r"/", FlashPolicyServer) line.
But you want to use port 80.
However, you can't do that, because you need to have another HTTP server at that port.
So what you can do is to use nginx and set up port 80 as reverse proxying for /websocket, serving a Flash policy file otherwise.
I solved this by monkey-patching IOStream and HTTPServer:
def prepend_to_buffer(self, chunk):
self._read_buffer.appendleft(chunk)
self._read_buffer_size += len(chunk)
if self._read_buffer_size >= self.max_buffer_size:
logging.error("Reached maximum read buffer size")
self.close()
raise IOError("Reached maximum read buffer size")
return len(chunk)
def first_data_handler(self, data):
if data == '<policy-file-request/>\0':
self.stream.write(policy_file + '\0')
else:
self.stream.prepend_to_buffer(data)
tornado.httpserver.HTTPConnection(self.stream, self.address, self.request_callback,self.no_keep_alive, self.xheaders)
def handle_stream(self, stream, address):
self.stream = stream
self.address = address
self.stream.read_bytes(23, self.first_data_handler)
tornado.iostream.IOStream.prepend_to_buffer = prepend_to_buffer
tornado.httpserver.HTTPServer.first_data_handler = first_data_handler
tornado.httpserver.HTTPServer.handle_stream = handle_stream