I'm completely new to network programming and event driven events. However I was able to successfully implement a pub-sub scheme using a TCP connection between my machine (server) and client machines for testing (command line). However, I need to actually use a UNIX socket with Twisted.
I am getting the following error when running the code:
Unhandled error in Deferred
Here is my code for pub_sub.py:
"""
a networking implementation of PubSub using Twisted.
=============
PubSub Server
=============
A PubSub server listens for subscription requests and publish commands, and, when
published to, sends data to subscribers. All incoming and outgoing requests are
encoded in JSON.
A Subscribe request looks like this:
{
"command": "subscribe",
"topic": "hello"
}
A Publish request looks like this:
{
"command": "publish",
"topic": "hello",
"data": {
"world": "WORLD"
}
}
When the server receives a Publish request, it will send the 'data' object to all
subscribers of 'topic'.
"""
import argparse
import json
import logging
from collections import defaultdict
from twisted.internet import reactor
from twisted.python import log
from twisted.python.filepath import FilePath
from twisted.internet.endpoints import UNIXClientEndpoint, UNIXServerEndpoint, \
connectProtocol
from twisted.internet.protocol import Protocol, Factory
class PubSubProtocol(Protocol):
def __init__(self, topics):
self.topics = topics
self.subscribed_topic = None
def connectionLost(self, reason):
print("Connection lost: {}".format(reason))
if self.subscribed_topic:
self.topics[self.subscribed_topic].remove(self)
def dataReceived(self, data):
print("Data received: {}".format(data))
try:
request = json.loads(data)
except ValueError:
logging.debug("ValueError on deconding incoming data. "
"Data: {}".format(data), exc_info=True)
self.transport.loseConnection()
return
if request['command'] == 'subscribe':
self.handle_subscribe(request['topic'])
elif request['command'] == 'publish':
self.handle_publish(request['topic'], request['data'])
def handle_subscribe(self, topic):
print("Subscribed to topic: {}".format(topic))
self.topics[topic].add(self)
self.subscribed_topic = topic
def handle_publish(self, topic, data):
request = json.dumps(data)
for protocol in self.topics[topic]:
protocol.transport.write(request)
print("Publish sent for topic: {}".format(topic))
class PubSubFactory(Factory):
def __init__(self):
self.topics = defaultdict(set)
def buildProtocol(self, addr):
return PubSubProtocol(self.topics)
class PublisherProtocol(Protocol):
"""
Publish protocol for sending data to client, i.e. front-end web GUI.
"""
def __init__(self, topic, **kwargs):
self.topic = topic
self.kwargs = kwargs
def connectionMade(self):
request = json.dumps({
'command': 'publish',
'topic': self.topic,
'data': self.kwargs,
})
self.transport.write(request)
self.transport.loseConnection()
class SubscriberProtocol(Protocol):
"""
Subscriber protocol for client sending a request to subscribe to a specific
topic.
"""
def __init__(self, topic, callback):
self.topic = topic
self.callback = callback
def connectionMade(self):
request = json.dumps({
'command': 'subscribe',
'topic': self.topic,
})
self.transport.write(request)
def dataReceived(self, data):
kwargs = json.loads(data)
self.callback(**kwargs)
class PubSub(object):
def __init__(self, path='./.sock'):
self.path = FilePath(path)
self.reactor = reactor
def _make_connection(self, protocol):
endpoint = UNIXClientEndpoint(reactor, self.path)
connection = connectProtocol(endpoint, protocol)
def subscribe(self, topic, callback):
"""
Subscribe 'callback' callable to 'topic'.
"""
sub = SubscriberProtocol(topic, callback)
self._make_connection(sub)
def publish(self, topic, **kwargs):
"""
Publish 'kwargs' to 'topic', calling all callables subscribed to 'topic'
with the arguments specified in '**kwargs'.
"""
pub = PublisherProtocol(topic, **kwargs)
self._make_connection(pub)
def run(self):
"""
Convenience method to start the Twisted event loop.
"""
self.reactor.run()
def main():
path = FilePath("./.sock")
endpoint = UNIXServerEndpoint(reactor, path)
endpoint.listen(PubSubFactory())
reactor.run()
if __name__ == '__main__':
main()
Any help would be greatly appreciated on what I am doing wrong.
Thank you,
Brian
You appear to be running your software on Windows. Alas, UNIX sockets are not available on Windows. If you want to use UNIX sockets, you need to use a more POSIX-ish environment - Linux, *BSD, OS X, etc.
Related
Im creating an application to receive data from multiple iot devices
Each iot device has a socket server
so i need to create multi socket client
Im using Twisted library for create multiple socket client
Problem is that my application can't detect connection failure and it wont try for reconnection
Herer is my code:
import time
import requests
from twisted.internet.protocol import Protocol, ReconnectingClientFactory as rcf, connectionDone
from twisted.internet import reactor
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.python import failure
client_list = [("device ip", device_port)]
endpoints = {}
class Client(Protocol):
"""
when receive a massage call this function
"""
def connectionMade(self):
self.transport.setTcpKeepAlive(True)
def dataReceived(self, data):
reactor.callInThread(self._dataReceived, data)
def _dataReceived(self, data):
if self.connected == 1:
data = data[:-1].decode("utf-8")
data = data.split('\n')
for tag in data:
reactor.callInThread(self._push_data, tag, self.transport.addr[0])
def _push_data(self, data, ip):
print("start pushing")
print(data[1:-1])
try:
pass
// send a web request
except Exception as xp:
print(xp, end="\n")
class ClientFactory(rcf):
def buildProtocol(self, addr):
print(addr, 'Connected.')
return Client()
if __name__ == "__main__":
for client in client_list:
endpoint = TCP4ClientEndpoint(reactor, client[0], client[1])
endpoints[client] = endpoint
endpoint.connect(ClientFactory())
reactor.run()
i writing some code for fix this problem but that's not worked
import time
import requests
from twisted.internet.protocol import Protocol, ReconnectingClientFactory as clf, connectionDone
from twisted.internet import reactor
from twisted.internet.endpoints import TCP4ClientEndpoint
from twisted.python import failure
client_list = [("device ip", device_port)]
endpoints = {}
class Client(Protocol):
"""
when receive a massage call this function
"""
def dataReceived(self, data):
reactor.callInThread(self._dataReceived, data)
def _dataReceived(self, data):
if self.connected == 1:
data = data[:-1].decode("utf-8")
data = data.split('\n')
for tag in data:
reactor.callInThread(self._push_data, tag, self.transport.addr[0])
def _push_data(self, data, ip):
print("start pushing")
print(data[1:-1])
try:
pass
// send a web request
except Exception as xp:
print(xp, end="\n")
def connectionLost(self, reason: failure.Failure = connectionDone):
try:
endpoints.pop(self.transport.addr[0])
except Exception as xp:
print(xp)
print(f"Protocol client {self.transport.addr[0]} connectionLost \n ", reason)
status = True
while status:
try:
endpoint = TCP4ClientEndpoint(reactor, self.transport.addr[0], 100)
endpoints[client] = endpoint
endpoint.connect(ClientFactory())
status = False
except Exception as xp:
print(xp)
class ClientFactory(clf):
def buildProtocol(self, addr):
print(addr, 'Connected.')
return Client()
def clientConnectionFailed(self, connector, reason):
print("can't start connect to the server")
print(reason)
clf.clientConnectionFailed(self, connector, reason)
def clientConnectionLost(self, connector, reason):
print(reason)
clf.clientConnectionLost(self, connector, reason)
if __name__ == "__main__":
for client in client_list:
endpoint = TCP4ClientEndpoint(reactor, client[0], client[1])
endpoints[client] = endpoint
endpoint.connect(ClientFactory())
reactor.run()
my question is how i can set an interval ping for check connection
I have started to learn Python twisted. I have a basic requirement to map the request and response for a TCP client. The client can send any request anytime based on some condition and it expects a specific response for that request from a server. How I could map a request-response pair. A sample code of what I want. Also, the server could send an arbitrary message which does not map to any of client request.
class MSOProtocol(Protocol):
def __init__(self):
self.client_id = uuid4()
self.messagePair = collections.OrderedDict()
def connectionMade(self):
log.msg("Device connected to TSIF")
self.doOperation()
def connectionLost(self, reason):
log.msg('Device lost connection because {}'.format(reason))
def sendMessage(self, data):
log.msg('Device sending...str(msg)'.format(data))
self.transport.write(data)
def dataReceived(self, data):
log.msg('Device received {}'.format(data))
#how do I map the response with my request
def doOperation(self):
CONDITION = "rainy"
if condition == "sunny":
self.sendMessage("sunny weather")
class DeviceFactory(ClientFactory):
def buildProtocol(self, addr):
return MSOProtocol()
def main():
log.startLogging(sys.stdout)
reactor.connectTCP('127.0.0.1', 16000, DeviceFactory())
reactor.run()
if __name__ == '__main__':
main()
I'm attempting to do the following:
connect as client to an existing websocket
process the streaming data received from this socket, and publish it on another websocket
I'm using twisted and autobahn to do so. I have managed to have the two parts working separately, by deriving a WebSocketClientProtocol for the client, and deriving an ApplicationSession in the second. The two run with the same reactor.
I am not sure however as to how to make them communicate. I would like to send a message on my server when the client receives a message, but I don't know how to get the running instance of the WebSocketClientProtocol...
Perhaps this isn't the right approach to do this either. What's the right way to do this?
I've been trying to solve similiar problem recently, here's what worked:
f = XLeagueBotFactory()
app = Application(f)
reactor.connectTCP("irc.gamesurge.net", 6667, f)
reactor.listenTCP(port, app, interface=host)
^ This is in if __name__ == "__main__":
class Application(web.Application):
def __init__(self, botfactory):
self.botfactory = botfactory
Define the instance as self, then in my instance I was sending it to another handler for http post request (using cyclone)
class requestvouch(web.RequestHandler):
def __init__(self, application, request, **kwargs):
super(requestvouch, self).__init__(application, request, **kwargs)
self.botfactory = application.botfactory
def msg(self, channel, msg):
bot = self.botfactory.getProtocolByName("XLeagueBot")
sendmsg(bot, channel, msg) # function that processed the msg through stuff like encoding and logging and then sent it to bot.msg() function that posts it to IRC (endpoint in my case)
def post(self):
msg = "What I'm sending to the protocol of the other thing"
self.msg("#xleague", msg)
Now the important part comes in factory
class XLeagueBotFactory(protocol.ClientFactory):
protocol = XLeagueBot
def __init__(self):
self.protocols = {}
def getProtocolByName(self, name):
return self.protocols.get(name)
def registerProtocol(self, protocol):
self.protocols[protocol.nickname] = protocol
def unregisterProtocol(self, protocol):
del self.protocols[protocol.nickname]
Finally in my client class:
class XLeagueBot(irc.IRCClient):
nickname = "XLeagueBot"
def connectionMade(self):
irc.IRCClient.connectionMade(self)
self.factory.registerProtocol(self)
def connectionLost(self, reason):
self.factory.unregisterProtocol(self)
irc.IRCClient.connectionLost(self, reason)
I'm not entirely sure that this code is perfect, or bugfree, but it should +- tell you how to deal with calling instance of protocol class. The problem afaik comes from name of instance protocol being generated inside of it's factory and not being sent elsewhere.
I've written a simple tornado application using web-sockets to manage daemons (just like transmission web-interface). One thing that bothers me is the implementation of status update.
My WebSocketHandler just recieves messages from client, performs neccessary actions, checks the status and sends it to client. It does not send status to client without request.
class WebSocketHandler(tornado.websocket.WebSocketHandler):
def open(self):
print('connected')
self.daemon = Daemon()
self.write_message(self.daemon.get_status())
def on_message(self, message):
if message == 'start':
self.daemon.start()
elif message == 'stop':
self.daemon.stop()
self.write_message(self.daemon.get_status())
def on_close(self):
print('disconnected')
I'm using setInterval function of javascript on client side to request for status update, like this:
ws = new WebSocket("ws://localhost:8080/websocket");
ws.onopen = function () {
setInterval(function() {
if (ws.bufferedAmount == 0)
ws.send('status');
}, 1000);
};
}
How can the same result be achieved on server side, so that tornado sends current status to client without blocking on_message method?
You can use tornado.ioloop.IOLoop.add_timeout to call a method every X number of seconds to update the client:
from tornado.ioloop import IOLoop
from datetime import timedelta
class WebSocketHandler(tornado.websocket.WebSocketHandler):
def open(self):
print('connected')
self.daemon = Daemon()
self.write_message(self.daemon.get_status())
self.schedule_update()
def schedule_update(self):
self.timeout = IOLoop.instance().add_timeout(timedelta(seconds=5),
self.update_client)
def update_client(self):
try:
self.write_message(self.daemon.get_status())
finally:
self.schedule_update()
def on_close(self):
IOLoop.instance().remove_timeout(self.timeout)
print('disconnected')
I am attempting to create a program that allows many clients to connect to 1 server simultaneously. These connections should be interactive on the server side, meaning that I can send requests from the server to the client, after the client has connected.
The following asyncore example code simply replies back with an echo, I need instead of an echo a way to interactively access each session. Somehow background each connection until I decided to interact with it. If I have 100 sessions I would like to chose a particular one or choose all of them or a subset of them to send a command to. Also I am not 100% sure that the asyncore lib is the way to go here, any help is appreciated.
import asyncore
import socket
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
data = self.recv(8192)
if data:
self.send(data)
class EchoServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
handler = EchoHandler(sock)
server = EchoServer('localhost', 8080)
asyncore.loop()
Here's a Twisted server:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import serverFromString
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
class HubConnection(LineReceiver, object):
def __init__(self, hub):
self.name = b'unknown'
self.hub = hub
def connectionMade(self):
self.hub.append(self)
def lineReceived(self, line):
words = line.split(" ", 1)
if words[0] == b'identify':
self.name = words[1]
else:
for connection in self.hub:
connection.sendLine("<{}> {}".format(
self.name, line
).encode("utf-8"))
def connectionLost(self, reason):
self.hub.remove(self)
def main(reactor, listen="tcp:4321"):
hub = []
endpoint = serverFromString(reactor, listen)
endpoint.listen(Factory.forProtocol(lambda: HubConnection(hub)))
return Deferred()
react(main, sys.argv[1:])
and command-line client:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import clientFromString
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.protocol import Factory
from twisted.internet.stdio import StandardIO
from twisted.protocols.basic import LineReceiver
from twisted.internet.fdesc import setBlocking
class HubClient(LineReceiver):
def __init__(self, name, output):
self.name = name
self.output = output
def lineReceived(self, line):
self.output.transport.write(line + b"\n")
def connectionMade(self):
self.sendLine("identify {}".format(self.name).encode("utf-8"))
def say(self, words):
self.sendLine("say {}".format(words).encode("utf-8"))
class TerminalInput(LineReceiver, object):
delimiter = "\n"
hubClient = None
def lineReceived(self, line):
if self.hubClient is None:
self.output.transport.write("Connecting, please wait...\n")
else:
self.hubClient.sendLine(line)
#inlineCallbacks
def main(reactor, name, connect="tcp:localhost:4321"):
endpoint = clientFromString(reactor, connect)
terminalInput = TerminalInput()
StandardIO(terminalInput)
setBlocking(0)
hubClient = yield endpoint.connect(
Factory.forProtocol(lambda: HubClient(name, terminalInput))
)
terminalInput.transport.write("Connecting...\n")
terminalInput.hubClient = hubClient
terminalInput.transport.write("Connected.\n")
yield Deferred()
react(main, sys.argv[1:])
which implement a basic chat server. Hopefully the code is fairly self-explanatory; you can run it to test with python hub_server.py in one terminal, python hub_client.py alice in a second and python hub_client.py bob in a third; then type into alice and bob's sessions and you can see what it does.
Review Requirements
You want
remote calls in client/server manner
probably using TCP communication
using sessions in the call
It is not very clear, how you really want to use sessions, so I will consider, that session is just one of call parameters, which has some meaning on server as well client side and will skip implementing it.
zmq as easy and reliable remote messaging platform
ZeroMQ is lightweight messaging platform, which does not require complex server infrastructure. It can handle many messaging patterns, following example showing request/reply pattern using multipart messages.
There are many alternatives, you can use simple messages encoded to some format like JSON and skip using multipart messages.
server.py
import zmq
class ZmqServer(object):
def __init__(self, url="tcp://*:5555"):
context = zmq.Context()
self.sock = context.socket(zmq.REP)
self.sock.bind(url)
self.go_on = False
def echo(self, message, priority=None):
priority = priority or "not urgent"
msg = "Echo your {priority} message: '{message}'"
return msg.format(**locals())
def run(self):
self.go_on = True
while self.go_on:
args = self.sock.recv_multipart()
if 1 <= len(args) <= 2:
code = "200"
resp = self.echo(*args)
else:
code = "401"
resp = "Bad request, 1-2 arguments expected."
self.sock.send_multipart([code, resp])
def stop(self):
self.go_on = False
if __name__ == "__main__":
ZmqServer().run()
client.py
import zmq
import time
class ZmqClient(object):
def __init__(self, url="tcp://localhost:5555"):
context = zmq.Context()
self.socket = context.socket(zmq.REQ)
self.socket.connect(url)
def call_echo(self, message, priority=None):
args = [message]
if priority:
args.append(priority)
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "200"
return resp
def too_long_call(self, message, priority, extrapriority):
args = [message, priority, extrapriority]
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "401"
return resp
def test_server(self):
print "------------------"
rqmsg = "Hi There"
print "rqmsg", rqmsg
print "response", self.call_echo(rqmsg)
print "------------------"
time.sleep(2)
rqmsg = ["Hi There", "very URGENT"]
print "rqmsg", rqmsg
print "response", self.call_echo(*rqmsg)
print "------------------"
time.sleep(2)
time.sleep(2)
rqmsg = []
print "too_short_call"
print "response", self.too_long_call("STOP", "VERY URGENT", "TOO URGENT")
print "------------------"
if __name__ == "__main__":
ZmqClient().test_server()
Play with the toy
Start the server:
$ python server.py
Now it runs and awaits requests.
Now start the client:
$ python client.py
------------------
rqmsg Hi There
response Echo your not urgent message: 'Hi There'
------------------
rqmsg ['Hi There', 'very URGENT']
response Echo your very URGENT message: 'Hi There'
------------------
too_short_call
response Bad request, 1-2 arguments expected.
------------------
Now experiment a bit:
start client first, then server
stop the server during processing, restart later on
start multiple clients
All these scenarios shall be handled by zmq without adding extra lines of Python code.
Conclusions
ZeroMQ provides very convenient remote messaging solution, try counting lines of messaging related code and compare with any other solution, providing the same level of stability.
Sessions (which were part of OP) can be considered just extra parameter of the call. As we saw, multiple parameters are not a problem.
Maintaining sessions, different backends can be used, they could live in memory (for single server instance), in database, or in memcache or Redis. This answer does not elaborate further on sessions, as it is not much clear, what use is expected.