I am trying to do asynchronous programming using Twisted. What I'm trying to do is create a client which will pass query parameters (in my case, hash type and hash value) and wait for the server's response. I am currently using NetstringReceiver for the parameters. However, am having these problems:
The client is not able to send its request to the server, and
The client hangs forever when I ran it. It seems that there is a callback that's not returning.
Below are the codes for the client and server.
This code is actually based on this tutorial by Dave Peticolas.
Client Code
import os, sys, argparse
from twisted.internet import defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.protocols.basic import NetstringReceiver
class QueryProtocol(Protocol):
response = ''
def dataReceived(self, data):
self.response = data
def connectionLost(self, reason):
self.responseReceived(self.response)
def responseReceived(self, response):
self.factory.response_finished(response)
class QueryFactory(ClientFactory):
protocol = QueryProtocol
def __init__(self, deferred):
self.deferred = deferred
def response_finished(self, response):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionFailed(self, connector, reason):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
class QueryNetProtocol(NetstringReceiver):
def connectionMade(self):
self.sendRequest(self.factory.hash_type, self.factory.hash_value)
def sendRequest(self, hash_type, hash_value):
self.sendString(hash_type + '.' + hash_value)
def stringReceived(self, s):
self.transport.loseConnection()
self.responseReceived(s)
def responseReceived(self, response):
self.factory.handleResponse(response)
class QueryNetFactory(ClientFactory):
protocol = QueryNetProtocol
def __init__(self, hash_type, hash_value):
self.hash_type = hash_type
self.hash_value = hash_value
self.deferred = defer.Deferred()
def handleResponse(self, response):
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionLost(self, _, reason):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
clientConnectionFailed = clientConnectionLost
class QueryProxy(object):
def __init__(self, host, port):
self.host = host
self.port = port
def query(self, hash_type, hash_value):
factory = QueryNetFactory(hash_type, hash_value)
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, factory)
return factory.deferred
def perform_query(host, port):
d = defer.Deferred()
from twisted.internet import reactor
factory = QueryFactory(d)
reactor.connectTCP(host, port, factory)
return d
def main(options):
done = False
query_result = ""
host = options.host
port = int(options.port)
sha1 = options.sha1
proxy = QueryProxy(host, port)
from twisted.internet import reactor
def process_query_result(response):
d = proxy.query('sha1', sha1)
def fail(err):
print "Problem in processing response : %s" % err
return response
return d.addErrback(fail)
def query_ok(response):
query_result = response
done = True
def query_failed(err):
print "Problem in query : %s" % err
done = True
def query_done(_):
if done == True: reactor.stop()
d = perform_query(host, port)
d.addCallback(process_query_result)
d.addCallbacks(query_ok, query_failed)
d.addBoth(query_done)
reactor.run()
print "The result of the query is : %s" % query_result
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("host", help="server host/ip")
parser.add_argument("port", help="server port number to listen to")
parser.add_argument("-sha1", help="sha1 value to be queried")
options = parser.parse_args()
main(options)
Server Code
import os, sys, argparse
from twisted.internet.protocol import ServerFactory
from twisted.protocols.basic import NetstringReceiver
class GridQueryService(object):
def query(self, hash_type, hash_value):
print "this is the query service. Type is %s and value is %s" % (hash_type, hash_value)
return hash_value
class GridQueryProtocol(NetstringReceiver):
def stringReceived(self, request):
print >>sys.stderr, request
if '.' not in request:
self.transport.loseConnection()
return
hash_type, hash_value = request.split('.')
self.formRequestReceived(hash_type, hash_value)
def formRequestReceived(self, hash_type, hash_value):
found_flag = self.factory.query(hash_type, hash_value)
if found_flag: self.sendString(str(found_flag))
self.transport.loseConnection()
class GridQueryFactory(ServerFactory):
protocol = GridQueryProtocol
def __init__(self, service):
self.service = service
def query(self, hash_type, hash_value):
return self.service.query(hash_type, hash_value)
def main(options):
grid_query_service = GridQueryService()
grid_query_factory = GridQueryFactory(grid_query_service)
from twisted.internet import reactor
port = reactor.listenTCP(int(options.port), grid_query_factory, interface=options.host)
print "Serving GRID query service on %s" % str(port.getHost())
reactor.run()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("host", help="server host/ip")
parser.add_argument("port", help="server port number to listen to")
options = parser.parse_args()
main(options)
Any ideas on how to resolve this? Thanks in advance. Help is greatly appreciated!
Your server may use netstrings and your client may contain code for using netstrings, but your client doesn't use netstrings.
main calls perform_query. perform_query makes a QueryFactory which connects a QueryProtocol which never sends any data and doesn't have any logic related to handling netstrings even if it did.
I've updated my client. Revised client code is below.
I am now able to send request. Here are the outputs:
Sent request!
String received!
Handling response!
I'm called!
Connection lost! - QueryNetFactory
As you can see, the callback process_query_result callback has been fired. However, the other callbacks were not and I can't still receive the data/result using the QueryFactory/QueryProtocol. I have a suspect that it's about the deferred and callbacks, but I'm quite confused.
In initializing the QueryFactory class, should I really create a new deferred? If yes, how will it realize/know the callbacks added to the original deferred (the one that was returned by the perform_query method)? Now if in case I should not create a new deferred, how can I fire the callbacks added in the original deferred in the 'responseReceived' method of the QueryProtocol?
Client Code
class QueryProtocol(Protocol):
response = ''
def dataReceived(self, data):
print "Data received!"
self.response = data
def connectionLost(self, reason):
print "Connection lost!"
self.responseReceived(self.response)
def responseReceived(self, response):
print "Response received!"
self.factory.response_finished(response)
class QueryFactory(ClientFactory):
protocol = QueryProtocol
def __init__(self):
self.deferred = defer.Deferred()
def response_finished(self, response):
print "Response finished!"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionFailed(self, connector, reason):
print "Client connection failed! - QueryFactory"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
class QueryNetProtocol(NetstringReceiver):
def connectionMade(self):
self.sendRequest(self.factory.hash_type, self.factory.hash_value)
def sendRequest(self, hash_type, hash_value):
print "Sent request!"
self.sendString(hash_type + '.' + hash_value)
def stringReceived(self, s):
print "String received!"
self.transport.loseConnection()
self.responseReceived(s)
def responseReceived(self, response):
self.factory.handleResponse(response)
class QueryNetFactory(ClientFactory):
protocol = QueryNetProtocol
def __init__(self, deferred, hash_type, hash_value):
self.hash_type = hash_type
self.hash_value = hash_value
self.deferred = deferred
def handleResponse(self, response):
print "Handling response!"
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionLost(self, _, reason):
print "Connection lost! - QueryNetFactory"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
clientConnectionFailed = clientConnectionLost
class QueryProxy(object):
def __init__(self, host, port):
self.host = host
self.port = port
def query(self):
factory = QueryFactory()
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, factory)
return factory.deferred
def perform_query(host, port, hash_type, hash_value):
d = defer.Deferred()
from twisted.internet import reactor
factory = QueryNetFactory(d, hash_type, hash_value)
reactor.connectTCP(host, port, factory)
return d
def main(options):
done = False
query_result = ""
host = options.host
port = int(options.port)
sha1 = options.sha1
proxy = QueryProxy(host, port)
from twisted.internet import reactor
def process_query_result(response):
print "I'm called!"
d = proxy.query()
def fail(err):
print "Process query result failure : %s" % err
return d.addErrback(fail)
def query_ok(response):
print "query ok!"
query_result = response
done = True
def query_failed(err):
print "Problem in query : %s" % err
done = True
def query_done(_):
if done == True: reactor.stop()
d = perform_query(host, port, "sha1", sha1)
d.addCallback(process_query_result)
d.addCallbacks(query_ok, query_failed)
d.addBoth(query_done)
reactor.run()
print "The result of the query is : %s" % query_result
Again, any help is greatly appreciated! Thanks!
Related
I have a code which is written in Twisted and I am having trouble with converting it to Asyncio. From my problem the problem is about protocol in Server class. The code which is written with Twisted is this:
from twisted.internet import reactor, protocol
class Client():
def __init__(self):
self.something = ""
def dataReceived(self, packet):
print(packet)
def makeConnection(self, transport):
self.transport = transport
self.server = self.factory
def connectionLost(self, args):
print("Connection lost.")
class Server(protocol.ServerFactory):
protocol = Client
if __name__ == '__main__':
validPort = []
S = Server()
try:
reactor.listenTCP(44444, S)
validPort.append(44444)
except:
print('Problem about port.')
if not validPort == []:
print('Server is running.')
reactor.run()
And this is the code which I converted to Asyncio and doesnt't work (I am not sure the problem is about protocol in Server Class):
import asyncio, logging, sys
SERVER_ADDRESS = ('localhost', 44444)
logging.basicConfig(
level=logging.DEBUG,
format='%(name)s: %(message)s',
stream=sys.stderr,
)
log = logging.getLogger('main')
event_loop = asyncio.get_event_loop()
class Client():
def __init__(self):
self.something = ""
def data_received(self, packet):
self.log.debug('received {!r}'.format(data))
print(packet)
def connection_made(self, transport):
self.transport = transport
self.server = self.factory
self.log = logging.getLogger(
'EchoServer_{}_{}'.format(*self.address)
)
self.log.debug('connection accepted')
def connection_lost(self, args):
if error:
self.log.error('ERROR: {}'.format(error))
else:
self.log.debug('closing')
super().connection_lost(error)
class Server(asyncio.Protocol):
protocol = Client
if __name__ == '__main__':
S = Server()
factory = event_loop.create_server(S, *SERVER_ADDRESS)
server = event_loop.run_until_complete(factory)
log.debug('starting up on {} port {}'.format(*SERVER_ADDRESS))
try:
event_loop.run_forever()
finally:
log.debug('closing server')
server.close()
event_loop.run_until_complete(server.wait_closed())
log.debug('closing event loop')
event_loop.close()
In asyncio there is no ProtocolFactory abstract interface. Just pass Client or lambda: Client to loop.create_server()
Here is a simple client which connects and sends a text message:
class Client(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket()
self.connect( (host, port) )
self.buffer = bytes("hello world", 'ascii')
def handle_connect(self):
pass
def handle_close(self):
self.close()
def handle_read(self):
print(self.recv(8192))
def writable(self):
return (len(self.buffer) > 0)
def writable(self):
return True
def handle_write(self):
sent = self.send(self.buffer)
print('Sent:', sent)
self.buffer = self.buffer[sent:]
client = Client('localhost', 8080)
asyncore.loop()
And here is the server which has to receive the message and echo it back:
class Server(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket()
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_read(self):
self.buffer = self.recv(4096)
while True:
partial = self.recv(4096)
print('Partial', partial)
if not partial:
break
self.buffer += partial
def readable(self):
return True
def handle_write(self):
pass
def handle_accepted(self, sock, addr):
print('Incoming connection from %s' % repr(addr))
self.handle_read()
print(self.buffer)
if __name__ == "__main__":
server = Server("localhost", 8080)
asyncore.loop()
The problem is that server isn't reading anything. When I print self.buffer the output is:
b''
What am I doing wrong?
First of all, you need two handlers: One for the server socket (where you expect only accept), and one for the actual communication sockets. In addition, you can only call read once in handle_read; if you call it twice, the second call may block, and that's not allowed in asyncore programming. Don't worry though; if your read did not get everything, you'll immediately be notified again once your read handler returns.
import asyncore
class Handler(asyncore.dispatcher):
def __init__(self, sock):
self.buffer = b''
super().__init__(sock)
def handle_read(self):
self.buffer += self.recv(4096)
print('current buffer: %r' % self.buffer)
class Server(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket()
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accepted(self, sock, addr):
print('Incoming connection from %s' % repr(addr))
Handler(sock)
if __name__ == "__main__":
server = Server("localhost", 1234)
asyncore.loop()
I'm trying to implement this code, in which each "node" is a self-contained "actor".
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor, defer
from twisted.protocols.basic import LineReceiver
class ChatProtocol(LineReceiver):
def __init__(self, factory):
self.factory = factory
self.name = None
self.state = "REGISTER"
def connectionMade(self):
self.sendLine("What's your name?")
def connectionLost(self, reason):
if self.name in self.factory.users:
del self.factory.users[self.name]
self.broadcastMessage("{} has left the channel.".format(self.name))
def lineReceived(self, line):
if self.state == "REGISTER":
self.handle_REGISTER(line)
else:
self.handle_CHAT(line)
def handle_REGISTER(self, name):
if name in self.factory.users:
self.sendLine("Name taken, please choose another!")
return
self.sendLine("Welcome, {}".format(name))
self.broadcastMessage("{} has joined the channel.".format(name))
self.name = name
self.factory.users[name] = self
self.state = "CHAT"
def handle_CHAT(self, message):
message = "[%s]>> %s" % (self.name, message)
self.broadcastMessage(message)
def broadcastMessage(self, message):
for name, protocol in self.factory.users.iteritems():
if protocol != self:
protocol.sendLine(message)
class ChatFactory(Factory):
"""Handle all the nodes' connection"""
def __init__(self):
self.users = {}
def buildProtocol(self, addr):
return ChatProtocol(self)
class Node:
def __init__(self, stop=None):
self.Factory = ChatFactory
self.reactor = reactor
self.d = defer.Deferred()
# with `stop` the node is bound to die
if stop:
self.reactor.callLater(stop, self.stop)
def listen(self, port):
self.reactor.listenTCP(port, self.Factory())
def run(self):
self.reactor.run()
def stop(self):
self.reactor.stop()
class Organization:
"""
An organization consists of several nodes, with one node as a leader
"""
def __init__(self):
self.nodes = []
def create_leader(self):
# create first node now with intentionally kill the leader's reactor after 5 seconds
leader_node = Node(5)
leader_node.listen(8000)
self.nodes.append(leader_node)
def create_more_nodes(self):
node_1 = Node()
node_2 = Node()
self.nodes.append(node_1)
self.nodes.append(node_2)
def activate(self):
self.nodes[1].listen(8001)
self.nodes[2].listen(8002)
"""
now leader_node listens at 8000
node_1 listens at 8001
node_2 listens at 8002
"""
# run each node's reactor
for n in self.nodes:
n.run()
if __name__ == '__main__':
org = Organization()
org.create_leader()
org.create_more_nodes()
org.activate()
After 5 seconds, the leader_node's reactor is stopped via Node.stop() that gets deferred. However, I have no idea why node_1 and node_2 listening at 8001 and 8002 were also stopped. If anyone with more experience with Twisted could point this out that'd be great!
reactor.run() means "run the entire program". While it does not forcibly terminate (the function does return), it only does so in order to allow you to clean up some state before exiting. So you should only ever run one reactor per process and exit shortly after it is finished.
If you want to have self-contained services which can shut down all of their incoming and outgoing connections as well as their listening ports, you have to track those connections in connectionMade and connectionLost. You also have to keep track of your listening port so that you can stopListening.
As a side note, listenTCP is a very low-level API that you probably shouldn't be calling directly; instead, use the high-level Endpoints API, which is much more flexible.
Here's a version of your code that tracks inbound and connections and listening ports and shuts them down as you'd like, while sharing a reactor between all the nodes.
from twisted.internet.protocol import Factory
from twisted.internet.endpoints import TCP4ServerEndpoint
from twisted.protocols.basic import LineReceiver
from twisted.internet.defer import Deferred
class ChatProtocol(LineReceiver):
def __init__(self, factory):
self.factory = factory
self.name = None
self.state = "REGISTER"
def connectionMade(self):
self.factory.node.activeTransports.append(self.transport)
self.sendLine("What's your name?")
def connectionLost(self, reason):
self.factory.node.activeTransports.remove(self.transport)
if self.name in self.factory.users:
del self.factory.users[self.name]
self.broadcastMessage("{} has left the channel.".format(self.name))
def lineReceived(self, line):
if self.state == "REGISTER":
self.handle_REGISTER(line)
else:
self.handle_CHAT(line)
def handle_REGISTER(self, name):
if name in self.factory.users:
self.sendLine("Name taken, please choose another!")
return
self.sendLine("Welcome, {}".format(name))
self.broadcastMessage("{} has joined the channel.".format(name))
self.name = name
self.factory.users[name] = self
self.state = "CHAT"
def handle_CHAT(self, message):
message = "[%s]>> %s" % (self.name, message)
self.broadcastMessage(message)
def broadcastMessage(self, message):
for name, protocol in self.factory.users.iteritems():
if protocol != self:
protocol.sendLine(message)
class ChatFactory(Factory):
"""Handle all the nodes' connection"""
def __init__(self, node):
self.users = {}
self.node = node
def buildProtocol(self, addr):
return ChatProtocol(self)
class Node:
def __init__(self, endpoint, clock, stop=None):
self.Factory = ChatFactory
self._endpoint = endpoint
self._listenStarting = None
self._listeningPort = None
self.activeTransports = []
if stop is not None:
print("Scheduling stop.", stop)
clock.callLater(stop, self.stop)
def listen(self):
self._listenStarting = self._endpoint.listen(self.Factory(self))
def setPort(port):
self._listeningPort = port
def clear(whatever):
self._listenStarting = None
return whatever
self._listenStarting.addCallback(setPort).addBoth(clear)
def stop(self):
if self._listenStarting is not None:
self._listenStarting.cancel()
if self._listeningPort is not None:
self._listeningPort.stopListening()
for transport in self.activeTransports[:]:
transport.abortConnection()
class Organization:
def __init__(self, reactor):
self.reactor = reactor
self.nodes = []
def port(self, number):
return TCP4ServerEndpoint(self.reactor, number)
def create_leader(self):
leader_node = Node(self.port(8000), self.reactor, 5)
leader_node.listen()
self.nodes.append(leader_node)
def create_more_nodes(self):
node_1 = Node(self.port(8001), self.reactor)
node_2 = Node(self.port(8002), self.reactor)
self.nodes.append(node_1)
self.nodes.append(node_2)
def activate(self):
self.nodes[1].listen()
self.nodes[2].listen()
def main(reactor):
org = Organization(reactor)
org.create_leader()
org.create_more_nodes()
org.activate()
return Deferred()
if __name__ == '__main__':
from twisted.internet.task import react
react(main)
I have 2 server file work with corona simulator. One is work but another isn't. Not sure what is the different between these 2 file. Below is my server code.
Non-working:
class Chat(Protocol):
def connectionMade(self):
self.factory.clients.append(self)
def connectionLost(self, reason):
self.factory.clients.remove(self)
def dataReceived(self,data):
for c in self.factory.clients:
c.message(data)
print data
def message(self, data):
self.transport.write(data)
factory = Factory()
factory.clients = []
factory.protocol = Chat
reactor.listenTCP(8080,factory)
reactor.run()
Working:
class IphoneChat(Protocol):
def connectionMade(self):
self.factory.clients.append(self)
print "Clients are " ,self.factory.clients
def connectionLost(self, reason):
self.factory.clients.remove(self)
def dataReceived(self, data):
print "The data is " ,data
for c in self.factory.clients:
c.message(data)
def message(self, message):
self.transport.write(message + '\n')
factory = Factory()
factory.clients = []
factory.protocol = IphoneChat
reactor.listenTCP(8080, factory)
print "Server Start!!!"
reactor.run()
I put all of my code because I afraid of missing something important about the code.
Thank you for incoming help.
You need to send "\n" at the end of message
class Chat(Protocol):
def connectionMade(self):
self.factory.clients.append(self)
def connectionLost(self, reason):
self.factory.clients.remove(self)
def dataReceived(self,data):
for c in self.factory.clients:
c.message(data)
print data
def message(self, data):
self.transport.write(data + '\n')
factory = Factory()
factory.clients = []
factory.protocol = Chat
reactor.listenTCP(8080,factory)
reactor.run()
Its HTTP protocol requirement.
Can you anyone please help me call the broadcast function from class BroadcastServerFactory in class test, as per attached code
I have tried so many methods of call a function from another class, but no solution
import sys
from twisted.internet import reactor
from twisted.python import log
from twisted.web.server import Site
from twisted.web.static import File
from autobahn.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, \
listenWS
class test():
//call broadcast function from here
class BroadcastServerProtocol(WebSocketServerProtocol):
def onOpen(self):
self.factory.register(self)
def onMessage(self, msg, binary):
if not binary:
self.factory.broadcast("'%s' from %s" % (msg, self.peerstr))
def connectionLost(self, reason):
WebSocketServerProtocol.connectionLost(self, reason)
self.factory.unregister(self)
class BroadcastServerFactory(WebSocketServerFactory):
"""
Simple broadcast server broadcasting any message it receives to all
currently connected clients.
"""
def __init__(self, url, debug = False, debugCodePaths = False):
WebSocketServerFactory.__init__(self, url, debug = debug, debugCodePaths = debugCodePaths)
self.clients = []
self.tickcount = 0
self.tick()
def tick(self):
self.tickcount += 1
self.broadcast("'tick %d' from server" % self.tickcount)
reactor.callLater(1, self.tick)
def register(self, client):
if not client in self.clients:
print "registered client " + client.peerstr
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print "unregistered client " + client.peerstr
self.clients.remove(client)
def broadcast(self, msg):
print "broadcasting message '%s' .." % msg
for c in self.clients:
c.sendMessage(msg)
print "message sent to " + c.peerstr
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'debug':
log.startLogging(sys.stdout)
debug = True
else:
debug = False
ServerFactory = BroadcastServerFactory
#ServerFactory = BroadcastPreparedServerFactory
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
factory.protocol = BroadcastServerProtocol
factory.setProtocolOptions(allowHixie76 = True)
listenWS(factory)
webdir = File(".")
web = Site(webdir)
reactor.listenTCP(8080, web)
reactor.run()
class test():
def __init__(self, factory):
factory.broadcast("I don't know what I'm doing!")
Meanwhile, in main...
factory = ServerFactory("ws://localhost:9000",
debug = debug,
debugCodePaths = debug)
test(factory)
This will do what you want, but it seems you're missing some core concepts about classes and instances. For the test class to call anything on another class, there needs to be an instance of it first (bar the case of static methods).