We Keep Coding

Asynchronous client in Twisted not sending / receiving request (using NetStringReceiver)

I am trying to do asynchronous programming using Twisted. What I'm trying to do is create a client which will pass query parameters (in my case, hash type and hash value) and wait for the server's response. I am currently using NetstringReceiver for the parameters. However, am having these problems:
The client is not able to send its request to the server, and
The client hangs forever when I ran it. It seems that there is a callback that's not returning.
Below are the codes for the client and server.
This code is actually based on this tutorial by Dave Peticolas.
Client Code
import os, sys, argparse
from twisted.internet import defer
from twisted.internet.protocol import Protocol, ClientFactory
from twisted.protocols.basic import NetstringReceiver
class QueryProtocol(Protocol):
response = ''
def dataReceived(self, data):
self.response = data
def connectionLost(self, reason):
self.responseReceived(self.response)
def responseReceived(self, response):
self.factory.response_finished(response)
class QueryFactory(ClientFactory):
protocol = QueryProtocol
def __init__(self, deferred):
self.deferred = deferred
def response_finished(self, response):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionFailed(self, connector, reason):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
class QueryNetProtocol(NetstringReceiver):
def connectionMade(self):
self.sendRequest(self.factory.hash_type, self.factory.hash_value)
def sendRequest(self, hash_type, hash_value):
self.sendString(hash_type + '.' + hash_value)
def stringReceived(self, s):
self.transport.loseConnection()
self.responseReceived(s)
def responseReceived(self, response):
self.factory.handleResponse(response)
class QueryNetFactory(ClientFactory):
protocol = QueryNetProtocol
def __init__(self, hash_type, hash_value):
self.hash_type = hash_type
self.hash_value = hash_value
self.deferred = defer.Deferred()
def handleResponse(self, response):
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionLost(self, _, reason):
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
clientConnectionFailed = clientConnectionLost
class QueryProxy(object):
def __init__(self, host, port):
self.host = host
self.port = port
def query(self, hash_type, hash_value):
factory = QueryNetFactory(hash_type, hash_value)
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, factory)
return factory.deferred
def perform_query(host, port):
d = defer.Deferred()
from twisted.internet import reactor
factory = QueryFactory(d)
reactor.connectTCP(host, port, factory)
return d
def main(options):
done = False
query_result = ""
host = options.host
port = int(options.port)
sha1 = options.sha1
proxy = QueryProxy(host, port)
from twisted.internet import reactor
def process_query_result(response):
d = proxy.query('sha1', sha1)
def fail(err):
print "Problem in processing response : %s" % err
return response
return d.addErrback(fail)
def query_ok(response):
query_result = response
done = True
def query_failed(err):
print "Problem in query : %s" % err
done = True
def query_done(_):
if done == True: reactor.stop()
d = perform_query(host, port)
d.addCallback(process_query_result)
d.addCallbacks(query_ok, query_failed)
d.addBoth(query_done)
reactor.run()
print "The result of the query is : %s" % query_result
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("host", help="server host/ip")
parser.add_argument("port", help="server port number to listen to")
parser.add_argument("-sha1", help="sha1 value to be queried")
options = parser.parse_args()
main(options)
Server Code
import os, sys, argparse
from twisted.internet.protocol import ServerFactory
from twisted.protocols.basic import NetstringReceiver
class GridQueryService(object):
def query(self, hash_type, hash_value):
print "this is the query service. Type is %s and value is %s" % (hash_type, hash_value)
return hash_value
class GridQueryProtocol(NetstringReceiver):
def stringReceived(self, request):
print >>sys.stderr, request
if '.' not in request:
self.transport.loseConnection()
return
hash_type, hash_value = request.split('.')
self.formRequestReceived(hash_type, hash_value)
def formRequestReceived(self, hash_type, hash_value):
found_flag = self.factory.query(hash_type, hash_value)
if found_flag: self.sendString(str(found_flag))
self.transport.loseConnection()
class GridQueryFactory(ServerFactory):
protocol = GridQueryProtocol
def __init__(self, service):
self.service = service
def query(self, hash_type, hash_value):
return self.service.query(hash_type, hash_value)
def main(options):
grid_query_service = GridQueryService()
grid_query_factory = GridQueryFactory(grid_query_service)
from twisted.internet import reactor
port = reactor.listenTCP(int(options.port), grid_query_factory, interface=options.host)
print "Serving GRID query service on %s" % str(port.getHost())
reactor.run()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("host", help="server host/ip")
parser.add_argument("port", help="server port number to listen to")
options = parser.parse_args()
main(options)
Any ideas on how to resolve this? Thanks in advance. Help is greatly appreciated!

Your server may use netstrings and your client may contain code for using netstrings, but your client doesn't use netstrings.
main calls perform_query. perform_query makes a QueryFactory which connects a QueryProtocol which never sends any data and doesn't have any logic related to handling netstrings even if it did.

I've updated my client. Revised client code is below.
I am now able to send request. Here are the outputs:
Sent request!
String received!
Handling response!
I'm called!
Connection lost! - QueryNetFactory
As you can see, the callback process_query_result callback has been fired. However, the other callbacks were not and I can't still receive the data/result using the QueryFactory/QueryProtocol. I have a suspect that it's about the deferred and callbacks, but I'm quite confused.
In initializing the QueryFactory class, should I really create a new deferred? If yes, how will it realize/know the callbacks added to the original deferred (the one that was returned by the perform_query method)? Now if in case I should not create a new deferred, how can I fire the callbacks added in the original deferred in the 'responseReceived' method of the QueryProtocol?
Client Code
class QueryProtocol(Protocol):
response = ''
def dataReceived(self, data):
print "Data received!"
self.response = data
def connectionLost(self, reason):
print "Connection lost!"
self.responseReceived(self.response)
def responseReceived(self, response):
print "Response received!"
self.factory.response_finished(response)
class QueryFactory(ClientFactory):
protocol = QueryProtocol
def __init__(self):
self.deferred = defer.Deferred()
def response_finished(self, response):
print "Response finished!"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionFailed(self, connector, reason):
print "Client connection failed! - QueryFactory"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
class QueryNetProtocol(NetstringReceiver):
def connectionMade(self):
self.sendRequest(self.factory.hash_type, self.factory.hash_value)
def sendRequest(self, hash_type, hash_value):
print "Sent request!"
self.sendString(hash_type + '.' + hash_value)
def stringReceived(self, s):
print "String received!"
self.transport.loseConnection()
self.responseReceived(s)
def responseReceived(self, response):
self.factory.handleResponse(response)
class QueryNetFactory(ClientFactory):
protocol = QueryNetProtocol
def __init__(self, deferred, hash_type, hash_value):
self.hash_type = hash_type
self.hash_value = hash_value
self.deferred = deferred
def handleResponse(self, response):
print "Handling response!"
d, self.deferred = self.deferred, None
d.callback(response)
def clientConnectionLost(self, _, reason):
print "Connection lost! - QueryNetFactory"
if self.deferred is not None:
d, self.deferred = self.deferred, None
d.errback(reason)
clientConnectionFailed = clientConnectionLost
class QueryProxy(object):
def __init__(self, host, port):
self.host = host
self.port = port
def query(self):
factory = QueryFactory()
from twisted.internet import reactor
reactor.connectTCP(self.host, self.port, factory)
return factory.deferred
def perform_query(host, port, hash_type, hash_value):
d = defer.Deferred()
from twisted.internet import reactor
factory = QueryNetFactory(d, hash_type, hash_value)
reactor.connectTCP(host, port, factory)
return d
def main(options):
done = False
query_result = ""
host = options.host
port = int(options.port)
sha1 = options.sha1
proxy = QueryProxy(host, port)
from twisted.internet import reactor
def process_query_result(response):
print "I'm called!"
d = proxy.query()
def fail(err):
print "Process query result failure : %s" % err
return d.addErrback(fail)
def query_ok(response):
print "query ok!"
query_result = response
done = True
def query_failed(err):
print "Problem in query : %s" % err
done = True
def query_done(_):
if done == True: reactor.stop()
d = perform_query(host, port, "sha1", sha1)
d.addCallback(process_query_result)
d.addCallbacks(query_ok, query_failed)
d.addBoth(query_done)
reactor.run()
print "The result of the query is : %s" % query_result
Again, any help is greatly appreciated! Thanks!

TCP client not able to write to socket but receive works fine

I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?

You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.

Python async socket recv not working

Here is a simple client which connects and sends a text message:
class Client(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket()
self.connect( (host, port) )
self.buffer = bytes("hello world", 'ascii')
def handle_connect(self):
pass
def handle_close(self):
self.close()
def handle_read(self):
print(self.recv(8192))
def writable(self):
return (len(self.buffer) > 0)
def writable(self):
return True
def handle_write(self):
sent = self.send(self.buffer)
print('Sent:', sent)
self.buffer = self.buffer[sent:]
client = Client('localhost', 8080)
asyncore.loop()
And here is the server which has to receive the message and echo it back:
class Server(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket()
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_read(self):
self.buffer = self.recv(4096)
while True:
partial = self.recv(4096)
print('Partial', partial)
if not partial:
break
self.buffer += partial
def readable(self):
return True
def handle_write(self):
pass
def handle_accepted(self, sock, addr):
print('Incoming connection from %s' % repr(addr))
self.handle_read()
print(self.buffer)
if __name__ == "__main__":
server = Server("localhost", 8080)
asyncore.loop()
The problem is that server isn't reading anything. When I print self.buffer the output is:
b''
What am I doing wrong?

First of all, you need two handlers: One for the server socket (where you expect only accept), and one for the actual communication sockets. In addition, you can only call read once in handle_read; if you call it twice, the second call may block, and that's not allowed in asyncore programming. Don't worry though; if your read did not get everything, you'll immediately be notified again once your read handler returns.
import asyncore
class Handler(asyncore.dispatcher):
def __init__(self, sock):
self.buffer = b''
super().__init__(sock)
def handle_read(self):
self.buffer += self.recv(4096)
print('current buffer: %r' % self.buffer)
class Server(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket()
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accepted(self, sock, addr):
print('Incoming connection from %s' % repr(addr))
Handler(sock)
if __name__ == "__main__":
server = Server("localhost", 1234)
asyncore.loop()

How to close Python socket running in a thread after CTRL+C?

I have a multithreaded program in python and I would like to close the socket after CTRL+C (or Z). I've tried this and this but none of them had worked.
When trying to re-run the program, error message appears:
Bind failed. Error code: 98 Message Address already in use called
Traceback (most recent call last): File "main.py", line 16, in
main.connection.close() NameError: name 'main' is not defined
from connection import Connection
class Main():
def __init__(self):
self.connection = Connection()
self.connection.start()
if __name__ == '__main__':
try:
main = Main()
except:
main.connection.close()
import socket
import sys
import threading
import time
class Connection(threading.Thread):
def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, verbose=None):
threading.Thread.__init__(self, group=group, target=target, name=name, args=args, kwargs=kwargs, verbose=verbose)
self.server = None
self.connection = self.start_connention()
self.data = "null"
self.lock = threading.Lock()
self.OK = True
def start_connention(self):
host = '192.168.42.1'
port = 8888
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
print 'Socket created'
#Bind socket to local host and port
try:
s.bind((host, port))
except socket.error, msg:
print 'Bind failed. Error code: ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
print 'Socket bind complete'
#Start listening on socket
s.listen(10)
print 'Socket now listening on ' + str(port)
connection, addr = s.accept()
print 'Connected with ' + addr[0] + ':' + str(addr[1])
self.server = s
return connection
def close(self):
print("closing")
self.OK = False
self.server.close()
def run(self):
while self.OK:
with self.lock:
self.data = self.connection.recv(4096)
print(str(self.data))
time.sleep(0.02)
def send(self, message):
self.connection.sendall(message)

From docs: The KeyboardInterrupt inherits from BaseException so as to not be accidentally caught by code that catches Exception and thus prevent the interpreter from exiting. docs
if __name__ == '__main__':
try:
main = Main()
except KeyboardInterrupt:
pass
finally:
main.connection.close()

I would suggest you to use atexit module to this stuff.
Just put this line in __init__ and in case any python process termination a connection will be close
atexit.register(self.close)

UDP client and server with Twisted Python

I want to create a server and client that sends and receives UDP packets from the network using Twisted. I've already written this with sockets in Python, but want to take advantage of Twisted's callback and threading features. However, I need help though with the design of Twisted.
I have multiple types of packets I want to receive, but let's pretend there is just one:
class Packet(object):
def __init__(self, data=None):
self.packet_type = 1
self.payload = ''
self.structure = '!H6s'
if data == None:
return
self.packet_type, self.payload = struct.unpack(self.structure, data)
def pack(self):
return struct.pack(self.structure, self.packet_type, self.payload)
def __str__(self):
return "Type: {0}\nPayload {1}\n\n".format(self.packet_type, self.payload)
I made a protocol class (almost direct copy of the examples), which seems to work when I send data from another program:
class MyProtocol(DatagramProtocol):
def datagramReceived(self, data, (host, port)):
p = Packet(data)
print p
reactor.listenUDP(3000, MyProtocol())
reactor.run()
What I don't know is how do I create a client which can send arbitrary packets on the network, which get picked up by the reactor:
# Something like this:
s = Sender()
p = Packet()
p.packet_type = 3
s.send(p.pack())
p.packet_type = 99
s.send(p.pack())
I also need to make sure to set the reuse address flag on the client and servers so I can run multiple instances of each at the same time on the same device (e.g. one script is sending heartbeats, another responds to heartbeats, etc).
Can someone show me how this could be done with Twisted?
Update:
This is how I do it with sockets in Python. I can run multiple listeners and senders at the same time and they all hear each other. How do I get this result with Twisted? (The listening portion need not be a separate process.)
class Listener(Process):
def __init__(self, ip='127.0.0.1', port=3000):
Process.__init__(self)
self.ip = ip
self.port = port
def run(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((self.ip, self.port))
data, from_ip = sock.recvfrom(4096)
p = Packet(data)
print p
class Sender(object):
def __init__(self, ip='127.255.255.255', port=3000):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.ip = (ip, port)
def send(self, data):
self.sock.sendto(data, self.ip)
if __name__ == "__main__":
l = Listener()
l.start()
s = Sender()
p = Packet()
p.packet_type = 4
p.payload = 'jake'
s.send(p.pack())
Working solution:
class MySender(DatagramProtocol):
def __init__(self, packet, host='127.255.255.255', port=3000):
self.packet = packet.pack()
self.host = host
self.port = port
def startProtocol(self):
self.transport.write(self.packet, (self.host, self.port))
if __name__ == "__main__":
packet = Packet()
packet.packet_type = 1
packet.payload = 'jake'
s = MySender(packet)
reactor.listenMulticast(3000, MyProtocol(), listenMultiple=True)
reactor.listenMulticast(3000, s, listenMultiple=True)
reactor.callLater(4, reactor.stop)
reactor.run()

Just like the server example above, there is a client example to.
This should help you get started:
https://twistedmatrix.com/documents/current/core/howto/udp.html
https://github.com/twisted/twisted/blob/trunk/docs/core/examples/echoclient_udp.py
Ok, here is a simple heart beat sender and receiver using datagram protocol.
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
from twisted.internet.task import LoopingCall
import sys, time
class HeartbeatSender(DatagramProtocol):
def __init__(self, name, host, port):
self.name = name
self.loopObj = None
self.host = host
self.port = port
def startProtocol(self):
# Called when transport is connected
# I am ready to send heart beats
self.loopObj = LoopingCall(self.sendHeartBeat)
self.loopObj.start(2, now=False)
def stopProtocol(self):
"Called after all transport is teared down"
pass
def datagramReceived(self, data, (host, port)):
print "received %r from %s:%d" % (data, host, port)
def sendHeartBeat(self):
self.transport.write(self.name, (self.host, self.port))
class HeartbeatReciever(DatagramProtocol):
def __init__(self):
pass
def startProtocol(self):
"Called when transport is connected"
pass
def stopProtocol(self):
"Called after all transport is teared down"
def datagramReceived(self, data, (host, port)):
now = time.localtime(time.time())
timeStr = str(time.strftime("%y/%m/%d %H:%M:%S",now))
print "received %r from %s:%d at %s" % (data, host, port, timeStr)
heartBeatSenderObj = HeartbeatSender("sender", "127.0.0.1", 8005)
reactor.listenMulticast(8005, HeartbeatReciever(), listenMultiple=True)
reactor.listenMulticast(8005, heartBeatSenderObj, listenMultiple=True)
reactor.run()
The broadcast example simply modifies the above approach:
from twisted.internet.protocol import DatagramProtocol
from twisted.internet import reactor
from twisted.internet.task import LoopingCall
import sys, time
class HeartbeatSender(DatagramProtocol):
def __init__(self, name, host, port):
self.name = name
self.loopObj = None
self.host = host
self.port = port
def startProtocol(self):
# Called when transport is connected
# I am ready to send heart beats
self.transport.joinGroup('224.0.0.1')
self.loopObj = LoopingCall(self.sendHeartBeat)
self.loopObj.start(2, now=False)
def stopProtocol(self):
"Called after all transport is teared down"
pass
def datagramReceived(self, data, (host, port)):
print "received %r from %s:%d" % (data, host, port)
def sendHeartBeat(self):
self.transport.write(self.name, (self.host, self.port))
class HeartbeatReciever(DatagramProtocol):
def __init__(self, name):
self.name = name
def startProtocol(self):
"Called when transport is connected"
self.transport.joinGroup('224.0.0.1')
pass
def stopProtocol(self):
"Called after all transport is teared down"
def datagramReceived(self, data, (host, port)):
now = time.localtime(time.time())
timeStr = str(time.strftime("%y/%m/%d %H:%M:%S",now))
print "%s received %r from %s:%d at %s" % (self.name, data, host, port, timeStr)
heartBeatSenderObj = HeartbeatSender("sender", "224.0.0.1", 8005)
reactor.listenMulticast(8005, HeartbeatReciever("listner1"), listenMultiple=True)
reactor.listenMulticast(8005, HeartbeatReciever("listner2"), listenMultiple=True)
reactor.listenMulticast(8005, heartBeatSenderObj, listenMultiple=True)
reactor.run()

Check out the echoclient_udp.py example.
Since UDP is pretty much symmetrical between client and server, you just want to run reactor.listenUDP there too, connect to the server (which really just sets the default destination for sent packets), then transport.write to send your packets.