We Keep Coding

How does Twisted reactor work with trial-based unit tests?

I have written a TCP/UDP intercepting proxy using Twisted and I want to add some unit tests to it. I want to setup an echo protocol, then send some data through my proxy, then check the returned response.
However, it seems like even for a simple test using a socket (let aside my intercepting proxy) to connect to the echoer, the reactor desn't seem to be spawned after setUp - the test hangs forever. If I add a timeout to the socket then a timeout exception is raised. I even tried to connect with ncat to make sure is not the manually created socket to blame - the echoer is listening indeed but I receive no echoed data back to the ncat client.
The test code I use is the following
import pytest
import socket
from twisted.trial import unittest
from twisted.internet import reactor, protocol
class EchoTCP(protocol.Protocol):
def dataReceived(self, data):
self.transport.write(data)
class EchoTCPFactory(protocol.Factory):
protocol = EchoTCP
class TestTCP(unittest.TestCase):
"""Twisted has its own unittest class
https://twistedmatrix.com/documents/15.2.0/core/howto/trial.html
"""
def setUp(self):
self.iface = "127.0.0.1"
self.data = b"Hello, World!"
# Setup twised echoer
self.port = reactor.listenTCP(
8080,
EchoTCPFactory(),
interface=self.iface
)
def tearDown(self):
self.port.stopListening()
def test_echo(self):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((self.iface, self.port.getHost().port))
sent = sock.send(self.data)
data = sock.recv(1024)
sock.close()
assert data == self.data
To run it I use the following command
PYTHONPATH="${PWD}" trial --reactor=default mymodule
The output is the following and stays like this until I kill the process
mymodule.test.test_network
TestTCP
test_echo ...
It seems like I'm missing something regarding how the reactor works. I've looked for similar examples but couldn't get it working.
How should I write the test to get the expected behavior?

It turned out I must run the test methods as Deffered, using inlineCallbacks, so they are called when the reactor is running. To test this behavior I've used the following snippet
from twisted.internet.defer import inlineCallbacks
# [...]
def check_reactor(self):
# time.sleep(100)
return reactor.running
#inlineCallbacks
def test_reactor(self):
reactor_running = yield threads.deferToThread(self.check_reactor)
assert reactor_running == True
...which makes the test successfully complete
mymodule.test.test_network
TestTCP
test_reactor ... [OK]
-------------------------------------------------------------------------------
Ran 1 tests in 0.007s
PASSED (successes=1)
If I enable the sleep(100) in the calledback method, and connect with ncat in that timespan, the data that I send to the listening port is indeed echoed back

Using asyncore to create interactive sessions with client/server model

I am attempting to create a program that allows many clients to connect to 1 server simultaneously. These connections should be interactive on the server side, meaning that I can send requests from the server to the client, after the client has connected.
The following asyncore example code simply replies back with an echo, I need instead of an echo a way to interactively access each session. Somehow background each connection until I decided to interact with it. If I have 100 sessions I would like to chose a particular one or choose all of them or a subset of them to send a command to. Also I am not 100% sure that the asyncore lib is the way to go here, any help is appreciated.
import asyncore
import socket
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
data = self.recv(8192)
if data:
self.send(data)
class EchoServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
handler = EchoHandler(sock)
server = EchoServer('localhost', 8080)
asyncore.loop()

Here's a Twisted server:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import serverFromString
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
class HubConnection(LineReceiver, object):
def __init__(self, hub):
self.name = b'unknown'
self.hub = hub
def connectionMade(self):
self.hub.append(self)
def lineReceived(self, line):
words = line.split(" ", 1)
if words[0] == b'identify':
self.name = words[1]
else:
for connection in self.hub:
connection.sendLine("<{}> {}".format(
self.name, line
).encode("utf-8"))
def connectionLost(self, reason):
self.hub.remove(self)
def main(reactor, listen="tcp:4321"):
hub = []
endpoint = serverFromString(reactor, listen)
endpoint.listen(Factory.forProtocol(lambda: HubConnection(hub)))
return Deferred()
react(main, sys.argv[1:])
and command-line client:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import clientFromString
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.protocol import Factory
from twisted.internet.stdio import StandardIO
from twisted.protocols.basic import LineReceiver
from twisted.internet.fdesc import setBlocking
class HubClient(LineReceiver):
def __init__(self, name, output):
self.name = name
self.output = output
def lineReceived(self, line):
self.output.transport.write(line + b"\n")
def connectionMade(self):
self.sendLine("identify {}".format(self.name).encode("utf-8"))
def say(self, words):
self.sendLine("say {}".format(words).encode("utf-8"))
class TerminalInput(LineReceiver, object):
delimiter = "\n"
hubClient = None
def lineReceived(self, line):
if self.hubClient is None:
self.output.transport.write("Connecting, please wait...\n")
else:
self.hubClient.sendLine(line)
#inlineCallbacks
def main(reactor, name, connect="tcp:localhost:4321"):
endpoint = clientFromString(reactor, connect)
terminalInput = TerminalInput()
StandardIO(terminalInput)
setBlocking(0)
hubClient = yield endpoint.connect(
Factory.forProtocol(lambda: HubClient(name, terminalInput))
)
terminalInput.transport.write("Connecting...\n")
terminalInput.hubClient = hubClient
terminalInput.transport.write("Connected.\n")
yield Deferred()
react(main, sys.argv[1:])
which implement a basic chat server. Hopefully the code is fairly self-explanatory; you can run it to test with python hub_server.py in one terminal, python hub_client.py alice in a second and python hub_client.py bob in a third; then type into alice and bob's sessions and you can see what it does.

Review Requirements
You want
remote calls in client/server manner
probably using TCP communication
using sessions in the call
It is not very clear, how you really want to use sessions, so I will consider, that session is just one of call parameters, which has some meaning on server as well client side and will skip implementing it.
zmq as easy and reliable remote messaging platform
ZeroMQ is lightweight messaging platform, which does not require complex server infrastructure. It can handle many messaging patterns, following example showing request/reply pattern using multipart messages.
There are many alternatives, you can use simple messages encoded to some format like JSON and skip using multipart messages.
server.py
import zmq
class ZmqServer(object):
def __init__(self, url="tcp://*:5555"):
context = zmq.Context()
self.sock = context.socket(zmq.REP)
self.sock.bind(url)
self.go_on = False
def echo(self, message, priority=None):
priority = priority or "not urgent"
msg = "Echo your {priority} message: '{message}'"
return msg.format(**locals())
def run(self):
self.go_on = True
while self.go_on:
args = self.sock.recv_multipart()
if 1 <= len(args) <= 2:
code = "200"
resp = self.echo(*args)
else:
code = "401"
resp = "Bad request, 1-2 arguments expected."
self.sock.send_multipart([code, resp])
def stop(self):
self.go_on = False
if __name__ == "__main__":
ZmqServer().run()
client.py
import zmq
import time
class ZmqClient(object):
def __init__(self, url="tcp://localhost:5555"):
context = zmq.Context()
self.socket = context.socket(zmq.REQ)
self.socket.connect(url)
def call_echo(self, message, priority=None):
args = [message]
if priority:
args.append(priority)
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "200"
return resp
def too_long_call(self, message, priority, extrapriority):
args = [message, priority, extrapriority]
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "401"
return resp
def test_server(self):
print "------------------"
rqmsg = "Hi There"
print "rqmsg", rqmsg
print "response", self.call_echo(rqmsg)
print "------------------"
time.sleep(2)
rqmsg = ["Hi There", "very URGENT"]
print "rqmsg", rqmsg
print "response", self.call_echo(*rqmsg)
print "------------------"
time.sleep(2)
time.sleep(2)
rqmsg = []
print "too_short_call"
print "response", self.too_long_call("STOP", "VERY URGENT", "TOO URGENT")
print "------------------"
if __name__ == "__main__":
ZmqClient().test_server()
Play with the toy
Start the server:
$ python server.py
Now it runs and awaits requests.
Now start the client:
$ python client.py
------------------
rqmsg Hi There
response Echo your not urgent message: 'Hi There'
------------------
rqmsg ['Hi There', 'very URGENT']
response Echo your very URGENT message: 'Hi There'
------------------
too_short_call
response Bad request, 1-2 arguments expected.
------------------
Now experiment a bit:
start client first, then server
stop the server during processing, restart later on
start multiple clients
All these scenarios shall be handled by zmq without adding extra lines of Python code.
Conclusions
ZeroMQ provides very convenient remote messaging solution, try counting lines of messaging related code and compare with any other solution, providing the same level of stability.
Sessions (which were part of OP) can be considered just extra parameter of the call. As we saw, multiple parameters are not a problem.
Maintaining sessions, different backends can be used, they could live in memory (for single server instance), in database, or in memcache or Redis. This answer does not elaborate further on sessions, as it is not much clear, what use is expected.

Too long response (self.transport.write) is formed in Twisted

Im trying to programming a simplest client-server application with SSL, and i have some issues with this:
1) How i can decrease time for serialization at JSON?
2) Maybe something exists for better than LineReciver, for creating communication between server and client? Or i can increase length of received lines?
Source code:
a) ServerSLL
import server
from twisted.internet.protocol import Factory
from twisted.internet import reactor
from OpenSSL import SSL
class ServerSSL(object):
def getContext(self):
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.use_certificate_file('server_cert.pem')
ctx.use_privatekey_file('server_key.pem')
return ctx
if __name__ == '__main__':
factory = Factory()
factory.protocol = server.Server
reactor.listenSSL(8000, factory, ServerSSL())
reactor.run()
b) Server
from json import dumps, loads
import sqlalchemy
from sqlalchemy.orm import sessionmaker
from db.create_db import Users
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor
engine = sqlalchemy.create_engine('postgresql://user:test#localhost/csan', pool_size=20, max_overflow=0)
class Server(Protocol):
def __init__(self):
self.Session = sessionmaker(bind=engine)
def __del__(self):
self.session.close()
def authorization(self, data):
"""
Checking user with DB
"""
session = self.Session()
result = session.execute(sqlalchemy.select([Users]).where(Users.name == data['user']))
result = result.fetchone()
if result is None:
data['error'] = 404
else:
if result['name'] == data['user']:
# correct users info --> real user
if result['password'] == data['pswd']:
data['auth'] = 1
# incorrect password --> fake user
else:
data['error'] = 403
session.close()
return data
def dataReceived(self, data):
"""
Processing request from user and send response
"""
new_data = loads(data)
if new_data['cmd'] == 'AUTH':
response = self.authorization(new_data)
self.transport.write(str(dumps(new_data)))
if __name__ == '__main__':
f = Factory()
f.protocol = Server
reactor.listenTCP(8000, f)
reactor.run()
c) client_console
from json import dumps, loads
from twisted.internet.protocol import ClientFactory
from twisted.protocols.basic import LineReceiver
from twisted.internet import ssl, reactor
class ServerClientSSL(LineReceiver):
"""
Basic client for talking with server under SSL
"""
def connectionMade(self):
"""
Send auth request to serverSLL.py
"""
login = raw_input('Login:')
password = raw_input('Password:')
hash_password = str(hash(password))
data = dumps({'cmd': 'AUTH', 'user': login, 'pswd': hash_password, 'auth': 0, 'error': 0})
self.sendLine(str(data))
def connectionLost(self, reason):
"""
Says to client, why we are close connection
"""
print 'connection lost (protocol)'
def lineReceived(self, data):
"""
Processing responses from serverSSL.py and send new requests to there
"""
new_data = loads(data)
if new_data['cmd'] == 'BBYE':
self.transport.loseConnection()
else:
print new_data
class ServerClientSLLFactory(ClientFactory):
protocol = ServerClientSSL
def clientConnectionFailed(self, connector, reason):
print 'connection failed:', reason.getErrorMessage()
reactor.stop()
def clientConnectionLost(self, connector, reason):
print 'connection lost:', reason.getErrorMessage()
reactor.stop()
if __name__ == '__main__':
import sys
if len(sys.argv) < 3:
print 'Using: python client_console.py [IP] [PORT] '
else:
ip = sys.argv[1]
port = sys.argv[2]
factory = ServerClientSLLFactory()
reactor.connectSSL(ip, int(port), factory, ssl.ClientContextFactory())
reactor.run()

class ServerSSL(object):
...
Don't write your own context factory. Use twisted.internet.ssl.CertificateOptions instead. It has fewer problems than what you have here.
def __del__(self):
self.session.close()
First rule of __del__: don't use __del__. Adding this method doesn't give you automatic session cleanup. Instead, it almost certainly guarantees your session will never be be cleaned up. Protocols have a method that gets called when they're done - it's called connectionLost. Use that instead.
result = session.execute(sqlalchemy.select([Users]).where(Users.name == data['user']))
result = result.fetchone()
Twisted is a single-threaded multi-tasking system. These statements block on network I/O and database operations. While they're running your server isn't doing anything else.
Use twisted.enterprise.adbapi or twext.enterprise.adbapi2 or alchimiato perform database interactions asynchronously instead.
class ServerClientSSL(LineReceiver):
...
There are lots of protocols better than LineReceiver. The simplest improvement you can make is to switch to Int32StringReceiver. A more substantial improvement would be to switch to twisted.protocols.amp.
1) How i can decrease time for serialization at JSON?
Use a faster JSON library. After you fix the blocking database code in your application, though, I doubt you'll still need a faster JSON library.
2) Maybe something exists for better than LineReciver, for creating communication between server and client? Or i can increase length of received lines?
LineReceiver.MAX_LENGTH. After you switch to Int32StringReceiver or AMP you won't need this anymore though.

why is a tcp message not getting written to the transport in twisted?

server.py
# This is the Twisted Fast Poetry Server, version 1.0
import optparse, os
from twisted.internet.protocol import ServerFactory, Protocol
def parse_args():
usage = """usage: %prog [options] poetry-file
This is the Fast Poetry Server, Twisted edition.
Run it like this:
python fastpoetry.py <path-to-poetry-file>
If you are in the base directory of the twisted-intro package,
you could run it like this:
python twisted-server-1/fastpoetry.py poetry/ecstasy.txt
to serve up John Donne's Ecstasy, which I know you want to do.
"""
parser = optparse.OptionParser(usage)
help = "The port to listen on. Default to a random available port."
parser.add_option('--port', type='int', help=help)
help = "The interface to listen on. Default is localhost."
parser.add_option('--iface', help=help, default='localhost')
options, args = parser.parse_args()
if len(args) != 1:
parser.error('Provide exactly one poetry file.')
poetry_file = args[0]
if not os.path.exists(args[0]):
parser.error('No such file: %s' % poetry_file)
return options, poetry_file
class PoetryProtocol(Protocol):
def __init__(self, factory):
self.factory = factory
def connectionMade(self):
self.factory.pushers.append(self)
#self.transport.write("self.factory.poem")
#self.transport.write(self.factory.poem)
#self.transport.loseConnection()
class PoetryFactory(ServerFactory):
#protocol = PoetryProtocol
def __init__(self, poem):
self.poem = poem
self.pushers = []#
def buildProtocol(self, addr):
return PoetryProtocol(self)
def main():
options, poetry_file = parse_args()
poem = open(poetry_file).read()
factory = PoetryFactory(poem)
from twisted.internet import reactor
port = reactor.listenTCP(options.port or 0, factory,
interface=options.iface)
print 'Serving %s on %s.' % (poetry_file, port.getHost())
reactor.run()
factory.pushers[0].transport.write("hey")#########why is this message not received on the client?
if __name__ == '__main__':
main()
I have created a list called pushers (in the factory) of the protocols, when a connection is made. When I try to write to it, the message doesnt arrive in the datareceived on the client side. why?

You calling factory.pushers[0].transport.write right after running reactor but protocol instance added to factory pushers list only when client connected to your server
Uncomment second line in connectionMade handler if you want to write to client when connection established:
def connectionMade(self):
self.factory.pushers.append(self)
self.transport.write("hey")

Python DNS Server

I am adding a feature to my current project that will allow network admins to install the software to the network. I need to code a DNS server in Python that will allow me to redirect to a certain page if the request address is in my list. I was able to write the server, just not sure how to redirect.
Thank you. I am using Python 2.6 on Windows XP.

There's little, simple example here that can easily be adapted to make all kinds of "mini fake dns servers". Note that absolutely no "redirect" is involved (that's not how DNS works): rather, the request is for a domain name, and the result of that request is an IP address. If what you want to do is drastically different from translating names to addresses, then maybe what you need is not actually a DNS server...?

Using circuits and dnslib here's a full recursive dns server written in Python in only 143 lines of code:
#!/usr/bin/env python
from __future__ import print_function
from uuid import uuid4 as uuid
from dnslib import CLASS, QR, QTYPE
from dnslib import DNSHeader, DNSQuestion, DNSRecord
from circuits.net.events import write
from circuits import Component, Debugger, Event
from circuits.net.sockets import UDPClient, UDPServer
class lookup(Event):
"""lookup Event"""
class query(Event):
"""query Event"""
class response(Event):
"""response Event"""
class DNS(Component):
def read(self, peer, data):
record = DNSRecord.parse(data)
if record.header.qr == QR["QUERY"]:
return self.fire(query(peer, record))
return self.fire(response(peer, record))
class ReturnResponse(Component):
def response(self, peer, response):
return response
class Client(Component):
channel = "client"
def init(self, server, port, channel=channel):
self.server = server
self.port = int(port)
self.transport = UDPClient(0, channel=self.channel).register(self)
self.protocol = DNS(channel=self.channel).register(self)
self.handler = ReturnResponse(channel=self.channel).register(self)
class Resolver(Component):
def init(self, server, port):
self.server = server
self.port = port
def lookup(self, qname, qclass="IN", qtype="A"):
channel = uuid()
client = Client(
self.server,
self.port,
channel=channel
).register(self)
yield self.wait("ready", channel)
self.fire(
write(
(self.server, self.port),
DNSRecord(
q=DNSQuestion(
qname,
qclass=CLASS[qclass],
qtype=QTYPE[qtype]
)
).pack()
)
)
yield (yield self.wait("response", channel))
client.unregister()
yield self.wait("unregistered", channel)
del client
class ProcessQuery(Component):
def query(self, peer, query):
qname = query.q.qname
qtype = QTYPE[query.q.qtype]
qclass = CLASS[query.q.qclass]
response = yield self.call(lookup(qname, qclass=qclass, qtype=qtype))
record = DNSRecord(
DNSHeader(id=query.header.id, qr=1, aa=1, ra=1),
q=query.q,
)
for rr in response.value.rr:
record.add_answer(rr)
yield record.pack()
class Server(Component):
def init(self, bind=("0.0.0.0", 53)):
self.bind = bind
self.transport = UDPServer(self.bind).register(self)
self.protocol = DNS().register(self)
self.handler = ProcessQuery().register(self)
class App(Component):
def init(self, bind=("0.0.0.0", 53), server="8.8.8.8", port=53,
verbose=False):
if verbose:
Debugger().register(self)
self.resolver = Resolver(server, port).register(self)
self.server = Server(bind).register(self)
def main():
App().run()
if __name__ == "__main__":
main()
Usage:
By default this example binds go 0.0.0.0:53 so you will need to do something like:
sudo ./dnsserver.py
Otherwise change the bind parameter.

Here is a dns serer/proxy that works for me written in python:
http://thesprawl.org/projects/dnschef/

I wrote a DNS Server using Python Twisted library for NameOcean.net. You can see examples on https://twistedmatrix.com/documents/16.5.0/names/howto/custom-server.html.