Im trying to programming a simplest client-server application with SSL, and i have some issues with this:
1) How i can decrease time for serialization at JSON?
2) Maybe something exists for better than LineReciver, for creating communication between server and client? Or i can increase length of received lines?
Source code:
a) ServerSLL
import server
from twisted.internet.protocol import Factory
from twisted.internet import reactor
from OpenSSL import SSL
class ServerSSL(object):
def getContext(self):
ctx = SSL.Context(SSL.SSLv23_METHOD)
ctx.use_certificate_file('server_cert.pem')
ctx.use_privatekey_file('server_key.pem')
return ctx
if __name__ == '__main__':
factory = Factory()
factory.protocol = server.Server
reactor.listenSSL(8000, factory, ServerSSL())
reactor.run()
b) Server
from json import dumps, loads
import sqlalchemy
from sqlalchemy.orm import sessionmaker
from db.create_db import Users
from twisted.internet.protocol import Protocol, Factory
from twisted.internet import reactor
engine = sqlalchemy.create_engine('postgresql://user:test#localhost/csan', pool_size=20, max_overflow=0)
class Server(Protocol):
def __init__(self):
self.Session = sessionmaker(bind=engine)
def __del__(self):
self.session.close()
def authorization(self, data):
"""
Checking user with DB
"""
session = self.Session()
result = session.execute(sqlalchemy.select([Users]).where(Users.name == data['user']))
result = result.fetchone()
if result is None:
data['error'] = 404
else:
if result['name'] == data['user']:
# correct users info --> real user
if result['password'] == data['pswd']:
data['auth'] = 1
# incorrect password --> fake user
else:
data['error'] = 403
session.close()
return data
def dataReceived(self, data):
"""
Processing request from user and send response
"""
new_data = loads(data)
if new_data['cmd'] == 'AUTH':
response = self.authorization(new_data)
self.transport.write(str(dumps(new_data)))
if __name__ == '__main__':
f = Factory()
f.protocol = Server
reactor.listenTCP(8000, f)
reactor.run()
c) client_console
from json import dumps, loads
from twisted.internet.protocol import ClientFactory
from twisted.protocols.basic import LineReceiver
from twisted.internet import ssl, reactor
class ServerClientSSL(LineReceiver):
"""
Basic client for talking with server under SSL
"""
def connectionMade(self):
"""
Send auth request to serverSLL.py
"""
login = raw_input('Login:')
password = raw_input('Password:')
hash_password = str(hash(password))
data = dumps({'cmd': 'AUTH', 'user': login, 'pswd': hash_password, 'auth': 0, 'error': 0})
self.sendLine(str(data))
def connectionLost(self, reason):
"""
Says to client, why we are close connection
"""
print 'connection lost (protocol)'
def lineReceived(self, data):
"""
Processing responses from serverSSL.py and send new requests to there
"""
new_data = loads(data)
if new_data['cmd'] == 'BBYE':
self.transport.loseConnection()
else:
print new_data
class ServerClientSLLFactory(ClientFactory):
protocol = ServerClientSSL
def clientConnectionFailed(self, connector, reason):
print 'connection failed:', reason.getErrorMessage()
reactor.stop()
def clientConnectionLost(self, connector, reason):
print 'connection lost:', reason.getErrorMessage()
reactor.stop()
if __name__ == '__main__':
import sys
if len(sys.argv) < 3:
print 'Using: python client_console.py [IP] [PORT] '
else:
ip = sys.argv[1]
port = sys.argv[2]
factory = ServerClientSLLFactory()
reactor.connectSSL(ip, int(port), factory, ssl.ClientContextFactory())
reactor.run()
class ServerSSL(object):
...
Don't write your own context factory. Use twisted.internet.ssl.CertificateOptions instead. It has fewer problems than what you have here.
def __del__(self):
self.session.close()
First rule of __del__: don't use __del__. Adding this method doesn't give you automatic session cleanup. Instead, it almost certainly guarantees your session will never be be cleaned up. Protocols have a method that gets called when they're done - it's called connectionLost. Use that instead.
result = session.execute(sqlalchemy.select([Users]).where(Users.name == data['user']))
result = result.fetchone()
Twisted is a single-threaded multi-tasking system. These statements block on network I/O and database operations. While they're running your server isn't doing anything else.
Use twisted.enterprise.adbapi or twext.enterprise.adbapi2 or alchimiato perform database interactions asynchronously instead.
class ServerClientSSL(LineReceiver):
...
There are lots of protocols better than LineReceiver. The simplest improvement you can make is to switch to Int32StringReceiver. A more substantial improvement would be to switch to twisted.protocols.amp.
1) How i can decrease time for serialization at JSON?
Use a faster JSON library. After you fix the blocking database code in your application, though, I doubt you'll still need a faster JSON library.
2) Maybe something exists for better than LineReciver, for creating communication between server and client? Or i can increase length of received lines?
LineReceiver.MAX_LENGTH. After you switch to Int32StringReceiver or AMP you won't need this anymore though.
Related
I'm trying to simulate a situation where data is received by a server periodically. In my set up, I run one process that sets up the server and another process that sets up a bunch of clients (suffices to think of a single client). I have set up some of the code by putting together bits and pieces mostly from here. The server/clients communicate by sending messages using transport.write. First, the server tells the clients to start (this works fine AFAIK). The clients report back to the server as they make progress. What has me confused is that I only get these intermittent messages at the very end when the client is done. This could be a problem with buffer flush and I tried (unsuccessfully) things like This. Also, each message is pretty large and I tried sending the same message multiple times so the buffer would get cleared.
I suspect what I am seeing is a problem with returning the control to the transport but I can't figure out how to do away with it.
Any help with this or any other issues that jump up to you is much appreciated.
Server:
from twisted.internet import reactor, protocol
import time
import serverSideAnalysis
import pdb
#import bson, json, msgpack
import _pickle as pickle # I expect the users to authenticate and not
# do anything malicious.
PORT = 9000
NUM = 1
local_scratch="/local/scratch"
class Hub(protocol.Protocol):
def __init__(self,factory, clients, nclients):
self.clients = clients
self.nclients = nclients
self.factory = factory
self.dispatcher = serverSideAnalysis.ServerTalker(NUM, self,
local_scratch)
def connectionMade(self):
print("connected to user" , (self))
if len(self.clients) < self.nclients:
self.factory.clients.append(self)
else:
self.factory.clients[self.nclients] = self
if len(self.clients) == NUM:
val = input("Looks like everyone is here, shall we start? (Y/N)")
while (val.upper() != "Y"):
time.sleep(20)
val = input("Looks like everyone is here, shall we start??? (Y/N)")
message = pickle.dumps({"TASK": "INIT", "SUBTASK":"STORE"})
self.message(message) # This reaches the client as I had expected
def message(self, command):
for c in self.factory.clients:
c.transport.write(command)
def connectionLost(self, reason):
self.factory.clients.remove(self)
self.nclients -= 1
def dataReceived(self, data):
if len(self.clients) == NUM:
self.dispatcher.dispatch(data)
class PauseTransport(protocol.Protocol):
def makeConnection(self, transport):
transport.pauseProducing()
class HubFactory(protocol.Factory):
def __init__(self, num):
self.clients = set([])
self.nclients = 0
self.totConnections = num
def buildProtocol(self, addr):
print(self.nclients)
if self.nclients < self.totConnections:
self.nclients += 1
return Hub(self, self.clients, self.nclients)
protocol = PauseTransport()
protocol.factory = self
return protocol
factory = HubFactory(NUM)
reactor.listenTCP(PORT, factory)
factory.clients = []
reactor.run()
Client:
from twisted.internet import reactor, protocol
import time
import clientSideAnalysis
import sys
HOST = 'localhost'
PORT = 9000
local_scratch="/local/scratch"
class MyClient(protocol.Protocol):
def connectionMade(self):
print("connected!")
self.factory.clients.append(self)
print ("clients are ", self.factory.clients)
self.cdispatcher = clientSideAnalysis.ServerTalker(analysis_file_name, local_scratch, self)
def clientConnectionLost(self, reason):
#TODO send warning
self.factory.clients.remove(self)
def dataReceived(self, data): #This is the problematic part I think
self.cdispatcher.dispatch(data)
print("1 sent")
time.sleep(10)
self.cdispatcher.dispatch(data)
print("2 sent")
time.sleep(10)
self.cdispatcher.dispatch(data)
time.sleep(10)
def message(self, data):
self.transport.write(data)
class MyClientFactory(protocol.ClientFactory):
protocol = MyClient
if __name__=="__main__":
analysis_file_name = sys.argv[1]
factory = MyClientFactory()
reactor.connectTCP(HOST, PORT, factory)
factory.clients = []
reactor.run()
The last bit of relevant information about what the dispatchers do.
In both cases, they load the message that has arrived (a dictionary) and do a few computations based on the content. Every once in a while, they use the message method to communicate with thier current values.
Finally, I'm using python 3.6. and twisted 18.9.0
The way you return control to the reactor from a Protocol.dataReceived method is you return from that method. For example:
def dataReceived(self, data):
self.cdispatcher.dispatch(data)
print("1 sent")
If you want more work to happen after this, you have some options. If you want the work to happen after some amount of time has passed, use reactor.callLater. If you want the work to happen after it is dispatched to another thread, use twisted.internet.threads.deferToThread. If you want the work to happen in response to some other event (for example, data being received), put it in the callback that handles that event (for example, dataReceived).
I am currently trying to create a small demo where I connect a web socket between my computer and my localhost ws://localhost:8080/ws. I want the web socket to monitor a file on my computer for changes. If there are changes, send a message. The connection and output is being monitored using Advanced Rest Client.
Is there a specific method I can use on the class to constantly check the contents of this file?
EDIT
I have implemented an observer using watchdog that detects any event for files in a specified directory. However, my message is not being sent in the sendFSEvent method and I also realized that my client isn't being registered when I connect to the web socket.
Here is my code in server.py
import sys
import os
from watchdog.observers import Observer
from twisted.web.static import File
from twisted.python import log
from twisted.web.server import Site
from twisted.internet import reactor, defer
from autobahn.twisted.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, listenWS
from MessangerEventHandler import MessangerEventHandler
class WsProtocol(WebSocketServerProtocol):
def connectionMade(self):
print("Connection made")
WebSocketServerProtocol.connectionMade(self)
def onOpen(self):
WebSocketServerProtocol.onOpen(self)
print("WebSocket connection open")
def onMessage(self, payload, isBinary):
print("Message was: {}".format(payload))
self.sendMessage("message received")
def sendFSEvent(self, json):
WebSocketProtocol.sendMessage(self, json)
print('Sent FS event')
def onClose(self, wasClean, code, reason):
print("Connection closed: {}".format(reason))
WebSocketServerProtocol.onClose(self, wasClean, code, reason)
class WsServerFactory(WebSocketServerFactory):
protocol = WsProtocol
def __init__(self, url='ws://localhost', port=8080):
addr = url + ':' + str(port)
print("Listening on: {}".format(addr))
WebSocketServerFactory.__init__(self, addr)
self.clients = []
def register(self, client):
if not client in self.clients:
print("Registered client: {}".format(client))
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print("Unregistered client: {}".format(client))
self.clients.remove(client)
self._printConnected()
def _printConnected(self):
print("Connected clients:[")
def notify_clients(self, message):
print("Broadcasting: {}".format(message))
for c in self.clients:
c.sendFSEvent(message)
print("\nSent messages")
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: python server_defer.py <dirs>")
sys.exit(1)
log.startLogging(sys.stdout)
ffactory = WsServerFactory("ws://localhost", 8080)
ffactory.protocol = WsProtocol
listenWS(ffactory)
observers = []
for arg in sys.argv[1:]:
dir_path = os.path.abspath(arg)
if not os.path.exists(dir_path):
print('{} does not exist.'.format(dir_path))
sys.exit(1)
if not os.path.isdir(dir_path):
print('{} is not a directory.'.format(dir_path))
sys.exit(1)
# Check for and handle events
event_handler = MessangerEventHandler(ffactory, reactor, os.getcwd())
observer = Observer()
observer.schedule(event_handler, path=dir_path, recursive=True)
observer.start()
observers.append(observer)
try:
reactor.run()
except KeyboardInterrupt:
for obs in observers:
obs.stop()
reactor.stop()
print("\nGoodbye")
sys.exit(1)
Any help would be greatly appreciated.
Thank you,
Brian
Most enterprise distros come with inotify which is really well suited for monitoring files and directories. The basic idea is to capture a list of connected web socket clients as they connect. Then create a callback that will execute when a change occurs on the files you're monitoring. Within this callback, you can iterate the clients and send them a message like 'file: "blah/blah.txt" has changed'. It's a little wonky, but the code snippet should clear things up for you.
from functools import partial
from twisted.internet import inotify
from twisted.python import filepath
# the rest of your imports ...
class SomeServerProtocol(WebSocketServerProtocol):
def onConnect(self, request):
self.factory.append(self) # <== append this client to the list in the factory
def notification_callback(ignored, filepath, mask, ws_clients):
"""
function that will execute when files are modified
"""
payload = "event on {0}".format(filepath)
for client in ws_clients:
client.sendMessage(
payload.encode('utf8'), # <== don't forget to encode the str to bytes before sending!
isBinary = False)
if __name__ == '__main__':
root = File(".")
factory = WebSocketServerFactory(u"ws://127.0.01:8080")
factory.protocol = SomeServerProtocol
factory.clients = [] # <== create a container for the clients that connect
# inotify stuff
notify = partial(notification_callback, ws_clients=factory.clients) # <== use functools.partial to pass extra params
notifier = inotify.INotify()
notifier.startReading()
notifier.watch(filepath.FilePath("/some/directory"), callbacks=[notify])
# the rest of your code ...
I am attempting to create a program that allows many clients to connect to 1 server simultaneously. These connections should be interactive on the server side, meaning that I can send requests from the server to the client, after the client has connected.
The following asyncore example code simply replies back with an echo, I need instead of an echo a way to interactively access each session. Somehow background each connection until I decided to interact with it. If I have 100 sessions I would like to chose a particular one or choose all of them or a subset of them to send a command to. Also I am not 100% sure that the asyncore lib is the way to go here, any help is appreciated.
import asyncore
import socket
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
data = self.recv(8192)
if data:
self.send(data)
class EchoServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
handler = EchoHandler(sock)
server = EchoServer('localhost', 8080)
asyncore.loop()
Here's a Twisted server:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import serverFromString
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
class HubConnection(LineReceiver, object):
def __init__(self, hub):
self.name = b'unknown'
self.hub = hub
def connectionMade(self):
self.hub.append(self)
def lineReceived(self, line):
words = line.split(" ", 1)
if words[0] == b'identify':
self.name = words[1]
else:
for connection in self.hub:
connection.sendLine("<{}> {}".format(
self.name, line
).encode("utf-8"))
def connectionLost(self, reason):
self.hub.remove(self)
def main(reactor, listen="tcp:4321"):
hub = []
endpoint = serverFromString(reactor, listen)
endpoint.listen(Factory.forProtocol(lambda: HubConnection(hub)))
return Deferred()
react(main, sys.argv[1:])
and command-line client:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import clientFromString
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.protocol import Factory
from twisted.internet.stdio import StandardIO
from twisted.protocols.basic import LineReceiver
from twisted.internet.fdesc import setBlocking
class HubClient(LineReceiver):
def __init__(self, name, output):
self.name = name
self.output = output
def lineReceived(self, line):
self.output.transport.write(line + b"\n")
def connectionMade(self):
self.sendLine("identify {}".format(self.name).encode("utf-8"))
def say(self, words):
self.sendLine("say {}".format(words).encode("utf-8"))
class TerminalInput(LineReceiver, object):
delimiter = "\n"
hubClient = None
def lineReceived(self, line):
if self.hubClient is None:
self.output.transport.write("Connecting, please wait...\n")
else:
self.hubClient.sendLine(line)
#inlineCallbacks
def main(reactor, name, connect="tcp:localhost:4321"):
endpoint = clientFromString(reactor, connect)
terminalInput = TerminalInput()
StandardIO(terminalInput)
setBlocking(0)
hubClient = yield endpoint.connect(
Factory.forProtocol(lambda: HubClient(name, terminalInput))
)
terminalInput.transport.write("Connecting...\n")
terminalInput.hubClient = hubClient
terminalInput.transport.write("Connected.\n")
yield Deferred()
react(main, sys.argv[1:])
which implement a basic chat server. Hopefully the code is fairly self-explanatory; you can run it to test with python hub_server.py in one terminal, python hub_client.py alice in a second and python hub_client.py bob in a third; then type into alice and bob's sessions and you can see what it does.
Review Requirements
You want
remote calls in client/server manner
probably using TCP communication
using sessions in the call
It is not very clear, how you really want to use sessions, so I will consider, that session is just one of call parameters, which has some meaning on server as well client side and will skip implementing it.
zmq as easy and reliable remote messaging platform
ZeroMQ is lightweight messaging platform, which does not require complex server infrastructure. It can handle many messaging patterns, following example showing request/reply pattern using multipart messages.
There are many alternatives, you can use simple messages encoded to some format like JSON and skip using multipart messages.
server.py
import zmq
class ZmqServer(object):
def __init__(self, url="tcp://*:5555"):
context = zmq.Context()
self.sock = context.socket(zmq.REP)
self.sock.bind(url)
self.go_on = False
def echo(self, message, priority=None):
priority = priority or "not urgent"
msg = "Echo your {priority} message: '{message}'"
return msg.format(**locals())
def run(self):
self.go_on = True
while self.go_on:
args = self.sock.recv_multipart()
if 1 <= len(args) <= 2:
code = "200"
resp = self.echo(*args)
else:
code = "401"
resp = "Bad request, 1-2 arguments expected."
self.sock.send_multipart([code, resp])
def stop(self):
self.go_on = False
if __name__ == "__main__":
ZmqServer().run()
client.py
import zmq
import time
class ZmqClient(object):
def __init__(self, url="tcp://localhost:5555"):
context = zmq.Context()
self.socket = context.socket(zmq.REQ)
self.socket.connect(url)
def call_echo(self, message, priority=None):
args = [message]
if priority:
args.append(priority)
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "200"
return resp
def too_long_call(self, message, priority, extrapriority):
args = [message, priority, extrapriority]
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "401"
return resp
def test_server(self):
print "------------------"
rqmsg = "Hi There"
print "rqmsg", rqmsg
print "response", self.call_echo(rqmsg)
print "------------------"
time.sleep(2)
rqmsg = ["Hi There", "very URGENT"]
print "rqmsg", rqmsg
print "response", self.call_echo(*rqmsg)
print "------------------"
time.sleep(2)
time.sleep(2)
rqmsg = []
print "too_short_call"
print "response", self.too_long_call("STOP", "VERY URGENT", "TOO URGENT")
print "------------------"
if __name__ == "__main__":
ZmqClient().test_server()
Play with the toy
Start the server:
$ python server.py
Now it runs and awaits requests.
Now start the client:
$ python client.py
------------------
rqmsg Hi There
response Echo your not urgent message: 'Hi There'
------------------
rqmsg ['Hi There', 'very URGENT']
response Echo your very URGENT message: 'Hi There'
------------------
too_short_call
response Bad request, 1-2 arguments expected.
------------------
Now experiment a bit:
start client first, then server
stop the server during processing, restart later on
start multiple clients
All these scenarios shall be handled by zmq without adding extra lines of Python code.
Conclusions
ZeroMQ provides very convenient remote messaging solution, try counting lines of messaging related code and compare with any other solution, providing the same level of stability.
Sessions (which were part of OP) can be considered just extra parameter of the call. As we saw, multiple parameters are not a problem.
Maintaining sessions, different backends can be used, they could live in memory (for single server instance), in database, or in memcache or Redis. This answer does not elaborate further on sessions, as it is not much clear, what use is expected.
Good Day to all,
I have a websocket server written in python that serves clients changes done to a postgresql database table. Presently it uses the standard ws protocol. I would Like to implement SSL wss but there seems to be no documentation on the tornadoweb site. Any help appreciated!
Code as follows:
import tornado.web
import tornado.websocket
import tornado.ioloop
import tornado.httpserver
import threading
import select
import psycopg2
import psycopg2.extensions
# This is a global variable to store all connected clients
websockets = []
# Connect to DB to listen for notifications
conn = psycopg2.connect("host=xxx.xxx.xxx.xxx dbname=mydb user=xxx port=yyy")
conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
curs = conn.cursor()
curs.execute("LISTEN notifier;")
# Create the websocket
class WebSocketHandler(tornado.websocket.WebSocketHandler):
#tornado.web.asynchronous
# Add to global variable the connected client
def open(self):
self.set_nodelay(True)
# on disconnect remove client
def on_close(self):
# Search for it in the list. If it matches with us (the client) then remove it as he has quit the con
try:
for websocket in websockets:
if websocket[0] == self:
websockets.remove(websocket)
except ValueError:
print ValueError
def on_message(self, message):
if self not in websockets:
websockets.append([self,message])
def on_pong(self, data):
print data
# This is the function that polls the database for changes, then it sends change to clients through websockets
def db_listener():
while 1:
if select.select([conn],[],[],5) == ([],[],[]):
for websocket in websockets:
# ping the client regularly to avoid disconnect
websocket[0].ping("ping")
else:
conn.poll()
while conn.notifies:
notify = conn.notifies.pop()
details = notify.payload.split(",")
if len(details) > 1:
for websocket in websockets:
if details[34] in websocket[1]:
websocket[0].write_message(notify.payload)
application = tornado.web.Application([
(r"/websocket", WebSocketHandler),
])
if __name__ == "__main__":
# Start a separate thread for every client so that we do not block the main websockets program!
threading.Thread(target=db_listener).start()
application.listen(5252)
tornado.ioloop.IOLoop.instance().start()
Any suggestions?
Thanks to all!!
try this:
server = tornado.httpserver.HTTPServer(application, ssl_options = {
"certfile": "path-to-crt-file",
"keyfile": "path-to-key-file",
})
server.listen(5252)
I'm fairly new with both Python and Twisted so I may just not be understanding things properly but I seem to be stuck at a point where I need help.
What I want to do is use ReconnectingClientFactory on an SSL connection. I have it all running, but if the connection is dropped all data sent to the transport's write() method is simply dropped without any error. The actual method called is twisted.protocols.tls.TLSMemoryBIOProtocol.write().
Here's what I think is happening (starting from a working connection):
connection is lost
call to write() method (source code here) with some data
self.disconnecting is False so data passes to _write() method
_write method gets to _lostTLSConnection which is True and then just runs return
connection is regained but no data sent because it's not buffered any where
Here's the a reduced version of the client:
from OpenSSL import SSL
from twisted.internet.protocol import (Protocol, ReconnectingClientFactory)
from twisted.internet import (reactor, ssl)
import struct
class MetricsServer(Protocol):
streambuffer = bytearray()
def connectionMade(self):
self.transport.setTcpKeepAlive(True) # maintain the TCP connection
self.transport.setTcpNoDelay(False) # allow Nagle algorithm
print("connected to server")
def dataReceived(self, data):
print("from server:", data)
def connectionLost(self, reason):
self.connected = 0
print("server connection lost:", reason)
class MetricsServerFactory(ReconnectingClientFactory):
protocol = MetricsServer
maxDelay = 300 # maximum seconds between retries
factor = 1.6180339887498948
packet_sequence_number = 0
active_connection = None
def buildProtocol(self, addr):
self.resetDelay()
if self.active_connection == None:
self.active_connection = self.protocol()
return self.active_connection
def get_packet_sequence_number(self):
self.packet_sequence_number += 1
return self.packet_sequence_number
def send_data(self):
print ("sending ssl packet")
packet = struct.pack("!I", self.get_packet_sequence_number())
self.active_connection.transport.write(packet)
reactor.callLater(1.0, metrics_server.send_data)
class CtxFactory(ssl.ClientContextFactory):
def getContext(self):
self.method = SSL.TLSv1_METHOD
ctx = ssl.ClientContextFactory.getContext(self)
ctx.use_certificate_file('keys/client.crt')
ctx.use_privatekey_file('keys/client.key')
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
ctx.set_verify(SSL.VERIFY_PEER, verifyCallback)
ctx.load_verify_locations("keys/ca.pem")
return ctx
if __name__ == "__main__":
metrics_server = MetricsServerFactory()
reactor.connectSSL('localhost', 8000, metrics_server, CtxFactory())
reactor.callLater(3.0, metrics_server.send_data)
reactor.run()
And here's a simple server that outputs the data it receives:
from OpenSSL import SSL
from twisted.internet import ssl, reactor
from twisted.internet.protocol import Factory, Protocol
class Echo(Protocol):
sent_back_data = False
def dataReceived(self, data):
print(' '.join("{0:02x}".format(x) for x in data))
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
if __name__ == '__main__':
factory = Factory()
factory.protocol = Echo
myContextFactory = ssl.DefaultOpenSSLContextFactory(
'keys/server.key', 'keys/server.crt'
)
ctx = myContextFactory.getContext()
ctx.set_verify(
SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
verifyCallback
)
ctx.load_verify_locations("keys/ca.pem")
reactor.listenSSL(8000, factory, myContextFactory)
reactor.run()
Process to recreate issue:
first you need to generate your own certs and CA for this to work
run the server first
run the client code
wait for some output on the server end and then end the program
notice the client continues to try and send data
restart the server end
notice the server end will continue receiving packets, but packets sent when the connection was lost are simply dropped
As a work-around I tried implementing my own buffer to send the data on re-connection, but ran into another issue. I want it to send the data when the connection is re-established and the only hook I can see is Protocol.connectionMade(). However, that method is called before the TLS handshaking is actually done so it ends up being caught by an exception handler in _write() and placed into another buffer to be sent later. But, that buffer only seems to be sent if data is received from the other end (which doesn't occur very often in my case, and would also mean that the data could arrive at the other end in the wrong order because a write() may be called before data is received). I also think another disconnection before data is received will also cause that buffer of data to just be wiped.
EDIT: added sample code for the first issue. It's probably odd that I have that active_connection in the Factory, but I am trying to make it work as a singleton.
Okay, I figured out the problem with my workaround... I was passing a bytearray to be written to the transport and then immediately cleared it afterwards not realizing that the write was being deferred until after I cleared the buffer. So, I pass a copy of the bytearray and it seems to be working now.
It still doesn't seem quite right that every call to the write has to be proceeded by a check to see if it's connected as the whole idea of the ReconnectingClientFactory is that it handles maintaining the connection for you. Also, I think it's possible to lose a connection between that if statement and when the write() is actually run, so it's still possible to lose data.
from OpenSSL import SSL
from twisted.internet.protocol import (Protocol, ReconnectingClientFactory)
from twisted.internet import (reactor, ssl)
import struct
class MetricsServer(Protocol):
streambuffer = bytearray()
def connectionMade(self):
self.transport.setTcpKeepAlive(True) # maintain the TCP connection
self.transport.setTcpNoDelay(False) # allow Nagle algorithm
print("connected to server")
if len(self.transport.factory.wrappedFactory.send_buffer) > 0:
self.transport.write(bytes(self.transport.factory.wrappedFactory.send_buffer))
self.transport.factory.wrappedFactory.send_buffer.clear()
def dataReceived(self, data):
print("from server:", data)
def connectionLost(self, reason):
self.connected = 0
print("server connection lost:", reason)
class MetricsServerFactory(ReconnectingClientFactory):
protocol = MetricsServer
maxDelay = 300 # maximum seconds between retries
factor = 1.6180339887498948
packet_sequence_number = 0
active_connection = None
send_buffer = bytearray()
def buildProtocol(self, addr):
self.resetDelay()
if self.active_connection == None:
self.active_connection = self.protocol()
return self.active_connection
def get_packet_sequence_number(self):
self.packet_sequence_number += 1
return self.packet_sequence_number
def send_data(self):
print ("sending ssl packet")
packet = struct.pack("!I", self.get_packet_sequence_number())
if self.active_connection and self.active_connection.connected:
self.active_connection.transport.write(packet)
else:
self.send_buffer.extend(packet)
reactor.callLater(1.0, metrics_server.send_data)
class CtxFactory(ssl.ClientContextFactory):
def getContext(self):
self.method = SSL.TLSv1_METHOD
ctx = ssl.ClientContextFactory.getContext(self)
ctx.use_certificate_file('keys/client.crt')
ctx.use_privatekey_file('keys/client.key')
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
ctx.set_verify(SSL.VERIFY_PEER, verifyCallback)
ctx.load_verify_locations("keys/ca.pem")
return ctx
if __name__ == "__main__":
metrics_server = MetricsServerFactory()
reactor.connectSSL('localhost', 8000, metrics_server, CtxFactory())
reactor.callLater(3.0, metrics_server.send_data)
reactor.run()