We Keep Coding

Python AsyncIO fails to create a protocol because of missing transport though transport is present

I am trying to write a proxy server using AsyncIO's asyncio.protocol's and that seems to work for the biggest part. But I am getting an error that confuses me pretty badly.
My code looks the following:
import asyncio
import socket
users = {}
class UserConnection(object):
def __init__(self, client_socket, proxy_socket, peername):
self.client_socket = client_socket
self.proxy_socket = proxy_socket
self.peername = peername
class ClientProxyProtocol(asyncio.Protocol):
def __init__(self):
self.transport = None
self.session = None
server_connection = loop.create_connection(
lambda: ProxyForwardProtocol,
host="127.0.0.1",
port=25575)
task = loop.create_task(server_connection)
task.add_done_callback(self.handle_proxy_connection)
def connection_made(self, transport):
# Get peername, client socket and create proxy socket
peername = transport.get_extra_info('peername')
client_socket = transport.get_extra_info('socket')
# Get session object for a user and add in dict keyed of proxy socket
self.session = UserConnection(client_socket, None, peername)
print('Connection from {}'.format(peername))
self.transport = transport
def handle_proxy_connection(self, task):
trans, proto = task.result()
proxy_socket = trans.get_extra_info("socket")
self.session.proxy_socket = proxy_socket
users[proxy_socket] = self.session
print(trans,"\n",proto)
def data_received(self, data):
message = str(data)
print('Data received: {!r}'.format(message))
def connection_lost(self, exc):
print('Close the client socket')
self.transport.close()
class ProxyForwardProtocol(asyncio.Protocol):
def __init__(self,transport):
self.transport = transport
def connection_made(self, transport):
proxy_socket = transport.get_extra_info("socket")
print("HELLO")
self.transport = transport
def connection_lost(self, exc):
print("closing forward connection")
self.transport.close()
loop = asyncio.get_event_loop()
# Each client connection will create a new protocol instance
coro = loop.create_server(ClientProxyProtocol, '127.0.0.1', 25565)
server = loop.run_until_complete(coro)
# Serve requests until Ctrl+C is pressed
print('Serving on {}'.format(server.sockets[0].getsockname()))
try:
loop.run_forever()
except KeyboardInterrupt:
pass
# Close the server
server.close()
loop.run_until_complete(server.wait_closed())
loop.close()
Running this script works perfectly fine to some extent, it accepts new connections and starts up a new Protocol instance, but it produces the following bug after it tries to establish a Protocol Connection with my target server. It produces the following error:
Exception in callback ProxyForwardProtocol.connection_made(<_SelectorSoc...e, bufsize=0>>) at .\asyncproto.py:53
handle: <Handle ProxyForwardProtocol.connection_made(<_SelectorSoc...e, bufsize=0>>) at .\asyncproto.py:53>
Traceback (most recent call last):
File "C:\Users\Huhn\AppData\Local\Programs\Python\Python36-32\lib\asyncio\events.py", line 126, in _run
self._callback(*self._args)
TypeError: connection_made() missing 1 required positional argument: 'transport'
My question is the following: The handle shows that the connection_made method takes in a SelectorSocketTransport, which I would assume to be a transport, yet still says it is missing a transport. That makes no sense to me.
I would really appreciate if someone could tell me where my mistake lies since I have not found anyone with a similar problem and am kind of stuck. If i have any kind of architectural misconception about proxies feel free to point them out. I am doing this as a teaching exercise, that is the reason why I am not using third-party alternatives which may lie out there.
Thanks

It seems you just made a typo in ClientProxyProtocol.__init__().
In create_connection first argument must be callable which returns Protocol object, instead your (lambda: ProxyForwardProtocol) returns Protocol class.
So, when the framework calls connection_made(), it will pass a single argument (transport), but the first argument must be self. Protocol is a class and not class instance -> method connection_made() is not bound -> self isn't supplied automatically.
Remove ProxyForwardProtocol.__init__ method and change your call like that:
server_connection = loop.create_connection(
ProxyForwardProtocol,
host="127.0.0.1",
port=25575)

Why is a parameter to __init__() missing?

Summary
I have a client-server application which makes use of Websockets. The backend (server) part is implemented in Python using autobahn.
The server, in addition to serving a Websockets endpoint, runs a series of threads which will feed the Websockets channel with data, though a queue.Queue().
One of these threads has a problem: it crashes at a missing parameter and hangs when resolving the exception.
Implementation details
The server implementation (cut down to highlight the problem):
from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory
import time
import threading
import arrow
import queue
import asyncio
import json
# backends of components
import dummy
class MyServerProtocol(WebSocketServerProtocol):
def __init__(self):
super().__init__()
print("webserver initialized")
# global queue to handle updates from modules
self.events = queue.Queue()
# consumer
threading.Thread(target=self.push).start()
threading.Thread(target=dummy.Dummy().dummy, args=(self.events,)).start()
def push(self):
""" consume the content of the queue and push it to the browser """
while True:
update = self.events.get()
print(update)
if update:
self.sendMessage(json.dumps(update).encode('utf-8'), False)
print(update)
time.sleep(1)
def worker(self):
print("started thread")
while True:
try:
self.sendMessage(arrow.now().isoformat().encode('utf-8'), False)
except AttributeError:
print("not connected?")
time.sleep(3)
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
def onOpen(self):
print("WebSocket connection open.")
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
factory = WebSocketServerFactory(u"ws://127.0.0.1:9100")
factory.protocol = MyServerProtocol
loop = asyncio.get_event_loop()
coro = loop.create_server(factory, '0.0.0.0', 9100)
loop.run_until_complete(coro)
loop.run_forever()
The dummy module imported in the code above:
import time
import arrow
class Dummy:
def __init__(self, events):
self.events = events
print("dummy initialized")
def dummy(self):
while True:
self.events.put({
'dummy': {
'time': arrow.now().isoformat()
}
})
time.sleep(1)
The problem
When running the code above and connecting from a client, I get on the output webserver initialized (which proves that the connection was initiated), and WebSocket connection to 'ws://127.0.0.1:9100/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED on the client.
When debugging the code, I see that the call to threading.Thread(target=dummy.Dummy().dummy, args=(self.events,)).start() crashes and the debugger (PyCharm) leads me to C:\Program Files (x86)\Python36-32\Lib\asyncio\selector_events.py, specifically to the line 236
# It's now up to the protocol to handle the connection.
except Exception as exc:
if self._debug:
The thread hangs when executing if self._debug but I see on the exceptline (thanks to Pycharm) that
exc: __init__() missing 1 required positional argument: 'events'
My question
Why is this parameter missing? It is provided via the threading.Thread(target=dummy.Dummy().dummy, args=(self.events,)).start() call.
As a side question: why does the thread hangs on the if condition?
Notes
there is never a Traceback thrown by my program (due to the hang)
removing this thread call resolves the issue (the client connects correctly)

The events arg is needed for the constructor, not the dummy method. I think you meant something more like:
d = Dummy(self.events)
threading.Thread(d.dummy).start()

issues with socket programming - python

I am doing a client-server project for my college project,
we have to allocate the login to the client.
Client system will request its status for every 2 seconds(to check whether the client is locked or unlocked). and server will accept the client request and reply the client status to the system.
But the problem is server thread is not responding to the client request.
CLIENT THREAD:
def checkPort():
while True:
try:
s = socket.socket()
s.connect((host, port))
s.send('pc1') # send PC name to the server
status = s.recv(1024) # receive the status from the server
if status == "unlock":
disableIntrrupts() # enable all the functions of system
else:
enableInterrupts() # enable all the functions of system
time.sleep(5)
s.close()
except Exception:
pass
SERVER THREAD:
def check_port():
while True:
try:
print "hello loop is repeating"
conn, addr = s.accept()
data = conn.recv(1024)
if exit_on_click == 1:
break
if (any(sublist[0] == data for sublist in available_sys)):
print "locked"
conn.send("lock")
elif (any(sublist[0] == data for sublist in occupied_sys)):
conn.send("unlock")
print "unlocked"
else:
print "added to gui for first time"
available_sys.append([data,addr[0],nameText,usnText,branchText])
availSysList.insert('end',data)
except Exception:
pass
But my problem is server thread is not executing more than 2 time,
So its unable to accept client request more than one time.
can't we handle multiple client sockets using single server socket?
How to handle multiple client request from server ?
Thanks for any help !!

Its because your server, will block waiting for a new connection on this line
conn, addr = s.accept()
This is because calls like .accept and .read are blocking calls that hold the process
You need to consider an alternative design, where in you either.
Have one process per connection (this idea is stupid)
One thread per connection (this idea is less stupid than the first but still mostly foolish)
Have a non blocking design that allows multiple clients and read/write without blocking execution.
To achieve the first, look at multiprocessing, the second is threading the third is slightly more complicated to get your head around but will yield the best results, the go to library for event driven code in Python is twisted but there are others like
gevent
tulip
tornado
And so so many more that I haven't listed here.

here's an full example of implementing a threaded server. it's fully functional and comes with the benefit of using SSL as well. further, i use threaded event objects to signal another class object after storing my received data in a database.
please note, _sni and _cams_db are additional modules purely of my own. if you want to see the _sni module (provides SNI support for pyOpenSSL), let me know.
what follows this, is a snippet from camsbot.py, there's a whole lot more that far exceeds the scope of this question. what i've built is a centralized message relay system. it listens to tcp/2345 and accepts SSL connections. each connection passes messages into the system. short lived connections will connect, pass message, and disconnect. long lived connections will pass numerous messages after connecting. messages are stored in a database and a threading.Event() object (attached to the DB class) is set to tell the bot to poll the database for new messages and relay them.
the below example shows
how to set up a threaded tcp server
how to pass information from the listener to the accept handler such as config data and etc
in addition, this example also shows
how to employ an SSL socket
how to do some basic certificate validations
how to cleanly wrap and unwrap SSL from a tcp socket
how to use poll() on the socket instead of select()
db.pending is a threading.Event() object in _cams_db.py
in the main process we start another thread that waits on the pending object with db.pending.wait(). this makes that thread wait until another thread does db.pending.set(). once it is set, our waiting thread immediately wakes up and continues to work. when our waiting thread is done, it calls db.pending.clear() and goes back to the beginning of the loop and starts waiting again with db.pending.wait()
while True:
db.pending.wait()
# after waking up, do code. for example, we wait for incoming messages to
# be stored in the database. the threaded server will call db.pending.set()
# which will wake us up. we'll poll the DB for new messages, relay them, clear
# our event flag and go back to waiting.
# ...
db.pending.clear()
snippet from camsbot.py:
import sys, os, sys, time, datetime, threading, select, logging, logging.handlers
import configparser, traceback, re, socket, hashlib
# local .py
sys.path.append('/var/vse/python')
import _util, _webby, _sni, _cams_db, _cams_threaded_server, _cams_bot
# ...
def start_courier(config):
# default values
host = '::'
port = 2345
configp = config['configp']
host = configp.get('main', 'relay msp hostport')
# require ipv6 addresses be specified in [xx:xx:xx] notation, therefore
# it is safe to look for :nnnn at the end
if ':' in host and not host.endswith(']'):
port = host.split(':')[-1]
try:
port = int(port, 10)
except:
port = 2345
host = host.split(':')[:-1][0]
server = _cams_threaded_server.ThreadedTCPServer((host, port), _cams_threaded_server.ThreadedTCPRequestHandler, config)
t = threading.Thread(target=server.serve_forever, name='courier')
t.start()
_cams_threaded_server.py:
import socket, socketserver, select, datetime, time, threading
import sys, struct
from OpenSSL.SSL import SSLv23_METHOD, SSLv3_METHOD, TLSv1_METHOD, OP_NO_SSLv2
from OpenSSL.SSL import VERIFY_NONE, VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, Context, Connection
from OpenSSL.SSL import FILETYPE_PEM
from OpenSSL.SSL import WantWriteError, WantReadError, WantX509LookupError, ZeroReturnError, SysCallError
from OpenSSL.crypto import load_certificate
from OpenSSL import SSL
# see note at beginning of answer
import _sni, _cams_db
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self, server_address, HandlerClass, config):
socketserver.BaseServer.__init__(self, server_address, HandlerClass)
self.address_family = socket.AF_INET6
self.connected = []
self.logger = config['logger']
self.config = config
self.socket = socket.socket(self.address_family, self.socket_type)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sc = Context(TLSv1_METHOD)
sc.set_verify(VERIFY_PEER|VERIFY_FAIL_IF_NO_PEER_CERT, _sni.verify_cb)
sc.set_tlsext_servername_callback(_sni.pick_certificate)
self.sc = sc
self.server_bind()
self.server_activate()
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
config = self.server.config
logger = self.server.logger
connected = self.server.connected
sc = self.server.sc
try:
self.peer_hostname = socket.gethostbyaddr(socket.gethostbyname(self.request.getpeername()[0]))[0]
except:
self.peer_hostname = '!'+self.request.getpeername()[0]
logger.info('peer: {}'.format(self.peer_hostname))
ssl_s = Connection(sc, self.request)
ssl_s.set_accept_state()
try:
ssl_s.do_handshake()
except:
t,v,tb = sys.exc_info()
logger.warn('handshake failed {}'.format(v))
ssl_s.setblocking(True)
self.ssl_s = ssl_s
try:
peercert = ssl_s.get_peer_certificate()
except:
peercert = False
t,v,tb = sys.exc_info()
logger.warn('SSL get peer cert failed: {}'.format(v))
if not peercert:
logger.warn('No peer certificate')
else:
acl = config['configp']['main'].get('client cn acl', '').split(' ')
cert_subject = peercert.get_subject().CN
logger.info('Looking for {} in acl: {}'.format(cert_subject,acl))
if cert_subject in acl:
logger.info('{} is permitted'.format(cert_subject))
else:
logger.warn('''client CN not approved''')
# it's ok to block here, every socket has its own thread
ssl_s.setblocking(True)
self.db = config['db']
msgcount = 0
p = select.poll()
# don't want writable, just readable
p.register(self.request, select.POLLIN|select.POLLPRI|select.POLLERR|select.POLLHUP|select.POLLNVAL)
peername = ssl_s.getpeername()
x = peername[0]
if x.startswith('::ffff:'):
x = x[7:]
peer_ip = x
try:
host = socket.gethostbyaddr(x)[0]
except:
host = peer_ip
logger.info('{}/{}:{} connected'.format(host, peer_ip, peername[1]))
connected.append( [host, peername[1]] )
if peercert:
threading.current_thread().setName('{}/port={}/CN={}'.format(host, peername[1], peercert.get_subject().CN))
else:
threading.current_thread().setName('{}/port={}'.format(host, peername[1]))
sockclosed = False
while not sockclosed:
keepreading = True
#logger.debug('starting 30 second timeout for poll')
pe = p.poll(30.0)
if not pe:
# empty list means poll timeout
# for SSL sockets it means WTF. we get an EAGAIN like return even if the socket is blocking
continue
logger.debug('poll indicates: {}'.format(pe))
#define SSL_NOTHING 1
#define SSL_WRITING 2
#define SSL_READING 3
#define SSL_X509_LOOKUP 4
while keepreading and not sockclosed:
data,sockclosed,keepreading = self._read_ssl_data(2, head=True)
if sockclosed or not keepreading:
time.sleep(5)
continue
plen = struct.unpack('H', data)[0]
data,sockclosed,keepreading = self._read_ssl_data(plen)
if sockclosed or not keepreading:
time.sleep(5)
continue
# send thank you, ignore any errors since we appear to have gotten
# the message
try:
self.ssl_s.sendall(b'ty')
except:
pass
# extract the timestamp
message_ts = data[0:8]
msgtype = chr(data[8])
message = data[9:].decode()
message_ts = struct.unpack('d', message_ts)[0]
message_ts = datetime.datetime.utcfromtimestamp(message_ts).replace(tzinfo=datetime.timezone.utc)
self.db.enqueue(config['group'], peer_ip, msgtype, message, message_ts)
self.db.pending.set()
# we're recommended to use the return socket object for any future operations rather than the original
try:
s = ssl_s.unwrap()
s.close()
except:
pass
connected.remove( [host, peername[1]] )
t_name = threading.current_thread().getName()
logger.debug('disconnect: {}'.format(t_name))
def _read_ssl_data(self, wantsize=16384, head=False):
_w = ['WANT_NOTHING','WANT_READ','WANT_WRITE','WANT_X509_LOOKUP']
logger = self.server.logger
data = b''
sockclosed = False
keepreading = True
while len(data) < wantsize and keepreading and not sockclosed:
rlen = wantsize - len(data)
try:
w,wr = self.ssl_s.want(),self.ssl_s.want_read()
#logger.debug(' want({}) want_read({})'.format(_w[w],wr))
x = self.ssl_s.recv(rlen)
#logger.debug(' recv(): {}'.format(x))
if not ( x or len(x) ):
raise ZeroReturnError
data += x
if not (len(x) == len(data) == wantsize):
logger.info(' read={}, len(data)={}, plen={}'.format(len(x),len(data),wantsize))
except WantReadError:
# poll(), when ready, read more
keepreading = False
logger.info(' got WantReadError')
continue
except WantWriteError:
# poll(), when ready, write more
keepreading = False
logger.info(' got WantWriteError')
continue
except ZeroReturnError:
# socket got closed, a '0' bytes read also means the same thing
keepreading = False
sockclosed = True
logger.info(' ZRE, socket closed normally')
continue
except SysCallError:
keepreading = False
sockclosed = True
t,v,tb = sys.exc_info()
if v.args[0] == -1: # normal EOF
logger.info(' EOF found, keepreading=False')
else:
logger.info('{} terminated session abruptly while reading plen'.format(self.peer_hostname))
logger.info('t: {}'.format(t))
logger.info('v: {}'.format(v))
continue
except:
t,v,tb = sys.exc_info()
logger.warning(' fucked? {}'.format(v))
raise
if not head and not len(data) == wantsize:
logger.warn(' short read {} of {}'.format(len(data), wantsize))
return data,sockclosed,keepreading

let's start with a bare bones threaded tcp server.
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self, server_address, HandlerClass):
socketserver.BaseServer.__init__(self, server_address, HandlerClass)
self.address_family = socket.AF_INET
self.socket = socket.socket(self.address_family, self.socket_type)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_bind()
self.server_activate()
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
# self.request is your accepted socket, do all your .read() and .wirte() on it
s = self.request
request = s.read(1024)
# decide locked or unlocked. this example arbitrarily writes back 'locked'
s.write('locked')
# we're done, close the socket and exit with a default return of None
s.close()
ok, start your threaded server with this in your main() function:
server = threading.ThreadedTCPServer(('127.0.0.1', 1234), ThreadedTCPRequestHandler)
t = threading.Thread(target=server.serve_forever, name='optional_name')
t.start()
now you can let the threading module handle the semantics of concurrency and not worry about it.

You might want to take a look at 0MQ and concurrent.futures. 0MQ has a Tornado event loop in the library and it reduces the complexity of socket programming. concurrent.futures is a high level interface over threading or multiprocessing.

You can see different concurrent server approaches at
https://bitbucket.org/arco_group/upper/src
These will help you to choose the better way for you.
Cheers

Twisted: Mixing ReconnectingClientFactory with SSL and have write() be "reliable"

I'm fairly new with both Python and Twisted so I may just not be understanding things properly but I seem to be stuck at a point where I need help.
What I want to do is use ReconnectingClientFactory on an SSL connection. I have it all running, but if the connection is dropped all data sent to the transport's write() method is simply dropped without any error. The actual method called is twisted.protocols.tls.TLSMemoryBIOProtocol.write().
Here's what I think is happening (starting from a working connection):
connection is lost
call to write() method (source code here) with some data
self.disconnecting is False so data passes to _write() method
_write method gets to _lostTLSConnection which is True and then just runs return
connection is regained but no data sent because it's not buffered any where
Here's the a reduced version of the client:
from OpenSSL import SSL
from twisted.internet.protocol import (Protocol, ReconnectingClientFactory)
from twisted.internet import (reactor, ssl)
import struct
class MetricsServer(Protocol):
streambuffer = bytearray()
def connectionMade(self):
self.transport.setTcpKeepAlive(True) # maintain the TCP connection
self.transport.setTcpNoDelay(False) # allow Nagle algorithm
print("connected to server")
def dataReceived(self, data):
print("from server:", data)
def connectionLost(self, reason):
self.connected = 0
print("server connection lost:", reason)
class MetricsServerFactory(ReconnectingClientFactory):
protocol = MetricsServer
maxDelay = 300 # maximum seconds between retries
factor = 1.6180339887498948
packet_sequence_number = 0
active_connection = None
def buildProtocol(self, addr):
self.resetDelay()
if self.active_connection == None:
self.active_connection = self.protocol()
return self.active_connection
def get_packet_sequence_number(self):
self.packet_sequence_number += 1
return self.packet_sequence_number
def send_data(self):
print ("sending ssl packet")
packet = struct.pack("!I", self.get_packet_sequence_number())
self.active_connection.transport.write(packet)
reactor.callLater(1.0, metrics_server.send_data)
class CtxFactory(ssl.ClientContextFactory):
def getContext(self):
self.method = SSL.TLSv1_METHOD
ctx = ssl.ClientContextFactory.getContext(self)
ctx.use_certificate_file('keys/client.crt')
ctx.use_privatekey_file('keys/client.key')
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
ctx.set_verify(SSL.VERIFY_PEER, verifyCallback)
ctx.load_verify_locations("keys/ca.pem")
return ctx
if __name__ == "__main__":
metrics_server = MetricsServerFactory()
reactor.connectSSL('localhost', 8000, metrics_server, CtxFactory())
reactor.callLater(3.0, metrics_server.send_data)
reactor.run()
And here's a simple server that outputs the data it receives:
from OpenSSL import SSL
from twisted.internet import ssl, reactor
from twisted.internet.protocol import Factory, Protocol
class Echo(Protocol):
sent_back_data = False
def dataReceived(self, data):
print(' '.join("{0:02x}".format(x) for x in data))
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
if __name__ == '__main__':
factory = Factory()
factory.protocol = Echo
myContextFactory = ssl.DefaultOpenSSLContextFactory(
'keys/server.key', 'keys/server.crt'
)
ctx = myContextFactory.getContext()
ctx.set_verify(
SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
verifyCallback
)
ctx.load_verify_locations("keys/ca.pem")
reactor.listenSSL(8000, factory, myContextFactory)
reactor.run()
Process to recreate issue:
first you need to generate your own certs and CA for this to work
run the server first
run the client code
wait for some output on the server end and then end the program
notice the client continues to try and send data
restart the server end
notice the server end will continue receiving packets, but packets sent when the connection was lost are simply dropped
As a work-around I tried implementing my own buffer to send the data on re-connection, but ran into another issue. I want it to send the data when the connection is re-established and the only hook I can see is Protocol.connectionMade(). However, that method is called before the TLS handshaking is actually done so it ends up being caught by an exception handler in _write() and placed into another buffer to be sent later. But, that buffer only seems to be sent if data is received from the other end (which doesn't occur very often in my case, and would also mean that the data could arrive at the other end in the wrong order because a write() may be called before data is received). I also think another disconnection before data is received will also cause that buffer of data to just be wiped.
EDIT: added sample code for the first issue. It's probably odd that I have that active_connection in the Factory, but I am trying to make it work as a singleton.

Okay, I figured out the problem with my workaround... I was passing a bytearray to be written to the transport and then immediately cleared it afterwards not realizing that the write was being deferred until after I cleared the buffer. So, I pass a copy of the bytearray and it seems to be working now.
It still doesn't seem quite right that every call to the write has to be proceeded by a check to see if it's connected as the whole idea of the ReconnectingClientFactory is that it handles maintaining the connection for you. Also, I think it's possible to lose a connection between that if statement and when the write() is actually run, so it's still possible to lose data.
from OpenSSL import SSL
from twisted.internet.protocol import (Protocol, ReconnectingClientFactory)
from twisted.internet import (reactor, ssl)
import struct
class MetricsServer(Protocol):
streambuffer = bytearray()
def connectionMade(self):
self.transport.setTcpKeepAlive(True) # maintain the TCP connection
self.transport.setTcpNoDelay(False) # allow Nagle algorithm
print("connected to server")
if len(self.transport.factory.wrappedFactory.send_buffer) > 0:
self.transport.write(bytes(self.transport.factory.wrappedFactory.send_buffer))
self.transport.factory.wrappedFactory.send_buffer.clear()
def dataReceived(self, data):
print("from server:", data)
def connectionLost(self, reason):
self.connected = 0
print("server connection lost:", reason)
class MetricsServerFactory(ReconnectingClientFactory):
protocol = MetricsServer
maxDelay = 300 # maximum seconds between retries
factor = 1.6180339887498948
packet_sequence_number = 0
active_connection = None
send_buffer = bytearray()
def buildProtocol(self, addr):
self.resetDelay()
if self.active_connection == None:
self.active_connection = self.protocol()
return self.active_connection
def get_packet_sequence_number(self):
self.packet_sequence_number += 1
return self.packet_sequence_number
def send_data(self):
print ("sending ssl packet")
packet = struct.pack("!I", self.get_packet_sequence_number())
if self.active_connection and self.active_connection.connected:
self.active_connection.transport.write(packet)
else:
self.send_buffer.extend(packet)
reactor.callLater(1.0, metrics_server.send_data)
class CtxFactory(ssl.ClientContextFactory):
def getContext(self):
self.method = SSL.TLSv1_METHOD
ctx = ssl.ClientContextFactory.getContext(self)
ctx.use_certificate_file('keys/client.crt')
ctx.use_privatekey_file('keys/client.key')
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
ctx.set_verify(SSL.VERIFY_PEER, verifyCallback)
ctx.load_verify_locations("keys/ca.pem")
return ctx
if __name__ == "__main__":
metrics_server = MetricsServerFactory()
reactor.connectSSL('localhost', 8000, metrics_server, CtxFactory())
reactor.callLater(3.0, metrics_server.send_data)
reactor.run()

RabbitMQ, Pika and reconnection strategy

I'm using Pika to process data from RabbitMQ.
As I seemed to run into different kind of problems I decided to write a small test application to see how I can handle disconnects.
I wrote this test app which does following:
Connect to Broker, retry until successful
When connected create a queue.
Consume this queue and put result into a python Queue.Queue(0)
Get item from Queue.Queue(0) and produce it back into the broker queue.
What I noticed were 2 issues:
When I run my script from one host connecting to rabbitmq on another host (inside a vm) then this scripts exits on random moments without producing an error.
When I run my script on the same host on which RabbitMQ is installed it runs fine and keeps running.
This might be explained because of network issues, packets dropped although I find the connection not really robust.
When the script runs locally on the RabbitMQ server and I kill the RabbitMQ then the script exits with error: "ERROR pika SelectConnection: Socket Error on 3: 104"
So it looks like I can't get the reconnection strategy working as it should be. Could someone have a look at the code so see what I'm doing wrong?
Thanks,
Jay
#!/bin/python
import logging
import threading
import Queue
import pika
from pika.reconnection_strategies import SimpleReconnectionStrategy
from pika.adapters import SelectConnection
import time
from threading import Lock
class Broker(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.logging = logging.getLogger(__name__)
self.to_broker = Queue.Queue(0)
self.from_broker = Queue.Queue(0)
self.parameters = pika.ConnectionParameters(host='sandbox',heartbeat=True)
self.srs = SimpleReconnectionStrategy()
self.properties = pika.BasicProperties(delivery_mode=2)
self.connection = None
while True:
try:
self.connection = SelectConnection(self.parameters, self.on_connected, reconnection_strategy=self.srs)
break
except Exception as err:
self.logging.warning('Cant connect. Reason: %s' % err)
time.sleep(1)
self.daemon=True
def run(self):
while True:
self.submitData(self.from_broker.get(block=True))
pass
def on_connected(self,connection):
connection.channel(self.on_channel_open)
def on_channel_open(self,new_channel):
self.channel = new_channel
self.channel.queue_declare(queue='sandbox', durable=True)
self.channel.basic_consume(self.processData, queue='sandbox')
def processData(self, ch, method, properties, body):
self.logging.info('Received data from broker')
self.channel.basic_ack(delivery_tag=method.delivery_tag)
self.from_broker.put(body)
def submitData(self,data):
self.logging.info('Submitting data to broker.')
self.channel.basic_publish(exchange='',
routing_key='sandbox',
body=data,
properties=self.properties)
if __name__ == '__main__':
format=('%(asctime)s %(levelname)s %(name)s %(message)s')
logging.basicConfig(level=logging.DEBUG, format=format)
broker=Broker()
broker.start()
try:
broker.connection.ioloop.start()
except Exception as err:
print err

The main problem with your script is that it is interacting with a single channel from both your main thread (where the ioloop is running) and the "Broker" thread (calls submitData in a loop). This is not safe.
Also, SimpleReconnectionStrategy does not seem to do anything useful. It does not cause a reconnect if the connection is interrupted. I believe this is a bug in Pika: https://github.com/pika/pika/issues/120
I attempted to refactor your code to make it work as I think you wanted it to, but ran into another problem. Pika does not appear to have a way to detect delivery failure, which means that data may be lost if the connection drops. This seems like such an obvious requirement! How can there be no way to detect that basic_publish failed? I tried all kinds of stuff including transactions and add_on_return_callback (all of which seemed clunky and overly complicated), but came up with nothing. If there truly is no way then pika only seems to be useful in situations that can tolerate loss of data sent to RabbitMQ, or in programs that only need to consume from RabbitMQ.
This is not reliable, but for reference, here's some code that solves your multi-thread problem:
import logging
import pika
import Queue
import sys
import threading
import time
from functools import partial
from pika.adapters import SelectConnection, BlockingConnection
from pika.exceptions import AMQPConnectionError
from pika.reconnection_strategies import SimpleReconnectionStrategy
log = logging.getLogger(__name__)
DEFAULT_PROPERTIES = pika.BasicProperties(delivery_mode=2)
class Broker(object):
def __init__(self, parameters, on_channel_open, name='broker'):
self.parameters = parameters
self.on_channel_open = on_channel_open
self.name = name
def connect(self, forever=False):
name = self.name
while True:
try:
connection = SelectConnection(
self.parameters, self.on_connected)
log.debug('%s connected', name)
except Exception:
if not forever:
raise
log.warning('%s cannot connect', name, exc_info=True)
time.sleep(10)
continue
try:
connection.ioloop.start()
finally:
try:
connection.close()
connection.ioloop.start() # allow connection to close
except Exception:
pass
if not forever:
break
def on_connected(self, connection):
connection.channel(self.on_channel_open)
def setup_submitter(channel, data_queue, properties=DEFAULT_PROPERTIES):
def on_queue_declared(frame):
# PROBLEM pika does not appear to have a way to detect delivery
# failure, which means that data could be lost if the connection
# drops...
channel.confirm_delivery(on_delivered)
submit_data()
def on_delivered(frame):
if frame.method.NAME in ['Confirm.SelectOk', 'Basic.Ack']:
log.info('submission confirmed %r', frame)
# increasing this value seems to cause a higher failure rate
time.sleep(0)
submit_data()
else:
log.warn('submission failed: %r', frame)
#data_queue.put(...)
def submit_data():
log.info('waiting on data queue')
data = data_queue.get()
log.info('got data to submit')
channel.basic_publish(exchange='',
routing_key='sandbox',
body=data,
properties=properties,
mandatory=True)
log.info('submitted data to broker')
channel.queue_declare(
queue='sandbox', durable=True, callback=on_queue_declared)
def blocking_submitter(parameters, data_queue,
properties=DEFAULT_PROPERTIES):
while True:
try:
connection = BlockingConnection(parameters)
channel = connection.channel()
channel.queue_declare(queue='sandbox', durable=True)
except Exception:
log.error('connection failure', exc_info=True)
time.sleep(1)
continue
while True:
log.info('waiting on data queue')
try:
data = data_queue.get(timeout=1)
except Queue.Empty:
try:
connection.process_data_events()
except AMQPConnectionError:
break
continue
log.info('got data to submit')
try:
channel.basic_publish(exchange='',
routing_key='sandbox',
body=data,
properties=properties,
mandatory=True)
except Exception:
log.error('submission failed', exc_info=True)
data_queue.put(data)
break
log.info('submitted data to broker')
def setup_receiver(channel, data_queue):
def process_data(channel, method, properties, body):
log.info('received data from broker')
data_queue.put(body)
channel.basic_ack(delivery_tag=method.delivery_tag)
def on_queue_declared(frame):
channel.basic_consume(process_data, queue='sandbox')
channel.queue_declare(
queue='sandbox', durable=True, callback=on_queue_declared)
if __name__ == '__main__':
if len(sys.argv) != 2:
print 'usage: %s RABBITMQ_HOST' % sys.argv[0]
sys.exit()
format=('%(asctime)s %(levelname)s %(name)s %(message)s')
logging.basicConfig(level=logging.DEBUG, format=format)
host = sys.argv[1]
log.info('connecting to host: %s', host)
parameters = pika.ConnectionParameters(host=host, heartbeat=True)
data_queue = Queue.Queue(0)
data_queue.put('message') # prime the pump
# run submitter in a thread
setup = partial(setup_submitter, data_queue=data_queue)
broker = Broker(parameters, setup, 'submitter')
thread = threading.Thread(target=
partial(broker.connect, forever=True))
# uncomment these lines to use the blocking variant of the submitter
#thread = threading.Thread(target=
# partial(blocking_submitter, parameters, data_queue))
thread.daemon = True
thread.start()
# run receiver in main thread
setup = partial(setup_receiver, data_queue=data_queue)
broker = Broker(parameters, setup, 'receiver')
broker.connect(forever=True)