I have written a test code which reads some coils/registers from a PLC's modbus server. When I call one request the code works. I unplugged the cable then Twisted calls clientConnectionLost function so my client will reconnected, when I plugged back the cable. If I do multiple requests, like in the code below, the handling breaks, nothing happens. I don't know what causes the problem.
#!/usr/bin/env python
from PyQt4 import QtCore, QtGui
from twisted.internet import reactor, protocol,defer
from pymodbus.constants import Defaults
from pymodbus.client.async import ModbusClientProtocol
from time import sleep
def logger():
import logging
logging.basicConfig()
log = logging.getLogger()
log.setLevel(logging.DEBUG)
logger()
class MyModbusClientProtocol(ModbusClientProtocol):
def connectionMade(self):
ModbusClientProtocol.connectionMade(self)
print 'Connected'
self.read()
def read(self):
deferred = self.read_coils(0,1999)
deferred.addCallbacks(self.requestFetched,self.requestNotFetched)
deferred = self.read_holding_registers(0,124)
deferred.addCallbacks(self.requestFetched,self.requestNotFetched)
def requestNotFetched(self,error):
print error
sleep(0.5)
def requestFetched(self,response):
try:
print ("Fetched %d" % response.getRegister(1))
except:
print ("Fetched %d" % response.getBit(1))
self.factory.counter += 1
if self.factory.counter == 2:
self.factory.counter = 0
reactor.callLater(0,self.read)
class MyModbusClientFactory(protocol.ClientFactory):
"""A factory.
A new protocol instance will be created each time we connect to the server.
"""
def __init__(self):
self.counter = 0
def buildProtocol(self, addr):
p = MyModbusClientProtocol()
p.factory = self
return p
def clientConnectionLost(self, connector, reason):
print "connection lost:", reason
connector.connect()
def clientConnectionFailed(self, connector, reason):
print "connection failed:", reason
connector.connect()
if __name__ == "__main__":
factoryinstance = MyModbusClientFactory()
reactor.connectTCP("192.168.2.69", 502, factoryinstance)
reactor.run()
I have tested your code and believe that you have seen a timing related red herring when your code was seen to work after commenting out one of your requests. The behavior you are seeing where clientConnectionLost is not called is covered in the twisted FAQ: Why isn't my connectionLost method called?
What you need to do is create your own protocol specific timeout as you can't always rely on TCP's timeouts to work in your favor. A simple way to fix your code would be to add this to the end of your read method:
self.timeout = reactor.callLater(5, self.transport.abortConnection)
Which will abort the connection after 5 seconds wait. You also need to cancel this timeout when your requests have completed successfully with:
self.timeout.cancel()
in your requestFetched method before you call your read again.
Related
I'm trying to simulate a situation where data is received by a server periodically. In my set up, I run one process that sets up the server and another process that sets up a bunch of clients (suffices to think of a single client). I have set up some of the code by putting together bits and pieces mostly from here. The server/clients communicate by sending messages using transport.write. First, the server tells the clients to start (this works fine AFAIK). The clients report back to the server as they make progress. What has me confused is that I only get these intermittent messages at the very end when the client is done. This could be a problem with buffer flush and I tried (unsuccessfully) things like This. Also, each message is pretty large and I tried sending the same message multiple times so the buffer would get cleared.
I suspect what I am seeing is a problem with returning the control to the transport but I can't figure out how to do away with it.
Any help with this or any other issues that jump up to you is much appreciated.
Server:
from twisted.internet import reactor, protocol
import time
import serverSideAnalysis
import pdb
#import bson, json, msgpack
import _pickle as pickle # I expect the users to authenticate and not
# do anything malicious.
PORT = 9000
NUM = 1
local_scratch="/local/scratch"
class Hub(protocol.Protocol):
def __init__(self,factory, clients, nclients):
self.clients = clients
self.nclients = nclients
self.factory = factory
self.dispatcher = serverSideAnalysis.ServerTalker(NUM, self,
local_scratch)
def connectionMade(self):
print("connected to user" , (self))
if len(self.clients) < self.nclients:
self.factory.clients.append(self)
else:
self.factory.clients[self.nclients] = self
if len(self.clients) == NUM:
val = input("Looks like everyone is here, shall we start? (Y/N)")
while (val.upper() != "Y"):
time.sleep(20)
val = input("Looks like everyone is here, shall we start??? (Y/N)")
message = pickle.dumps({"TASK": "INIT", "SUBTASK":"STORE"})
self.message(message) # This reaches the client as I had expected
def message(self, command):
for c in self.factory.clients:
c.transport.write(command)
def connectionLost(self, reason):
self.factory.clients.remove(self)
self.nclients -= 1
def dataReceived(self, data):
if len(self.clients) == NUM:
self.dispatcher.dispatch(data)
class PauseTransport(protocol.Protocol):
def makeConnection(self, transport):
transport.pauseProducing()
class HubFactory(protocol.Factory):
def __init__(self, num):
self.clients = set([])
self.nclients = 0
self.totConnections = num
def buildProtocol(self, addr):
print(self.nclients)
if self.nclients < self.totConnections:
self.nclients += 1
return Hub(self, self.clients, self.nclients)
protocol = PauseTransport()
protocol.factory = self
return protocol
factory = HubFactory(NUM)
reactor.listenTCP(PORT, factory)
factory.clients = []
reactor.run()
Client:
from twisted.internet import reactor, protocol
import time
import clientSideAnalysis
import sys
HOST = 'localhost'
PORT = 9000
local_scratch="/local/scratch"
class MyClient(protocol.Protocol):
def connectionMade(self):
print("connected!")
self.factory.clients.append(self)
print ("clients are ", self.factory.clients)
self.cdispatcher = clientSideAnalysis.ServerTalker(analysis_file_name, local_scratch, self)
def clientConnectionLost(self, reason):
#TODO send warning
self.factory.clients.remove(self)
def dataReceived(self, data): #This is the problematic part I think
self.cdispatcher.dispatch(data)
print("1 sent")
time.sleep(10)
self.cdispatcher.dispatch(data)
print("2 sent")
time.sleep(10)
self.cdispatcher.dispatch(data)
time.sleep(10)
def message(self, data):
self.transport.write(data)
class MyClientFactory(protocol.ClientFactory):
protocol = MyClient
if __name__=="__main__":
analysis_file_name = sys.argv[1]
factory = MyClientFactory()
reactor.connectTCP(HOST, PORT, factory)
factory.clients = []
reactor.run()
The last bit of relevant information about what the dispatchers do.
In both cases, they load the message that has arrived (a dictionary) and do a few computations based on the content. Every once in a while, they use the message method to communicate with thier current values.
Finally, I'm using python 3.6. and twisted 18.9.0
The way you return control to the reactor from a Protocol.dataReceived method is you return from that method. For example:
def dataReceived(self, data):
self.cdispatcher.dispatch(data)
print("1 sent")
If you want more work to happen after this, you have some options. If you want the work to happen after some amount of time has passed, use reactor.callLater. If you want the work to happen after it is dispatched to another thread, use twisted.internet.threads.deferToThread. If you want the work to happen in response to some other event (for example, data being received), put it in the callback that handles that event (for example, dataReceived).
Summary
I have a client-server application which makes use of Websockets. The backend (server) part is implemented in Python using autobahn.
The server, in addition to serving a Websockets endpoint, runs a series of threads which will feed the Websockets channel with data, though a queue.Queue().
One of these threads has a problem: it crashes at a missing parameter and hangs when resolving the exception.
Implementation details
The server implementation (cut down to highlight the problem):
from autobahn.asyncio.websocket import WebSocketServerProtocol, WebSocketServerFactory
import time
import threading
import arrow
import queue
import asyncio
import json
# backends of components
import dummy
class MyServerProtocol(WebSocketServerProtocol):
def __init__(self):
super().__init__()
print("webserver initialized")
# global queue to handle updates from modules
self.events = queue.Queue()
# consumer
threading.Thread(target=self.push).start()
threading.Thread(target=dummy.Dummy().dummy, args=(self.events,)).start()
def push(self):
""" consume the content of the queue and push it to the browser """
while True:
update = self.events.get()
print(update)
if update:
self.sendMessage(json.dumps(update).encode('utf-8'), False)
print(update)
time.sleep(1)
def worker(self):
print("started thread")
while True:
try:
self.sendMessage(arrow.now().isoformat().encode('utf-8'), False)
except AttributeError:
print("not connected?")
time.sleep(3)
def onConnect(self, request):
print("Client connecting: {0}".format(request.peer))
def onOpen(self):
print("WebSocket connection open.")
def onClose(self, wasClean, code, reason):
print("WebSocket connection closed: {0}".format(reason))
if __name__ == '__main__':
factory = WebSocketServerFactory(u"ws://127.0.0.1:9100")
factory.protocol = MyServerProtocol
loop = asyncio.get_event_loop()
coro = loop.create_server(factory, '0.0.0.0', 9100)
loop.run_until_complete(coro)
loop.run_forever()
The dummy module imported in the code above:
import time
import arrow
class Dummy:
def __init__(self, events):
self.events = events
print("dummy initialized")
def dummy(self):
while True:
self.events.put({
'dummy': {
'time': arrow.now().isoformat()
}
})
time.sleep(1)
The problem
When running the code above and connecting from a client, I get on the output webserver initialized (which proves that the connection was initiated), and WebSocket connection to 'ws://127.0.0.1:9100/' failed: Error in connection establishment: net::ERR_CONNECTION_REFUSED on the client.
When debugging the code, I see that the call to threading.Thread(target=dummy.Dummy().dummy, args=(self.events,)).start() crashes and the debugger (PyCharm) leads me to C:\Program Files (x86)\Python36-32\Lib\asyncio\selector_events.py, specifically to the line 236
# It's now up to the protocol to handle the connection.
except Exception as exc:
if self._debug:
The thread hangs when executing if self._debug but I see on the exceptline (thanks to Pycharm) that
exc: __init__() missing 1 required positional argument: 'events'
My question
Why is this parameter missing? It is provided via the threading.Thread(target=dummy.Dummy().dummy, args=(self.events,)).start() call.
As a side question: why does the thread hangs on the if condition?
Notes
there is never a Traceback thrown by my program (due to the hang)
removing this thread call resolves the issue (the client connects correctly)
The events arg is needed for the constructor, not the dummy method. I think you meant something more like:
d = Dummy(self.events)
threading.Thread(d.dummy).start()
I'm currently working to add support for gevent-socketio to an existing django project. I'm finding that gevent.monkey.patch_all() call is breaking the cancellation mechanism of a thread which is responsible for receiving data from a socket, we'll call the class SocketReadThread for now.
SocketReadThread is pretty simple, it calls recv() on a blocking socket. When it receives data is processes it and calls recv() again. The thread stops when an exception occurs or when recv() returns 0 bytes as occurs when socket.shutdown(SHUT_RDWR) is called in SocketReadThread.stop_reading()
The problem occurs when the gevent.monkey.patch_all() replaces the default socket implementation. Instead of shutting down nicely I get the following exception:
error: [Errno 9] File descriptor was closed in another greenlet
I'm assuming this is occurring because gevent makes my socket non-blocking in order to work its magic. This means that when I call socket.shutdown(socket.SHUT_RDWR) the greenlet that was doing the work for the monkey patched socket.recv call tried to read from the closed file descriptor.
I coded an example to isolate this issue:
from gevent import monkey
monkey.patch_all()
import socket
import sys
import threading
import time
class SocketReadThread(threading.Thread):
def __init__(self, socket):
super(SocketReadThread, self).__init__()
self._socket = socket
def run(self):
connected = True
while connected:
try:
print "calling socket.recv"
data = self._socket.recv(1024)
if (len(data) < 1):
print "received nothing, assuming socket shutdown"
connected = False
else :
print "Recieved something: {}".format(data)
except socket.timeout as e:
print "Socket timeout: {}".format(e)
connected = false
except :
ex = sys.exc_info()[1]
print "Unexpected exception occurrred: {}".format(str(ex))
raise ex
def stop_reading(self):
self._socket.shutdown(socket.SHUT_RDWR)
self._socket.close()
if __name__ == '__main__':
sock = socket.socket()
sock.connect(('127.0.0.1', 4242))
st = SocketReadThread(sock)
st.start()
time.sleep(3)
st.stop_reading()
st.join()
If you open a terminal an run nc -lp 4242 & (to give this program something to connect to) and then run this program you will see the exception mentioned above. If you remove the call to monkey.patch_all() you will see that it works just fine.
My question is: How can support cancellation of the SocketReadThread in a way that works with or without gevent monkey patching and doesn't require the use of an arbitrary timeout that would make cancellation slow (i.e. calling recv() with a timeout and checking a conditional)?
I found that there were two different workarounds for this. The first was to simply catch and suppress the exception. This appears to work fine since it is common practice for one thread to close a socket in order to cause another thread to exit from a blocking read. I don't know or understand why greenlets would complain about this other than a debugging aid. It is really just an annoyance.
The second option was to use the self-pipe trick (a quick search yields many explanations) as a mechanism to wake up a blocked thread. Essentially we create a second file descriptor (a socket is like a type of file descriptor to the OS) for signaling cancellation. We then use select as our blocking to wait for either incoming data on the socket or a cancellation request to come in on the cancellation file descriptor. See the example code below.
from gevent import monkey
monkey.patch_all()
import os
import select
import socket
import sys
import threading
import time
class SocketReadThread(threading.Thread):
def __init__(self, socket):
super(SocketReadThread, self).__init__()
self._socket = socket
self._socket.setblocking(0)
r, w = os.pipe()
self._cancelpipe_r = os.fdopen(r, 'r')
self._cancelpipe_w = os.fdopen(w, 'w')
def run(self):
connected = True
read_fds = [self._socket, self._cancelpipe_r]
while connected:
print "Calling select"
read_list, write_list, x_list = select.select(read_fds, [], [])
print "Select returned"
if self._cancelpipe_r in read_list :
print "exiting"
self._cleanup()
connected = False
elif self._socket in read_list:
print "calling socket.recv"
data = self._socket.recv(1024)
if (len(data) < 1):
print "received nothing, assuming socket shutdown"
connected = False
self._cleanup()
else :
print "Recieved something: {}".format(data)
def stop_reading(self):
print "writing to pipe"
self._cancelpipe_w.write("\n")
self._cancelpipe_w.flush()
print "joining"
self.join()
print "joined"
def _cleanup(self):
self._cancelpipe_r.close()
self._cancelpipe_w.close()
self._socket.shutdown(socket.SHUT_RDWR)
self._socket.close()
if __name__ == '__main__':
sock = socket.socket()
sock.connect(('127.0.0.1', 4242))
st = SocketReadThread(sock)
st.start()
time.sleep(3)
st.stop_reading()
Again, before running the above program run netcat -lp 4242 & to give it a listening socket to connect to.
I'm fairly new with both Python and Twisted so I may just not be understanding things properly but I seem to be stuck at a point where I need help.
What I want to do is use ReconnectingClientFactory on an SSL connection. I have it all running, but if the connection is dropped all data sent to the transport's write() method is simply dropped without any error. The actual method called is twisted.protocols.tls.TLSMemoryBIOProtocol.write().
Here's what I think is happening (starting from a working connection):
connection is lost
call to write() method (source code here) with some data
self.disconnecting is False so data passes to _write() method
_write method gets to _lostTLSConnection which is True and then just runs return
connection is regained but no data sent because it's not buffered any where
Here's the a reduced version of the client:
from OpenSSL import SSL
from twisted.internet.protocol import (Protocol, ReconnectingClientFactory)
from twisted.internet import (reactor, ssl)
import struct
class MetricsServer(Protocol):
streambuffer = bytearray()
def connectionMade(self):
self.transport.setTcpKeepAlive(True) # maintain the TCP connection
self.transport.setTcpNoDelay(False) # allow Nagle algorithm
print("connected to server")
def dataReceived(self, data):
print("from server:", data)
def connectionLost(self, reason):
self.connected = 0
print("server connection lost:", reason)
class MetricsServerFactory(ReconnectingClientFactory):
protocol = MetricsServer
maxDelay = 300 # maximum seconds between retries
factor = 1.6180339887498948
packet_sequence_number = 0
active_connection = None
def buildProtocol(self, addr):
self.resetDelay()
if self.active_connection == None:
self.active_connection = self.protocol()
return self.active_connection
def get_packet_sequence_number(self):
self.packet_sequence_number += 1
return self.packet_sequence_number
def send_data(self):
print ("sending ssl packet")
packet = struct.pack("!I", self.get_packet_sequence_number())
self.active_connection.transport.write(packet)
reactor.callLater(1.0, metrics_server.send_data)
class CtxFactory(ssl.ClientContextFactory):
def getContext(self):
self.method = SSL.TLSv1_METHOD
ctx = ssl.ClientContextFactory.getContext(self)
ctx.use_certificate_file('keys/client.crt')
ctx.use_privatekey_file('keys/client.key')
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
ctx.set_verify(SSL.VERIFY_PEER, verifyCallback)
ctx.load_verify_locations("keys/ca.pem")
return ctx
if __name__ == "__main__":
metrics_server = MetricsServerFactory()
reactor.connectSSL('localhost', 8000, metrics_server, CtxFactory())
reactor.callLater(3.0, metrics_server.send_data)
reactor.run()
And here's a simple server that outputs the data it receives:
from OpenSSL import SSL
from twisted.internet import ssl, reactor
from twisted.internet.protocol import Factory, Protocol
class Echo(Protocol):
sent_back_data = False
def dataReceived(self, data):
print(' '.join("{0:02x}".format(x) for x in data))
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
if __name__ == '__main__':
factory = Factory()
factory.protocol = Echo
myContextFactory = ssl.DefaultOpenSSLContextFactory(
'keys/server.key', 'keys/server.crt'
)
ctx = myContextFactory.getContext()
ctx.set_verify(
SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT,
verifyCallback
)
ctx.load_verify_locations("keys/ca.pem")
reactor.listenSSL(8000, factory, myContextFactory)
reactor.run()
Process to recreate issue:
first you need to generate your own certs and CA for this to work
run the server first
run the client code
wait for some output on the server end and then end the program
notice the client continues to try and send data
restart the server end
notice the server end will continue receiving packets, but packets sent when the connection was lost are simply dropped
As a work-around I tried implementing my own buffer to send the data on re-connection, but ran into another issue. I want it to send the data when the connection is re-established and the only hook I can see is Protocol.connectionMade(). However, that method is called before the TLS handshaking is actually done so it ends up being caught by an exception handler in _write() and placed into another buffer to be sent later. But, that buffer only seems to be sent if data is received from the other end (which doesn't occur very often in my case, and would also mean that the data could arrive at the other end in the wrong order because a write() may be called before data is received). I also think another disconnection before data is received will also cause that buffer of data to just be wiped.
EDIT: added sample code for the first issue. It's probably odd that I have that active_connection in the Factory, but I am trying to make it work as a singleton.
Okay, I figured out the problem with my workaround... I was passing a bytearray to be written to the transport and then immediately cleared it afterwards not realizing that the write was being deferred until after I cleared the buffer. So, I pass a copy of the bytearray and it seems to be working now.
It still doesn't seem quite right that every call to the write has to be proceeded by a check to see if it's connected as the whole idea of the ReconnectingClientFactory is that it handles maintaining the connection for you. Also, I think it's possible to lose a connection between that if statement and when the write() is actually run, so it's still possible to lose data.
from OpenSSL import SSL
from twisted.internet.protocol import (Protocol, ReconnectingClientFactory)
from twisted.internet import (reactor, ssl)
import struct
class MetricsServer(Protocol):
streambuffer = bytearray()
def connectionMade(self):
self.transport.setTcpKeepAlive(True) # maintain the TCP connection
self.transport.setTcpNoDelay(False) # allow Nagle algorithm
print("connected to server")
if len(self.transport.factory.wrappedFactory.send_buffer) > 0:
self.transport.write(bytes(self.transport.factory.wrappedFactory.send_buffer))
self.transport.factory.wrappedFactory.send_buffer.clear()
def dataReceived(self, data):
print("from server:", data)
def connectionLost(self, reason):
self.connected = 0
print("server connection lost:", reason)
class MetricsServerFactory(ReconnectingClientFactory):
protocol = MetricsServer
maxDelay = 300 # maximum seconds between retries
factor = 1.6180339887498948
packet_sequence_number = 0
active_connection = None
send_buffer = bytearray()
def buildProtocol(self, addr):
self.resetDelay()
if self.active_connection == None:
self.active_connection = self.protocol()
return self.active_connection
def get_packet_sequence_number(self):
self.packet_sequence_number += 1
return self.packet_sequence_number
def send_data(self):
print ("sending ssl packet")
packet = struct.pack("!I", self.get_packet_sequence_number())
if self.active_connection and self.active_connection.connected:
self.active_connection.transport.write(packet)
else:
self.send_buffer.extend(packet)
reactor.callLater(1.0, metrics_server.send_data)
class CtxFactory(ssl.ClientContextFactory):
def getContext(self):
self.method = SSL.TLSv1_METHOD
ctx = ssl.ClientContextFactory.getContext(self)
ctx.use_certificate_file('keys/client.crt')
ctx.use_privatekey_file('keys/client.key')
def verifyCallback(connection, x509, errnum, errdepth, ok):
return bool(ok)
ctx.set_verify(SSL.VERIFY_PEER, verifyCallback)
ctx.load_verify_locations("keys/ca.pem")
return ctx
if __name__ == "__main__":
metrics_server = MetricsServerFactory()
reactor.connectSSL('localhost', 8000, metrics_server, CtxFactory())
reactor.callLater(3.0, metrics_server.send_data)
reactor.run()
I'm trying to write a simple Echo client in Twisted that sends keyboard input to the server, and is terminated by the user entering 'q' on it's own. In short, I'm just trying to modify the simple echo client (and variants) found on this page. Nothing sexy at all, just the basics.
I'm struggling with the very basic event loop. It looks like I can't start/stop the reactor within the loop as a stopped reactor cannot be restarted. If I don't stop the reactor, then I'll never get to the next line that gets keyboard input.
Any help in getting my echo client working would be much appreciated.
from twisted.internet.protocol import ClientFactory
from twisted.protocols.basic import LineReceiver
from twisted.internet import reactor
class EchoClient(LineReceiver):
end="Bye-bye!"
def connectionMade(self):
#only write and end transmission if the message isn't empty
if len(self.factory.message) > 0:
self.sendLine(self.factory.message)
self.sendLine(self.end)
else:
#Else just terminate the connection
self.transport.loseConnection()
def lineReceived(self, line):
print "receive:", line
if line==self.end:
self.transport.loseConnection()
class EchoClientFactory(ClientFactory):
message = ""
def buildProtocol(self, address):
p = EchoClient()
p.factory = self
return p
def clientConnectionFailed(self, connector, reason):
reactor.stop()
def clientConnectionLost(self, connector, reason):
reactor.stop()
def main():
s = raw_input('Text to send (''q'' to terminate): ')
while s != 'q':
factory = EchoClientFactory()
factory.message = s
reactor.connectTCP('localhost', 8000, factory)
#This is bad because reactor cannot be restarted once it's been stopped
reactor.run()
s = raw_input('Text to send(''q'' to terminate): ')
if __name__ == '__main__':
main()
As a rule of a thumb - there are very rare instances where you would want to restart or stop reactor, unless you are terminating your program alltogeather. If your encounter a piece of code which will cause block e.g. database access, long computation or in your case raw_input, you have to either: find a twisted alternative (twisted.enterprise.adabi in case of database) or make it twisted compatible.
The easiest way to 'deblock' your code is move the blocking bits into thread by utilizing deferToThread from twisted.internet.threads.
Consider this example:
from twisted.internet.threads import deferToThread as __deferToThread
from twisted.internet import reactor
def mmprint(s):
print(s)
class TwistedRAWInput(object):
def start(self,callable,terminator):
self.callable=callable
self.terminator=terminator
self.startReceiving()
def startReceiving(self,s=''):
if s!=self.terminator:
self.callable(s)
__deferToThread(raw_input,':').addCallback(self.startReceiving)
tri = TwistedRAWInput()
reactor.callWhenRunning(tri.start,mmprint,'q')
reactor.run()
You would never have to stop reactor, as raw_input would happen in an outside thread, callbacking deferred on every new line.