What I'm trying to do is fairly simple: send a file from client to server. First, the client sends information about the file - the size of it that is. Then it sends the actual file.
This is what I've done so far:
Server.py
from twisted.internet import reactor, protocol
from twisted.protocols.basic import LineReceiver
import pickle
import sys
class Echo(LineReceiver):
def connectionMade(self):
self.factory.clients.append(self)
self.setRawMode()
def connectionLost(self, reason):
self.factory.clients.remove(self)
def lineReceived(self, data):
print "line", data
def rawDataReceived(self, data):
try:
obj = pickle.loads(data)
print obj
except:
print data
#self.transport.write("wa2")
def main():
"""This runs the protocol on port 8000"""
factory = protocol.ServerFactory()
factory.protocol = Echo
factory.clients = []
reactor.listenTCP(8000,factory)
reactor.run()
# this only runs if the module was *not* imported
if __name__ == '__main__':
main()
Client.py
import pickle
from twisted.internet import reactor, protocol
import time
import os.path
from twisted.protocols.basic import LineReceiver
class EchoClient(LineReceiver):
def connectionMade(self):
file = "some file that is a couple of megs"
filesize = os.path.getsize(file)
self.sendLine(pickle.dumps({"size":filesize}))
f = open(file, "rb")
contents = f.read()
print contents[:20]
self.sendLine(contents[:20])
f.close()
# self.sendLine("hej")
# self.sendLine("wa")
def connectionLost(self, reason):
print "connection lost"
class EchoFactory(protocol.ClientFactory):
protocol = EchoClient
def clientConnectionFailed(self, connector, reason):
print "Connection failed - goodbye!"
reactor.stop()
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
reactor.stop()
# this connects the protocol to a server runing on port 8000
def main():
f = EchoFactory()
reactor.connectTCP("localhost", 8000, f)
reactor.run()
# this only runs if the module was *not* imported
if __name__ == '__main__':
main()
The server will only output the deserialized object:
{'size': 183574528L}
How come? What happend to the 20 chars from the file I wanted to send?
If use the "hej" and "wa" sends instead, I will get them both (in the same message, not twice).
Somebody?
You've set your server to raw mode with setRawMode(), so the callback rawDataReceived is being called with the incoming data (not lineReceived). If you print the data you receive in rawDataReceived, you see everything including the file content, but as you call pickle to deserialize the data, it's being ignored.
Either you change the way you send data to the server (I would suggest the netstring format) or you pass the content inside the pickle serialized object, and do this in one call.
self.sendLine(pickle.dumps({"size":filesize, 'content': contents[:20]}))
Related
I am currently trying to create a small demo where I connect a web socket between my computer and my localhost ws://localhost:8080/ws. I want the web socket to monitor a file on my computer for changes. If there are changes, send a message. The connection and output is being monitored using Advanced Rest Client.
Is there a specific method I can use on the class to constantly check the contents of this file?
EDIT
I have implemented an observer using watchdog that detects any event for files in a specified directory. However, my message is not being sent in the sendFSEvent method and I also realized that my client isn't being registered when I connect to the web socket.
Here is my code in server.py
import sys
import os
from watchdog.observers import Observer
from twisted.web.static import File
from twisted.python import log
from twisted.web.server import Site
from twisted.internet import reactor, defer
from autobahn.twisted.websocket import WebSocketServerFactory, \
WebSocketServerProtocol, listenWS
from MessangerEventHandler import MessangerEventHandler
class WsProtocol(WebSocketServerProtocol):
def connectionMade(self):
print("Connection made")
WebSocketServerProtocol.connectionMade(self)
def onOpen(self):
WebSocketServerProtocol.onOpen(self)
print("WebSocket connection open")
def onMessage(self, payload, isBinary):
print("Message was: {}".format(payload))
self.sendMessage("message received")
def sendFSEvent(self, json):
WebSocketProtocol.sendMessage(self, json)
print('Sent FS event')
def onClose(self, wasClean, code, reason):
print("Connection closed: {}".format(reason))
WebSocketServerProtocol.onClose(self, wasClean, code, reason)
class WsServerFactory(WebSocketServerFactory):
protocol = WsProtocol
def __init__(self, url='ws://localhost', port=8080):
addr = url + ':' + str(port)
print("Listening on: {}".format(addr))
WebSocketServerFactory.__init__(self, addr)
self.clients = []
def register(self, client):
if not client in self.clients:
print("Registered client: {}".format(client))
self.clients.append(client)
def unregister(self, client):
if client in self.clients:
print("Unregistered client: {}".format(client))
self.clients.remove(client)
self._printConnected()
def _printConnected(self):
print("Connected clients:[")
def notify_clients(self, message):
print("Broadcasting: {}".format(message))
for c in self.clients:
c.sendFSEvent(message)
print("\nSent messages")
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: python server_defer.py <dirs>")
sys.exit(1)
log.startLogging(sys.stdout)
ffactory = WsServerFactory("ws://localhost", 8080)
ffactory.protocol = WsProtocol
listenWS(ffactory)
observers = []
for arg in sys.argv[1:]:
dir_path = os.path.abspath(arg)
if not os.path.exists(dir_path):
print('{} does not exist.'.format(dir_path))
sys.exit(1)
if not os.path.isdir(dir_path):
print('{} is not a directory.'.format(dir_path))
sys.exit(1)
# Check for and handle events
event_handler = MessangerEventHandler(ffactory, reactor, os.getcwd())
observer = Observer()
observer.schedule(event_handler, path=dir_path, recursive=True)
observer.start()
observers.append(observer)
try:
reactor.run()
except KeyboardInterrupt:
for obs in observers:
obs.stop()
reactor.stop()
print("\nGoodbye")
sys.exit(1)
Any help would be greatly appreciated.
Thank you,
Brian
Most enterprise distros come with inotify which is really well suited for monitoring files and directories. The basic idea is to capture a list of connected web socket clients as they connect. Then create a callback that will execute when a change occurs on the files you're monitoring. Within this callback, you can iterate the clients and send them a message like 'file: "blah/blah.txt" has changed'. It's a little wonky, but the code snippet should clear things up for you.
from functools import partial
from twisted.internet import inotify
from twisted.python import filepath
# the rest of your imports ...
class SomeServerProtocol(WebSocketServerProtocol):
def onConnect(self, request):
self.factory.append(self) # <== append this client to the list in the factory
def notification_callback(ignored, filepath, mask, ws_clients):
"""
function that will execute when files are modified
"""
payload = "event on {0}".format(filepath)
for client in ws_clients:
client.sendMessage(
payload.encode('utf8'), # <== don't forget to encode the str to bytes before sending!
isBinary = False)
if __name__ == '__main__':
root = File(".")
factory = WebSocketServerFactory(u"ws://127.0.01:8080")
factory.protocol = SomeServerProtocol
factory.clients = [] # <== create a container for the clients that connect
# inotify stuff
notify = partial(notification_callback, ws_clients=factory.clients) # <== use functools.partial to pass extra params
notifier = inotify.INotify()
notifier.startReading()
notifier.watch(filepath.FilePath("/some/directory"), callbacks=[notify])
# the rest of your code ...
How do i connect the stdout of a spawnProcess to the stdin of another spawnProcess in twisted, where spawnProcess with the stdout is on the client and the stdin spawnprocess on the server? The CLI command in bash is btrfs send #mysubvol | btrfs receive /some/path/. With rfd, wrd = os.pipe() I managed to pipe process1 to process2 on the server side. (Now I want to pipe it from the client to the server instead). Following code shows the processes piped on the same side:
Following code
from twisted.internet import protocol
from twisted.internet import reactor
import os
class Writer(protocol.ProcessProtocol):
def connectionMade(self):
print "Writer -- connection made"
self.transport.closeChildFD(0)
def childDataReceived(self, fd):
pass
def processEnded(self, status):
pass
class Reader(protocol.ProcessProtocol):
def __init__(self):
pass
def connectionMade(self):
print "Reader -- connection made"
pass
def childDataReceived(self, fd):
print "Reader -- childDataReceived"
def processEnded(self, status):
print "process ended, got:"
def test2():
rfd, wfd = os.pipe()
p1 = reactor.spawnProcess(Writer(), "btrfs", ["btrfs", "send", "/#mySubvol"],env=None, childFDs={0:"w", 1: wfd })
p2 = reactor.spawnProcess(Reader(), "btrfs", ["btrfs", "receive", "/subvolContainer/"], env=None, childFDs={0: rfd, 1: "r"})
os.close(rfd)
os.close(wfd)
reactor.run()
test2()
I tried:
server.py
from twisted.internet.protocol import Protocol, Factory, ClientFactory
from twisted.internet import protocol
from twisted.internet import reactor
import os
class Reader(protocol.ProcessProtocol):
def __init__(self):
pass
def connectionMade(self):
print "Reader -- connection made"
pass
def childDataReceived(self, fd):
print "Reader -- childDataReceived"
def processEnded(self, status):
print "process ended, got:"
class EchoClientFactory(ClientFactory):
protocol = Reader
def clientConnectionFailed(self, connector, reason):
print 'connection failed:', reason.getErrorMessage()
reactor.stop()
def clientConnectionLost(self, connector, reason):
print 'connection lost:', reason.getErrorMessage()
reactor.stop()
def main():
f = Factory()
reactor.listenTCP(8000, f)
rfd = os.pipe()
p2 = reactor.spawnProcess(Reader(), "btrfs", ["btrfs", "receive", "/"], env=None, childFDs={0: rfd, 1: "r"})
os.close(rfd)
reactor.run()
if __name__ == '__main__':
main()
client.py
from twisted.internet import reactor
from twisted.internet import protocol
import sys
import os
class Writer(protocol.ProcessProtocol):
def connectionMade(self):
print "Writer -- connection made"
self.transport.closeChildFD(0)
def childDataReceived(self, fd):
pass
def processEnded(self, status):
pass
class EchoClientFactory(protocol.ClientFactory):
protocol = Writer
def clientConnectionFailed(self, connector, reason):
print 'connection failed:', reason.getErrorMessage()
reactor.stop()
def clientConnectionLost(self, connector, reason):
print 'connection lost:', reason.getErrorMessage()
reactor.stop()
def main():
factory = EchoClientFactory()
rfd, wfd = os.pipe()
p1 = reactor.spawnProcess(Writer(), "btrfs", ["btrfs", "send", "/home/philipp/testEnv/a2a/#a2"], env=None, childFDs={0:"w", 1: wfd })
reactor.connectTCP('localhost', 8000, factory)
os.close(wfd)
reactor.run()
if __name__ == '__main__':
main()
Obviously my attempt is wrong, because the server does not know about the client's stdout pipe, but I don't know how to pipe the client's spawnProcess stdout to the server.
Update 01:
Following Jean-Paul's answer I created two protocols on the client and server (ProcessProtocol and TCP-Protocol on each side). I could successfully send a snapshot from the client to the server. On the client I had to start the ProcessProtocol with the instance of my TCP-Protocol, so that they are both interconnected. See: http://twistedmatrix.com/trac/wiki/FrequentlyAskedQuestions#HowdoImakeinputononeconnectionresultinoutputonanother
client.py
from twisted.internet.protocol import Protocol, Factory, ClientFactory, ProcessProtocol
from twisted.internet import reactor
import sys
import os
class Writer(Protocol):
def connectionMade(self): # Called when a connection is made
print "connec made"
proc = MyProcessProtocol(self)
p1 = reactor.spawnProcess(proc, "btrfs", ["btrfs", "send", "/home/user/testEnv/a2a/#a2"])
class EchoClientFactory(ClientFactory):
protocol = Writer
def clientConnectionFailed(self, connector, reason):
print 'connection failed:', reason.getErrorMessage()
reactor.stop()
def clientConnectionLost(self, connector, reason):
print 'connection lost:', reason.getErrorMessage()
reactor.stop()
class MyProcessProtocol(ProcessProtocol):
def __init__(self, instance):
self.w = instance
def outReceived(self, data): # Some data was received from stdout
self.w.transport.write(data) # Write some data to the physical connection, in sequence, in a non-blocking fashion
def main():
factory = EchoClientFactory()
reactor.connectTCP('localhost', 8000, factory)
reactor.run()
if __name__ == '__main__':
main()
server.py
from twisted.internet.protocol import Protocol, Factory, ClientFactory, ProcessProtocol, ServerFactory
from twisted.internet import reactor
import os
class Reader(Protocol):
def connectionMade(self):
print "connected"
self.r2 = Reader2()
p1 = reactor.spawnProcess(self.r2, "btrfs", ["btrfs", "receive", "/"])
def dataReceived(self, data):
print "dataReceived"
self.r2.transport.write(data)
class Reader2(ProcessProtocol):
def connectionMade(self):
print "connectionMade!"
def processEnded(self, reason):
print "quitting"
def main():
f = ServerFactory()
f.protocol = Reader
reactor.listenTCP(8000, f)
reactor.run()
if __name__ == '__main__':
main()
You cannot set up a pipe across machines. You can't do this in a shell either. The shell expression:
btrfs send #mysubvol | btrfs receive /some/path/
runs two btrfs processes on a single machine with a pipe connecting them.
Pipes are purely local. They cannot be shared across machines. For this, you need something else. For example, a TCP connection.
You have taken a couple steps in the right direction. Your server starts a TCP server. Your client attempts to establish a new connection to that server.
But your server doesn't define any behavior for handling connections it accepts. And your client uses a ProcessProtocol to define its behavior - when a TCP connection is not a process.
One idea you may have missed is that on your client you will need two protocols. One protocol is connected to the btrfs send process and reads its stdout. Another protocol is connected to your server and can write that btrfs send output to the TCP connection.
And your server will need two protocols as well. One of them handles the connection from the client and reads the data the client is writing to that connection. The other is connected to a btrfs receive process and writes the data read from the TCP connection to that process's stdin.
You've found the childFDs feature of spawnProcess which is a nice local optimization. And while you can technically use it to help you connect two processes on two different machines, it involves extra steps that you probably don't want to bother with (at least not until you're comfortable with a "regular" forwarder).
Instead, you just want a handful of protocols that receive data from objects nearer the sending side of your setup (via childDataReceived and dataReceived) and hand it off to objects nearer the receiving side of your setup (via transport.write).
I'm messing around with the python twisted library, and I can't seem to figure out how to get my client to detect a server closing its socket. My client continues to let me send data to the non existent server. Here is my server:
test_server.py
from twisted.internet import protocol, reactor, endpoints, stdio
from twisted.protocols.basic import LineReceiver
class ConnectionProtocol(LineReceiver):
from os import linesep as delimiter
def lineReceived(self, line):
print 'got line: %s' % line
self.sendLine(line)
class ConnectionFactory(protocol.Factory):
def buildProtocol(self, addr):
return ConnectionProtocol()
def main():
endpoint = endpoints.UNIXServerEndpoint(reactor, './test.sock', 5, 0777, False)
endpoint.listen(ConnectionFactory())
print 'starting the reactor'
reactor.run()
main()
All it does is send each line it gets back to the connecting client.
Here is the client:
test_client.py
import os
from twisted.internet import protocol, reactor, endpoints, stdio
from twisted.protocols.basic import LineReceiver
class CommandProtocol(protocol.Protocol):
def dataReceived(self, data):
print data,
class StdinProtocol(LineReceiver):
from os import linesep as delimiter
def __init__(self, client):
self._client = client
def connectionMade(self):
self.transport.write('>>> ')
def lineReceived(self, line):
print 'writing line: %s' % line
self._client.transport.write(line + os.linesep)
def printError(failure):
print str(failure)
def main():
point = endpoints.UNIXClientEndpoint(reactor, './test.sock')
proto = CommandProtocol()
d = endpoints.connectProtocol(point, proto)
d.addErrback(printError)
stdio.StandardIO(StdinProtocol(proto))
reactor.run()
main()
If I run the server and then the client, and then kill the server, the client still writes to the CommandProtocol's transport like nothing happened. I thought the errback function would at least report something. In the case where the client is run before the server, the errback function is called with a ConnectError, but I'm specifically looking to detect the situation where the client has already connected to a server, and the server exits.
How can I detect that the server has shutdown?
ITransport.write is a silent no-op if called on a transport that has been disconnected.
If you want to learn that a connection was lost, override the connectionLost method. Once you know that the connection has been lost you can arrange for your program to stop accepting input or do something else with the input it receives.
i guess os.stat method and stat module can help you. Can you change your client codes to like below:
import os
import stat
from twisted.internet import protocol, reactor, endpoints, stdio
from twisted.protocols.basic import LineReceiver
class CommandProtocol(protocol.Protocol):
def dataReceived(self, data):
print data,
class StdinProtocol(LineReceiver):
from os import linesep as delimiter
def __init__(self, client):
self._client = client
def connectionMade(self):
self.transport.write('>>> ')
def lineReceived(self, line):
print 'writing line: %s' % line
self._client.transport.write(line + os.linesep)
def printError(failure):
print str(failure)
def main():
if os.stat('./test.sock').st_mode & (stat.S_IRGRP | stat.S_IRUSR | stat.S_IROTH):
print "1"
print "using control socket"
point = endpoints.UNIXClientEndpoint(reactor, './test.sock')
proto = CommandProtocol()
d = endpoints.connectProtocol(point, proto)
d.addErrback(printError)
stdio.StandardIO(StdinProtocol(proto))
reactor.run()
else:
print "not ready"
main()
I have recently started developing a server-client chat protocol for learning purposes (later I would like to do more with this communication, but for now this suffices. Needless to say, I am still early into the learning phase of this portion of Python, but I have modified some examples to a server and a client that I found online. The communication works well from what I've seen so far, but I have to relaunch the client every time I want to send a message to the server.
Here is the code:
server:
from twisted.internet import reactor, protocol
from twisted.protocols import basic
class Echo(protocol.Protocol):
def dataReceived(self, data):
"As soon as any data is received, write it back."
self.transport.write(data)
class MyChat(basic.LineReceiver):
def connectionMade(self):
print "Got new client!"
self.factory.clients.append(self)
def connectionLost(self, reason):
print "Lost a client!"
self.factory.clients.remove(self)
def dataReceived(self, data):
print "received", repr(data)
for c in self.factory.clients:
c.message(data)
def message(self, message):
self.transport.write(message + '\n')
def main():
"""This runs the protocol on port 8000"""
factory = protocol.ServerFactory()
factory.protocol = MyChat
factory.clients = []
reactor.listenTCP(8000,factory)
reactor.run()
# this only runs if the module was *not* imported
if __name__ == '__main__':
main()
client:
from twisted.internet import reactor, protocol
# a client protocol
class EchoClient(protocol.Protocol):
"""Once connected, send a message, then print the result."""
def connectionMade(self):
self.transport.write("hello, world!")
def dataReceived(self, data):
"As soon as any data is received, write it back."
print "Server said:", data
self.transport.loseConnection()
def connectionLost(self, reason):
print "connection lost"
class EchoFactory(protocol.ClientFactory):
protocol = EchoClient
def clientConnectionFailed(self, connector, reason):
connector.connect()
print "Connection failed - goodbye!"
reactor.stop()
def clientConnectionLost(self, connector, reason):
connector.connect()
print "Connection lost - goodbye!"
reactor.stop()
# this connects the protocol to a server runing on port 8000
def main():
f = EchoFactory()
client = EchoClient()
reactor.connectTCP("localhost", 8000, f)
reactor.run()
# this only runs if the module was *not* imported
if __name__ == '__main__':
main()
What am I forgetting to add so that I can have multiple clients connected to the server and stay connected?
I looked here and here (the first one seems to be the same type of question) but I'm still confused as to how to fix this issue. Any suggestions are appreciated. Thanks!
I have this simple Twisted Client which connects to a Twisted server & queries an index.
If you see fn. connectionMade() in class SpellClient, the query is hard-coded. Did that for testing purposes. How would one pass this query from outside to this class?
The code -
from twisted.internet import reactor
from twisted.internet import protocol
# a client protocol
class SpellClient(protocol.Protocol):
"""Once connected, send a message, then print the result."""
def connectionMade(self):
query = 'abased'
self.transport.write(query)
def dataReceived(self, data):
"As soon as any data is received, write it back."
print "Server said:", data
self.transport.loseConnection()
def connectionLost(self, reason):
print "connection lost"
class SpellFactory(protocol.ClientFactory):
protocol = SpellClient
def clientConnectionFailed(self, connector, reason):
print "Connection failed - goodbye!"
reactor.stop()
def clientConnectionLost(self, connector, reason):
print "Connection lost - goodbye!"
reactor.stop()
# this connects the protocol to a server runing on port 8000
def main():
f = SpellFactory()
reactor.connectTCP("localhost", 8090, f)
reactor.run()
# this only runs if the module was *not* imported
if __name__ == '__main__':
main()
Protocols, like SpellClient, have access to their factory as self.factory.
...so there would be a number of ways to do this, but one way would be to create another method on SpellFactory, such as setQuery, and then access that from the client...
#...in SpellFactory:
def setQuery(self, query):
self.query = query
#...and in SpellClient:
def connectionMade(self):
self.transport.write(self.factory.query)
...so in main:
f = SpellFactory()
f.setQuery('some query')
...
...or you could just create an _init_ method for SpellFactory, and pass it in there.