We Keep Coding

Python import async tcp server as a module: data exchange not working, Multithreading with modules

I am working on the following project: I created a async tcp tcp ip client/ server connection. The client (c# wpf GUI) writes messages to the server (Raspberry pi, Python) and waits for sensor data. The server receive the message from the client and sends back the current sensor data in a loop.
The server/ client application standalone works fine. In the next step I want to use modules in Python. Here are the main parts shown that I want to implement in a seperate Python file:
Main.py
start the async tcp server and run it in the background (thread)
print the received data from the client
read sensor data (thread)
evaluation (optional)
hand over the sensor data to the async server
Here is the code in python that I have so fare (start the async tcp server):
Main.py
import thread
import logging
import socket
import asyncore
#Import async server
from AsyncServerPi_Test import Server
if __name__=='__main__':
logging.basicConfig(level=logging.DEBUG,format='%(name)s: %(message)s',)
log = logging.getLogger('main')
#parameter
ip = socket.gethostname()
port = 12345
address = (ip, port) # port 0 means the kernel gives port
dataFromPi= 'SensorData'
server = Server(address,dataFromPi) #issue with parameter dataFromPI
#dataFromClient = server.getDataFromClient() Code is not working!
#log.debug(dataFromGUI)
asyncore.loop(timeout=1) # checks all client's readable/writable every second
AsyncServerPi_Test.py
import asyncore
import logging
import time
import socket
# Create one or more network channels -- instances of class asyncore.dispatcher.
# These channels are automatically added to a global map, used by the loop() function.
class Server(asyncore.dispatcher):
"""Receives connections and establishes handlers for each client."""
def __init__(self, address, dataFromPi):
self.dataFromPi = dataFromPi
self.logger = logging.getLogger('Server')
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(address)
self.address = self.socket.getsockname()
self.logger.debug('binding to %s', self.address)
self.listen(1)
def timein(self):
self.start_time = time.time()
self.logger.debug('init start_time -> %e', self.start_time)
def handle_accept(self):
# Called when a client connects to our socket
client_info = self.accept()
if client_info is not None:
# start the timeout clock right away
self.timein()
self.logger.debug('handle_accept() -> %s', client_info[1])
Handler(client_info[0], self.start_time, self.dataFromPi)
def handle_close(self):
self.logger.debug('server_handle_close()')
self.close()
class Handler(asyncore.dispatcher):
"""Handles echoing messages from a single client. """
def __init__(self, sock, start_time, dataFromPi, chunk_size=1024):
self.dataFromPi = dataFromPi
self.start_time = start_time
# send data per 5 seconds
self.timeout = 5
self.chunk_size = chunk_size
self.logger = logging.getLogger('Handler%s' % str(sock.getsockname()))
asyncore.dispatcher.__init__(self, sock=sock)
self.data_to_write = []
def timeout_check(self):
#Time event
delta_t = time.time() - self.start_time
if delta_t > self.timeout:
self.logger.debug('timeout! -> %e %e', delta_t, self.timeout)
return True
else:
self.logger.debug('no timeout -> %e %e', delta_t, self.timeout)
return False
def trigger_close(self):
return self.timeout_check()
def writable(self):
"""We want to write if we have received data and when sendSensor data is on."""
# the trigger_close here is a hack
response = bool(self.data_to_write) or self.trigger_close()
self.logger.debug('writable() -> %s', response)
return response
def handle_write(self):
"""Write as much as possible of the reversed recent message we received."""
self.logger.debug('write data to GUI')
if self.trigger_close(): # hack to timeout socket
self.start_time = time.time()
#data = self.data_to_write.pop()
sent = self.send(dataFromPi)
return
data = self.data_to_write.pop()
sent = self.send(dataFromPi)
self.logger.debug('Send data to GUI:'+ str(sent))
#if sent < len(data)
#remaining = data[sent:]
#self.data_to_write.append(remaining)
#self.logger.debug('handle_write() -> (%d) "%s"', sent, data[:sent])
#if not self.writable(): # empty recv
#self.handle_close()
def reverse(self, s):
s = list(s)
s.reverse()
s = "".join(s)
return s
def handle_read(self):
"""Read an incoming message from the client and put it into our outgoing queue."""
data = self.recv(self.chunk_size)
self.logger.debug('handle_read() -> (%d) "%s"', len(data), data)
data = self.reverse(data)
self.data_to_write.insert(0, data)
#return self.data --> didnt work!
def handle_close(self):
"""server close only gets called if client decides or after timeout"""
self.logger.debug('handle_close()')
self.close()
I have some issues with the parameter dataFromPi and dataFromClient. dataFromPi is a input parameter that I handover the instance of the server (server = Server(address,dataFromPi)). I get the following error when I execute the main.py file:
error: uncaptured python exception, closing channel <AsyncServerPi_Test.Handler connected 192.168.0.108:51028 at 0x31ba7d8> (<type 'exceptions.NameError'>:global name 'dataFromPi' is not defined [C:\Python27\lib\asyncore.py|write|91] [C:\Python27\lib\asyncore.py|handle_write_event|468] [C:\Python27\lib\AsyncServerPi_Test.py|handle_write|75])
Handler('192.168.0.108', 12345): handle_close()
I checked the handover of dataFromPi in the AsyncServerPi_Test.py file but I couldnt find a solution.
The next issue I have is that I am not able to print the dataFromClient in the main.py file. Within the AsyncServerPi_Test.py file I can read the dataFromClient (handle_read). I tried to return the received data in handle_read, but I think the return of the data must happen in the class server(). How can I archieve that the server class returns the data from the client so I can print the received data in main.py?
Furthermore the server should run in the background so that I can run other operations parallel (read sensor data from ultrasonic, evaluation...). As an example I want that the main.py should look similar like that:
import thread
import logging
import socket
import asyncore
#Import US
import Ultrasonic
#Import async tcp ip server
from AsyncServerPi_Test import Server
def main():
#Logging
logging.basicConfig(level=logging.DEBUG,format='%(name)s: %(message)s',)
log = logging.getLogger('main')
#Parameter for server
ip = socket.gethostname()
port = 12345
address = (ip, port) # port 0 means the kernel gives port
ultrasonicTest = Ultrasonic() # run in a seperate thread?
dataFromSensor = ultrasonicTest.readSensorData
serverTest = Server(address,dataFromSensor) # run in a seperate thread?
dataFromClient = serverTest.receivedData()
asyncore.loop(timeout=1) # checks all client's readable/writable every second
if __name__=='__main__':
main()
In this context I have the following questions:
The instance of the server (serverTest = Server(address,dataFromSensor)) must run in a seperate thread?
Create a new thread for the sensor (ultrasonicTest = Ultrasonic()):
Is it better to run threads within a module (e.g. start threading within the ultrasonic.py and not in the main.py)
Synchronize the threads?
I am relatie new in python and I dont have much experience in multithreading. It would be helpful if someone has done some similar projects in python and could give me some tips or reference to similar projects.

Fail on second call

I need your help please.
This code only works once, a second wget gives timeout (attached file).
wget http://localhost:9090
#!/usr/bin/env python
import trollius as asyncio
from trollius import From
import os
class Client(asyncio.Protocol):
def connection_made(self, transport):
self.connected = True
# save the transport
self.transport = transport
def data_received(self, data):
# forward data to the server
self.server_transport.write(data)
def connection_lost(self, *args):
self.connected = False
class Server(asyncio.Protocol):
clients = {}
def connection_made(self, transport):
# save the transport
self.transport = transport
#asyncio.coroutine
def send_data(self, data):
# get a client by its peername
peername, port = self.transport.get_extra_info('peername')
client = self.clients.get(peername)
# create a client if peername is not known or the client disconnect
if client is None or not client.connected:
protocol, client = yield From(loop.create_connection(Client, 'google.com', 80))
client.server_transport = self.transport
self.clients[peername] = client
# forward data to the client
client.transport.write(data)
def data_received(self, data):
# use a task so this is executed async
asyncio.Task(self.send_data(data))
#asyncio.coroutine
def initialize(loop):
# use a coroutine to use yield from and get the async result of
# create_server
server = yield From(loop.create_server(Server, '127.0.0.1', 9090))
loop = asyncio.get_event_loop()
# main task to initialize everything
asyncio.Task(initialize(loop))
# run
loop.run_forever()
Does anyone know the reason?
Thanks!

You need a real 'loop' in servers when you are writing socket servers in asyncio. Note that despite 'sync' coding, infinite loops do not block execution here. You need an infinite while loop within your server. There are many samples,I recommend websockets library samples!

Using asyncore to create interactive sessions with client/server model

I am attempting to create a program that allows many clients to connect to 1 server simultaneously. These connections should be interactive on the server side, meaning that I can send requests from the server to the client, after the client has connected.
The following asyncore example code simply replies back with an echo, I need instead of an echo a way to interactively access each session. Somehow background each connection until I decided to interact with it. If I have 100 sessions I would like to chose a particular one or choose all of them or a subset of them to send a command to. Also I am not 100% sure that the asyncore lib is the way to go here, any help is appreciated.
import asyncore
import socket
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
data = self.recv(8192)
if data:
self.send(data)
class EchoServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
handler = EchoHandler(sock)
server = EchoServer('localhost', 8080)
asyncore.loop()

Here's a Twisted server:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import serverFromString
from twisted.internet.defer import Deferred
from twisted.internet.protocol import Factory
from twisted.protocols.basic import LineReceiver
class HubConnection(LineReceiver, object):
def __init__(self, hub):
self.name = b'unknown'
self.hub = hub
def connectionMade(self):
self.hub.append(self)
def lineReceived(self, line):
words = line.split(" ", 1)
if words[0] == b'identify':
self.name = words[1]
else:
for connection in self.hub:
connection.sendLine("<{}> {}".format(
self.name, line
).encode("utf-8"))
def connectionLost(self, reason):
self.hub.remove(self)
def main(reactor, listen="tcp:4321"):
hub = []
endpoint = serverFromString(reactor, listen)
endpoint.listen(Factory.forProtocol(lambda: HubConnection(hub)))
return Deferred()
react(main, sys.argv[1:])
and command-line client:
import sys
from twisted.internet.task import react
from twisted.internet.endpoints import clientFromString
from twisted.internet.defer import Deferred, inlineCallbacks
from twisted.internet.protocol import Factory
from twisted.internet.stdio import StandardIO
from twisted.protocols.basic import LineReceiver
from twisted.internet.fdesc import setBlocking
class HubClient(LineReceiver):
def __init__(self, name, output):
self.name = name
self.output = output
def lineReceived(self, line):
self.output.transport.write(line + b"\n")
def connectionMade(self):
self.sendLine("identify {}".format(self.name).encode("utf-8"))
def say(self, words):
self.sendLine("say {}".format(words).encode("utf-8"))
class TerminalInput(LineReceiver, object):
delimiter = "\n"
hubClient = None
def lineReceived(self, line):
if self.hubClient is None:
self.output.transport.write("Connecting, please wait...\n")
else:
self.hubClient.sendLine(line)
#inlineCallbacks
def main(reactor, name, connect="tcp:localhost:4321"):
endpoint = clientFromString(reactor, connect)
terminalInput = TerminalInput()
StandardIO(terminalInput)
setBlocking(0)
hubClient = yield endpoint.connect(
Factory.forProtocol(lambda: HubClient(name, terminalInput))
)
terminalInput.transport.write("Connecting...\n")
terminalInput.hubClient = hubClient
terminalInput.transport.write("Connected.\n")
yield Deferred()
react(main, sys.argv[1:])
which implement a basic chat server. Hopefully the code is fairly self-explanatory; you can run it to test with python hub_server.py in one terminal, python hub_client.py alice in a second and python hub_client.py bob in a third; then type into alice and bob's sessions and you can see what it does.

Review Requirements
You want
remote calls in client/server manner
probably using TCP communication
using sessions in the call
It is not very clear, how you really want to use sessions, so I will consider, that session is just one of call parameters, which has some meaning on server as well client side and will skip implementing it.
zmq as easy and reliable remote messaging platform
ZeroMQ is lightweight messaging platform, which does not require complex server infrastructure. It can handle many messaging patterns, following example showing request/reply pattern using multipart messages.
There are many alternatives, you can use simple messages encoded to some format like JSON and skip using multipart messages.
server.py
import zmq
class ZmqServer(object):
def __init__(self, url="tcp://*:5555"):
context = zmq.Context()
self.sock = context.socket(zmq.REP)
self.sock.bind(url)
self.go_on = False
def echo(self, message, priority=None):
priority = priority or "not urgent"
msg = "Echo your {priority} message: '{message}'"
return msg.format(**locals())
def run(self):
self.go_on = True
while self.go_on:
args = self.sock.recv_multipart()
if 1 <= len(args) <= 2:
code = "200"
resp = self.echo(*args)
else:
code = "401"
resp = "Bad request, 1-2 arguments expected."
self.sock.send_multipart([code, resp])
def stop(self):
self.go_on = False
if __name__ == "__main__":
ZmqServer().run()
client.py
import zmq
import time
class ZmqClient(object):
def __init__(self, url="tcp://localhost:5555"):
context = zmq.Context()
self.socket = context.socket(zmq.REQ)
self.socket.connect(url)
def call_echo(self, message, priority=None):
args = [message]
if priority:
args.append(priority)
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "200"
return resp
def too_long_call(self, message, priority, extrapriority):
args = [message, priority, extrapriority]
self.socket.send_multipart(args)
code, resp = self.socket.recv_multipart()
assert code == "401"
return resp
def test_server(self):
print "------------------"
rqmsg = "Hi There"
print "rqmsg", rqmsg
print "response", self.call_echo(rqmsg)
print "------------------"
time.sleep(2)
rqmsg = ["Hi There", "very URGENT"]
print "rqmsg", rqmsg
print "response", self.call_echo(*rqmsg)
print "------------------"
time.sleep(2)
time.sleep(2)
rqmsg = []
print "too_short_call"
print "response", self.too_long_call("STOP", "VERY URGENT", "TOO URGENT")
print "------------------"
if __name__ == "__main__":
ZmqClient().test_server()
Play with the toy
Start the server:
$ python server.py
Now it runs and awaits requests.
Now start the client:
$ python client.py
------------------
rqmsg Hi There
response Echo your not urgent message: 'Hi There'
------------------
rqmsg ['Hi There', 'very URGENT']
response Echo your very URGENT message: 'Hi There'
------------------
too_short_call
response Bad request, 1-2 arguments expected.
------------------
Now experiment a bit:
start client first, then server
stop the server during processing, restart later on
start multiple clients
All these scenarios shall be handled by zmq without adding extra lines of Python code.
Conclusions
ZeroMQ provides very convenient remote messaging solution, try counting lines of messaging related code and compare with any other solution, providing the same level of stability.
Sessions (which were part of OP) can be considered just extra parameter of the call. As we saw, multiple parameters are not a problem.
Maintaining sessions, different backends can be used, they could live in memory (for single server instance), in database, or in memcache or Redis. This answer does not elaborate further on sessions, as it is not much clear, what use is expected.

Very simple python client/server working, but strange delays

I am trying to learn how to use sockets and a useful asynchronous backend. I've started in python with asyncore. After reading various online posts I've written a very simple chat server and connection client, reproduced below.
It seems to work. I open a python interactive session and type
> import chatserver
> server = chatserver.EchoServer('localhost', 7667)
> server.serve()
Then I open another IPython interactive session and type
> import chatserver
> cxn = chatserver.Connection()
> cxn._connect('localhost', 7667)
When I do that, I get a log output in the server window indicating that a connection has been made. Good. Then I type
> cxn.say('hi')
Nothing happens for a while, and then log messages show up for the server and client as expected.
Why is this delay ocurring?
Am I using the log functionality correctly?
I used threading to make it so that I could use the interactive session while the asyncore loop does it's thing for the Connection. Did I do this in a reasonable way?
(optional) If I don't include the line self.out_buffer="" in the Connection._connect function I get an error saying that .out_buffer does not exist. What's up with this?
import asyncore
import socket
import logging
import threading
logging.basicConfig(level=logging.DEBUG, format="%(created)-15s %(msecs)d %(levelname)8s %(thread)d %(name)s %(message)s")
log = logging.getLogger(__name__)
class Connection(asyncore.dispatcher_with_send):
def __init__(self):
asyncore.dispatcher.__init__(self)
def _connect(self, host, port, timeout=5, password=None):
self.host = host
self.port = port
self.out_buffer=""
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
#Run the asyncore loop in its own thread so that we can use the interactive session
self.loop = threading.Thread(target=asyncore.loop)
self.loop.daemon = True
self.loop.start()
def say(self, msg):
self.out_buffer = msg
def handle_read(self):
data = self.recv(4096)
log.debug('Received %s'%data)
class EchoHandler(asyncore.dispatcher_with_send):
def handle_read(self):
log.debug("handle_read")
data = self.recv(1024)
log.debug("after recv")
if data:
log.debug("got data: %s"%data)
self.out_buffer = data
else:
log.debug("got null data")
class EchoServer(asyncore.dispatcher):
SOCKET_TYPE = socket.SOCK_STREAM
ADDRESS_FAMILY = socket.AF_INET
def __init__(self, host, port):
self.address = (host,port)
asyncore.dispatcher.__init__(self)
self.create_socket(self.ADDRESS_FAMILY, self.SOCKET_TYPE)
log.debug("bind address=%s %s"%(host,port))
self.bind(self.address)
self.listen(1)
def fileno(self):
return self.socket.fileno()
def serve(self):
asyncore.loop()
#Start asyncore loop in new thread
# self.loop = threading.Thread(target=asyncore.loop)
# self.loop.daemon = True
# self.loop.start()
def handle_accept(self):
"""Deal with a newly accepted client"""
(connSock, clientAddress) = self.accept()
log.info("conn made: clientAddress=%s %s"%(clientAddress[0], clientAddress[1]))
#Make a handler for this connection
EchoHandler(connSock)
def handle_close(self):
self.close()

Looking at the asyncore docs you are relying on asyncore.dispatcher_with_send to call send() and the default timeout for asyncore.loop() is 30 seconds. This may explain the delay.

It turns out the problem was as Eero suggested.
I made two changes:
In EchoServer
asyncore.loop() to asyncore.loop(timeout=0.1)
In Connection
self.loop = threading.Thread(target=asyncore.loop) to self.loop = threading.Thread(target=asyncore.loop, kwargs={'timeout':0.1})
The response is now much faster. This seems like a hack though so if someone can explain a way to get the same effect in a proper way please contribute.

Python DNS Server

I am adding a feature to my current project that will allow network admins to install the software to the network. I need to code a DNS server in Python that will allow me to redirect to a certain page if the request address is in my list. I was able to write the server, just not sure how to redirect.
Thank you. I am using Python 2.6 on Windows XP.

There's little, simple example here that can easily be adapted to make all kinds of "mini fake dns servers". Note that absolutely no "redirect" is involved (that's not how DNS works): rather, the request is for a domain name, and the result of that request is an IP address. If what you want to do is drastically different from translating names to addresses, then maybe what you need is not actually a DNS server...?

Using circuits and dnslib here's a full recursive dns server written in Python in only 143 lines of code:
#!/usr/bin/env python
from __future__ import print_function
from uuid import uuid4 as uuid
from dnslib import CLASS, QR, QTYPE
from dnslib import DNSHeader, DNSQuestion, DNSRecord
from circuits.net.events import write
from circuits import Component, Debugger, Event
from circuits.net.sockets import UDPClient, UDPServer
class lookup(Event):
"""lookup Event"""
class query(Event):
"""query Event"""
class response(Event):
"""response Event"""
class DNS(Component):
def read(self, peer, data):
record = DNSRecord.parse(data)
if record.header.qr == QR["QUERY"]:
return self.fire(query(peer, record))
return self.fire(response(peer, record))
class ReturnResponse(Component):
def response(self, peer, response):
return response
class Client(Component):
channel = "client"
def init(self, server, port, channel=channel):
self.server = server
self.port = int(port)
self.transport = UDPClient(0, channel=self.channel).register(self)
self.protocol = DNS(channel=self.channel).register(self)
self.handler = ReturnResponse(channel=self.channel).register(self)
class Resolver(Component):
def init(self, server, port):
self.server = server
self.port = port
def lookup(self, qname, qclass="IN", qtype="A"):
channel = uuid()
client = Client(
self.server,
self.port,
channel=channel
).register(self)
yield self.wait("ready", channel)
self.fire(
write(
(self.server, self.port),
DNSRecord(
q=DNSQuestion(
qname,
qclass=CLASS[qclass],
qtype=QTYPE[qtype]
)
).pack()
)
)
yield (yield self.wait("response", channel))
client.unregister()
yield self.wait("unregistered", channel)
del client
class ProcessQuery(Component):
def query(self, peer, query):
qname = query.q.qname
qtype = QTYPE[query.q.qtype]
qclass = CLASS[query.q.qclass]
response = yield self.call(lookup(qname, qclass=qclass, qtype=qtype))
record = DNSRecord(
DNSHeader(id=query.header.id, qr=1, aa=1, ra=1),
q=query.q,
)
for rr in response.value.rr:
record.add_answer(rr)
yield record.pack()
class Server(Component):
def init(self, bind=("0.0.0.0", 53)):
self.bind = bind
self.transport = UDPServer(self.bind).register(self)
self.protocol = DNS().register(self)
self.handler = ProcessQuery().register(self)
class App(Component):
def init(self, bind=("0.0.0.0", 53), server="8.8.8.8", port=53,
verbose=False):
if verbose:
Debugger().register(self)
self.resolver = Resolver(server, port).register(self)
self.server = Server(bind).register(self)
def main():
App().run()
if __name__ == "__main__":
main()
Usage:
By default this example binds go 0.0.0.0:53 so you will need to do something like:
sudo ./dnsserver.py
Otherwise change the bind parameter.

Here is a dns serer/proxy that works for me written in python:
http://thesprawl.org/projects/dnschef/

I wrote a DNS Server using Python Twisted library for NameOcean.net. You can see examples on https://twistedmatrix.com/documents/16.5.0/names/howto/custom-server.html.