I'm trying to create a small program that will log information output from a device via TCP
Basically this just streams data out, that i want to capture, and dump into a database for dealing with later
but the device reboots so i need to be able to reconnect when the socket closes with out any human interference
so this is what i have so far
import socket, time, logging, sys, smtplib # Import socket module
logging.basicConfig(filename='Tcplogger.log',level=logging.DEBUG,format='%(asctime)s : %(levelname)s : %(message)s')
logging.info('|--------------------------------------|')
logging.info('|--------------- TCP Logger Starting---|')
logging.info('|--------------------------------------|')
host = '127.0.0.01' # host or Ip address
port = 12345 # output port
retrytime = 1 # reconnect time
reconnectattemps = 10 # Number of time to try and reconnect
class TPCLogger:
def __init__(self):
logging.debug('****Trying connection****')
print('****Trying connection****')
self.initConnection()
def initConnection(self):
s = socket.socket()
try:
s.connect((host, port))
logging.debug('****Connected****')
except IOError as e:
while 1:
reconnectcount = 0;
logging.error(format(e.errno)+' : '+format(e.strerror))
while 1:
reconnectcount = reconnectcount + 1
logging.error('Retrying connection to Mitel attempt : '+str(reconnectcount))
try:
s.connect((host, port))
connected = True
logging.debug('****Connected****')
except IOError as e:
connected = False
logging.error(format(e.errno)+' : '+format(e.strerror))
if reconnectcount == reconnectattemps:
logging.error('******####### Max Reconnect attempts reached logger will Terminate ######******')
sys.exit("could Not connect")
time.sleep(retrytime)
if connected == True:
break
break
while 1:
s.recv(1034)
LOGGER= TCPLogger()
Which all works fine on start up if a try to connect and its not there it will retry the amount of times set by reconnectattemps
but he is my issue
while 1:
s.recv(1034)
when this fails i want to try to reconnect
i could of course type out or just copy my connection part again but what i want to be able todo is call a function that will handle the connection and retry and hand me back the connection object
for example like this
class tcpclient
#set some var
host, port etc....
def initconnection:
connect to socket and retry if needed
RETURN SOCKET
def dealwithdata:
initconnection()
while 1:
try:
s.recv
do stuff here copy to db
except:
log error
initconnection()
I think this is possible but im really not geting how the class/method system works in python so i think im missing something here
FYI just in case you didn't notice iv very new to python. any other comments on what i already have are welcome too :)
Thanks
Aj
Recommendation
For this use-case I would recommend something higher-level than sockets. Why? Controlling all these exceptions and errors for yourself can be irritating when you just want to retrieve or send data and maintain connection.
Of course you can achieve what you want with your plain solution, but you mess with code a bit more, methinks. Anyway it'll look similarly to class amustafa wrote, with handling socket errors to close/reconnect method, etc.
Example
I made some example for easier solution using asyncore module:
import asyncore
import socket
from time import sleep
class Client(asyncore.dispatcher_with_send):
def __init__(self, host, port, tries_max=5, tries_delay=2):
asyncore.dispatcher.__init__(self)
self.host, self.port = host, port
self.tries_max = tries_max
self.tries_done = 0
self.tries_delay = tries_delay
self.end = False # Flag that indicates whether socket should reconnect or quit.
self.out_buffer = '' # Buffer for sending.
self.reconnect() # Initial connection.
def reconnect(self):
if self.tries_done == self.tries_max:
self.end = True
return
print 'Trying connecting in {} sec...'.format(self.tries_delay)
sleep(self.tries_delay)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.connect((self.host, self.port))
except socket.error:
pass
if not self.connected:
self.tries_done += 1
print 'Could not connect for {} time(s).'.format(self.tries_done)
def handle_connect(self):
self.tries_done = 0
print 'We connected and can get the stuff done!'
def handle_read(self):
data = self.recv(1024)
if not data:
return
# Check for terminator. Can be any action instead of this clause.
if 'END' in data:
self.end = True # Everything went good. Shutdown.
else:
print data # Store to DB or other thing.
def handle_close(self):
print 'Connection closed.'
self.close()
if not self.end:
self.reconnect()
Client('localhost', 6666)
asyncore.loop(timeout=1)
reconnnect() method is somehow core of your case - it's called when connection is needed to be made: when class initializes or connection brokes.
handle_read() operates any recieved data, here you log it or something.
You can even send data using buffer (self.out_buffer += 'message'), which will remain untouched after reconnection, so class will resume sending when connected again.
Setting self.end to True will inform class to quit when possible.
asyncore takes care of exceptions and calls handle_close() when such events occur, which is convenient way of dealing with connection failures.
You should look at the python documentation to understand how classes and methods work. The biggest difference between python methods and methods in most other languages is the addition of the "self" tag. The self represents the instance that a method is called against and is automatically fed in by the python system. So:
class TCPClient():
def __init__(self, host, port, retryAttempts=10 ):
#this is the constructor that takes in host and port. retryAttempts is given
# a default value but can also be fed in.
self.host = host
self.port = port
self.retryAttempts = retryAttempts
self.socket = None
def connect(self, attempt=0):
if attempts<self.retryAttempts:
#put connecting code here
if connectionFailed:
self.connect(attempt+1)
def diconnectSocket(self):
#perform all breakdown operations
...
self.socket = None
def sendDataToDB(self, data):
#send data to db
def readData(self):
#read data here
while True:
if self.socket is None:
self.connect()
...
Just make sure you properly disconnect the socket and set it to None.
Related
A "direct-tcpip" request (commonly known as port-forwarding) occurs when you run SSH as ssh user#host -L <local port>:<remote host>:<remote port> and then try to connect over the local port.
I'm trying to implement direct-tcpip on a custom SSH server, and Paramiko offers the check_channel_direct_tcpip_request in the ServerInterface class in order to check if the "direct-tcpip" request should be allowed, which can be implemented as follows:
class Server(paramiko.ServerInterface):
# ...
def check_channel_direct_tcpip_request(self, chanid, origin, destination):
return paramiko.OPEN_SUCCEEDED
However, when I use the aforementioned SSH command, and connect over the local port, nothing happens, probably because I need to implement the connection handling myself.
Reading the documentation, it also appears that the channel is only opened after OPEN_SUCCEDED has been returned.
How can I handle the direct-tcpip request after returning OPEN_SUCCEEDED for the request?
You indeed do need to set up your own connection handler. This is a lengthy answer to explain the steps I took - some of it you will not need if your server code already works. The whole running server example in its entirety is here: https://controlc.com/25439153
I used the Paramiko example server code from here https://github.com/paramiko/paramiko/blob/master/demos/demo_server.py as a basis and implanted some socket code on that. This does not have any error handling, thread related niceties or anything else "proper" for that matter but it allows you to use the port forwarder.
This also has a lot of things you do not need as I did not want to start tidying up a dummy example code. Apologies for that.
To start with, we need the forwarder tools. This creates a thread to run the "tunnel" forwarder. This also answers to your question where you get your channel. You accept() it from the transport but you need to do that in the forwarder thread. As you stated in your OP, it is not there yet in the check_channel_direct_tcpip_request() function but it will be eventually available to the thread.
def tunnel(sock, chan, chunk_size=1024):
while True:
r, w, x = select.select([sock, chan], [], [])
if sock in r:
data = sock.recv(chunk_size)
if len(data) == 0:
break
chan.send(data)
if chan in r:
data = chan.recv(chunk_size)
if len(data) == 0:
break
sock.send(data)
chan.close()
sock.close()
class ForwardClient(threading.Thread):
daemon = True
# chanid = 0
def __init__(self, address, transport, chanid):
threading.Thread.__init__(self)
self.socket = socket.create_connection(address)
self.transport = transport
self.chanid = chanid
def run(self):
while True:
chan = self.transport.accept(10)
if chan == None:
continue
print("Got new channel (id: %i).", chan.get_id())
if chan.get_id() == self.chanid:
break
peer = self.socket.getpeername()
try:
tunnel(self.socket, chan)
except:
pass
Back to the example server code. Your server class needs to have transport as a parameter, unlike in the example code:
class Server(paramiko.ServerInterface):
# 'data' is the output of base64.b64encode(key)
# (using the "user_rsa_key" files)
data = (
b"AAAAB3NzaC1yc2EAAAABIwAAAIEAyO4it3fHlmGZWJaGrfeHOVY7RWO3P9M7hp"
b"fAu7jJ2d7eothvfeuoRFtJwhUmZDluRdFyhFY/hFAh76PJKGAusIqIQKlkJxMC"
b"KDqIexkgHAfID/6mqvmnSJf0b5W8v5h2pI/stOSwTQ+pxVhwJ9ctYDhRSlF0iT"
b"UWT10hcuO4Ks8="
)
good_pub_key = paramiko.RSAKey(data=decodebytes(data))
def __init__(self, transport):
self.transport = transport
self.event = threading.Event()
Then you will override the relevant method and create the forwarder there:
def check_channel_direct_tcpip_request(self, chanid, origin, destination):
print(chanid, origin, destination)
f = ForwardClient(destination, self.transport, chanid)
f.start()
return paramiko.OPEN_SUCCEEDED
You need to add transport parameter to the creation of the server class:
t.add_server_key(host_key)
server = Server(t)
This example server requires you to have a RSA private key in the directory named test_rsa.key. Create any RSA key there, you do not need it but I did not bother to strip the use of it off the code.
You can then run your server (runs on port 2200) and issue
ssh -p 2200 -L 2300:www.google.com:80 robey#localhost
(password is foo)
Now when you try
telnet localhost 2300
and type something there, you will get a response from Google.
I am doing a client-server project for my college project,
we have to allocate the login to the client.
Client system will request its status for every 2 seconds(to check whether the client is locked or unlocked). and server will accept the client request and reply the client status to the system.
But the problem is server thread is not responding to the client request.
CLIENT THREAD:
def checkPort():
while True:
try:
s = socket.socket()
s.connect((host, port))
s.send('pc1') # send PC name to the server
status = s.recv(1024) # receive the status from the server
if status == "unlock":
disableIntrrupts() # enable all the functions of system
else:
enableInterrupts() # enable all the functions of system
time.sleep(5)
s.close()
except Exception:
pass
SERVER THREAD:
def check_port():
while True:
try:
print "hello loop is repeating"
conn, addr = s.accept()
data = conn.recv(1024)
if exit_on_click == 1:
break
if (any(sublist[0] == data for sublist in available_sys)):
print "locked"
conn.send("lock")
elif (any(sublist[0] == data for sublist in occupied_sys)):
conn.send("unlock")
print "unlocked"
else:
print "added to gui for first time"
available_sys.append([data,addr[0],nameText,usnText,branchText])
availSysList.insert('end',data)
except Exception:
pass
But my problem is server thread is not executing more than 2 time,
So its unable to accept client request more than one time.
can't we handle multiple client sockets using single server socket?
How to handle multiple client request from server ?
Thanks for any help !!
Its because your server, will block waiting for a new connection on this line
conn, addr = s.accept()
This is because calls like .accept and .read are blocking calls that hold the process
You need to consider an alternative design, where in you either.
Have one process per connection (this idea is stupid)
One thread per connection (this idea is less stupid than the first but still mostly foolish)
Have a non blocking design that allows multiple clients and read/write without blocking execution.
To achieve the first, look at multiprocessing, the second is threading the third is slightly more complicated to get your head around but will yield the best results, the go to library for event driven code in Python is twisted but there are others like
gevent
tulip
tornado
And so so many more that I haven't listed here.
here's an full example of implementing a threaded server. it's fully functional and comes with the benefit of using SSL as well. further, i use threaded event objects to signal another class object after storing my received data in a database.
please note, _sni and _cams_db are additional modules purely of my own. if you want to see the _sni module (provides SNI support for pyOpenSSL), let me know.
what follows this, is a snippet from camsbot.py, there's a whole lot more that far exceeds the scope of this question. what i've built is a centralized message relay system. it listens to tcp/2345 and accepts SSL connections. each connection passes messages into the system. short lived connections will connect, pass message, and disconnect. long lived connections will pass numerous messages after connecting. messages are stored in a database and a threading.Event() object (attached to the DB class) is set to tell the bot to poll the database for new messages and relay them.
the below example shows
how to set up a threaded tcp server
how to pass information from the listener to the accept handler such as config data and etc
in addition, this example also shows
how to employ an SSL socket
how to do some basic certificate validations
how to cleanly wrap and unwrap SSL from a tcp socket
how to use poll() on the socket instead of select()
db.pending is a threading.Event() object in _cams_db.py
in the main process we start another thread that waits on the pending object with db.pending.wait(). this makes that thread wait until another thread does db.pending.set(). once it is set, our waiting thread immediately wakes up and continues to work. when our waiting thread is done, it calls db.pending.clear() and goes back to the beginning of the loop and starts waiting again with db.pending.wait()
while True:
db.pending.wait()
# after waking up, do code. for example, we wait for incoming messages to
# be stored in the database. the threaded server will call db.pending.set()
# which will wake us up. we'll poll the DB for new messages, relay them, clear
# our event flag and go back to waiting.
# ...
db.pending.clear()
snippet from camsbot.py:
import sys, os, sys, time, datetime, threading, select, logging, logging.handlers
import configparser, traceback, re, socket, hashlib
# local .py
sys.path.append('/var/vse/python')
import _util, _webby, _sni, _cams_db, _cams_threaded_server, _cams_bot
# ...
def start_courier(config):
# default values
host = '::'
port = 2345
configp = config['configp']
host = configp.get('main', 'relay msp hostport')
# require ipv6 addresses be specified in [xx:xx:xx] notation, therefore
# it is safe to look for :nnnn at the end
if ':' in host and not host.endswith(']'):
port = host.split(':')[-1]
try:
port = int(port, 10)
except:
port = 2345
host = host.split(':')[:-1][0]
server = _cams_threaded_server.ThreadedTCPServer((host, port), _cams_threaded_server.ThreadedTCPRequestHandler, config)
t = threading.Thread(target=server.serve_forever, name='courier')
t.start()
_cams_threaded_server.py:
import socket, socketserver, select, datetime, time, threading
import sys, struct
from OpenSSL.SSL import SSLv23_METHOD, SSLv3_METHOD, TLSv1_METHOD, OP_NO_SSLv2
from OpenSSL.SSL import VERIFY_NONE, VERIFY_PEER, VERIFY_FAIL_IF_NO_PEER_CERT, Context, Connection
from OpenSSL.SSL import FILETYPE_PEM
from OpenSSL.SSL import WantWriteError, WantReadError, WantX509LookupError, ZeroReturnError, SysCallError
from OpenSSL.crypto import load_certificate
from OpenSSL import SSL
# see note at beginning of answer
import _sni, _cams_db
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self, server_address, HandlerClass, config):
socketserver.BaseServer.__init__(self, server_address, HandlerClass)
self.address_family = socket.AF_INET6
self.connected = []
self.logger = config['logger']
self.config = config
self.socket = socket.socket(self.address_family, self.socket_type)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sc = Context(TLSv1_METHOD)
sc.set_verify(VERIFY_PEER|VERIFY_FAIL_IF_NO_PEER_CERT, _sni.verify_cb)
sc.set_tlsext_servername_callback(_sni.pick_certificate)
self.sc = sc
self.server_bind()
self.server_activate()
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
config = self.server.config
logger = self.server.logger
connected = self.server.connected
sc = self.server.sc
try:
self.peer_hostname = socket.gethostbyaddr(socket.gethostbyname(self.request.getpeername()[0]))[0]
except:
self.peer_hostname = '!'+self.request.getpeername()[0]
logger.info('peer: {}'.format(self.peer_hostname))
ssl_s = Connection(sc, self.request)
ssl_s.set_accept_state()
try:
ssl_s.do_handshake()
except:
t,v,tb = sys.exc_info()
logger.warn('handshake failed {}'.format(v))
ssl_s.setblocking(True)
self.ssl_s = ssl_s
try:
peercert = ssl_s.get_peer_certificate()
except:
peercert = False
t,v,tb = sys.exc_info()
logger.warn('SSL get peer cert failed: {}'.format(v))
if not peercert:
logger.warn('No peer certificate')
else:
acl = config['configp']['main'].get('client cn acl', '').split(' ')
cert_subject = peercert.get_subject().CN
logger.info('Looking for {} in acl: {}'.format(cert_subject,acl))
if cert_subject in acl:
logger.info('{} is permitted'.format(cert_subject))
else:
logger.warn('''client CN not approved''')
# it's ok to block here, every socket has its own thread
ssl_s.setblocking(True)
self.db = config['db']
msgcount = 0
p = select.poll()
# don't want writable, just readable
p.register(self.request, select.POLLIN|select.POLLPRI|select.POLLERR|select.POLLHUP|select.POLLNVAL)
peername = ssl_s.getpeername()
x = peername[0]
if x.startswith('::ffff:'):
x = x[7:]
peer_ip = x
try:
host = socket.gethostbyaddr(x)[0]
except:
host = peer_ip
logger.info('{}/{}:{} connected'.format(host, peer_ip, peername[1]))
connected.append( [host, peername[1]] )
if peercert:
threading.current_thread().setName('{}/port={}/CN={}'.format(host, peername[1], peercert.get_subject().CN))
else:
threading.current_thread().setName('{}/port={}'.format(host, peername[1]))
sockclosed = False
while not sockclosed:
keepreading = True
#logger.debug('starting 30 second timeout for poll')
pe = p.poll(30.0)
if not pe:
# empty list means poll timeout
# for SSL sockets it means WTF. we get an EAGAIN like return even if the socket is blocking
continue
logger.debug('poll indicates: {}'.format(pe))
#define SSL_NOTHING 1
#define SSL_WRITING 2
#define SSL_READING 3
#define SSL_X509_LOOKUP 4
while keepreading and not sockclosed:
data,sockclosed,keepreading = self._read_ssl_data(2, head=True)
if sockclosed or not keepreading:
time.sleep(5)
continue
plen = struct.unpack('H', data)[0]
data,sockclosed,keepreading = self._read_ssl_data(plen)
if sockclosed or not keepreading:
time.sleep(5)
continue
# send thank you, ignore any errors since we appear to have gotten
# the message
try:
self.ssl_s.sendall(b'ty')
except:
pass
# extract the timestamp
message_ts = data[0:8]
msgtype = chr(data[8])
message = data[9:].decode()
message_ts = struct.unpack('d', message_ts)[0]
message_ts = datetime.datetime.utcfromtimestamp(message_ts).replace(tzinfo=datetime.timezone.utc)
self.db.enqueue(config['group'], peer_ip, msgtype, message, message_ts)
self.db.pending.set()
# we're recommended to use the return socket object for any future operations rather than the original
try:
s = ssl_s.unwrap()
s.close()
except:
pass
connected.remove( [host, peername[1]] )
t_name = threading.current_thread().getName()
logger.debug('disconnect: {}'.format(t_name))
def _read_ssl_data(self, wantsize=16384, head=False):
_w = ['WANT_NOTHING','WANT_READ','WANT_WRITE','WANT_X509_LOOKUP']
logger = self.server.logger
data = b''
sockclosed = False
keepreading = True
while len(data) < wantsize and keepreading and not sockclosed:
rlen = wantsize - len(data)
try:
w,wr = self.ssl_s.want(),self.ssl_s.want_read()
#logger.debug(' want({}) want_read({})'.format(_w[w],wr))
x = self.ssl_s.recv(rlen)
#logger.debug(' recv(): {}'.format(x))
if not ( x or len(x) ):
raise ZeroReturnError
data += x
if not (len(x) == len(data) == wantsize):
logger.info(' read={}, len(data)={}, plen={}'.format(len(x),len(data),wantsize))
except WantReadError:
# poll(), when ready, read more
keepreading = False
logger.info(' got WantReadError')
continue
except WantWriteError:
# poll(), when ready, write more
keepreading = False
logger.info(' got WantWriteError')
continue
except ZeroReturnError:
# socket got closed, a '0' bytes read also means the same thing
keepreading = False
sockclosed = True
logger.info(' ZRE, socket closed normally')
continue
except SysCallError:
keepreading = False
sockclosed = True
t,v,tb = sys.exc_info()
if v.args[0] == -1: # normal EOF
logger.info(' EOF found, keepreading=False')
else:
logger.info('{} terminated session abruptly while reading plen'.format(self.peer_hostname))
logger.info('t: {}'.format(t))
logger.info('v: {}'.format(v))
continue
except:
t,v,tb = sys.exc_info()
logger.warning(' fucked? {}'.format(v))
raise
if not head and not len(data) == wantsize:
logger.warn(' short read {} of {}'.format(len(data), wantsize))
return data,sockclosed,keepreading
let's start with a bare bones threaded tcp server.
class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
def __init__(self, server_address, HandlerClass):
socketserver.BaseServer.__init__(self, server_address, HandlerClass)
self.address_family = socket.AF_INET
self.socket = socket.socket(self.address_family, self.socket_type)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server_bind()
self.server_activate()
class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler):
def handle(self):
# self.request is your accepted socket, do all your .read() and .wirte() on it
s = self.request
request = s.read(1024)
# decide locked or unlocked. this example arbitrarily writes back 'locked'
s.write('locked')
# we're done, close the socket and exit with a default return of None
s.close()
ok, start your threaded server with this in your main() function:
server = threading.ThreadedTCPServer(('127.0.0.1', 1234), ThreadedTCPRequestHandler)
t = threading.Thread(target=server.serve_forever, name='optional_name')
t.start()
now you can let the threading module handle the semantics of concurrency and not worry about it.
You might want to take a look at 0MQ and concurrent.futures. 0MQ has a Tornado event loop in the library and it reduces the complexity of socket programming. concurrent.futures is a high level interface over threading or multiprocessing.
You can see different concurrent server approaches at
https://bitbucket.org/arco_group/upper/src
These will help you to choose the better way for you.
Cheers
I have to send data only to a connection, as I can do?
server:
import asyncore, socket, threading
class EchoHandler(asyncore.dispatcher_with_send):
def __init__(self,sock):
asyncore.dispatcher.__init__(self,sock=sock);
self.out_buffer = ''
def handle_read(self):
datos = self.recv(1024);
if datos:
print(datos);
self.sock[0].send("signal");
class Server(asyncore.dispatcher):
def __init__(self,host='',port=6666):
asyncore.dispatcher.__init__(self);
self.create_socket(socket.AF_INET, socket.SOCK_STREAM);
self.set_reuse_addr();
self.bind((host,port));
self.listen(1);
def handle_accept(self):
self.sock,self.addr = self.accept();
if self.addr:
print self.addr[0];
handler = EchoHandler(self.sock);
def handle_close(self):
self.close();
cliente = Server();
asyncore.loop()
this line is an example fails, but I want to send data to zero sock:
self.sock[0].send("probando");
for example, if I have 5 sockets choose who to send the data
Explanation
You tried to get sock from list and execute its send method. This causes error, because EchoHandler neither has sock attribute nor it's a list of sockets. The right method is to get instance of EchoHandler you want (based on, eg. IP address, or slots assigned by some user-defined protocol) and then use its send method - here (with dispatcher_with_send) its also better to use special buffer for that than send.
EchoHandler instantion is created on every accept of connection - from then it is an established channel for communication with the given host. Server listens for any non-established connection, while EchoHandlers use socks (given by Server in handle_accept) for established ones, so there are as many EchoHandler instances as connections.
Solution
You need to make some list of connections (EchoHandler instantions; we'll use buffer, not socket's send() directly) and give them opportunity to delete their entries on close:
class Server(asyncore.dispatcher):
def __init__(self, host='', port=6666):
...
self.connections = []
def handle_accept(self):
...
handler = EchoHandler(self.sock, self);
self.connections.append(self.sock)
...
def remove_channel(self, sock):
if sock in self.connections:
self.connections.remove(sock)
class EchoHandler(asyncore.dispatcher_with_send):
def __init__(self, sock, server):
...
self.server = server
def handle_read(self):
datos = self.recv(1024);
if datos:
print(datos);
self.out_buffer += 'I echo you: ' + datos
def handle_close(self):
self.server.remove_channel(self)
self.close()
EchoHandler is now aware of server instance and can remove its socket from list. This echo example is now fully functional, and with working socket list we can proceed to asynchronous sending.
But, at this point you can use this list as you wanted - cliente.connections[0].out_buffer += 'I am data' will do the work, but probably you'd want some better controlling of this. If yes, go ahead.
'For whom, by me'
In order to send data asynchronously, we need to separate asyncore from our control thread, in which we'll enter what to send and to whom.
class ServerThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.daemon = True # if thread is a daemon, it'll be killed when main program exits
self.cliente = Server()
self.start()
def run(self):
print 'Starting server thread...'
asyncore.loop()
thread = ServerThread()
while True:
msg = raw_input('Enter IP and message divided by semicolon: ')
if msg == 'exit':
break
ip, data = msg.split('; ')
for sock in thread.cliente.connections:
if sock.addr[0] == ip:
sock.out_buffer += data
break
This will work and wait for destination IP and data. Remember to have client connected.
As I said, you can use anything to indicate which socket is which. It can be a class with fields for eg. IP and username, so you could send data only to peers whose usernames start with 'D'.
But...
This solution is a bit rough and needs better knowledge of asyncore module if you want to send data nicely (here it has some delay due to how select() works) and make good use of this socket wrapper.
Here and here are some resources.
Syntax note
Although your code will now work, your code has some not-nice things. Semicolons on instructions ends don't cause errors, but making nearly every variable of class attribute can lead to them. For example here:
def handle_accept(self):
self.sock,self.addr = self.accept();
if self.addr:
print self.addr[0];
handler = EchoHandler(self.sock);
self.sock and self.addr might be used in that class for something other (eg. socket-related thing; addresses) and overriding them could make trouble. Methods used for requests should never save state of previous actions.
I hope Python will be good enough for you to stay with it!
Edit: sock.addr[0] can be used instead of sock.socket.getpeername()[0] but it requires self.addr not to be modified, so handle_accept() should look like this:
def handle_accept(self):
sock, addr = self.accept()
if addr:
print addr[0]
handler = EchoHandler(sock, self)
self.connections.append(handler)
I have the following situation:
SomeServer(S) <-> (C)MyApp(S) <-> (C)User
(S) represents a server socket
(C) represents a client socket
Essentially, MyApp initiates communication with SomeServer (SomeServer(S) <-> (C)MyApp) and once some authentication routines are successful MyApp(S) starts waiting for (C)User to connect. As soon as User connects, MyApp relays data from SomeServer to User. This happens in both directions.
I have SomeServer(S) <-> (C)MyApp working perfectly, but I'm not able to get MyApp(S) <-> (C)User working. I get as far as User connecting to MyApp(S), but can't get data relayed!
Ok, I hope that's some what clear ;) Now let me show my code for MyApp. Btw the implementation of SomeServer and User are not relevant for solving my question, as neither can be modified.
I have commented my code indicating where I'm experiencing issues. Oh, I should also mention that I have no problem scrapping the whole "Server Section" for some other code if necessary. This is a POC, so my main focus is getting the functionality working rather than writing efficient code. Thanks for you time.
''' MyApp.py module '''
import asyncore, socket
import SSL
# Client Section
# Connects to SomeServer
class MyAppClient(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect((host, port))
connectionPhase = 1
def handle_read(self):
print "connectionPhase =", self.connectionPhase
# The following IF statements may not make sense
# as I have removed code irrelevant to this question
if self.connectionPhase < 3: # authentication phase
data = self.recv(1024)
print 'Received:', data
# Client/Server authentication is handled here
# Everything from this point on happens over
# an encrypted socket using SSL
# Start the RelayServer listening on localhost 8080
# self.socket is encrypted and is the socket communicating
# with SomeServer
rs = RelayServer(('localhost', 8080), self.socket)
print 'RelayServer started'
# connectionPhase = 3 when this IF loop is done
elif self.connectionPhase == 3: # receiving data for User
data = self.recv(1024)
print 'Received data - forward to User:', data
# Forward this data to User
# Don't understand why data is being read here
# when the RelayServer was instantiated above
# Server Section
# Connects to User
class RelayConnection(asyncore.dispatcher):
def __init__(self, client, sock):
asyncore.dispatcher.__init__(self)
self.client = client
print "connecting to %s..." % str(sock)
def handle_connect(self):
print "connected."
# Allow reading once the connection
# on the other side is open.
self.client.is_readable = True
# For some reason this never runs, i.e. data from SomeServer
# isn't read here, but instead in MyAppClient.handle_read()
# don't know how to make it arrive here instead as it should
# be relayed to User
def handle_read(self):
self.client.send(self.recv(1024))
class RelayClient(asyncore.dispatcher):
def __init__(self, server, client, sock):
asyncore.dispatcher.__init__(self, client)
self.is_readable = False
self.server = server
self.relay = RelayConnection(self, sock)
def handle_read(self):
self.relay.send(self.recv(1024))
def handle_close(self):
print "Closing relay..."
# If the client disconnects, close the
# relay connection as well.
self.relay.close()
self.close()
def readable(self):
return self.is_readable
class RelayServer(asyncore.dispatcher):
def __init__(self, bind_address, MyAppClient_sock):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.bind(bind_address)
self.MyAppClient_sock = MyAppClient_sock
print self.MyAppClient_sock
self.listen(1)
def handle_accept(self):
conn, addr = self.accept()
RelayClient(self, conn, self.MyAppClient_sock)
if __name__ == "__main__":
# Connect to host
# First connection stage
connectionPhase = 1
c = MyAppClient('host', port) # SomeServer's host and port
asyncore.loop()
EDIT:
#samplebias I replaced my complete module with your code (not shown) and I have re-added all the bits and pieces that I need for authentication etc.
At this point I'm getting the same result, as with my own code above. What I mean is that MyApp (or Server in your code) is connected to SomeServer and passing data back and forth. Everything is fine thus far. When User (or client application) connects to localhost 8080, this code is run:
if not self.listener:
self.listener = Listener(self.listener_addr, self)
BUT, this is not run
# if user is attached, send data
elif self.user:
print 'self.user'
self.user.send(data)
So, Server is not relaying data to User. I added print statements throughout the User class to see what is run and init is the only thing. handle_read() never runs.
Why is this?
The code is a bit hard to follow, and I'm sure there are a few bugs. For
example in handle_read() you're passing MyAppClient's raw socket self.socket to
RelayServer. You end up with both MyAppClient and RelayConnection working on the same socket.
Rather than attempt to suggest bug fixes to the original code I put together
an example which does what your code intents and is cleaner and easier to follow.
I've tested it talking to an IMAP server and it works, but omits some
things for brevity (error handling, proper close() handling in all cases, etc).
Server initiates the connection to "someserver". Once it connects
it starts the Listener.
Listener listens on port 8080 and accepts only 1 connection, creates a User,
and passes it a reference to Server. Listener rejects all other
client connections while User is active.
User forwards all data to Server, and vice versa. The comments
indicate where the authentication should be plugged in.
Source:
import asyncore
import socket
class User(asyncore.dispatcher_with_send):
def __init__(self, sock, server):
asyncore.dispatcher_with_send.__init__(self, sock)
self.server = server
def handle_read(self):
data = self.recv(4096)
# parse User auth protocol here, authenticate, set phase flag, etc.
# if authenticated, send data to server
if self.server:
self.server.send(data)
def handle_close(self):
if self.server:
self.server.close()
self.close()
class Listener(asyncore.dispatcher_with_send):
def __init__(self, listener_addr, server):
asyncore.dispatcher_with_send.__init__(self)
self.server = server
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind(listener_addr)
self.listen(1)
def handle_accept(self):
conn, addr = self.accept()
# this listener only accepts 1 client. while it is serving 1 client
# it will reject all other clients.
if not self.server.user:
self.server.user = User(conn, self.server)
else:
conn.close()
class Server(asyncore.dispatcher_with_send):
def __init__(self, server_addr, listener_addr):
asyncore.dispatcher_with_send.__init__(self)
self.server_addr = server_addr
self.listener_addr = listener_addr
self.listener = None
self.user = None
def start(self):
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.connect(self.server_addr)
def handle_error(self, *n):
self.close()
def handle_read(self):
data = self.recv(4096)
# parse SomeServer auth protocol here, set phase flag, etc.
if not self.listener:
self.listener = Listener(self.listener_addr, self)
# if user is attached, send data
elif self.user:
self.user.send(data)
def handle_close(self):
if self.user:
self.user.server = None
self.user.close()
self.user = None
if self.listener:
self.listener.close()
self.listener = None
self.close()
self.start()
if __name__ == '__main__':
app = Server(('someserver', 143), ('localhost', 8080))
app.start()
asyncore.loop()
I try to program a TCPServer with threads (ThreadingMixIn) in Python. The problem is that I can't shut it down properly as I get the socket.error: [Errno 48] Address already in use when I try to run it again. This is a minimal example of the python code that triggers the problem:
import socket
import threading
import SocketServer
class FakeNetio230aHandler(SocketServer.BaseRequestHandler):
def send(self,message):
self.request.send(message+N_LINE_ENDING)
def handle(self):
self.request.send("Hello\n")
class FakeNetio230a(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
def __init__(self, server_address, RequestHandlerClass):
self.allow_reuse_address = True
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass)
if __name__ == '__main__':
for i in range(2):
fake_server = FakeNetio230a(("", 1234), FakeNetio230aHandler)
server_thread = threading.Thread(target=fake_server.serve_forever)
server_thread.setDaemon(True)
server_thread.start()
# might add some client connection here
fake_server.shutdown()
All the main code should do is to start the server, shut it down and run it again. But it triggers the error stated above because the socket has not been released after the first shutdown.
I thought that setting self.allow_reuse_address = True could solve the problem, but that did not work. When the python program finishes I can run it again straight away and it can start the server once (but again not twice).
However the problem is gone when I randomize the port (replace 1234 by 1234+i for example) as no other server is listening on that address.
There is a similar SO Q Shutting down gracefully from ThreadingTCPServer but the solution (set allow_reuse_address to True does not work for my code and I don't use ThreadingTCPServer).
How do I have to modify my code in order to be able to start the server twice in my code?
Some more information: The reason why I'm doing this is that I want to run some unit tests for my python project. This requires to provide a (fake) server that my software should to connect to.
edit:
I just found the most correct answer to my problem: I have to add fake_server.server_close() at the end of my main execution code (right after fake_server.shutdown()). I found it in the source file of the TCPServer implementation. All it does is self.socket.close().
Somehow, fake_server doesn't unbind when you assign to it (in first line in for statement).
To fix that, just remove fake_server at the end of loop:
del fake_server # force server to unbind
This post helped me get over the un-closed socket problem.
I had the same problem and wanted to post here my simple implementation for TCP server class (and client method).
I made a TCPThreadedServer class. In order to use it is needed to be inherited, and the method process(msg) must be overridden. the overridden method invokes every time the server gets a message msg, and if it returns a not None object, it will be returned as string to the connected client.
from SocketServer import TCPServer, StreamRequestHandler, ThreadingMixIn
import threading
class TCPThreadedServer(TCPServer, ThreadingMixIn):
class RequstHandler(StreamRequestHandler):
def handle(self):
msg = self.rfile.readline().strip()
reply = self.server.process(msg)
if reply is not None:
self.wfile.write(str(reply) + '\n')
def __init__(self, host, port, name=None):
self.allow_reuse_address = True
TCPServer.__init__(self, (host, port), self.RequstHandler)
if name is None: name = "%s:%s" % (host, port)
self.name = name
self.poll_interval = 0.5
def process(self, msg):
"""
should be overridden
process a message
msg - string containing a received message
return - if returns a not None object, it will be sent back
to the client.
"""
raise NotImplemented
def serve_forever(self, poll_interval=0.5):
self.poll_interval = poll_interval
self.trd = threading.Thread(target=TCPServer.serve_forever,
args = [self, self.poll_interval],
name = "PyServer-" + self.name)
self.trd.start()
def shutdown(self):
TCPServer.shutdown(self)
TCPServer.server_close(self)
self.trd.join()
del self.trd
I found it quite easy to use:
class EchoServerExample(TCPThreadedServer):
def __init__(self):
TCPThreadedServer.__init__(self, "localhost", 1234, "Server")
def process(self, data):
print "EchoServer Got: " + data
return str.upper(data)
for i in range(10):
echo = EchoServerExample()
echo.serve_forever()
response = client("localhost", 1234, "hi-%i" % i)
print "Client received: " + response
echo.shutdown()
I used the method:
import socket
def client(ip, port, msg, recv_len=4096,
timeout=socket._GLOBAL_DEFAULT_TIMEOUT):
msg = str(msg)
response = None
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((ip, port))
if timeout != socket._GLOBAL_DEFAULT_TIMEOUT:
sock.settimeout(timeout)
sock.send(msg + "\n")
if recv_len > 0:
response = sock.recv(recv_len)
finally:
sock.close()
return response
Enjoy it!
Change your FakeNetio230a definition to this:
class FakeNetio230a(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
def __init__(self, server_address, RequestHandlerClass):
self.allow_reuse_address = True
SocketServer.TCPServer.__init__(self,
server_address,
RequestHandlerClass,
False) # do not implicitly bind
Then, add these two lines in your entry point below your FakeNetio230a instantiation:
fake_server.server_bind() # explicitly bind
fake_server.server_activate() # activate the server
Here's an example:
if __name__ == '__main__':
for i in range(2):
fake_server = FakeNetio230a(("", 1234), FakeNetio230aHandler)
fake_server.server_bind() # explicitly bind
fake_server.server_activate() # activate the server
server_thread = threading.Thread(target=fake_server.serve_forever)
server_thread.setDaemon(True)
server_thread.start()
# might add some client connection here
fake_server.shutdown()