I want 2 processes to communicate on a given port without either one having a defined client or server role. Either of the processes may be running alone. Either may stop and restart at any time, in any order. When they are both running they need to communicate (when only one is running, communication is just discarded).
I want non-blocking sockets and Windows/Linux support.
Here's a rather crude class that actually works to some extent, which might get you started.
The main trick here is not to bother with listen at all: these are pure peer to peer connections, fully specified by the <local-addr, remote-addr> pair.
Note that the sockets are left in non-blocking mode. I caught the recv exception but there can be a send one as well (plus, you get broken-pipe errors when sending to a dead peer, etc). You'll also need to handle EOF-from-terminated-peer (when recv returns '' instead of failing with EAGAIN).
import errno
import os
import select
import socket
class Peer(object):
def __init__(self, local_addr, peer_addr):
self._local_addr = local_addr
self._peer_addr = peer_addr
self._renew()
self.reopen()
def _renew(self):
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._sock.bind(self._local_addr)
self._sock.setblocking(False)
self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self._state = 'bound'
def is_open(self):
return self._state == 'open'
def is_opening(self):
return self._state == 'opening'
def reopen(self):
if self._state == 'open':
raise ValueError('already open')
if self._state == 'opening':
raise ValueError('open in progress')
print 'try connect to:', self._peer_addr
error = self._sock.connect_ex(self._peer_addr)
print 'result:', error
if error == 0:
self._state = 'open'
print 'connected immediately'
elif error in (errno.EINPROGRESS, errno.EINTR):
self._state = 'opening'
print 'connection in progress'
else:
raise socket.error(error, os.strerror(error))
def _check_open(self):
if self._state != 'opening':
raise ValueError('improper call to _check_open')
print 'check connect to:', self._peer_addr
_, wfds, _ = select.select([], [self._sock], [])
if len(wfds) == 0:
# connection still in progress
return
# we have a result: fail or succeed, either way a result
try:
peer = self._sock.getpeername()
except socket.error as err:
print 'caught err:', err
if err.errno == errno.ENOTCONN:
print 'connection failed, no peer available'
self.close()
return
raise
print 'got a peer:', peer
self._state = 'open'
print 'connection finished'
def close(self):
if self._state in ('open', 'opening'):
self._sock.close()
self._renew()
# self.reopen() - or leave to caller
def send_if_connected(self, data):
# to do: add check for send to dead peer, and if so, _renew etc
if self._state == 'bound':
self.reopen()
if self._state == 'opening':
self._check_open()
if self._state == 'open':
self._sock.send(data)
def recv_if_connected(self):
# to do: add check for send to dead peer, and if so, _renew etc
if self._state == 'bound':
self.reopen()
if self._state == 'opening':
self._check_open()
if self._state == 'open':
try:
return self._sock.recv(1024)
except socket.error as err:
# still connected but no data avail
if err.errno == errno.EAGAIN:
return ''
raise
else:
return None
if __name__ == '__main__':
import argparse
import time
parser = argparse.ArgumentParser(description='test Peer()')
parser.add_argument('-l', '--localhost', default='')
parser.add_argument('-p', '--port', type=int, default=9001)
parser.add_argument('-R', '--remote-host', default='')
parser.add_argument('-r', '--remote-port', type=int, default=9002)
args = parser.parse_args()
x = Peer((args.localhost, args.port), (args.remote_host, args.remote_port))
for i in range(1, 10):
print 'attempt to send %d' % i
x.send_if_connected('send %d' % i)
got = x.recv_if_connected()
if got is not None:
print 'got: "%s"' % got
time.sleep(1)
Run with: $ python peerish.py -p 9001 -r 9002 & python peerish.py -p 9002 -r 9001 & for instance.
Related
I have implemented a server in Python 3.8.6 using the socket library (running on a Raspberry Pi 4 w/ 8G RAM & Raspberry Pi OS: Buster) which, by design, only handles a single connection at a time. When a connection is gained, the server spawns two threads, one for sending data, and one for receiving it. When the connection is lost (in theory, it will never be intentionally closed), these threads are terminated and a new thread is spawned to accept a new connection. The problem is, the receiving thread never actually terminates, instead getting stuck infinitely on socket.recv(). This persists even after calling socket.shutdown(SHUT_RDWR) and socket.close(). The code is essentially as follows (I've paired it down to only the pertinent bits and left the heartbeat management code, as I've found this is the only way I can consistently tell that the connection has dropped, as the errors that are supposed to be thrown don't always):
import dataclasses
import logging
import pickle
import queue
import socket
import threading
import time
import traceback
log = logging.getLogger()
class Server(threading.Thread):
#dataclasses.dataclass
class Heartbeat:
failures: int = 0
reciprocate: bool = True
time: float = 0
__CLIENT_IP = '10.1.1.58'
__EOF = "\r\nEOF\r\n".encode()
__PORT = 33368
def __init__(self, kill_flag:threading.Event) -> None:
self.__kill_flag: threading.Event = kill_flag
self.__server = socket.create_server(("", self.__PORT), backlog=0, reuse_port=True)
self.__server.settimeout(5)
self.__client = None
self.__queue_out = queue.Queue(-1)
self.__queue_in = queue.Queue(-1)
self._connected = False
self.__connecting_lock = threading.Lock()
self.__receive_thread = None
self.__transmit_thread = None
threading.Thread(target=self.__connect).start()
def __connect(self) -> None:
with self.__connecting_lock:
if self._connected:
return
addr = ("", 0)
if self.__client:
try:
self.__client.shutdown(socket.SHUT_RDWR)
except OSError:
pass
finally:
self.__client.close()
self.__client = None
while (self.__receive_thread and self.__receive_thread.is_alive()) or (self.__transmit_thread and self.__transmit_thread.is_alive()):
log.debug("connected: %s, recv alive: %s, trans alive: %s", self._connected, self.__receive_thread.is_alive(), self.__transmit_thread.is_alive())
time.sleep(1)
log.info("Waiting for connection...")
while addr[0] != self.__CLIENT_IP and not self.__kill_flag.is_set():
try:
self.__client, addr = self.__server.accept()
if addr[0] != self.__CLIENT_IP:
self.__client.close()
log.warning("Connection from %s:%s - Rejected", addr[0], addr[1])
time.sleep(1)
except socket.timeout:
continue
if self.__kill_flag.is_set():
self.__del__()
else:
log.info("Connection from %s:%s - Accepted", addr[0], addr[1])
self.heartbeat = self.Heartbeat()
self._connected = True
self.__receive_thread = threading.Thread(target=self.__receive)
self.__transmit_thread = threading.Thread(target=self.__transmit)
self.__queue_in.queue.clear()
self.__receive_thread.start()
self.__transmit_thread.start()
def get_package(self) -> tuple:
return self.__queue_out.get(True)
def queue_message(self, content:tuple = ()) -> None:
self.__queue_in.put_nowait(content)
def __receive(self) -> None:
buffer = bytearray()
while self._connected and not self.__kill_flag.is_set():
try:
if self.__EOF in buffer:
payload = pickle.loads(buffer.split(self.__EOF, maxsplit=1)[0])
self.__queue_out.put_nowait(payload)
buffer = bytearray(self.__EOF.join(buffer.split(self.__EOF)[1:]))
else:
try:
log.debug("looping into recv")
data = self.__client.recv(1024)
log.debug("looping past recv")
if len(data) == 0:
raise ConnectionResetError
else:
buffer += data
except OSError as error:
log.error(f"Receive error: {error}")
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
except AttributeError:
break
except socket.timeout:
continue
except Exception as error:
log.error(f"Receive error: {error}")
traceback.print_tb(error.__traceback__)
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
def __transmit(self) -> None:
while self._connected and not self.__kill_flag.is_set():
try:
payload = self.__queue_in.get(True, 5)
except queue.Empty:
if self.heartbeat.reciprocate:
payload = (time.time(),)
self.heartbeat.reciprocate = False
elif time.time() - 7 >= self.heartbeat.time:
self.heartbeat.failures += 1
log.debug("Heartbeat failure")
if self.heartbeat.failures >= 3:
log.warning("Lost connection to client: No heartbeat detected")
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
self.heartbeat.reciprocate = True
continue
else:
continue
self.heartbeat.time = time.time()
try:
self.__client.sendall(pickle.dumps(payload))
log.debug("Package sent: %s", payload)
except BrokenPipeError:
log.warning("Lost connection to client: Broken pipe")
self.__queue_in.put_nowait(payload)
if self._connected:
self._connected = False
threading.Thread(target=self.__connect).start()
except AttributeError:
break
def __del__(self) -> None:
if self.__client:
self.__client.close()
self.__server.close()
After the connection drops, the __receive method/thread hangs specifically on this line: data = self.__client.recv(1024).
Perhaps the strangest part is that the client, whose __transmit and __receive methods are virtually identical, does not have this problem.
Any insight anyone can offer into as why this is happening, and how to remedy it, would be greatly appreciated.
Many thanks!
I did debug the code below, line-by-line using some print statements.
class Timeout(Exception):
pass
def getSource(comm):
source = comm.split('#')
params = source[1].split(':')
debug = '--debug' in sys.argv
if source[0] == 'serial':
try:
return Serial(params[0], int(params[1]), flush=True, debug=debug)
except:
print ("ERROR: Unable to initialize a serial connection to", comm)
raise Exception
Everything looks OK until the line:
return Serial(params[0], int(params[1]), flush=True, debug=debug)
this line is supposed to be compiled since all the objects in the Serial like params[0], etc are obtained. But it returns an error jumping to the except and printing the statement "ERROR: Unable to initialize a serial connection to ..."
I am using Python 3.6.8 on a Docker container.
Any kind of help would be appreciated. I'm ready for any further info, if needed.
Here, I post the Serial. I hope this helps you guys to better understand my issue.
class Serial:
def __init__(self, port, baudrate, flush=False, debug=False, readTimeout=None, ackTimeout=0.02):
self.debug = debug
self.readTimeout = readTimeout
self.ackTimeout = ackTimeout
self._ts = None
if port.startswith('COM') or port.startswith('com'):
port = int(port[3:]) - 1
elif port.isdigit():
port = int(port) - 1
self._s = serial.Serial(port, int(baudrate), rtscts=0, timeout=0.5)
self._s.flushInput()
if flush:
print >>sys.stdout, "Flushing the serial port",
endtime = time.time() + 1
while time.time() < endtime:
self._s.read()
sys.stdout.write(".")
if not self.debug:
sys.stdout.write("\n")
self._s.close()
self._s = serial.Serial(port, baudrate, rtscts=0, timeout=readTimeout)
def getByte(self):
c = self._s.read()
if c == '':
raise Timeout
#print 'Serial:getByte: 0x%02x' % ord(c)
return ord(c)
def putBytes(self, data):
#print "DEBUG: putBytes:", data
for b in data:
self._s.write(struct.pack('B', b))
time.sleep(0.000001)
def getTimeout(self):
return self._s.timeout
def setTimeout(self, timeout):
self._s.timeout = timeout
I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?
You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.
I have a Twisted application that's listening for Int32StringReceiver messages and then re-sending them to another app. Basically, it's a router, but it has some intelligence, introspecting where the data is going.
My problem is with the outbound side, getting lot of error messages, etc.
Inbound is a class Receiver(Int32StringReceiver):
def doActualForwarding(self, data):
self.stats.recvBits += 8 * (4 + len(data))
self.stats.recvMsgs += 1
dlen = len(data)
if dlen > 1024*256:
self.logger.info("router.Receiver.doActualForwarding(): data len: %s" % (dlen))
self.router.forward(data)
def stringReceived(self, data):
d = threads.deferToThread(self.doActualForwarding, data)
d.addCallback(self.forwardingDoneOkay)
d.addErrback(self.forwardingDoneError)
The self.router is instantiated object that needs to send these messages out via socket comms in the same format. So, it just turns around and does this in the Router class:
def connect(self):
if self.sock:
try:
self.sock.close()
except:
pass
try:
self.stats.connectAttempts += 1
self.sock = socket.socket()
self.sock.settimeout(self.CONNECT_TIMEOUT)
self.sock.connect(self.destination)
self.sock.settimeout(self.SEND_TIMEOUT)
self.set_keepalive_linux(self.sock)
self.connected = True
self.log.info("connected to %s" % (self.destination,))
self.stats.reconnects += 1
self.stats.connectCompletes += 1
return True
except Exception, e:
self.connected = False
if not self.drop_ok:
self.log.error("connect %s: %s" % (self.destination, e))
return False
def send(self, msg):
trynum = 0
while trynum < self.MAX_SEND_ATTEMPTS:
self.logSent()
if not self.connected:
if not self.connect():
self.stats.badSends += 1
time.sleep(self.DELAY_BEFORE_RECONNECT)
continue
try:
if ((time.time() - self.lastReconnectTime) > self.RECONNECT_EVERY):
self.lastReconnectTime = time.time()
assert False, "Reconnecting with destination to redistribute load."
self.sock.sendall(msg)
#self.closeSocket()
self.stats.events += 1
return True
except Exception, e:
whichKind = None
if 'Broken pipe' in str(e):
self.stats.brokenPipe += 1
elif 'Resource temporarily unavilable' in str(e):
self.stats.resourceTempUnavail += 1
elif 'Bad file descriptor' in str(e):
self.stats.badFileDescriptor += 1
self.log.error("send: %s %s" % (str(self.destination), str(e)))
try:
self.sock.close()
except:
pass
self.connected = False
self.stats.badSends += 1
trynum += 1
if trynum == 1:
self.stats.eventsWithRetry += 1
if trynum > 1:
self.log.warning("recon_sender.send(): Trynum non-singular, was: %s" % (trynum))
return False
def __del__(self):
try:
self.sock.close()
except:
pass
QUESTIONS:
Is Python's Socket library threadsafe? That is, functionally, two or more threads have a pointer to the object Router. Both threads are calling self.sock.sendall(msg) and I'm concerned they'll step on each other.
One symptom is that it might be that successive messages are appended to each other. I'm not sure about this, but it looks that way.
I'm seeing a lot of resource temp. unavail (meaning destination is busy), about the same number of broken pipes, and a small number of bad file descriptor.
[Errno 9] Bad file descriptor
[Errno 11] Resource temporarily unavailable
[Errno 32] Broken pipe
These messages correspond to maybe 0.5% (.005) of the number of messages going through this thing.
I tried to have each send do a connect/sendall/shutdown/close, but that resulted in a ton of messages about 'connection reset by peer'.
Everyone seems to be intent on code that handles multi-threaded receiving on sockets, but not so many comment on multi-threaded SENDING on sockets.
I also tried to use (possibly incorrectly):
import threading
self.lock = threading.Lock()
with self.lock:
sock.sendall(msg)
but this resulted in error messages about timing out (yuck).
Can someone point me in the direction of some good examples (Or PROVIDE SOME?!?!?!?) that demonstrate multithreaded socket sendall()?
I would say that if the processes do not have to communicate with eachother, your best solution will be to spawn a new process to handle each incoming connection. This way you don't have to worry about locking as each connection will be handled separately.
Simple implementation would be:
import socket
import multiprocessing
import pdb
import random
from pycurl import Curl
import os
import time
import re
class query(object):
pid, addr, conn, url, ua, ref = [None for i in range(6)]
compression = True
def __init__(self, conn, addr):
self.pid = addr[1]
self.addr = addr
self.conn = conn
self.process()
def process(self):
#do your socket stuff here
class ProxyServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
def start(self):
logging.info("Server started on %s:%i" % (self.host, self.port))
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.bind((self.host, self.port))
self.sock.listen(0)
while True:
conn, addr = self.sock.accept()
logging.info('Connection made from %s' % conn)
proc = multiprocessing.Process(target=query, args=(conn, addr))
proc.daemon = True
proc.start()
logging.info('Started processing query %r for %s' % (proc, addr))
if __name__ == "__main__":
serv = ProxyServer(host, port)
try:
serv.start()
except:
finally:
for proc in multiprocessing.active_children():
proc.terminate()
proc.join()
Keep in mind that this is an example that I cut from old proof-of-concept code, you will have to tweak it a bit before it's ready for production.
I'm working on a project wich consist on testing a board connection with a JTAG connector and OpenOCD server.
Here is the connection class I've coded, it's simply using pexpect :
"""
Communication with embedded board
"""
import sys
import time
import threading
import Queue
import pexpect
import serial
import fdpexpect
from pexpect import EOF, TIMEOUT
class ModTelnet():
def __init__(self):
self.is_running = False
self.HOST = 'localhost'
self.port = '4444'
def receive(self):
#receive data (= msg) from telnet stdout
data = [ EOF, TIMEOUT, '>' ]
index = self._tn.expect(data, 2)
if index == 0:
return 'eof', None
elif index == 1:
return 'timeout', None
elif index == 2:
print 'success', self._tn.before.split('\r\n')[1:]
return 'success',self._tn.before
def send(self, command):
print 'sending command: ', command
self._tn.sendline(command)
def stop(self):
print 'Connection stopped !'
self._ocd.sendcontrol('c')
def connect(self):
#connect to MODIMX27 with JTAG and OpenOCD
self.is_running = True
password = 'xxxx'
myfile = 'openocd.cfg'
self._ocd = pexpect.spawn('sudo openocd -f %s' % (myfile))
i = self._ocd.expect(['password', EOF, TIMEOUT])
if i == 0:
self._ocd.sendline(password)
time.sleep(1.0)
self._connect_to_tn()
elif i == 1:
print ' *** OCD Connection failed *** '
raise Disconnected()
elif i == 2:
print ' *** OCD Connection timeout *** '
raise Timeout()
def _connect_to_tn(self):
#connect to telnet session # localhost port 4444
self._tn = pexpect.spawn('telnet %s %s' % (self.HOST, self.port))
condition = self._tn.expect(['>', EOF, TIMEOUT])
if condition == 0:
print 'Telnet opened with success'
elif condition == 1:
print self._tn.before
raise Disconnected()
elif condition == 2:
print self._tn.before
raise Timeout()
if __name__ =='__main__':
try:
tn = ModTelnet()
tn.connect()
except :
print 'Cannot connect to board!'
exit(0)
The problem is when I'm trying to use send, receive and stop command in ohter modules doing this :
>>> from OCDConnect import *
>>> import time
>>> tn = ModTelnet()
>>> tn.connect()
Telnet opened with success
>>> time.sleep(2.0)
>>> self.send('soft_reset_halt')
MMU: disabled, D-Cache: disabled, I-Cache: disabled
>>> self.stop()
It give me an error like : "ModTelnet has no send attribute"
How can I fix this??
Thanks for you help !
try
'send' in dir(tn)
if it's False, you haven't implemented the send method.
The problem was the syntax of my class definition :
class ModTelnet:
And not :
class ModTelnet():
wich is a useless because I don't inherit from an other class ... :D
Thanks anyway !