I'm having problems detecting a broken socket when a broken pipe exception occurs. See the below code for an example:
The Server:
import errno, select, socket, time, SocketServer
class MetaServer(object):
def __init__(self):
self.server = Server(None, Handler, bind_and_activate=False)
def run(self, sock, addr):
rfile = sock.makefile('rb', 1)
self.server.process_request(sock, addr)
while 1:
r, _, _ = select.select([rfile], [], [], 1.0)
if r:
print 'Got %s' % rfile.readline()
else:
print 'nothing to read'
class Server(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
allow_reuse_address = True
daemon_threads = True
class Handler(SocketServer.StreamRequestHandler):
def handle(self):
print 'connected!'
try:
while 1:
self.wfile.write('testing...')
time.sleep(1)
except socket.error as e:
if e.errno == errno.EPIPE:
print 'Broken pipe!'
self.finish()
self.request.close()
if __name__ == '__main__':
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind(('127.0.0.1', 8081))
s.listen(5)
ms = MetaServer()
while 1:
client, address = s.accept()
ms.run(client, address)
The Client:
import select, socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('127.0.0.1', 8081))
while 1:
r, _, _ = select.select([s], [], [], 1.0)
if not r:
continue
msg = s.recv(1024)
print 'Got %s' % (msg,)
Now, if I run the server and client, all is well, and I get a "nothing is read" message every second. As soon as I CTRL-C out of the client, the server goes crazy and starts to "read" from what should be a busted socket, dumping a lot of "Got " messages.
Is there some way to detect this broken socket in the MetaServer.run() function to avoid the above said behavior?
Yes, that's something which is not really in the documentation but old Un*x behavior: You need to abort when you get an empty string.
Related
I am fairly new to Python network programming. Recently I am trying to achieve make two programs talk to each other(i.e., send and receive information bi-laterally).
In program A, I have:
server_ip = ('', 4001)
server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server.bind(server_ip)
while True:
#continuously send and receive info to program B until some breaking condition reached
server.sendto(json.dumps(some_data).encode("utf-8"), server_ip)
recv_data = server.recv(1024)
# ...
In Program B, I have:
ADDR=('', 4001)
class Task()
"""
"""
def __init__(self):
self.client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
print('trying to connect to XXX')
while True:
try:
self.client.connect(ADDR)
break
except:
pass
print('connected to XXX')
def step(self):
"""
This method will be called repeatedly
"""
#...
self.send_udp_data()
self.get_data()
def send_udp_data(self):
#...
self.client.sendall(bytes(control_cmd, encoding='utf-8'))
print("Sending CMD")
def get_data(self):
while True:
try:
data = self.client.recv(10240)
data = bytearray(data)
data_dict=json.loads(data.decode('utf-8'))
except Exception as e:
#some error handling
I got countless errors while trying to achieve aforementioned functionality. How can I ensure these two programs properly communicate to each other?
This works:
Program A:
import json
import socket
ADDR_A = ('', 4001)
ADDR_B = ('', 4002)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.bind(ADDR_A)
while True:
#continuously send and receive info to program B until some breaking condition reached
print("A sending...")
some_data = "This is some data sent by A"
# Note: this will be silently dropped if the client is not up and running yet
# And even if the the client is running, it may still be silently dropped since UDP is unreliable.
sock.sendto(json.dumps(some_data).encode("utf-8"), ADDR_B)
print("A receiving...")
recv_data = sock.recv(1024)
print(f"A received {recv_data}")
Program B:
import json
import socket
ADDR_A = ('', 4001)
ADDR_B = ('', 4002)
class Task():
"""
"""
def __init__(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.bind(ADDR_B)
def step(self):
"""
This method will be called repeatedly
"""
print("B step")
self.send_data()
self.receive_data()
def send_data(self):
control_cmd = "This is a control command sent by B"
print("B sending...")
self.sock.sendto(bytes(control_cmd, encoding='utf-8'), ADDR_A)
print(f"B sent {control_cmd}")
def receive_data(self):
try:
data = self.sock.recv(10240)
print(f"B received raw data {data}")
data = bytearray(data)
data_dict=json.loads(data.decode('utf-8'))
print(f"B received JSON {data_dict}")
except Exception as e:
print(f"B exception {e} in receive_data")
task = Task()
while True:
task.step()
The issue is the following.
I have the following server:
import socket
class Receiver:
TCP_IP = '127.0.0.1'
TCP_PORT = 2999
BUFFER_SIZE = 20
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((TCP_IP, TCP_PORT))
s.listen(1)
conn, addr = s.accept()
print('Connection address:', addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send('0x55'.encode()) # echo
conn.close()
And the client:
import socket import logging
class NvgClient:
_instance = None
def __init__(self):
self.s = socket.socket()
self.s.settimeout(3)
self.connect()
return
def __del__(self):
try:
self.s.close()
finally:
return
#staticmethod
def getInstance():
if(NvgClient._instance == None):
NvgClient._instance = NvgClient()
return NvgClient._instance
def connect(self):
try:
print("****** TRYING_TO_CONNECT_TO_SOCKET ********")
self.s.connect(('127.0.0.0', 2999))
except socket.error:
self.s.close()
self.s = socket.socket()
self.s.settimeout(3)
self.connect()
logging.error("Socket can`t connect! Reconnected.")
return
def send(self, data: bytearray):
try:
print("****** TRYING_TO_SEND_DATA ********")
self.s.send(data)
logging.info(str(data))
rdata = self.s.recv(1024)
if(rdata[0] == 0x55 and rdata[1:5] == data[0:4]):
logging.info('NVG OK')
return True
else:
logging.info('NVG BAD')
except socket.timeout:
self.s.close()
self.connect()
except IndexError:
logging.info('Server returns nothing. Reconnecting.')
self.s.close()
self.s = socket.socket()
self.s.settimeout(3)
self.connect()
return False
But when I try to send some data, it is impossible to connect to server:
self.s.connect(('127.0.0.0', 2999)). I get socket.error.
Is there any mistakes or something wrong in code? For other simple examples or telnet, server works well.
You need to connect to localhost which is:
127.0.0.1
and not
127.0.0.0
as you wrote for your client (server is okay though)
self.handlers.append(ConnHandler(sock, self.handlers))I'm new to python and I tried to write a simple socket server to test stuff out and get to know the language better.
import asyncore
import socket
import json
class ConnHandler(asyncore.dispatcher_with_send):
def __init__(self, conn, handlerlist):
asyncore.dispatcher_with_send.__init__(self, conn)
self.handlers = handlerlist
def handle_close(self):
self.close()
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
data += self.recv(1024)
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message
class TestServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
self.handlers = []
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
self.handlers.append(ConnHandler(sock, self.handlers))
def sendToAll(self, string):
for h in self.handlers:
h.send(string + '\n')
server = TestServer('localhost', 55555)
asyncore.loop()
My problem is the following.
If I connect to the server with telnet and then quit telnet to close the connection, python just spams 'Socket closed' in the terminal. How can I fix this? Are there obvious beginner mistakes in the above code?
handle_close is not meant to detect if peer is disconnected, you get this information in handle_read if .recv call returns an empty string. Then you can close the socket, and then handle_close is called.
Here is a modified version of your code:
def handle_close(self):
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
new_data = self.recv(1024)
if not new_data:
self.close()
return
else:
data += new_data
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message
I have a Twisted application that's listening for Int32StringReceiver messages and then re-sending them to another app. Basically, it's a router, but it has some intelligence, introspecting where the data is going.
My problem is with the outbound side, getting lot of error messages, etc.
Inbound is a class Receiver(Int32StringReceiver):
def doActualForwarding(self, data):
self.stats.recvBits += 8 * (4 + len(data))
self.stats.recvMsgs += 1
dlen = len(data)
if dlen > 1024*256:
self.logger.info("router.Receiver.doActualForwarding(): data len: %s" % (dlen))
self.router.forward(data)
def stringReceived(self, data):
d = threads.deferToThread(self.doActualForwarding, data)
d.addCallback(self.forwardingDoneOkay)
d.addErrback(self.forwardingDoneError)
The self.router is instantiated object that needs to send these messages out via socket comms in the same format. So, it just turns around and does this in the Router class:
def connect(self):
if self.sock:
try:
self.sock.close()
except:
pass
try:
self.stats.connectAttempts += 1
self.sock = socket.socket()
self.sock.settimeout(self.CONNECT_TIMEOUT)
self.sock.connect(self.destination)
self.sock.settimeout(self.SEND_TIMEOUT)
self.set_keepalive_linux(self.sock)
self.connected = True
self.log.info("connected to %s" % (self.destination,))
self.stats.reconnects += 1
self.stats.connectCompletes += 1
return True
except Exception, e:
self.connected = False
if not self.drop_ok:
self.log.error("connect %s: %s" % (self.destination, e))
return False
def send(self, msg):
trynum = 0
while trynum < self.MAX_SEND_ATTEMPTS:
self.logSent()
if not self.connected:
if not self.connect():
self.stats.badSends += 1
time.sleep(self.DELAY_BEFORE_RECONNECT)
continue
try:
if ((time.time() - self.lastReconnectTime) > self.RECONNECT_EVERY):
self.lastReconnectTime = time.time()
assert False, "Reconnecting with destination to redistribute load."
self.sock.sendall(msg)
#self.closeSocket()
self.stats.events += 1
return True
except Exception, e:
whichKind = None
if 'Broken pipe' in str(e):
self.stats.brokenPipe += 1
elif 'Resource temporarily unavilable' in str(e):
self.stats.resourceTempUnavail += 1
elif 'Bad file descriptor' in str(e):
self.stats.badFileDescriptor += 1
self.log.error("send: %s %s" % (str(self.destination), str(e)))
try:
self.sock.close()
except:
pass
self.connected = False
self.stats.badSends += 1
trynum += 1
if trynum == 1:
self.stats.eventsWithRetry += 1
if trynum > 1:
self.log.warning("recon_sender.send(): Trynum non-singular, was: %s" % (trynum))
return False
def __del__(self):
try:
self.sock.close()
except:
pass
QUESTIONS:
Is Python's Socket library threadsafe? That is, functionally, two or more threads have a pointer to the object Router. Both threads are calling self.sock.sendall(msg) and I'm concerned they'll step on each other.
One symptom is that it might be that successive messages are appended to each other. I'm not sure about this, but it looks that way.
I'm seeing a lot of resource temp. unavail (meaning destination is busy), about the same number of broken pipes, and a small number of bad file descriptor.
[Errno 9] Bad file descriptor
[Errno 11] Resource temporarily unavailable
[Errno 32] Broken pipe
These messages correspond to maybe 0.5% (.005) of the number of messages going through this thing.
I tried to have each send do a connect/sendall/shutdown/close, but that resulted in a ton of messages about 'connection reset by peer'.
Everyone seems to be intent on code that handles multi-threaded receiving on sockets, but not so many comment on multi-threaded SENDING on sockets.
I also tried to use (possibly incorrectly):
import threading
self.lock = threading.Lock()
with self.lock:
sock.sendall(msg)
but this resulted in error messages about timing out (yuck).
Can someone point me in the direction of some good examples (Or PROVIDE SOME?!?!?!?) that demonstrate multithreaded socket sendall()?
I would say that if the processes do not have to communicate with eachother, your best solution will be to spawn a new process to handle each incoming connection. This way you don't have to worry about locking as each connection will be handled separately.
Simple implementation would be:
import socket
import multiprocessing
import pdb
import random
from pycurl import Curl
import os
import time
import re
class query(object):
pid, addr, conn, url, ua, ref = [None for i in range(6)]
compression = True
def __init__(self, conn, addr):
self.pid = addr[1]
self.addr = addr
self.conn = conn
self.process()
def process(self):
#do your socket stuff here
class ProxyServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
def start(self):
logging.info("Server started on %s:%i" % (self.host, self.port))
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.bind((self.host, self.port))
self.sock.listen(0)
while True:
conn, addr = self.sock.accept()
logging.info('Connection made from %s' % conn)
proc = multiprocessing.Process(target=query, args=(conn, addr))
proc.daemon = True
proc.start()
logging.info('Started processing query %r for %s' % (proc, addr))
if __name__ == "__main__":
serv = ProxyServer(host, port)
try:
serv.start()
except:
finally:
for proc in multiprocessing.active_children():
proc.terminate()
proc.join()
Keep in mind that this is an example that I cut from old proof-of-concept code, you will have to tweak it a bit before it's ready for production.
In Python 3.3.3， i create a thread to listen some connection to the socket.It likes this:
import threading
import socket
import time
Host = ''
Port = 50000
flag = False
class ServerThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def run(self):
try:
self._sock.bind((Host, Port))
self._sock.listen(5)
while True:
conn, addr = self._sock.accept()
print('Connected by', addr)
except socket.error as msg:
print(msg)
except Exception as e:
print(str(e))
finally:
self._sock.close()
def exit(self):
self._sock.close()
def TargetFunc(vlock):
vlock.acquire()
flag = True
vlock.release()
def main():
sthread = ServerThread()
sthread.start()
vlock = threading.Lock()
time.sleep(10)
vthread = threading.Thread(target = TargetFunc, args = (vlock, ))
vthread.start()
while True:
vlock.acquire()
if flag:
sthread.exit()
vlock.release()
break
vlock.release()
sthread.join()
vthread.join()
if __name__ == '__main__':
main()
There are two threads, one is listening socket, the other is to set a flag. When the flag is True, close the socket, then raise a socket error and catch it, so the listening socket terminates.But why it does not work this.
Thanks!
self._sock.accept() is blocking. So it will wait until somebody connects. You should use a nonblocking variant (or blocking but with a time-out). So that you can check the exit conditions.
Alternatively you could force an exception in the ServerThread.