We Keep Coding

TCP client not able to write to socket but receive works fine

I have client program written in python that talks to some server.
[Client]
import asyncore
import logging
import socket
import sys, threading, traceback
from cStringIO import StringIO
class Client(threading.Thread, asyncore.dispatcher):
def __init__(self, host, port):
self.logger = logging.getLogger()
threading.Thread.__init__(self)
self._thread_sockets = dict()
asyncore.dispatcher.__init__(self, map=self._thread_sockets)
# data members for the module
self.host = host
self.port = port
self.write_buffer = ""
self.is_connected = False
self.read_buffer = StringIO()
# Ok now to run the thread !!
self.start()
def run(self) :
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
address = (self.host, self.port)
self.logger.debug('connecting to %s', address)
# wait until server is up
while not self.is_connected :
try :
self.connect(address)
except Exception as ex :
pass #do nothing, proceed forward !!
asyncore.loop(map=self._thread_sockets)
def handle_connect(self):
self.is_connected = True
self.logger.debug('handle_connect()')
def handle_close(self):
self.logger.debug('handle_close()')
self.close()
def handle_error(self):
traceback.print_exc(sys.stderr)
self.close()
def writable(self):
self.logger.debug('writable() : len is %d bytes', len(self.write_buffer))
is_writable = (len(self.write_buffer) > 0)
if is_writable:
self.logger.debug('writable() -> %s', is_writable)
return is_writable
def readable(self):
self.logger.debug('readable() -> True')
return True
def handle_write(self):
sent = self.send(self.write_buffer)
self.logger.debug('data len written to socket -> %s', sent)
self.logger.debug('handle_write() -> "%s"', self.write_buffer[:sent])
#self.write_buffer = self.write_buffer[sent:]
def handle_read(self):
data = self.recv(8192)
self.logger.debug('handle_read() -> %d bytes', len(data))
self.read_buffer.write(data)
self.logger.debug('data received from socket -> %s', self.read_buffer.getvalue())
self.read_buffer.truncate(0)
def send(self, data) :
self.write_buffer = data
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(name)s: %(message)s',
)
try :
client = Client("127.0.0.1", 8182)
client.send('sending data from client')
except Exception as ex :
logging.exception(ex)
sys.exit(1)
I am able to receive data from server correctly but send call to the server always fails. As from the log the send always return 'None'.
Am i missing anything ?

You override the send method of asyncore.dispatcher with code that does not send any data and returns no value:
def send(self, data) :
self.write_buffer = data
At the least, you need to change your code to look similar to this:
def send_data(self, data):
self.write_buffer = data
and this:
client.send_data('sending data from client')
The asyncore.dispatcher class already has a send method which is a wrapper around the socket.send method. From asyncore.py:
def send(self, data):
try:
result = self.socket.send(data)
return result
except socket.error, why:
if why.args[0] == EWOULDBLOCK:
return 0
elif why.args[0] in _DISCONNECTED:
self.handle_close()
return 0
else:
raise
Because you override this method, your send method gets called in your handle_write method, and no data is sent to the server.

How to close Python socket running in a thread after CTRL+C?

I have a multithreaded program in python and I would like to close the socket after CTRL+C (or Z). I've tried this and this but none of them had worked.
When trying to re-run the program, error message appears:
Bind failed. Error code: 98 Message Address already in use called
Traceback (most recent call last): File "main.py", line 16, in
main.connection.close() NameError: name 'main' is not defined
from connection import Connection
class Main():
def __init__(self):
self.connection = Connection()
self.connection.start()
if __name__ == '__main__':
try:
main = Main()
except:
main.connection.close()
import socket
import sys
import threading
import time
class Connection(threading.Thread):
def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, verbose=None):
threading.Thread.__init__(self, group=group, target=target, name=name, args=args, kwargs=kwargs, verbose=verbose)
self.server = None
self.connection = self.start_connention()
self.data = "null"
self.lock = threading.Lock()
self.OK = True
def start_connention(self):
host = '192.168.42.1'
port = 8888
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
print 'Socket created'
#Bind socket to local host and port
try:
s.bind((host, port))
except socket.error, msg:
print 'Bind failed. Error code: ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
print 'Socket bind complete'
#Start listening on socket
s.listen(10)
print 'Socket now listening on ' + str(port)
connection, addr = s.accept()
print 'Connected with ' + addr[0] + ':' + str(addr[1])
self.server = s
return connection
def close(self):
print("closing")
self.OK = False
self.server.close()
def run(self):
while self.OK:
with self.lock:
self.data = self.connection.recv(4096)
print(str(self.data))
time.sleep(0.02)
def send(self, message):
self.connection.sendall(message)

From docs: The KeyboardInterrupt inherits from BaseException so as to not be accidentally caught by code that catches Exception and thus prevent the interpreter from exiting. docs
if __name__ == '__main__':
try:
main = Main()
except KeyboardInterrupt:
pass
finally:
main.connection.close()

I would suggest you to use atexit module to this stuff.
Just put this line in __init__ and in case any python process termination a connection will be close
atexit.register(self.close)

asyncores's handle_close keeps firing continuously

self.handlers.append(ConnHandler(sock, self.handlers))I'm new to python and I tried to write a simple socket server to test stuff out and get to know the language better.
import asyncore
import socket
import json
class ConnHandler(asyncore.dispatcher_with_send):
def __init__(self, conn, handlerlist):
asyncore.dispatcher_with_send.__init__(self, conn)
self.handlers = handlerlist
def handle_close(self):
self.close()
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
data += self.recv(1024)
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message
class TestServer(asyncore.dispatcher):
def __init__(self, host, port):
asyncore.dispatcher.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind((host, port))
self.listen(5)
self.handlers = []
def handle_accept(self):
pair = self.accept()
if pair is not None:
sock, addr = pair
print 'Incoming connection from %s' % repr(addr)
self.handlers.append(ConnHandler(sock, self.handlers))
def sendToAll(self, string):
for h in self.handlers:
h.send(string + '\n')
server = TestServer('localhost', 55555)
asyncore.loop()
My problem is the following.
If I connect to the server with telnet and then quit telnet to close the connection, python just spams 'Socket closed' in the terminal. How can I fix this? Are there obvious beginner mistakes in the above code?

handle_close is not meant to detect if peer is disconnected, you get this information in handle_read if .recv call returns an empty string. Then you can close the socket, and then handle_close is called.
Here is a modified version of your code:
def handle_close(self):
print 'Socket closed'
if(self.handlers.count(self) > 0):
self.handlers.remove(self);
def handle_read(self):
data = ''
more = True
while more:
try:
new_data = self.recv(1024)
if not new_data:
self.close()
return
else:
data += new_data
except socket.error, e:
more = False
if data == '':
return
try:
message = json.loads(data)
except ValueError:
self.send('Invalid JSON\n')
return
print message

Multithreaded Python Socket Sender/Client

I have a Twisted application that's listening for Int32StringReceiver messages and then re-sending them to another app. Basically, it's a router, but it has some intelligence, introspecting where the data is going.
My problem is with the outbound side, getting lot of error messages, etc.
Inbound is a class Receiver(Int32StringReceiver):
def doActualForwarding(self, data):
self.stats.recvBits += 8 * (4 + len(data))
self.stats.recvMsgs += 1
dlen = len(data)
if dlen > 1024*256:
self.logger.info("router.Receiver.doActualForwarding(): data len: %s" % (dlen))
self.router.forward(data)
def stringReceived(self, data):
d = threads.deferToThread(self.doActualForwarding, data)
d.addCallback(self.forwardingDoneOkay)
d.addErrback(self.forwardingDoneError)
The self.router is instantiated object that needs to send these messages out via socket comms in the same format. So, it just turns around and does this in the Router class:
def connect(self):
if self.sock:
try:
self.sock.close()
except:
pass
try:
self.stats.connectAttempts += 1
self.sock = socket.socket()
self.sock.settimeout(self.CONNECT_TIMEOUT)
self.sock.connect(self.destination)
self.sock.settimeout(self.SEND_TIMEOUT)
self.set_keepalive_linux(self.sock)
self.connected = True
self.log.info("connected to %s" % (self.destination,))
self.stats.reconnects += 1
self.stats.connectCompletes += 1
return True
except Exception, e:
self.connected = False
if not self.drop_ok:
self.log.error("connect %s: %s" % (self.destination, e))
return False
def send(self, msg):
trynum = 0
while trynum < self.MAX_SEND_ATTEMPTS:
self.logSent()
if not self.connected:
if not self.connect():
self.stats.badSends += 1
time.sleep(self.DELAY_BEFORE_RECONNECT)
continue
try:
if ((time.time() - self.lastReconnectTime) > self.RECONNECT_EVERY):
self.lastReconnectTime = time.time()
assert False, "Reconnecting with destination to redistribute load."
self.sock.sendall(msg)
#self.closeSocket()
self.stats.events += 1
return True
except Exception, e:
whichKind = None
if 'Broken pipe' in str(e):
self.stats.brokenPipe += 1
elif 'Resource temporarily unavilable' in str(e):
self.stats.resourceTempUnavail += 1
elif 'Bad file descriptor' in str(e):
self.stats.badFileDescriptor += 1
self.log.error("send: %s %s" % (str(self.destination), str(e)))
try:
self.sock.close()
except:
pass
self.connected = False
self.stats.badSends += 1
trynum += 1
if trynum == 1:
self.stats.eventsWithRetry += 1
if trynum > 1:
self.log.warning("recon_sender.send(): Trynum non-singular, was: %s" % (trynum))
return False
def __del__(self):
try:
self.sock.close()
except:
pass
QUESTIONS:
Is Python's Socket library threadsafe? That is, functionally, two or more threads have a pointer to the object Router. Both threads are calling self.sock.sendall(msg) and I'm concerned they'll step on each other.
One symptom is that it might be that successive messages are appended to each other. I'm not sure about this, but it looks that way.
I'm seeing a lot of resource temp. unavail (meaning destination is busy), about the same number of broken pipes, and a small number of bad file descriptor.
[Errno 9] Bad file descriptor
[Errno 11] Resource temporarily unavailable
[Errno 32] Broken pipe
These messages correspond to maybe 0.5% (.005) of the number of messages going through this thing.
I tried to have each send do a connect/sendall/shutdown/close, but that resulted in a ton of messages about 'connection reset by peer'.
Everyone seems to be intent on code that handles multi-threaded receiving on sockets, but not so many comment on multi-threaded SENDING on sockets.
I also tried to use (possibly incorrectly):
import threading
self.lock = threading.Lock()
with self.lock:
sock.sendall(msg)
but this resulted in error messages about timing out (yuck).
Can someone point me in the direction of some good examples (Or PROVIDE SOME?!?!?!?) that demonstrate multithreaded socket sendall()?

I would say that if the processes do not have to communicate with eachother, your best solution will be to spawn a new process to handle each incoming connection. This way you don't have to worry about locking as each connection will be handled separately.
Simple implementation would be:
import socket
import multiprocessing
import pdb
import random
from pycurl import Curl
import os
import time
import re
class query(object):
pid, addr, conn, url, ua, ref = [None for i in range(6)]
compression = True
def __init__(self, conn, addr):
self.pid = addr[1]
self.addr = addr
self.conn = conn
self.process()
def process(self):
#do your socket stuff here
class ProxyServer(object):
def __init__(self, host, port):
self.host = host
self.port = port
def start(self):
logging.info("Server started on %s:%i" % (self.host, self.port))
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.bind((self.host, self.port))
self.sock.listen(0)
while True:
conn, addr = self.sock.accept()
logging.info('Connection made from %s' % conn)
proc = multiprocessing.Process(target=query, args=(conn, addr))
proc.daemon = True
proc.start()
logging.info('Started processing query %r for %s' % (proc, addr))
if __name__ == "__main__":
serv = ProxyServer(host, port)
try:
serv.start()
except:
finally:
for proc in multiprocessing.active_children():
proc.terminate()
proc.join()
Keep in mind that this is an example that I cut from old proof-of-concept code, you will have to tweak it a bit before it's ready for production.

How can i stop a socket thread in Python?

In Python 3.3.3， i create a thread to listen some connection to the socket.It likes this:
import threading
import socket
import time
Host = ''
Port = 50000
flag = False
class ServerThread(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def run(self):
try:
self._sock.bind((Host, Port))
self._sock.listen(5)
while True:
conn, addr = self._sock.accept()
print('Connected by', addr)
except socket.error as msg:
print(msg)
except Exception as e:
print(str(e))
finally:
self._sock.close()
def exit(self):
self._sock.close()
def TargetFunc(vlock):
vlock.acquire()
flag = True
vlock.release()
def main():
sthread = ServerThread()
sthread.start()
vlock = threading.Lock()
time.sleep(10)
vthread = threading.Thread(target = TargetFunc, args = (vlock, ))
vthread.start()
while True:
vlock.acquire()
if flag:
sthread.exit()
vlock.release()
break
vlock.release()
sthread.join()
vthread.join()
if __name__ == '__main__':
main()
There are two threads, one is listening socket, the other is to set a flag. When the flag is True, close the socket, then raise a socket error and catch it, so the listening socket terminates.But why it does not work this.
Thanks!

self._sock.accept() is blocking. So it will wait until somebody connects. You should use a nonblocking variant (or blocking but with a time-out). So that you can check the exit conditions.
Alternatively you could force an exception in the ServerThread.