We Keep Coding

Thread a server to accept more than one client doesn't work properly

Well, I'm trying to make a simple Network TCP chatting program to dive deeper in python, threading and networking. The program worked but with just one user, I looked this up, I found that I need threading to make the server accept more than one user. I threaded the server but now when you connect the second user it disconnect the first one. Source code may not be that good..
#!/usr/bin/python
import socket, sys, threading
from time import sleep
# Global Stuff
localhost = socket.gethostbyname(socket.gethostname())
#localhost = '192.168.x.x'
serverPort = 5003
buffer = 1024 #Bytes
backlog = 5
userThread= []
count = 0
class server(object):
''' Constructor to Establish Bind server once an object made'''
def __init__(self, localhost, serverPort): # Connect Tcp
global backlog, count
self.servSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.servSock.bind((localhost, serverPort))# bind((host,Port))
self.servSock.listen(backlog)
print count
except Exception, e:
print "[Bind ]", e
sys.exit()
def accept(self):
global userThread, conn, addr, count
"""
- PROBLEM IS IN HERE SOMEWHERE SERVER DOESN'T ADD THE OTHER CLIENT EXCEPT ONCE.
- THREAD DOEN'T KEEP THE CLIENT.
- THE SECOND CLIENT FREEZES WHILE SENDING THE VERY FIRST MESSAGE TILL THE FIRST
CLIENT SEND A MESSAGE THEN IT CAN SEND MESSAGES AND THE FIRST CLIENT CAN'T SEND SHIT.
"""
count+=1
while True:
print count
self.conn, self.addr = self.servSock.accept()
conn = self.conn
print("This is a connection: ", conn)
#acceptThread = threading.start_new_thread(target=serverObj.accept, args=(conn))
#addr = self.addr
print "[Listening..]"
if(self.addr not in userThread):
userThread.append(self.addr)
print "Client's added Successfully"
else:
pass
def redirect(self):
global buffer, userThread, conn, count
count+=1
while True:
try:
print "Redirecting " + str(count)
self.data = conn.recv(buffer)
if self.data:
for user in userThread:
#conn.send(b'Recieved by server!\n')
conn.sendto("Sent!\n"+self.data+"\n", user)
print "Server: Data sent[" +self.data+"] to ["+str(user)+"]"
else:
self.data = conn.recv(buffer)
print "No dataa found"
except Exception, e:
print "[Redirect ] ",e
sleep(7)
print "OUT"# testing if it's getting out this infinite loop.
def exit(self):
self.server.close()
def main():
global localhost, serverPort, conn
try:
serverObj = server(localhost, serverPort)
print("[+] Server is UP!")
except Exception, e:
print "[Main ] ",e
exit()
acceptThread = threading.Thread(name = "Accepting Connections", target=serverObj.accept)
redirThread = threading.Thread(name = "Redirecting Data", target=serverObj.redirect)
acceptThread.start()
redirThread.start()
print userThread
main()
######################################### Client ##########################################
#!/usr/bin/python
# This is client file
"""
http://eli.thegreenplace.net/2011/05/18/code-sample-socket-client-thread-in-python
https://docs.python.org/2/library/threading.html
"""
import socket
import threading
from time import sleep
# Client Info
#clientSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#localhost = '192.168.x.x'
# Global Stuff
serverIP = socket.gethostbyname(socket.gethostname())
#serverIP = '192.168.x.x'
serverPort, MsgSendError, MsgSendSucc, clientPort, data, buffer =\
5003, False, True, 12345, '',1024 #Bytes
class client(object):
global MsgSendError, MsgSendSucc, buffer, data
''' Constructor to Establish Connection once client is up'''
def __init__(self, serverIP, serverPort): # Connect Tcp
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.sock.connect((serverIP, serverPort))
except Exception, e:
return "[Connecting to Server]", e
def send(self, data):
try:
self.sock.send(data) # covnert it from string into byte streams to be in proper format.
#print str(data)
return MsgSendSucc
except:
return MsgSendError
def receive2(self):
try:
data = self.sock.recv(buffer)
#print "Function: Receive2."# testing
#print(str(data))# testing
#print "Received!"# testing
return str(data)
except Exception, e:
return "[In receive2]", e
def main():
global serverIP, serverPort, data#, sock
clientObj = client(serverIP, serverPort)
alias = raw_input("Your Name USER! ")
sentData = ''
while sentData is not 'Quit':
sentData = raw_input("Data>> ")
data = alias + ": "+sentData
if clientObj.send(data) == MsgSendSucc:
#print "Sent!"
#print "Fetching..\n"# testing
print(clientObj.receive2())
# testing
main()

python socket tchat issue

I started to code in python with sockets and I have a little problem for my chat script.
Server script
import pickle, socket, struct, sys, threading
SERVERADDRESS = ("localhost", 6030)
class helloChatServer(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
self.__server = socket.socket()
self.users = []
try:
self.__server.bind(SERVERADDRESS)
except socket.error:
print('Bind failed {}'.format(socket.error))
self.__server.listen(10)
def exit(self):
self.__server.close()
def run(self):
print( "Listening... {}".format(SERVERADDRESS))
while True:
client, addr = self.__server.accept()
try:
threading.Thread(target=self._handle, args=(client, addr)).start()
except OSError:
print('Error during processing the message')
def _handle(self, client, addr):
print('Client connected with {}:{}'.format(addr[0], str(addr[1])))
self.users.append(addr)
while True:
data = client.recv(1024)
print(data)
client.send(data)
client.close()
if __name__ == '__main__':
helloChatServer().run()
Client script
import pickle, socket, struct, sys, threading
SERVERADDRESS = (socket.gethostname(), 6050)
class helloChatClient():
def __init__(self, host='localhost', port=5000, pseudo="Visitor"):
self.__socket = socket.socket()
self.__socket.bind((host, port))
self.__pseudo = pseudo
print('Listening on {}:{}'.format(host, port))
def run(self):
handlers = {
'/exit': self._exit,
'/quit': self._quit,
'/join': self._join,
'/send': self._send
}
self.__running = True
self.__address = None
threading.Thread(target=self._receive).start()
while self.__running:
line = sys.stdin.readline().rstrip() + ' '
# Extract the command and the param
command = line[:line.index(' ')]
param = line[line.index(' ')+1:].rstrip()
# Call the command handler
if command in handlers:
try:
handlers[command]() if param == '' else handlers[command](param)
except:
print("Error during the execution of the message")
else:
print('Command inconnue:', command)
def _exit(self):
self.__running = False
self.__address = None
self.__socket.close()
def _quit(self):
self.__address = None
def _join(self, param):
if self.__pseudo == "Visitor":
self.__pseudo = input("Choose a username: ")
tokens = param.split(' ')
if len(tokens) == 2:
try:
self.__address = (tokens[0], int(tokens[1]))
self.__socket.connect(self.__address)
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
print('Connected at {}:{}'.format(*self.__address))
print('~~~~~~~~~~~~~~~~~~~~~~~~~~')
except OSError:
print("Error during the sending of the message")
self.__socket.send(self.__pseudo.encode())
def _send(self, param):
if self.__address is not None:
try:
message = param.encode()
totalsent = 0
while totalsent < len(message):
sent = self.__socket.send(message[totalsent:])
totalsent += sent
print(self.__socket.recv(1024).decode())
except OSError:
print('Error during the reception of the message')
def _receive(self):
while self.__running:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
if __name__ == '__main__':
if len(sys.argv) == 4:
helloChatClient(sys.argv[1], int(sys.argv[2]), sys.argv[3]).run()
else:
helloChatClient().run()
Well when I run the script on the terminal, I see this.
Server
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatServer.py
En écoute sur... ('MacBook-Pro-de-Saikou.local', 6030)
Client connected with 127.0.0.1:5004
Il y a actuellement 1 connecté
b'bluebeel'
b'hello'
Client
MacBook-Pro-de-Saikou:labo2 saikouah$ python3.4 helloChatClient.py localhost 5004 bluebeel
Écoute sur localhost:5004
/join MacBook-Pro-de-Saikou.local 6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
Connecté à MacBook-Pro-de-Saikou.local:6030
~~~~~~~~~~~~~~~~~~~~~~~~~~
/send hello
bluebeel
On the client terminal he doesn't print me hello but bluebeel. I made several test and he took me every time the previous one message. Looks like he is late.
Someone can help me? :)

PROBLEM ANALYSIS
Your code fails in _receive function:
data = self.__socket.recv(1024).decode()
This line throws OSError because you try to call .recv before connecting to the server. Thus the exception handler is fired and the function exits. So what happens is that after calling
threading.Thread(target=self._receive).start()
function _receive exits before you call /join. So watch what happens
You call /join.
bluebeel is send to the server
Server receives it and sends it back to the client
But _receive function is no longer there. So the message is "stacked" on the socket (it will wait for next .recv() call)
You call /send hello
Server receives hello and sends it back
Client calls print(self.__socket.recv(1024).decode()) in _send method
But .recv retrieves the first message that is stacked on the socket. In that case it is not hello, it is bluebeel.
Now this schema continues to work. You send message, server pings it back but there's always 1 message in front of the received one. The "late" message.
SOLUTION
One way of solving this issue is to call
threading.Thread(target=self._receive).start()
in ._join method after .connect. Remember to remove print(self.__socket.recv(1024).decode()) from _send method, otherwise it will block stdin.
Of course you will have problems when issuing multiple /join commands. To properly address that you would have to keep track of _receive thread and kill it at the begining of ._join method. This however is beyond the scope of this question IMHO.
SIDE NOTE
Don't ever handle exceptions like you did. This is wrong:
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
pass
except OSError:
return
At least do this:
import traceback
try:
data = self.__socket.recv(1024).decode()
print(data)
except socket.timeout:
traceback.print_exc()
except OSError:
traceback.print_exc()
return

Incomplete data transfer over python socket

I'm running into issues transferring data over TCP with a remote client and server written in Python. The server is located in a pretty remote region with relatively slow internet connection (<2Mb/sec). When the client is run on the LAN with the server the complete string is transferred (2350 bytes); however, when I run the client outside of the LAN sometimes the string is truncated (1485 bytes) and sometimes the full string comes through (2350 bytes). The size of the truncated string always seems to be 1485 bytes. The full size of the string is well below the set buffer size for the client and server.
I've copied abbreviated versions of the client and server code below, where I have tried to edit out all extraneous details:
Client
import socket
from time import sleep
class FTIRdataClient():
def __init__(self,TCP_IP="xxx.xxx.xxx.xxx",TCP_Port=xxx,BufferSize=4096):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def writeTCP(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.send(message)
incomming = sock.recv(self.RECV_BUFFER)
sock.close()
except:
print "Unable to connect to data server!!"
incomming = False
return incomming
if __name__ == "__main__":
#----------------------------------
# Initiate remote data client class
#----------------------------------
dataClass = FTIRdataClient(TCP_IP=dataServer_IP,TCP_Port=portNum,BufferSize=4096)
#--------------------------------
# Ask database for all parameters
#--------------------------------
allParms = dataClass.writeTCP("LISTALL")
Server
import os
import sys
import socket
import select
import smtplib
import datetime as dt
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
#------------------------------------------------
# Three types of call to server:
# 1) set -- sets the value of a data parameter
# 2) get -- gets the value of a data parameter
# 3) write -- write data to a file
#------------------------------------------------
splitVals = data.strip().split()
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#---------------------------------------------------
# Remove client from socket list after disconnection
#---------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
def closeServer(self):
''' Close the TCP data server '''
self.server_socket.close()
Your help is greatly appreciated!!!

For anyone who is interested I found the solution to this problem. John Nielsen has a pretty good explanation here. Basically, TCP stream only guarantees that bytes will not arrive out of order or be duplicated; however, it does not guarantee how many groups the data will be sent in. So one needs to continually read (socket.recv) until all the data is sent. The previous code work on the LAN because the server was sending the entire string in one group. Over a remote connection the string was split into several groups.
I modified the client to continually loop on socket.recv() until the socket is closed and I modified the server to immediately close the socket after sending the data. There are several other ways to do this mentioned in the above link. The new code looks like:
Client
class FTIRdataClient(object):
def __init__(self,TCP_IP="xxx.xxx.xx.xxx",TCP_Port=xxxx,BufferSize=4024):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def setParam(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.sendall("set "+message)
#-------------------------
# Loop to recieve all data
#-------------------------
incommingTotal = ""
while True:
incommingPart = sock.recv(self.RECV_BUFFER)
if not incommingPart: break
incommingTotal += incommingPart
sock.close()
except:
print "Unable to connect to data server!!"
incommingTotal = False
return incommingTotal
Server
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
elif splitVals[0].upper() == 'LISTALLTS': # List all time stamps
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParamTS:
msgLst.append("{0:}={1:}".format(k,self.dataParamTS[k]))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#------------------------
# Close socket connection
#------------------------
sock.close()
self.connection_list.remove(sock)
#------------------------------------------------------
# Remove client from socket list if client discconnects
#------------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
Whatever. This is probably common knowledge and I'm just a little slow.

Sending numerous packets instead of 1 SSL Python

I have set up an experiment where I pass Modbus traffic over a SSL tunnel (this being the first thing I've ever done in python). I am able to send and receive data but when I send one request numerous requests are actually sent (see screenshot)
I've tried numerous configurations including (in both client and server):
send()--no change
sendall() --no change
setblocking(1)
setblocking(0)--doesn't read all the data
On the server side:
if data == Read_Coils_Answer-- I don't think I'm converting the big endian properly for comparison and this didn't work
while data: --the while loop seems to be the only way to prevent either side from stopping short with a "Broken Pipe" error. So this is what I'm using.
I eventually plan to use a for loop (now commented out and set to 4).
My Server code:
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def do_something(connstream, data):
readCoilsReq = str('\x01\x01\x00')
answer = str(ModbusPDU01_Read_Coils_Answer)
while data:
print ("Request Recevied from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
return False
def deal_with_client(connstream):
data = connstream.recv(64)
while data:
if not do_something(connstream, data):
break
data = connstream.recv(64)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Recieved\n")
sys.exit(1)
My Client Side code:
from ModLib import *
from time import sleep
import socket, ssl, pprint
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_sock = ssl.wrap_socket(s,
ca_certs="server.crt",
cert_reqs=ssl.CERT_REQUIRED)
ssl_sock.connect(('localhost', 502))
ssl_sock.setblocking(1)
readCoils = ModbusPDU01_Read_Coils()
#for i in range(4):
sleep(2)
ssl_sock.sendall(str(readCoils))
print ("Request for Read Coils Sent")
#start receive
data = ssl_sock.recv(64)
print ("Response from Server:")
print pprint.pformat(data)
if False: #from the python docs
ssl_sock.write("""GET / HTTP/1.0\r
Host: www.verisign.com\n\n""")
data = ssl_sock.read()
ssl_sock.close()

The do_something() loop was not necessary, as the deal_with_client() loop was doing the same thing. I removed do_something() and put the code in deal_with_client() which allows me to keep the connection open (see below)
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def deal_with_client(connstream):
data = connstream.recv(1120)
answer = str(ModbusPDU01_Read_Coils_Answer())
while data:
print ("Request Received from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
data = connstream.recv(1120)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Received\n")
sys.exit(1)

Python socket can't receive data

I'm writing interprocess communication using localhost sockets in Python 3.2 and testing it on Windows. Here is some test code with a server and a client, sending messages to each other. Strangely, it fails randomly with the RuntimeError error, raised in the receive function, somewhere around the 5th or 10th connection.
#!/usr/bin/python
# -*- coding: utf-8 -*-
import socket
import pickle
import time
import logging
from multiprocessing import Process
def receive(conn):
def ensure_receive(length):
parts = []
received = 0
while received < length:
chunk = conn.recv(length - received)
if not chunk:
raise RuntimeError("Connection broken")
parts.append(chunk)
received += len(chunk)
return b''.join(parts)
lengthString = ensure_receive(8)
serialized = ensure_receive(int(lengthString))
return pickle.loads(serialized)
def send(conn, message):
def ensure_send(message):
sent = 0
while sent < len(message):
sent += conn.send(message[sent:])
# logging.warning("Now sending")
serialized = pickle.dumps(message, 1)
messageLength = len(serialized)
ensure_send("{:8}".format(messageLength).encode('Latin-1'))
ensure_send(serialized)
def client_function(clientLimit):
for index in range(1, clientLimit + 1):
print ("Client", index)
try:
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect(('localhost', 12333))
send(conn, list(range(100000)))
message = receive(conn)
send(conn, list(range(100)))
# time.sleep(0.01)
conn.shutdown(socket.SHUT_WR)
conn.close()
except Exception:
logging.exception("Socket error in client")
def server_function(clientLimit):
newSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
newSocket.bind(('localhost', 12333))
newSocket.listen(16)
for _ in range(clientLimit):
(conn, address) = newSocket.accept()
time.sleep(0.01)
message = receive(conn)
send(conn, list(range(10)))
message = receive(conn)
conn.shutdown(socket.SHUT_WR)
conn.close()
def test(clientLimit):
server = Process(target = server_function, args = (clientLimit,))
server.start()
time.sleep(1)
client = Process(target = client_function, args = (clientLimit,))
client.start()
client.join()
server.join()
if __name__ == "__main__":
test(100)
However, there are no errors if I uncomment time.sleep(0.01) in client_function, or if I change message order a bit.
Is there a way to make it work, without putting in random waits, and allowing for arbitrary protocols?

It is because of conn.shutdown(socket.SHUT_WR) in your server_function. What you need is socket.SHUT_RD, or better yet, get rid of the shutdown() call at all.