We Keep Coding

Attempting secure socket communication between client and server and getting An operation was attempted on something that is not a socket error

I am attempting to piece together a secure socket client server communication solution. I do not have experience in doing so, so have cobbled together what I believe are relevant sections. The idea is that the Server waits for connections, the client creates a connection that is secure and then communication can take place.
The code also utilizes secure communication in authorization with client and server keys and certificates.
client code:
class Client:
def __init__(self):
try:
self.host, self.port = "127.0.0.1", 65416
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self._context = ssl.SSLContext()
self._context.load_cert_chain(self.client_cert, self.client_key)
self._sock = None
self._ssock = None
except Exception as e:
print("Error in Initializing")
def checkvalidclient(self):
# ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock,)
self._ssock.connect((HOST, PORT))
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
Response = self._ssock.recv(1024)
while True:
Input = input('Say Something: ')
# s.send(str.encode(Input))
send_msg(self._ssock, str.encode(Input))
# Response = s.recv(1024)
Response = recv_msg(self._ssock)
if Response is not None:
print(Response.decode('utf-8'))
def closesockconnection(self):
self._ssock.close()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
client = Client()
client.checkvalidclient()
Server code:
import socket
import os
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
try:
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
print("Exception in recvall : " + str(e))
# ---- Server Communication Setup
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_verify_locations(self.client_cert)
self.sock = None
def connect(self):
try: # create socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
print(str(e))
print('Waiting for a Connection..')
self.sock.listen(3)
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
print(data)
if data is not None:
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, str.encode(reply))
#conn.close()
def waitforconnection(self):
while True:
Client, addr = self.sock.accept()
self._context.wrap_socket(Client, server_side=True)
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
start_new_thread(self.threaded_client, (Client, )) # Calling threaded_client() on a new thread
self.ThreadCount += 1
print('Thread Number: ' + str(self.ThreadCount))
#self.sock.close()
server = Server()
server.connect()
server.waitforconnection()
The lines:
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
result in the error:
[WinError 10038] An operation was attempted on something that is not a socket
When I removed the certificate related lines in client:
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self._context = ssl.SSLContext()
self._context.load_cert_chain(self.client_cert, self.client_key)
and the certificate related lines in the server:
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_verify_locations(self.client_cert)
self.sock = None
and a couple of small changes to remove the certificate related functionality, everything seemed to work, the client could send messages to the server and the server could respond (and the client displayed the response).
When however I added the context related certificates I start getting the error:
An operation was attempted on something that is not a socket
The server waits at:
Client, addr = self.sock.accept()
and continues to run once the client has called (in the client.py file):
self._ssock.connect((HOST, PORT))
The server then reaches the lines:
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
where it fails on this error.
Printing the terminal, a traceback and exception error results in:
Socket successfully created
Waiting for a Connection..
Connected to: 127.0.0.1:57434
Thread Number: 1
Traceback (most recent call last):
File "c:\testcode\Server.py", line 71, in threaded_client
conn.send(str.encode('Welcome to the Server'))
OSError: [WinError 10038] An operation was attempted on something that is not a socket
My knowledge is limited and I cannot find more examples of secure multi threaded two way communication client to server socket code. The idea is to ensure the client is authorized to communicate with the server before transmission happens.
Any ideas on where I am failing?
Thanks

Ok, It seems like I was close, but had a couple of tweaks to do.
The solution of:
SSL/TLS client certificate verification with Python v3.4+ SSLContext
and the commenters here, helped me get over the finish line.
Server code:
import socket
import os
from socket import AF_INET, SOCK_STREAM, SO_REUSEADDR, SOL_SOCKET, SHUT_RDWR
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
import traceback
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
try:
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
print("Exception in recvall : " + str(e))
# ---- Server Communication Setup
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(certfile=self.server_cert, keyfile=self.server_key)
self._context.load_verify_locations(cafile=self.client_cert)
self.sock = None
def connect(self):
try: # create socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) ###<-- socket.socket() ???
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
print(str(e))
print('Waiting for a Connection..')
self.sock.listen(3)
def threaded_client(self, conn: socket):
try:
conn.send(str.encode('Welcome to the Server'))
while True:
data = recv_msg(conn)
print("data")
print(data)
if data is not None:
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
send_msg(conn, str.encode(reply))
except Exception as e:
print(traceback.format_exc())
print(str(e))
finally:
print("Closing connection")
conn.shutdown(socket.SHUT_RDWR)
conn.close()
#conn.close()
def waitforconnection(self):
while True:
Client, addr = self.sock.accept()
conn = self._context.wrap_socket(Client, server_side=True)
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
print("SSL established. Peer: {}".format(conn.getpeercert()))
start_new_thread(self.threaded_client, (conn, )) # Calling threaded_client() on a new thread
self.ThreadCount += 1
print('Thread Number: ' + str(self.ThreadCount))
#self.sock.close()
server = Server()
server.connect()
server.waitforconnection()
Client code:
import socket
import struct # Here to convert Python data types into byte streams (in string) and back
import sys
import ssl
import socket
import selectors
import types
import io
import os
import time
import requests
from pathlib import Path
import mysql.connector as mysql
from loguru import logger as log
from utils.misc import read_py_config
import json
import rsa
import base64
class Client:
def __init__(self):
self.host, self.port = "127.0.0.1", 65416
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self.server_crt = os.path.join(os.path.dirname(__file__), "server.crt")
self.sni_hostname = "example.com"
self._context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=self.server_crt)
self._context.load_cert_chain(certfile=self.client_cert, keyfile=self.client_key)
self._sock = None
self._ssock = None
def checkvalidclient(self):
# ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock, server_side=False, server_hostname=self.sni_hostname)
self._ssock.connect((HOST, PORT))
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
Response = self._ssock.recv(1024)
if Response is not None:
print(Response.decode('utf-8'))
while True:
Input = input('Say Something: ')
send_msg(self._ssock, str.encode(Input))
Response = recv_msg(self._ssock)
if Response is not None:
print(Response.decode('utf-8'))
def closesockconnection(self):
self._ssock.close()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
Also ensure (as per the link) that the certificate creation is correct.
There is also another useful link at:
Exploring HTTPS With Python
Which covers HTTPS, specifically the Wireshark section allows you to monitor the traffic from client to server. After completing the above and deploying Wireshark I see that the data is encrypted. Any editing of the certificates (manually) causes the app to fail.
There still needs to be additions of try and except if the communication is halted midway etc. But hoping it will smooth the journey for others.
Thanks to the commenters, helped lead me on the way to solution.

P2P Network: Python sockets not receiving data from other nodes

I have checked the documentation online and several Stackoverflow posts, but cannot debug my code. I think I am incorrectly sending bytes, so the other nodes connected to the socket cannot 'hear' the calls.
Is there something wrong with my use of sendall(), or something wrong with how I'm packing my bytestring?
import logging
from random import randint
import socket
import sys
import time
import threading
class Server:
connections = list()
peers = list()
def __init__(self):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('0.0.0.0', 10000))
s.listen(1)
logging.info('Successfully started the server on port 10000.')
while True:
c, a = s.accept()
logging.info('New client node joins pool.')
t = threading.Thread(target=self.handler, args=(c, a))
t.daemon = True
t.start()
self.connections.append(c)
self.peers.append(a[0])
logging.info('New node connected: {}'.format(str(a[0])))
self.send_peers()
def handler(self, c, a):
while True:
data = c.recv(1024)
logging.info('Sending data: {}'.format(data))
for connection in self.connections:
connection.sendall(data)
if not data:
logging.info('Node disconnected: {}:{}'.format(str(a[0]), str(a[1])))
self.connections.remove(c)
self.peers.remove(a[0])
c.close()
self.send_peers()
break
def send_peers(self):
"""Sends the list of peers in the swarm to all connected peers."""
p = ''
for peer in self.peers:
p = p + peer + ','
for connection in self.connections:
connection.sendall(b'\x11' + p.encode())
class Client:
def __init__(self, address):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.connect((address, 10000))
logging.info('Successfully made a client connection at: {}. Listening..'.format(address))
t = threading.Thread(target=self.send_message, args=(s,))
t.daemon = True
t.start()
while True:
data = s.recv(1024)
print(data)
if not data:
break
if data[0:1] == b'\x11':
# Message prefixed with \x11, so we know it is a new node message
self.update_peers(data[1:])
else:
# Since the message received isn't prefixed with \x11, we know it is a message
logging.info('Received message: {}'.format(data.decode()))
#staticmethod
def send_message(s):
"""Sends a message to all connected nodes."""
while True:
message = str(input('Enter message: ')).encode()
s.sendall(message)
logging.info('Sending message: {}'.format(message.decode()))
#staticmethod
def update_peers(peer_data):
"""Refreshes the local copy of all connected nodes."""
logging.info('Node dropped/connected. Updating connected peers list.')
Tracker.peers = peer_data.decode.split(',')[:-1]
class Tracker:
"""Tracks the Peers connected within the swarm.
This is a separate class due to data accessibility. We want to be able to access the peers list
from outside the Client and Server classes.
"""
peers = ['127.0.0.1']
if __name__ == '__main__':
logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s',
level=logging.INFO,
datefmt='%Y-%m-%d %H:%M:%S')
logging.info('Connecting..')
while True:
try:
time.sleep(randint(1, 5))
for peer in Tracker.peers:
try:
server = Server()
except KeyboardInterrupt:
sys.exit(0)
except:
logging.warning('Server is already started. Becoming a client node.')
try:
client = Client(peer)
except KeyboardInterrupt:
sys.exit(0)
except:
logging.warning('Could not become a client node.')
pass
except KeyboardInterrupt:
sys.exit(0)

Thread a server to accept more than one client doesn't work properly

Well, I'm trying to make a simple Network TCP chatting program to dive deeper in python, threading and networking. The program worked but with just one user, I looked this up, I found that I need threading to make the server accept more than one user. I threaded the server but now when you connect the second user it disconnect the first one. Source code may not be that good..
#!/usr/bin/python
import socket, sys, threading
from time import sleep
# Global Stuff
localhost = socket.gethostbyname(socket.gethostname())
#localhost = '192.168.x.x'
serverPort = 5003
buffer = 1024 #Bytes
backlog = 5
userThread= []
count = 0
class server(object):
''' Constructor to Establish Bind server once an object made'''
def __init__(self, localhost, serverPort): # Connect Tcp
global backlog, count
self.servSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.servSock.bind((localhost, serverPort))# bind((host,Port))
self.servSock.listen(backlog)
print count
except Exception, e:
print "[Bind ]", e
sys.exit()
def accept(self):
global userThread, conn, addr, count
"""
- PROBLEM IS IN HERE SOMEWHERE SERVER DOESN'T ADD THE OTHER CLIENT EXCEPT ONCE.
- THREAD DOEN'T KEEP THE CLIENT.
- THE SECOND CLIENT FREEZES WHILE SENDING THE VERY FIRST MESSAGE TILL THE FIRST
CLIENT SEND A MESSAGE THEN IT CAN SEND MESSAGES AND THE FIRST CLIENT CAN'T SEND SHIT.
"""
count+=1
while True:
print count
self.conn, self.addr = self.servSock.accept()
conn = self.conn
print("This is a connection: ", conn)
#acceptThread = threading.start_new_thread(target=serverObj.accept, args=(conn))
#addr = self.addr
print "[Listening..]"
if(self.addr not in userThread):
userThread.append(self.addr)
print "Client's added Successfully"
else:
pass
def redirect(self):
global buffer, userThread, conn, count
count+=1
while True:
try:
print "Redirecting " + str(count)
self.data = conn.recv(buffer)
if self.data:
for user in userThread:
#conn.send(b'Recieved by server!\n')
conn.sendto("Sent!\n"+self.data+"\n", user)
print "Server: Data sent[" +self.data+"] to ["+str(user)+"]"
else:
self.data = conn.recv(buffer)
print "No dataa found"
except Exception, e:
print "[Redirect ] ",e
sleep(7)
print "OUT"# testing if it's getting out this infinite loop.
def exit(self):
self.server.close()
def main():
global localhost, serverPort, conn
try:
serverObj = server(localhost, serverPort)
print("[+] Server is UP!")
except Exception, e:
print "[Main ] ",e
exit()
acceptThread = threading.Thread(name = "Accepting Connections", target=serverObj.accept)
redirThread = threading.Thread(name = "Redirecting Data", target=serverObj.redirect)
acceptThread.start()
redirThread.start()
print userThread
main()
######################################### Client ##########################################
#!/usr/bin/python
# This is client file
"""
http://eli.thegreenplace.net/2011/05/18/code-sample-socket-client-thread-in-python
https://docs.python.org/2/library/threading.html
"""
import socket
import threading
from time import sleep
# Client Info
#clientSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#localhost = '192.168.x.x'
# Global Stuff
serverIP = socket.gethostbyname(socket.gethostname())
#serverIP = '192.168.x.x'
serverPort, MsgSendError, MsgSendSucc, clientPort, data, buffer =\
5003, False, True, 12345, '',1024 #Bytes
class client(object):
global MsgSendError, MsgSendSucc, buffer, data
''' Constructor to Establish Connection once client is up'''
def __init__(self, serverIP, serverPort): # Connect Tcp
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.sock.connect((serverIP, serverPort))
except Exception, e:
return "[Connecting to Server]", e
def send(self, data):
try:
self.sock.send(data) # covnert it from string into byte streams to be in proper format.
#print str(data)
return MsgSendSucc
except:
return MsgSendError
def receive2(self):
try:
data = self.sock.recv(buffer)
#print "Function: Receive2."# testing
#print(str(data))# testing
#print "Received!"# testing
return str(data)
except Exception, e:
return "[In receive2]", e
def main():
global serverIP, serverPort, data#, sock
clientObj = client(serverIP, serverPort)
alias = raw_input("Your Name USER! ")
sentData = ''
while sentData is not 'Quit':
sentData = raw_input("Data>> ")
data = alias + ": "+sentData
if clientObj.send(data) == MsgSendSucc:
#print "Sent!"
#print "Fetching..\n"# testing
print(clientObj.receive2())
# testing
main()

Incomplete data transfer over python socket

I'm running into issues transferring data over TCP with a remote client and server written in Python. The server is located in a pretty remote region with relatively slow internet connection (<2Mb/sec). When the client is run on the LAN with the server the complete string is transferred (2350 bytes); however, when I run the client outside of the LAN sometimes the string is truncated (1485 bytes) and sometimes the full string comes through (2350 bytes). The size of the truncated string always seems to be 1485 bytes. The full size of the string is well below the set buffer size for the client and server.
I've copied abbreviated versions of the client and server code below, where I have tried to edit out all extraneous details:
Client
import socket
from time import sleep
class FTIRdataClient():
def __init__(self,TCP_IP="xxx.xxx.xxx.xxx",TCP_Port=xxx,BufferSize=4096):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def writeTCP(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.send(message)
incomming = sock.recv(self.RECV_BUFFER)
sock.close()
except:
print "Unable to connect to data server!!"
incomming = False
return incomming
if __name__ == "__main__":
#----------------------------------
# Initiate remote data client class
#----------------------------------
dataClass = FTIRdataClient(TCP_IP=dataServer_IP,TCP_Port=portNum,BufferSize=4096)
#--------------------------------
# Ask database for all parameters
#--------------------------------
allParms = dataClass.writeTCP("LISTALL")
Server
import os
import sys
import socket
import select
import smtplib
import datetime as dt
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
#------------------------------------------------
# Three types of call to server:
# 1) set -- sets the value of a data parameter
# 2) get -- gets the value of a data parameter
# 3) write -- write data to a file
#------------------------------------------------
splitVals = data.strip().split()
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#---------------------------------------------------
# Remove client from socket list after disconnection
#---------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
def closeServer(self):
''' Close the TCP data server '''
self.server_socket.close()
Your help is greatly appreciated!!!

For anyone who is interested I found the solution to this problem. John Nielsen has a pretty good explanation here. Basically, TCP stream only guarantees that bytes will not arrive out of order or be duplicated; however, it does not guarantee how many groups the data will be sent in. So one needs to continually read (socket.recv) until all the data is sent. The previous code work on the LAN because the server was sending the entire string in one group. Over a remote connection the string was split into several groups.
I modified the client to continually loop on socket.recv() until the socket is closed and I modified the server to immediately close the socket after sending the data. There are several other ways to do this mentioned in the above link. The new code looks like:
Client
class FTIRdataClient(object):
def __init__(self,TCP_IP="xxx.xxx.xx.xxx",TCP_Port=xxxx,BufferSize=4024):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def setParam(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.sendall("set "+message)
#-------------------------
# Loop to recieve all data
#-------------------------
incommingTotal = ""
while True:
incommingPart = sock.recv(self.RECV_BUFFER)
if not incommingPart: break
incommingTotal += incommingPart
sock.close()
except:
print "Unable to connect to data server!!"
incommingTotal = False
return incommingTotal
Server
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
elif splitVals[0].upper() == 'LISTALLTS': # List all time stamps
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParamTS:
msgLst.append("{0:}={1:}".format(k,self.dataParamTS[k]))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#------------------------
# Close socket connection
#------------------------
sock.close()
self.connection_list.remove(sock)
#------------------------------------------------------
# Remove client from socket list if client discconnects
#------------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
Whatever. This is probably common knowledge and I'm just a little slow.

Python socket can't receive data

I'm writing interprocess communication using localhost sockets in Python 3.2 and testing it on Windows. Here is some test code with a server and a client, sending messages to each other. Strangely, it fails randomly with the RuntimeError error, raised in the receive function, somewhere around the 5th or 10th connection.
#!/usr/bin/python
# -*- coding: utf-8 -*-
import socket
import pickle
import time
import logging
from multiprocessing import Process
def receive(conn):
def ensure_receive(length):
parts = []
received = 0
while received < length:
chunk = conn.recv(length - received)
if not chunk:
raise RuntimeError("Connection broken")
parts.append(chunk)
received += len(chunk)
return b''.join(parts)
lengthString = ensure_receive(8)
serialized = ensure_receive(int(lengthString))
return pickle.loads(serialized)
def send(conn, message):
def ensure_send(message):
sent = 0
while sent < len(message):
sent += conn.send(message[sent:])
# logging.warning("Now sending")
serialized = pickle.dumps(message, 1)
messageLength = len(serialized)
ensure_send("{:8}".format(messageLength).encode('Latin-1'))
ensure_send(serialized)
def client_function(clientLimit):
for index in range(1, clientLimit + 1):
print ("Client", index)
try:
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect(('localhost', 12333))
send(conn, list(range(100000)))
message = receive(conn)
send(conn, list(range(100)))
# time.sleep(0.01)
conn.shutdown(socket.SHUT_WR)
conn.close()
except Exception:
logging.exception("Socket error in client")
def server_function(clientLimit):
newSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
newSocket.bind(('localhost', 12333))
newSocket.listen(16)
for _ in range(clientLimit):
(conn, address) = newSocket.accept()
time.sleep(0.01)
message = receive(conn)
send(conn, list(range(10)))
message = receive(conn)
conn.shutdown(socket.SHUT_WR)
conn.close()
def test(clientLimit):
server = Process(target = server_function, args = (clientLimit,))
server.start()
time.sleep(1)
client = Process(target = client_function, args = (clientLimit,))
client.start()
client.join()
server.join()
if __name__ == "__main__":
test(100)
However, there are no errors if I uncomment time.sleep(0.01) in client_function, or if I change message order a bit.
Is there a way to make it work, without putting in random waits, and allowing for arbitrary protocols?

It is because of conn.shutdown(socket.SHUT_WR) in your server_function. What you need is socket.SHUT_RD, or better yet, get rid of the shutdown() call at all.