We Keep Coding

Attempting secure socket communication between client and server and getting An operation was attempted on something that is not a socket error

I am attempting to piece together a secure socket client server communication solution. I do not have experience in doing so, so have cobbled together what I believe are relevant sections. The idea is that the Server waits for connections, the client creates a connection that is secure and then communication can take place.
The code also utilizes secure communication in authorization with client and server keys and certificates.
client code:
class Client:
def __init__(self):
try:
self.host, self.port = "127.0.0.1", 65416
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self._context = ssl.SSLContext()
self._context.load_cert_chain(self.client_cert, self.client_key)
self._sock = None
self._ssock = None
except Exception as e:
print("Error in Initializing")
def checkvalidclient(self):
# ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock,)
self._ssock.connect((HOST, PORT))
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
Response = self._ssock.recv(1024)
while True:
Input = input('Say Something: ')
# s.send(str.encode(Input))
send_msg(self._ssock, str.encode(Input))
# Response = s.recv(1024)
Response = recv_msg(self._ssock)
if Response is not None:
print(Response.decode('utf-8'))
def closesockconnection(self):
self._ssock.close()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
client = Client()
client.checkvalidclient()
Server code:
import socket
import os
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
try:
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
print("Exception in recvall : " + str(e))
# ---- Server Communication Setup
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_verify_locations(self.client_cert)
self.sock = None
def connect(self):
try: # create socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
print(str(e))
print('Waiting for a Connection..')
self.sock.listen(3)
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
print(data)
if data is not None:
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, str.encode(reply))
#conn.close()
def waitforconnection(self):
while True:
Client, addr = self.sock.accept()
self._context.wrap_socket(Client, server_side=True)
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
start_new_thread(self.threaded_client, (Client, )) # Calling threaded_client() on a new thread
self.ThreadCount += 1
print('Thread Number: ' + str(self.ThreadCount))
#self.sock.close()
server = Server()
server.connect()
server.waitforconnection()
The lines:
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
result in the error:
[WinError 10038] An operation was attempted on something that is not a socket
When I removed the certificate related lines in client:
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self._context = ssl.SSLContext()
self._context.load_cert_chain(self.client_cert, self.client_key)
and the certificate related lines in the server:
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(self.server_cert, self.server_key)
self._context.load_verify_locations(self.client_cert)
self.sock = None
and a couple of small changes to remove the certificate related functionality, everything seemed to work, the client could send messages to the server and the server could respond (and the client displayed the response).
When however I added the context related certificates I start getting the error:
An operation was attempted on something that is not a socket
The server waits at:
Client, addr = self.sock.accept()
and continues to run once the client has called (in the client.py file):
self._ssock.connect((HOST, PORT))
The server then reaches the lines:
def threaded_client(self, conn: socket):
conn.send(str.encode('Welcome to the Server'))
where it fails on this error.
Printing the terminal, a traceback and exception error results in:
Socket successfully created
Waiting for a Connection..
Connected to: 127.0.0.1:57434
Thread Number: 1
Traceback (most recent call last):
File "c:\testcode\Server.py", line 71, in threaded_client
conn.send(str.encode('Welcome to the Server'))
OSError: [WinError 10038] An operation was attempted on something that is not a socket
My knowledge is limited and I cannot find more examples of secure multi threaded two way communication client to server socket code. The idea is to ensure the client is authorized to communicate with the server before transmission happens.
Any ideas on where I am failing?
Thanks

Ok, It seems like I was close, but had a couple of tweaks to do.
The solution of:
SSL/TLS client certificate verification with Python v3.4+ SSLContext
and the commenters here, helped me get over the finish line.
Server code:
import socket
import os
from socket import AF_INET, SOCK_STREAM, SO_REUSEADDR, SOL_SOCKET, SHUT_RDWR
import ssl
from os import path
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
import traceback
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
try:
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
except Exception as e:
print("Exception in recvall : " + str(e))
# ---- Server Communication Setup
class Server:
def __init__(self):
self.HOST = '127.0.0.1' # Standard loopback interface address (localhost)
self.PORT = 65416 # Port to listen on (non-privileged ports are > 1023)
self.ThreadCount = 0
self.server_cert = path.join(path.dirname(__file__), "server.crt")
self.server_key = path.join(path.dirname(__file__), "server.key")
self.client_cert = path.join(path.dirname(__file__), "client.crt")
self._context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
self._context.verify_mode = ssl.CERT_REQUIRED
self._context.load_cert_chain(certfile=self.server_cert, keyfile=self.server_key)
self._context.load_verify_locations(cafile=self.client_cert)
self.sock = None
def connect(self):
try: # create socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) ###<-- socket.socket() ???
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
self.sock.bind((self.HOST, self.PORT))
except socket.error as e:
print(str(e))
print('Waiting for a Connection..')
self.sock.listen(3)
def threaded_client(self, conn: socket):
try:
conn.send(str.encode('Welcome to the Server'))
while True:
data = recv_msg(conn)
print("data")
print(data)
if data is not None:
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
send_msg(conn, str.encode(reply))
except Exception as e:
print(traceback.format_exc())
print(str(e))
finally:
print("Closing connection")
conn.shutdown(socket.SHUT_RDWR)
conn.close()
#conn.close()
def waitforconnection(self):
while True:
Client, addr = self.sock.accept()
conn = self._context.wrap_socket(Client, server_side=True)
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
print("SSL established. Peer: {}".format(conn.getpeercert()))
start_new_thread(self.threaded_client, (conn, )) # Calling threaded_client() on a new thread
self.ThreadCount += 1
print('Thread Number: ' + str(self.ThreadCount))
#self.sock.close()
server = Server()
server.connect()
server.waitforconnection()
Client code:
import socket
import struct # Here to convert Python data types into byte streams (in string) and back
import sys
import ssl
import socket
import selectors
import types
import io
import os
import time
import requests
from pathlib import Path
import mysql.connector as mysql
from loguru import logger as log
from utils.misc import read_py_config
import json
import rsa
import base64
class Client:
def __init__(self):
self.host, self.port = "127.0.0.1", 65416
self.client_cert = os.path.join(os.path.dirname(__file__), "client.crt")
self.client_key = os.path.join(os.path.dirname(__file__), "client.key")
self.server_crt = os.path.join(os.path.dirname(__file__), "server.crt")
self.sni_hostname = "example.com"
self._context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=self.server_crt)
self._context.load_cert_chain(certfile=self.client_cert, keyfile=self.client_key)
self._sock = None
self._ssock = None
def checkvalidclient(self):
# ---- Client Communication Setup ----
HOST = self.host # The server's hostname or IP address
PORT = self.port # The port used by the server
try:
self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self._ssock = self._context.wrap_socket(self._sock, server_side=False, server_hostname=self.sni_hostname)
self._ssock.connect((HOST, PORT))
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
Response = self._ssock.recv(1024)
if Response is not None:
print(Response.decode('utf-8'))
while True:
Input = input('Say Something: ')
send_msg(self._ssock, str.encode(Input))
Response = recv_msg(self._ssock)
if Response is not None:
print(Response.decode('utf-8'))
def closesockconnection(self):
self._ssock.close()
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
Also ensure (as per the link) that the certificate creation is correct.
There is also another useful link at:
Exploring HTTPS With Python
Which covers HTTPS, specifically the Wireshark section allows you to monitor the traffic from client to server. After completing the above and deploying Wireshark I see that the data is encrypted. Any editing of the certificates (manually) causes the app to fail.
There still needs to be additions of try and except if the communication is halted midway etc. But hoping it will smooth the journey for others.
Thanks to the commenters, helped lead me on the way to solution.

Sending and receiving using sockets python

I am trying to create a function to send and receive information over a socket client & server. It appears that my code is somehow blocking. In the code the first command iteration in my for loop is carried out but then the process becomes blocked. Does anyone have any suggestions how to do this using threading or multithreading?
My code is below:
import socket
import json
import sys
import time
import select
import queue
Ni_Rio_IP= "172.22.11.2"
Ni_Base_IP= "172.22.11.1"
class AliceRio:
def __init__(self, ip_rio, ip_pc):
self.ip_rio = ip_rio
AliceRio.udp_port_rio = 60006
self.ip_pc = ip_pc
AliceRio.udp_port_pc = 50005
AliceRio.json= '{"Dest":"","Name":"","Time":"","Val":{"Str":[],"Pos":[[]],"Data":[[]]},"IP":0,"Port":0,"RT error":{"status":false,"code":0,"source":""}}'
AliceRio.dict= json.loads(self.json)
def PrintUDP(self):
print("RIO IP: %s" % self.ip_rio)
print("RIO UDP port: %s" % self.udp_port_rio)
print("PC IP: %s" % self.ip_pc)
print("PC UDP port: %s" % self.udp_port_pc)
def SendRec(self, send_str):
# Set up socket for sending
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # Internet, UDP
sock.sendto(bytes(send_str, 'utf-8'), (self.ip_rio, self.udp_port_rio))
sock.close()
print('got here')
# Set up socket for receiving
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # Internet, UDP
sock.bind((self.ip_pc, self.udp_port_pc))
rec_str, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
print('got here2')
sock.close()
return rec_str
def Receive(self, rec_str):
sock = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) # Internet, UDP
sock.bind((self.ip_pc, self.udp_port_pc))
rec_str, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
sock.close()
return rec_str
def Send(self, send_str):
# Set up socket for sending
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # Internet, UDP
sock.sendto(bytes(send_str, 'utf-8'), (self.ip_rio, self.udp_port_rio))
sock.close()
#return rec_str
def Aim(self, aim_perc):
if aim_perc < 0 or aim_perc > 100: return "aim_perc out of range"
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='Laser Control'
Laser_Mode=1
Simmer_A=0
Pulse_A= 0
Pulse_ms= 20
send_dict["Val"]["Str"]=[str(Laser_Mode), str(aim_perc), str(Simmer_A), str(Pulse_A), str(Pulse_ms)]
send_json=json.dumps(send_dict)
# send it out
self.SendRec(send_json)
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
return "Aim laser now at " + rec_dict["Val"]["Str"][1] +'%'
def PWM_Laser_Fan(self, fan_perc):
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='PWM Laser'
send_dict["Val"]["Str"][0]=str(fan_perc)
send_json=json.dumps(send_dict)
# send it out
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
return rec_dict["Val"]["Str"][0]
def Poll(self):
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='Poll'
send_json=json.dumps(send_dict)
# send it out
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
if rec_dict["Val"]["Data"][0][0]==0: pid_mode='off'
else: pid_mode='PID'
print('PID mode:', pid_mode)
print('Pos X:', rec_dict["Val"]["Data"][0][1])
print('Pos Y:', rec_dict["Val"]["Data"][0][2])
print('Home:', rec_dict["Val"]["Data"][0][3])
print('Enabled:', rec_dict["Val"]["Data"][0][4])
def PIDControl(self, pid_mode,pid_center):
if pid_mode=="off": mode= 0
elif pid_mode=="PID":mode =1
else: return "pid_mode not valid"
if pid_center[0] not in range(-2048,2048): return "center x-pos not in range"
if pid_center[1] not in range(-2048,2048): return "center y-pos not in range"
send_dict=AliceRio.dict
send_dict["Dest"]='Rio'
send_dict["Name"]='PID Control'
send_dict["Val"]["Str"]=[str(mode), str(pid_center[0]), str(pid_center[1])]
send_json=json.dumps(send_dict)
# send it out
rec_json= self.SendRec(send_json)
rec_dict=json.loads(rec_json)
return "PID mode now at " + rec_dict["Val"]["Str"][0]
Alice1 = AliceRio(Ni_Rio_IP, Ni_Base_IP)
Alice1.PrintUDP()
for i in range(10):
Alice1.Aim((i*10)+10)
time.sleep(0.2)

I would suggest learning to use Pdb and trace through the execution of your program to find where it is getting caught.
Also when learning/developing with sockets I've found that it helps to have separate programs for your client and server in the beginning so you can see how both sides are handling exchanges instead of going the threading route to start since the logging can get confusing, best of luck!

Module threading does help in this scenario.
We can create a thread to receiving incoming messages. And when new message received the thread trigger an event to notify the waiting method SendRec.
import sys
import socket
import json
import threading
import time
class AliceRio:
def __init__(self, .....):
# .........
self.s_in = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.s_in.bind((self.ip_pc, self.udp_port_pc))
self.evt = threading.Event()
self.last_msg = None
def _recv(self):
while True:
msg, _ = self.s_in.recvfrom(1024)
self.last_msg = msg
self.evt.set()
def SendRec(self, send_str):
if not hasattr(self, 'th_recv'):
th = threading.Thread(target=self._recv)
th.setDaemon(True)
th.start()
self.th_recv = th
self.evt.clear()
rio_endpoint = (self.ip_rio, self.udp_port_rio)
s_out = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s_out.sendto(bytes(send_str, 'utf-8'), rio_endpoint)
s_out.close()
if self.evt.wait(timeout=15.0) and self.last_msg:
return self.last_msg
raise Exception('timeout waiting for response.')

How to receive all data after TCP connection has been closed by the peer?

Running some production code I faced the following problem:
When sending HTTP requests to some server, server immediately closes the connection after sending response, which, for some reason, results in data loss.
Analyzing TCP dumps i can see that conversation goes as this:
client request
server ack
server push
server fin, ack (sent after ~0.000020 secs after previous push)
As the result my code can't get data sent by the server, (i'm guessing because of the small delay after push POLLHUP event might go before POLLIN?)
Trying to mimic the problem I've written the following code:
(It mimics the client behaviour on my side)
client:
import time
import socket
from errno import EAGAIN
from select import poll, POLLIN, POLLPRI, POLLERR, POLLHUP, POLLNVAL
def main(buf=""):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.setblocking(False)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
polling_object = poll()
polling_object.register(client, POLLPRI | POLLIN | POLLERR | POLLHUP)
in_buf = ""
sock_closed = False
try:
client.connect(("127.0.0.1", 8877))
except socket.error, e:
pass
while True and not sock_closed:
events = polling_object.poll(0)
for _, e in events:
if e & (POLLIN | POLLPRI):
while True:
try:
data = client.recv(1024)
if data:
in_buf += data
elif data == "":
client.close()
sock_closed = True
break
except socket.error, e:
if e.args[0] == EAGAIN:
break
else:
raise
elif e & (POLLERR|POLLHUP|POLLNVAL):
client.close()
sock_closed = True
if buf and not sock_closed:
try:
b_sent = client.send(buf)
if b_sent == len(buf):
buf = ""
else:
buf = buf[b_sent:]
except socket.error, e:
if e.args[0] != EAGAIN:
client.close()
sock_closed = True
time.sleep(0.5)
if sock_closed:
return in_buf
if __name__ == '__main__':
import sys
if len(sys.argv) > 1:
buf = sys.argv[1]
else:
buf = 'hello'
print main(buf)
server
import datetime
import time
import socket
def main():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", 8877))
server.listen(0)
client, _ = server.accept()
t1 = time.time()
data = ""
while not data:
data += client.recv(1024)
print "recv data %s" % data
client.sendall('{"ok": 1}')
t2 = time.time()
client.close()
t3 = time.time()
server.close()
return t1, t2, t3
if __name__ == '__main__':
c_r, d_s, c_c = main()
print "Connection received at ", datetime.datetime.fromtimestamp(c_r)
print "All Data sent after %.12f secs" % (d_s - c_r)
print "Connection closed after %.12f secs" % (c_c - d_s)
Running this code won't help me reproduce the problem because my client still can get data from socket buffer, which is kind of obviously by just following the code. The only difference is that in tcp dump it goes like this:
client request
server ack
server push
client ack
server fin, ack
I'm wondering is there a way to send fin, ack right after push without "letting" client to sent ack? Can it be done using python?

How can I make multiclient server in python using multithreading?

This is the code that I have used.But I don't get actual result that I want.When I execute code ChatServer file works properly,but ChatClient gives only one line(Usage : python telnet.py hostname port).Please Help me.I am new in python.
The server code:
#!/usr/bin/env python
#!/usr/bin/env python
"""
A basic, multiclient 'chat server' using Python's select module
with interrupt handling.
Entering any line of input at the terminal will exit the server.
"""
import select
import socket
import sys
import signal
from communication import send, receive
BUFSIZ = 1024
class ChatServer(object):
""" Simple chat server using select """
def __init__(self, port=3490, backlog=5):
self.clients = 0
# Client map
self.clientmap = {}
# Output socket list
self.outputs = []
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server.bind(('',port))
print 'Listening to port',port,'...'
self.server.listen(backlog)
# Trap keyboard interrupts
signal.signal(signal.SIGINT, self.sighandler)
def sighandler(self, signum, frame):
# Close the server
print 'Shutting down server...'
# Close existing client sockets
for o in self.outputs:
o.close()
self.server.close()
def getname(self, client):
# Return the printable name of the
# client, given its socket...
info = self.clientmap[client]
host, name = info[0][0], info[1]
return '#'.join((name, host))
def serve(self):
inputs = [self.server,sys.stdin]
self.outputs = []
running = 1
while running:
try:
inputready,outputready,exceptready = select.select(inputs, self.outputs, [])
except select.error, e:
break
except socket.error, e:
break
for s in inputready:
if s == self.server:
# handle the server socket
client, address = self.server.accept()
print 'chatserver: got connection %d from %s' % (client.fileno(), address)
# Read the login name
cname = receive(client).split('NAME: ')[1]
# Compute client name and send back
self.clients += 1
send(client, 'CLIENT: ' + str(address[0]))
inputs.append(client)
self.clientmap[client] = (address, cname)
# Send joining information to other clients
msg = '\n(Connected: New client (%d) from %s)' % (self.clients, self.getname(client))
for o in self.outputs:
# o.send(msg)
send(o, msg)
self.outputs.append(client)
elif s == sys.stdin:
# handle standard input
junk = sys.stdin.readline()
running = 0
else:
# handle all other sockets
try:
# data = s.recv(BUFSIZ)
data = receive(s)
if data:
# Send as new client's message...
msg = '\n#[' + self.getname(s) + ']>> ' + data
# Send data to all except ourselves
for o in self.outputs:
if o != s:
# o.send(msg)
send(o, msg)
else:
print 'chatserver: %d hung up' % s.fileno()
self.clients -= 1
s.close()
inputs.remove(s)
self.outputs.remove(s)
# Send client leaving information to others
msg = '\n(Hung up: Client from %s)' % self.getname(s)
for o in self.outputs:
# o.send(msg)
send(o, msg)
except socket.error, e:
# Remove
inputs.remove(s)
self.outputs.remove(s)
self.server.close()
if __name__ == "__main__":
ChatServer().serve()
The chat client:
#! /usr/bin/env python
"""
Simple chat client for the chat server. Defines
a simple protocol to be used with chatserver.
"""
import socket
import sys
import select
from communication import send, receive
BUFSIZ = 1024
class ChatClient(object):
""" A simple command line chat client using select """
def __init__(self, name, host='127.0.0.1', port=3490):
self.name = name
# Quit flag
self.flag = False
self.port = int(port)
self.host = host
# Initial prompt
self.prompt='[' + '#'.join((name, socket.gethostname().split('.')[0])) + ']> '
# Connect to server at port
try:
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect((host, self.port))
print 'Connected to chat server#%d' % self.port
# Send my name...
send(self.sock,'NAME: ' + self.name)
data = receive(self.sock)
# Contains client address, set it
addr = data.split('CLIENT: ')[1]
self.prompt = '[' + '#'.join((self.name, addr)) + ']> '
except socket.error, e:
print 'Could not connect to chat server #%d' % self.port
sys.exit(1)
def cmdloop(self):
while not self.flag:
try:
sys.stdout.write(self.prompt)
sys.stdout.flush()
# Wait for input from stdin & socket
inputready, outputready,exceptrdy = select.select([0, self.sock], [],[])
for i in inputready:
if i == 0:
data = sys.stdin.readline().strip()
if data: send(self.sock, data)
elif i == self.sock:
data = receive(self.sock)
if not data:
print 'Shutting down.'
self.flag = True
break
else:
sys.stdout.write(data + '\n')
sys.stdout.flush()
except KeyboardInterrupt:
print 'Interrupted.'
self.sock.close()
break
if __name__ == "__main__":
import sys
if len(sys.argv)<3:
sys.exit('Usage: %s chatid host portno' % sys.argv[0])
client = ChatClient(sys.argv[1],sys.argv[2], int(sys.argv[3]))
client.cmdloop()
###############################################################################
# The communication module (communication.py)
###############################################################################
import cPickle
import socket
import struct
marshall = cPickle.dumps
unmarshall = cPickle.loads
def send(channel, *args):
buf = marshall(args)
value = socket.htonl(len(buf))
size = struct.pack("L",value)
channel.send(size)
channel.send(buf)
def receive(channel):
size = struct.calcsize("L")
size = channel.recv(size)
try:
size = socket.ntohl(struct.unpack("L", size)[0])
except struct.error, e:
return ''
buf = ""
while len(buf) < size:
buf = channel.recv(size - len(buf))
return unmarshall(buf)[0]

Python socket can't receive data

I'm writing interprocess communication using localhost sockets in Python 3.2 and testing it on Windows. Here is some test code with a server and a client, sending messages to each other. Strangely, it fails randomly with the RuntimeError error, raised in the receive function, somewhere around the 5th or 10th connection.
#!/usr/bin/python
# -*- coding: utf-8 -*-
import socket
import pickle
import time
import logging
from multiprocessing import Process
def receive(conn):
def ensure_receive(length):
parts = []
received = 0
while received < length:
chunk = conn.recv(length - received)
if not chunk:
raise RuntimeError("Connection broken")
parts.append(chunk)
received += len(chunk)
return b''.join(parts)
lengthString = ensure_receive(8)
serialized = ensure_receive(int(lengthString))
return pickle.loads(serialized)
def send(conn, message):
def ensure_send(message):
sent = 0
while sent < len(message):
sent += conn.send(message[sent:])
# logging.warning("Now sending")
serialized = pickle.dumps(message, 1)
messageLength = len(serialized)
ensure_send("{:8}".format(messageLength).encode('Latin-1'))
ensure_send(serialized)
def client_function(clientLimit):
for index in range(1, clientLimit + 1):
print ("Client", index)
try:
conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
conn.connect(('localhost', 12333))
send(conn, list(range(100000)))
message = receive(conn)
send(conn, list(range(100)))
# time.sleep(0.01)
conn.shutdown(socket.SHUT_WR)
conn.close()
except Exception:
logging.exception("Socket error in client")
def server_function(clientLimit):
newSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
newSocket.bind(('localhost', 12333))
newSocket.listen(16)
for _ in range(clientLimit):
(conn, address) = newSocket.accept()
time.sleep(0.01)
message = receive(conn)
send(conn, list(range(10)))
message = receive(conn)
conn.shutdown(socket.SHUT_WR)
conn.close()
def test(clientLimit):
server = Process(target = server_function, args = (clientLimit,))
server.start()
time.sleep(1)
client = Process(target = client_function, args = (clientLimit,))
client.start()
client.join()
server.join()
if __name__ == "__main__":
test(100)
However, there are no errors if I uncomment time.sleep(0.01) in client_function, or if I change message order a bit.
Is there a way to make it work, without putting in random waits, and allowing for arbitrary protocols?

It is because of conn.shutdown(socket.SHUT_WR) in your server_function. What you need is socket.SHUT_RD, or better yet, get rid of the shutdown() call at all.