I am developing a python socket server. The client send each message to start with a STX (\x02) and end with ETX (\x03). My code can receive message successfully but I can't implement receiving full string using STX and ETX condition. Need help in resolving this issue. Below I have sharing my code for better understanding.
import socket
import time
# Start New RnD
# Global Veriable
enq = chr(5)
ack = chr(6)
stx = chr(2)
etx = chr(3)
# Connect to the server with `telnet $HOSTNAME 5000`.
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setblocking(False)
server.bind(('0.0.0.0', 5000))
server.listen(1)
connections = []
while True:
try:
connection, address = server.accept()
connection.setblocking(False)
connections.append(connection)
except BlockingIOError:
pass
# Incoming Data Processing
for connection in connections:
try:
full_message = ''
data = ""
while True:
try:
received = connection.recv(1)
if received == enq.encode('utf-8'):
print("Received <ENQ>, Sending <ACK>")
connection.sendall(ack.encode('utf-8'))
if not received:
raise RuntimeError("unexpected end-of-message", data)
data += received.decode('utf-8')
#print("Received: {!r}".format(data))
if "\x03" in received.decode("utf-8") :
break
except BlockingIOError:
pass
print("Full Received: {!r}".format(data))
print("Data Received, Sending <ACK>")
connection.sendall(ack.encode('utf-8'))
except BlockingIOError:
continue
Related
Backstory: what I have done
{Codes at the bottom} I've already coded the multithreaded client and server programs using python socket, with the help of the following sites:
I. Echo Client and Server
II. Socket Server with Multiple Clients | Multithreading | Python
III. Python Socket Receive Large Amount of Data
Regarding Encryption & Decryption
(1) Exactly at what places in my codes should I encrypt/decrypt my message? Do
I encrypt the messages themselves after the user inputs or do I encrypt the byte streams after the input messages have been encoded?
(2) And how am I supposed to encrypt/decrypt the communication properly and efficiently? (It'd be nice to see code solutions with explanation, many thanks)
My Codes Currently
_server.py
import socket
import os
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
# ---- Server Communication Setup
HOST = '127.0.0.1' # Standard loopback interface address (localhost)
PORT = 65432 # Port to listen on (non-privileged ports are > 1023)
ThreadCount = 0
try: # create socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
s.bind((HOST, PORT))
except socket.error as e:
print(str(e))
print('Waitiing for a Connection..')
s.listen(3)
def threaded_client(conn: socket):
conn.send(str.encode('Welcome to the Server'))
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, str.encode(reply))
conn.close()
while True:
Client, addr = s.accept()
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
start_new_thread(threaded_client, (Client, )) # Calling threaded_client() on a new thread
ThreadCount += 1
print('Thread Number: ' + str(ThreadCount))
s.close()
_client.py
import socket
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
# ---- Client Communication Setup ----
HOST = '127.0.0.1' # The server's hostname or IP address
PORT = 65432 # The port used by the server
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
try:
s.connect((HOST, PORT))
except socket.error as e:
print(str(e))
Response = s.recv(1024)
while True:
Input = input('Say Something: ')
# s.send(str.encode(Input))
send_msg(s, str.encode(Input))
# Response = s.recv(1024)
Response = recv_msg(s)
print(Response.decode('utf-8'))
s.close()
You only need to Encrypt the Message itself. I would use RSA to Encrypt the Messages.
If you plan on using multiple Servers, you can open a second Port and if someone connects to it, the Server/Host. Sends the Public Key to the Client.
After that, the Client sends his Public key to the Server.
Now Server and Client switch Ports, and Communicate over there while using the Public key of the other to Encrypt the Messages and decrypt them with their Private key.
You can also Hard-code the Public Keys, if you only have one Server and one Client.
A good Module for RSA Encryption is PyCrytodome.
Here is an Example of to encrypt messages with PyCrytodome.
from Crypto.PublicKey import RSA
from Crypto.Cipher import PKCS1_OAEP
def encrypt(pk_receiver, message):
key = RSA.import_key(pk_receiver)
cipher = PKCS1_OAEP.new(key)
c = cipher.encrypt(message.encode())
return c
def decrypt(sk, c):
key = RSA.import_key(sk)
cipher = PKCS1_OAEP.new(key)
m = cipher.decrypt(c)
return m
def generate_sk(key_length):
key = RSA.generate(key_length)
with open('./secret_key.pem', 'wb') as f:
f.write(key.export_key(format='PEM'))
return key
def generate_pk(sk):
pk = sk.public_key()
with open('./public_key.pem', 'wb') as f:
f.write(pk.export_key(format='PEM'))
return
I'm trying to send a 2D array across a socket, it needs to be made out of multiple socket.recv() otherwise I will get a _pickle.UnpicklingError: pickle data was truncated error.
I've tried to do this with a while loop that receives packets and appends them to a list until all data is received:
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
Just doing:
def receive(self):
data = self.socket.recv(1024)
data = pickle.loads(data)
return data
Works if I send something smaller than the 2D array e.g. a coordinate pair tuple (2, 3). But not with the 2D array.
I get a OSError: [WinError 10022] when attempting the while loop approach.
I have seen that a OSError: [WinError 10022] could be an issue with not binding the socket but I think I have done that.
I can't figure out where connections are closing and am very confused.
Rest of the code:
Server:
import socket
from _thread import start_new_thread
clients = []
def threaded(client):
while True:
data = client.recv(1024)
if not data:
print('Not Data.')
break
# Send the data received from one client to all the other clients.
for c in clients:
if c != client:
c.send(data)
client.close()
def Main():
host = ""
port = 5555
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
print("Socket binded to port:", port)
s.listen(2)
print("Socket is listening.")
while True:
c, addr = s.accept()
clients.append(c)
print(f"Connected to: {addr[0]}:{addr[1]}")
start_new_thread(threaded, (c,))
s.close()
Client:
import socket
import pickle
class Client:
def __init__(self):
self.host = 'localhost'
self.port = 5555
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((self.host, self.port))
def send(self, data):
message = pickle.dumps(data)
try:
self.socket.send(message)
except socket.error as e:
return str(e)
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
The client-server I want to make will have 2 clients and each client will send data to the other constantly. How can I correct my code and properly implement this?
After doing more research and figuring out what was going wrong I got a solution.
The code never got out of this while loop - because data was constantly being sent and some packet was always incoming.
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
A solution I found was to send a message indicating how big the message being received should be, so I can tell when all the bytes for a particular message have arrived. Links: Sockets Python 3.5: Socket server hangs forever on file receive, Python Socket Receive Large Amount of Data
def send(self, data):
message = pickle.dumps(data)
msg_len = len(message)
try:
# Send what the total length of the message to be sent is in bytes.
self.socket.send(msg_len.to_bytes(4, 'big'))
self.socket.sendall(message)
except socket.error as e:
return str(e)
def receive(self):
remaining = int.from_bytes(self.socket.recv(4), 'big')
chunks = []
while remaining:
# until there are bytes left...
# fetch remaining bytes or 4096 (whatever smaller)
chunk = self.socket.recv(min(remaining, 4096))
remaining -= len(chunk)
# write to file
chunks.append(chunk)
chunks = b"".join(chunks)
data = pickle.loads(chunks)
return data
Running some production code I faced the following problem:
When sending HTTP requests to some server, server immediately closes the connection after sending response, which, for some reason, results in data loss.
Analyzing TCP dumps i can see that conversation goes as this:
client request
server ack
server push
server fin, ack (sent after ~0.000020 secs after previous push)
As the result my code can't get data sent by the server, (i'm guessing because of the small delay after push POLLHUP event might go before POLLIN?)
Trying to mimic the problem I've written the following code:
(It mimics the client behaviour on my side)
client:
import time
import socket
from errno import EAGAIN
from select import poll, POLLIN, POLLPRI, POLLERR, POLLHUP, POLLNVAL
def main(buf=""):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.setblocking(False)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
polling_object = poll()
polling_object.register(client, POLLPRI | POLLIN | POLLERR | POLLHUP)
in_buf = ""
sock_closed = False
try:
client.connect(("127.0.0.1", 8877))
except socket.error, e:
pass
while True and not sock_closed:
events = polling_object.poll(0)
for _, e in events:
if e & (POLLIN | POLLPRI):
while True:
try:
data = client.recv(1024)
if data:
in_buf += data
elif data == "":
client.close()
sock_closed = True
break
except socket.error, e:
if e.args[0] == EAGAIN:
break
else:
raise
elif e & (POLLERR|POLLHUP|POLLNVAL):
client.close()
sock_closed = True
if buf and not sock_closed:
try:
b_sent = client.send(buf)
if b_sent == len(buf):
buf = ""
else:
buf = buf[b_sent:]
except socket.error, e:
if e.args[0] != EAGAIN:
client.close()
sock_closed = True
time.sleep(0.5)
if sock_closed:
return in_buf
if __name__ == '__main__':
import sys
if len(sys.argv) > 1:
buf = sys.argv[1]
else:
buf = 'hello'
print main(buf)
server
import datetime
import time
import socket
def main():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", 8877))
server.listen(0)
client, _ = server.accept()
t1 = time.time()
data = ""
while not data:
data += client.recv(1024)
print "recv data %s" % data
client.sendall('{"ok": 1}')
t2 = time.time()
client.close()
t3 = time.time()
server.close()
return t1, t2, t3
if __name__ == '__main__':
c_r, d_s, c_c = main()
print "Connection received at ", datetime.datetime.fromtimestamp(c_r)
print "All Data sent after %.12f secs" % (d_s - c_r)
print "Connection closed after %.12f secs" % (c_c - d_s)
Running this code won't help me reproduce the problem because my client still can get data from socket buffer, which is kind of obviously by just following the code. The only difference is that in tcp dump it goes like this:
client request
server ack
server push
client ack
server fin, ack
I'm wondering is there a way to send fin, ack right after push without "letting" client to sent ack? Can it be done using python?
I have to write a code that allows non-blocking recv (along with non-blocking accept)
The following is what I got so far:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_address = ('', int(PortEntry_Page2_Text))
sock.bind(server_address)
sock.setblocking(0)
sock.settimeout(1)
sock.listen(1)
EnterToTerminal2("Listening to port "+str(PortEntry_Page2_Text)+"..")
EnterToTerminal2("Waiting for a connection. Press STOP to cancel.")
connected = 0
while (StopButtonStatePage2 == 0 and connected == 0):
try:
connection, client_address = sock.accept()
EnterToTerminal2("Connection from "+ str(client_address)+"..")
connected = 1
except Exception as inst:
template = "An exception of type {0} occured. Arguments:\n{1!r}"
message = template.format(type(inst).__name__,inst.args)
print message
finally:
if (connected == 0):
EnterToTerminal2("No client connected..")
StopButtonStatePage2 = GetStopButtonStatePage2()
###########################################
StopButtonStatePage2 = 0
sock.setblocking(0) ##?
print "xx0 "+str(connected)+" "+str(StopButtonStatePage2)
while (connected == 1 and StopButtonStatePage2 == 0):
a = select.select([sock], [], [], 1) # steelkiwi.com/blog/working-tcp-sockets
if a[0]:##?
data = sock.recv(64) # or connection.recv ??
EnterToTerminal2("Data recv: "+str(data))
StopButtonStatePage2 = GetStopButtonStatePage2()
Until sock.recv part, it works as expected. Now, the problem is that with the select approach, it does not enter the a[0] condition at all, even though I send stuff from client. Basically what I'm trying to achieve is to make recv part non-blocking/or with timeout so that I can stop its operation with a button, whenever I want.
Any help is appreaciated. Thanks in advance.
When a client connects to the pipe, and sends data I can receive this fine and I can keep receiving the data. Trouble comes when the client disconnects and the while loop is still active, connection.recv() doesn't block and therefore keeps looping frantically! So I need a way to detect if a client is still connected.
I have the following code:
pipe = './pipes/uds_defzone-lrecv'
try:
os.unlink(pipe)
except OSError:
if os.path.exists(pipe):
raise
self.logger.debug('Created UDS pipe: ' + pipe)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(pipe)
sock.listen(1)
self.logger.debug('Waiting for connection: ' + pipe)
connection, client_address = sock.accept()
self.logger.debug('Connection from: ' + client_address)
while True:
self.logger.debug('Waiting for data')
data = connection.recv(4096)
self.logger.debug('Received: ' + str(data))
For reference, the sender.py code:
# Create a UDS socket
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Connect the socket to the port where the server is listening
pipe = './pipes/uds_defzone-lrecv'
logger.debug('connecting to: ' + pipe)
try:
sock.connect(pipe)
except socket.error, msg:
logger.debug(msg)
sys.exit(1)
try:
message = 'THIS IS A TEST'
logger.debug('sending: ' + message)
sock.sendall(message)
time.sleep(2)
finally:
logger.debug('closing socket')
sock.close()
TIA!
UPDATE
I can slow it down with the following code I suppose, but not exactly what I want.
while True:
try:
self.logger.debug('Waiting for data')
data_present = select.select([sock], [], [], 30)
if data_present[0]:
data = connection.recv(4096)
self.logger.debug('Received: ' + data)
except select.timeout:
pass
UPDATE 2
For reference this is the code I came up with:
while True:
logger.debug('Waiting for data')
data = connection.recv(4096)
if not data == '':
logger.debug('Received: ' + data)
else:
logger.debug('Nothing received')
break
A hack I came up with in the process... Might be usable where it is legitimate that a client might send empty data, for signalling perhaps?
while True:
try:
logger.debug('Waiting for data')
data = connection.recv(4096)
# *** This throws an exception when client has disconnected
x = connection.getpeername()
logger.debug('Received: ' + data)
except:
logger.debug('Client disconnected')
break
connection.recv() doesn't block and therefore keeps looping frantically! So I need a way to detect if a client is still connected.
If the peer disconnects recv data will return empty data (''). You need to check this and exit the loop.