We Keep Coding

Can't implement receiving full string using STX and ETX condition

I am developing a python socket server. The client send each message to start with a STX (\x02) and end with ETX (\x03). My code can receive message successfully but I can't implement receiving full string using STX and ETX condition. Need help in resolving this issue. Below I have sharing my code for better understanding.
import socket
import time
# Start New RnD
# Global Veriable
enq = chr(5)
ack = chr(6)
stx = chr(2)
etx = chr(3)
# Connect to the server with `telnet $HOSTNAME 5000`.
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setblocking(False)
server.bind(('0.0.0.0', 5000))
server.listen(1)
connections = []
while True:
try:
connection, address = server.accept()
connection.setblocking(False)
connections.append(connection)
except BlockingIOError:
pass
# Incoming Data Processing
for connection in connections:
try:
full_message = ''
data = ""
while True:
try:
received = connection.recv(1)
if received == enq.encode('utf-8'):
print("Received <ENQ>, Sending <ACK>")
connection.sendall(ack.encode('utf-8'))
if not received:
raise RuntimeError("unexpected end-of-message", data)
data += received.decode('utf-8')
#print("Received: {!r}".format(data))
if "\x03" in received.decode("utf-8") :
break
except BlockingIOError:
pass
print("Full Received: {!r}".format(data))
print("Data Received, Sending <ACK>")
connection.sendall(ack.encode('utf-8'))
except BlockingIOError:
continue

How do I achieve Python TCP Socket Client/Server with Communications Encrypted?

Backstory: what I have done
{Codes at the bottom} I've already coded the multithreaded client and server programs using python socket, with the help of the following sites:
I. Echo Client and Server
II. Socket Server with Multiple Clients | Multithreading | Python
III. Python Socket Receive Large Amount of Data
Regarding Encryption & Decryption
(1) Exactly at what places in my codes should I encrypt/decrypt my message? Do
I encrypt the messages themselves after the user inputs or do I encrypt the byte streams after the input messages have been encoded?
(2) And how am I supposed to encrypt/decrypt the communication properly and efficiently? (It'd be nice to see code solutions with explanation, many thanks)
My Codes Currently
_server.py
import socket
import os
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
# ---- Server Communication Setup
HOST = '127.0.0.1' # Standard loopback interface address (localhost)
PORT = 65432 # Port to listen on (non-privileged ports are > 1023)
ThreadCount = 0
try: # create socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
s.bind((HOST, PORT))
except socket.error as e:
print(str(e))
print('Waitiing for a Connection..')
s.listen(3)
def threaded_client(conn: socket):
conn.send(str.encode('Welcome to the Server'))
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, str.encode(reply))
conn.close()
while True:
Client, addr = s.accept()
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
start_new_thread(threaded_client, (Client, )) # Calling threaded_client() on a new thread
ThreadCount += 1
print('Thread Number: ' + str(ThreadCount))
s.close()
_client.py
import socket
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
# ---- Client Communication Setup ----
HOST = '127.0.0.1' # The server's hostname or IP address
PORT = 65432 # The port used by the server
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
try:
s.connect((HOST, PORT))
except socket.error as e:
print(str(e))
Response = s.recv(1024)
while True:
Input = input('Say Something: ')
# s.send(str.encode(Input))
send_msg(s, str.encode(Input))
# Response = s.recv(1024)
Response = recv_msg(s)
print(Response.decode('utf-8'))
s.close()

You only need to Encrypt the Message itself. I would use RSA to Encrypt the Messages.
If you plan on using multiple Servers, you can open a second Port and if someone connects to it, the Server/Host. Sends the Public Key to the Client.
After that, the Client sends his Public key to the Server.
Now Server and Client switch Ports, and Communicate over there while using the Public key of the other to Encrypt the Messages and decrypt them with their Private key.
You can also Hard-code the Public Keys, if you only have one Server and one Client.
A good Module for RSA Encryption is PyCrytodome.
Here is an Example of to encrypt messages with PyCrytodome.
from Crypto.PublicKey import RSA
from Crypto.Cipher import PKCS1_OAEP
def encrypt(pk_receiver, message):
key = RSA.import_key(pk_receiver)
cipher = PKCS1_OAEP.new(key)
c = cipher.encrypt(message.encode())
return c
def decrypt(sk, c):
key = RSA.import_key(sk)
cipher = PKCS1_OAEP.new(key)
m = cipher.decrypt(c)
return m
def generate_sk(key_length):
key = RSA.generate(key_length)
with open('./secret_key.pem', 'wb') as f:
f.write(key.export_key(format='PEM'))
return key
def generate_pk(sk):
pk = sk.public_key()
with open('./public_key.pem', 'wb') as f:
f.write(pk.export_key(format='PEM'))
return

infinite loop when receiving a large pickled object with sockets

I'm using a remote linux server and I want to send an array via sockets from client with python so I used this code :
message = pickle.dumps(faceBlob)
message_header = bytes(f"{len(message):<{HEADER_LENGTH}}", "utf-8")
client_socket.send(message_header + message)
to receive it in the server I used a while loop to catch all the message since it is > 4096 :
def receive_blop(client_socket):
# Receive our "header" containing message length, it's size is defined and constant
message_header = client_socket.recv(HEADER_LENGTH)
# If we received no data, client gracefully closed a connection, for example using socket.close() or socket.shutdown(socket.SHUT_RDWR)
if not len(message_header):
return False
# Convert header to int value
message_length = int(message_header.decode('utf-8').strip())
fragments = []
print(message_length)
while True:
# this loop is infinite
print("I arrived her")
chunk = client_socket.recv(4096)
if not chunk:
break
fragments.append(chunk)
data_arr = b"".join(fragments)
# Return an object of message header and message data
return {'header': message_header, 'data': data_arr}
the server still printing the 'I arrived here' but receive the message until the connection is ended from the client

Your loop will continue until the client closes the connection. If they don't close until they get a response from you, you've got a deadlock.
Since you know the message length, you can stop the loop when you've received that many bytes.
received_length = 0
while received_len < message_length:
print("I arrived her")
chunk = client_socket.recv(message_length - received_length)
if not chunk:
break
fragments.append(chunk)
received_length += len(chunk)

Socket Programming three way handshake (PYTHON) with TCP

So I am working on a project that requires me to simulate a TCP three handshake protocol using UDP. I want to divided the project into two parts: the first is to establish and close connection; the second is to implement the flow control protocol without any package loss. Since the client and server will be interacting with each other, so I need two sockets, which means I also need two ports - 19990 and 19991 - to send and receive data between client and server.
I am having trouble with the three way handshake
Client -> (Send SYN)
Sever -> (Receive SYN and Send SYN ACK)
Client -> Receives SYN ACK and Sends ACK with Data
Server then establishes a connection and says that data has been received
Client -> Server ACK the connection & received data. So send FIN to close the connection
Server -> Receives FIN and closes the connection (with server_socket.close)
output on the client side:
Send: a,1
Receive: b, num:2
Send: c,3
Receive: d, num:4
Client done
output on server side:
Receive: a, num:1
Send: b,2
Receive c, num:3
Send: d,4
Server done.
However, what I am hoping for them to look like is like so:
client.py
Send SYN
Send: ,1
Receive: num,2
Received SYN-ACK. Send ACK & Data
Send: I love ABC University in New York City.,3
Receive: ,num:4
Received ACK for data. Send FIN. Connection closed.
Send: ,4
server.py
Receive: ,num: 1
Received SYN. Send SYN-ACK
Send: ,2
Receive: I love ABC University in New York City.,num:3
Receive ACK. Connection Established. Received Data. Send ACK
Send: ,4
Receive: ,num:4
Connection Closed
Here is the code I have for server.py:
import socket
import sys
import struct
server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_socket.bind(("0.0.0.0", 19990))
def send(c,num):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
ss = struct.pack("!50si",c.encode(),num)
s.sendto(ss,("127.0.0.1",19991))
s.close()
print("Send:%s,%d" % (c,num))
def recv():
global server_socket
data, addr = server_socket.recvfrom(1024)
str,num = struct.unpack("!50si",data)
str = str.decode("utf-8").replace("\0","")
print("Receive:%s,num:%d" % (str,num))
return str,num
while True:
str,num = recv()
num = num + 1
str = chr(ord(str)+1)
send(str,num)
if str == "d":
break
print("Server Done.")
server_socket.close()
Here is the code I have for client.py
import socket
import sys
import struct
server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_socket.bind(("0.0.0.0", 19991))
def send(c,num):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
ss = struct.pack("!50si",c.encode(),num)
s.sendto(ss,("127.0.0.1",19990))
s.close()
print("Send:%s,%d" % (c,num))
def recv():
global server_socket
data, addr = server_socket.recvfrom(1024)
str,num = struct.unpack("!50si",data)
str = str.decode("utf-8").replace("\0","")
print("Receive:%s,num:%d" % (str,num))
return str,num
str = 'a'
num = 1
while True:
send(str,num)
str,num = recv()
if str == "d":
break
str = chr(ord(str)+1)
num = num + 1
print("Client done.")
server_socket.close()
#If the number = 1, sender will send and the receiver will receive
#c is the data and num is the sequence number, for the first three msgs it is the flag
#The payload we want to send is seven characters with one sentence
#Window size is 4 with 4 being four characters
#First package is "i lo"
#I is sent as a package, then space as a package, then l as a package, and o as a package as a window
#First byte is i, second is space, l is third, o is forth.
#Send out 4 bytes, receive 4 acknowledgements.
#When the sender sends out the last byte "."
#To do: Change 'a' to past establishment
#Instead of Send: a, 1
#professor wants to see "Send Think message"
#Receive: Think ack message
#Send: Ack msg
#a b c d are just give you some examples

python socket and epoll

I use python's socket and epoll to make a web server.
My operating system is Linux CentOS 6,
My python version is python 2.7.8.
My source code is:
# -*-coding:utf-8-*-
import socket
import select
import time
EOL1 = b'\n\n'
EOL2 = b'\n\r\n'
response = b'HTTP/1.0 200 OK\r\nDate: Mon, 1 Jan 1996 01:01:01 GMT\r\n'
response += b'Content-Type: text/plain\r\nContent-Length: 13\r\n\r\n'
response += b'<html><head><title>title</title></head><body><p>Hello, world!</p></body></html>'
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversocket.bind(('0.0.0.0', 8080))
serversocket.listen(1) # the number of client that connect to server
serversocket.setblocking(0) # set 0 not block other block
serversocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
epoll = select.epoll()
epoll.register(serversocket.fileno(), select.EPOLLIN)
try:
connections = {}
requests = {}
responses = {}
while True:
events = epoll.poll(1)
for fileno, event in events:
if fileno == serversocket.fileno(): # if request come
connection, address = serversocket.accept() # waiting income connection
connection.setblocking(0) # none block
epoll.register(connection.fileno(), select.EPOLLIN) # register socket read event to epoll
connections[connection.fileno()] = connection # add connection to connections dict
requests[connection.fileno()] = b''
responses[connection.fileno()] = response # write data to responses dict
elif event & select.EPOLLIN: # when data in os's read buffer area
requests[fileno] += connections[fileno].recv(1024) # read data from connections
if EOL1 in requests[fileno] or EOL2 in requests[fileno]: # if http message
print('-' * 40 + '\n' + requests[fileno].decode()[:-2])
responses[fileno] += str(time.time())
epoll.modify(fileno, select.EPOLLOUT) # change file number to epoll out mode
elif event & select.EPOLLOUT: # if out mode
byteswritten = connections[fileno].send(responses[fileno]) # write data to os's write buffer
responses[fileno] = responses[fileno][byteswritten:] # get http response message
if len(responses[fileno]) == 0: # if file sent
epoll.modify(fileno, 0) # change file number to hup mode
connections[fileno].shutdown(socket.SHUT_RDWR) # set socket read and write mode shutdown
elif event & select.EPOLLHUP: # if message sent and file number in epoll is hup
epoll.unregister(fileno) # remove file number from epoll
connections[fileno].close() # close connection
del connections[fileno] # delete connection from connections dict
finally:
epoll.unregister(serversocket.fileno())
epoll.close()
serversocket.close()
But when I open web browser and visit "http://localhost:8080/", I get some data like these <html><head><,it is not full data, it just a part of my data.What's the matter in my project.
view more info please look this picture.

You have "Content-Length: 13" in your code and hence only first 13 characters are showing up!