I am trying to implement UDP stop-and-wait protocol using python socket programming. I have been trying out my code with different retransmission times. And sometimes I get all the files correctly received at the receiver, but sometimes some packets get lost. For example when I ran it with 40ms five times, twice the file was received correctly and three times incorrectly. Why is this variability happening?
Here is my code for the sender and receiver:
senderSocket = socket(AF_INET, SOCK_DGRAM) # Create UDP socket for sender
r = 0 # Number of retransmissions
for i in range(0, len(messages)):
senderSocket.settimeout(retryTimeout) # After the message is sent, set retransmission timeout to listen for acknowledgement
while True:
senderSocket.sendto(messages[i], (hostName, portNumber))
acknowledged = False
while not acknowledged:
try:
ack, receiverAddress = senderSocket.recvfrom(2) # Receive ACK
acknowledged = True
except: # Socket timeout exception occurs when timeout expires but no ACK received
senderSocket.sendto(messages[i], (hostName, portNumber)) # Retransmit the message
r = r + 1 # Increment the number of retransmissions
break # On to the next message
senderSocket.close()
while True:
message, senderAddress = receiverSocket.recvfrom(1027) # Read from UDP socket into message, getting sender's address (sender IP and port)
header = message[:3] # Header is the first 3 bytes (index 0, 1, 2)
data = message[3:] # Rest is the data
first_byte = '{0:08b}'.format(header[0])
second_byte = '{0:08b}'.format(header[1])
seq_num = int(first_byte + second_byte, 2) # Convert bytes to decimal
if seq_num not in seq_nums: # Detect duplicates
seq_nums.append(seq_num)
file_content.extend(data)
ack = header[:2] # ACK is the receipt of the received message (sequence number)
receiverSocket.sendto(ack, senderAddress) # Send ACK
if header[2] == 1: # Sent multiple ACKs at lat message to make sure it receives and the sender closes
receiverSocket.sendto(ack, senderAddress)
receiverSocket.sendto(ack, senderAddress)
receiverSocket.sendto(ack, senderAddress)
break
receiverSocket.close()
Backstory: what I have done
{Codes at the bottom} I've already coded the multithreaded client and server programs using python socket, with the help of the following sites:
I. Echo Client and Server
II. Socket Server with Multiple Clients | Multithreading | Python
III. Python Socket Receive Large Amount of Data
Regarding Encryption & Decryption
(1) Exactly at what places in my codes should I encrypt/decrypt my message? Do
I encrypt the messages themselves after the user inputs or do I encrypt the byte streams after the input messages have been encoded?
(2) And how am I supposed to encrypt/decrypt the communication properly and efficiently? (It'd be nice to see code solutions with explanation, many thanks)
My Codes Currently
_server.py
import socket
import os
from _thread import *
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
# ---- Server Communication Setup
HOST = '127.0.0.1' # Standard loopback interface address (localhost)
PORT = 65432 # Port to listen on (non-privileged ports are > 1023)
ThreadCount = 0
try: # create socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
try: # bind socket to an address
s.bind((HOST, PORT))
except socket.error as e:
print(str(e))
print('Waitiing for a Connection..')
s.listen(3)
def threaded_client(conn: socket):
conn.send(str.encode('Welcome to the Server'))
while True:
# data = conn.recv(2048) # receive message from client
data = recv_msg(conn)
reply = 'Server Says: ' + data.decode('utf-8')
if not data:
break
# conn.sendall(str.encode(reply))
send_msg(conn, str.encode(reply))
conn.close()
while True:
Client, addr = s.accept()
print('Connected to: ' + addr[0] + ':' + str(addr[1]))
start_new_thread(threaded_client, (Client, )) # Calling threaded_client() on a new thread
ThreadCount += 1
print('Thread Number: ' + str(ThreadCount))
s.close()
_client.py
import socket
import struct # Here to convert Python data types into byte streams (in string) and back
# ---- To Avoid Message Boundary Problem on top of TCP protocol ----
def send_msg(sock: socket, msg): # ---- Use this to send
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock: socket): # ---- Use this to receive
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock: socket, n: int):
# Helper function to receive n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
# ---- Client Communication Setup ----
HOST = '127.0.0.1' # The server's hostname or IP address
PORT = 65432 # The port used by the server
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print ("Socket successfully created")
except socket.error as err:
print ("socket creation failed with error %s" %(err))
print('Waiting for connection')
try:
s.connect((HOST, PORT))
except socket.error as e:
print(str(e))
Response = s.recv(1024)
while True:
Input = input('Say Something: ')
# s.send(str.encode(Input))
send_msg(s, str.encode(Input))
# Response = s.recv(1024)
Response = recv_msg(s)
print(Response.decode('utf-8'))
s.close()
You only need to Encrypt the Message itself. I would use RSA to Encrypt the Messages.
If you plan on using multiple Servers, you can open a second Port and if someone connects to it, the Server/Host. Sends the Public Key to the Client.
After that, the Client sends his Public key to the Server.
Now Server and Client switch Ports, and Communicate over there while using the Public key of the other to Encrypt the Messages and decrypt them with their Private key.
You can also Hard-code the Public Keys, if you only have one Server and one Client.
A good Module for RSA Encryption is PyCrytodome.
Here is an Example of to encrypt messages with PyCrytodome.
from Crypto.PublicKey import RSA
from Crypto.Cipher import PKCS1_OAEP
def encrypt(pk_receiver, message):
key = RSA.import_key(pk_receiver)
cipher = PKCS1_OAEP.new(key)
c = cipher.encrypt(message.encode())
return c
def decrypt(sk, c):
key = RSA.import_key(sk)
cipher = PKCS1_OAEP.new(key)
m = cipher.decrypt(c)
return m
def generate_sk(key_length):
key = RSA.generate(key_length)
with open('./secret_key.pem', 'wb') as f:
f.write(key.export_key(format='PEM'))
return key
def generate_pk(sk):
pk = sk.public_key()
with open('./public_key.pem', 'wb') as f:
f.write(pk.export_key(format='PEM'))
return
What's the best way to stream bytes from client to server in chunks of determined size?
Right now I'm encoding an audio file with base64, then compressing it with zlib and sending through the socket connection. My problem is trying to rebuild the original within the server.
I thought and tested using an empty string that is added with all the bytes the server is receiving. Seemed alright, but the " b' " in the beginning was being kept, which left it unable to recover the original audio file.
I've just tried to decode the bytes and deleting the " b' " from the beginning and " " " from the end (data[2:-1]) of each set of strings received by the server, but this cut a few characters from the original.
client side:
with open(arquivo, 'rb') as input_file:
abc = base64.b64encode(input_file.read())
try1 = zlib.compress(abc)
n = 338
result = [try1[i:i+n] for i in range(0, len(try1), n)]
HOST = ''
PORT = 9999
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.connect((HOST,PORT))
i = 0
for item in result:
item = str(item)
print(item)
s.send(item.encode())
i += 1
print('i = ', i)
time.sleep(2)
Server side:
HOST = ''
PORT = 9999
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
print('Servidor Inicializado')
s.bind((HOST,PORT))
s.listen()
audiofile = ''
i = 0
conn, addr = s.accept()
while True:
data1 = conn.recv(2048)
print('data1 undecoded = ', data1)
text = data1.decode()
data = text[2:-1]
print('data EDITADO = ', data)
audiofile = audiofile + data
i += 1
print('i = ', i)
print('audiofile = ', audiofile)
if not data:
print('No Data Received!')
print('Recebeu tratado :', data)
No idea how to proceed, any help is appreciated. Thanks!
Here is an example of how I send and receive data with sockets.
Typically I'll pickle them. If you're not familiar with pickle it's used to serialize python objects to store or send over connections such as sockets.
Client Side:
import pickle
with open(arquivo, 'rb') as input_file:
abc = base64.b64encode(input_file.read())
# I haven't used these libraries so I'm assuming you know how to unpack it from here
try1 = zlib.compress(abc)
HOST = ''
PORT = 9999
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.connect((HOST,PORT))
# serialize the python object
message = pickle.dumps(try1)
# get the length of the pickled object
length = len(message)
# convert into a fixed width string
length = str(length).rjust(8, '0')
# send the length of the object we will send
s.sendall(bytes(length, 'utf-8'))
# send the object
self.client.sendall(message)
Server Side:
import pickle
HOST = ''
PORT = 9999
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((HOST,PORT))
s.listen()
conn, addr = s.accept()
# get the length of the object we are about to receive
length = conn.recv(8)
# turn it back into an int
length = int(length.decode('utf-8'))
# I keep this to determine if we've received everything or not
full_length = length
message = None
# loop until we've zeroed out our length
while length > 0:
# only receive what we need
# at a maximum of 128 bit chunks
chunk_len = min(128, length)
length -= chunk_len
chunk = conn.recv(chunk_len)
if message is None:
message = chunk
else:
message = message + chunk
# Edit: I've had issues with slow connections not receiving the full data
# for those cases adding something like this works
while len(message) < full_length:
chunk_len = min(128, full_length - len(message))
chunk = conn.recv(chunk_len)
message = message + chunk
# now that we've received everything, we turn it back into a python object
try1 = pickle.loads(message)
# this should be the same try1 you sent
Disclaimer: I did not test any of this, nor do I know what the try1 object is or what you want to do with it. this is just getting it from point a to point b.
Explanation
I'm currently trying to control a smart power strip using a python script. To accomplish this, I'm using a TCP connection with the socket module. Around 75% of the time, I get the response/data I was looking for and everything works perfectly. However, around 25% of the time, the response is cut off at the exact same length, 1024 bytes. This doesn't make any sense to me, as my buffer size is actually set to 2048 bytes. The speed at which I wait in between using recv() doesn't seem to effect/cause this either. Altough TCP is a stream of bytes, is it still possible that this could have to do with packet fragmentation?
Code
Main Code
ip='192.168.0.62'
port=9999
sock_tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock_tcp.connect((ip, port))
sock_tcp.send(encrypt('{"system":{"get_sysinfo":{}}}'))
data = sock_tcp.recv(2048)
sock_tcp.close()
print len(data) #On succesful runs output is 1221, on unsuccesful runs it is 1024
rec = decrypt(data[4:])
print str(rec) #See output below
Encrypt Function
def encrypt(string):
key = 171
result = pack('>I', len(string))
for i in string:
a = key ^ ord(i)
key = a
result += chr(a)
return result
Decrypt Function
def decrypt(string):
key = 171
result = ""
for i in string:
a = key ^ ord(i)
key = ord(i)
result += chr(a)
return result
Output
The string itself that I recieve. It's most likeley not relevant, but I thought I would include it anyway. This is value of the variable rec.
Desired and regular output
Full desired output
{"system":{"get_sysinfo":{"sw_ver":"1.0.6 Build 180627
Rel.081000","hw_ver":"1.0","model":"HS300(US)","deviceId":"80067B24A755F99C4D6C1807455E09F91AB7B2AA","oemId":"5C9E6254BEBAED63B2B6102966D24C17","hwId":"34C41AA028022D0CCEA5E678E8547C54","rssi":-60,"longitude_i":-1222955,"latitude_i":379078,"alias":"TP-LINK_Power
Strip_4F01","mic_type":"IOT.SMARTPLUGSWITCH","feature":"TIM:ENE","mac":"B0:BE:76:12:4F:01","updating":0,"led_off":0,"children":[{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA00","state":0,"alias":"CezHeat","on_time":0,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA01","state":1,"alias":"CezUVB","on_time":191208,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA02","state":1,"alias":"CyanHeat","on_time":191208,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA03","state":1,"alias":"ZanderHeat","on_time":191208,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA04","state":1,"alias":"CairoHeat","on_time":191208,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA05","state":1,"alias":"KodaMister","on_time":191208,"next_action":{"type":-1}}],"child_num":6,"err_code":0}}}
Abnormal and rarer output
Cut off output
{"system":{"get_sysinfo":{"sw_ver":"1.0.6 Build 180627
Rel.081000","hw_ver":"1.0","model":"HS300(US)","deviceId":"80067B24A755F99C4D6C1807455E09F91AB7B2AA","oemId":"5C9E6254BEBAED63B2B6102966D24C17","hwId":"34C41AA028022D0CCEA5E678E8547C54","rssi":-59,"longitude_i":-1222955,"latitude_i":379078,"alias":"TP-LINK_Power
Strip_4F01","mic_type":"IOT.SMARTPLUGSWITCH","feature":"TIM:ENE","mac":"B0:BE:76:12:4F:01","updating":0,"led_off":0,"children":[{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA00","state":0,"alias":"CezHeat","on_time":0,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA01","state":1,"alias":"CezUVB","on_time":191207,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA02","state":1,"alias":"CyanHeat","on_time":191207,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA03","state":1,"alias":"ZanderHeat","on_time":191207,"next_action":{"type":-1}},{"id":"80067B24A755F99C4D6C1807455E09F91AB7B2AA04","state":1,"alias":"CairoHeat","on
Conclusion
If anyone could provide me with a solution or explanation as to why the output/stream gets cut off, it would be much appreciated. I used a lot of the code from this open source module. I'm also looking to understand more of how this all works, so if you could explain a bit more I would really appreciate it.
As per the documentation, the bufsize argument only specifies the maximum amount of data to be read:
socket.recv(bufsize[, flags])
Receive data from the socket. The return
value is a bytes object representing the data received. The maximum
amount of data to be received at once is specified by bufsize. See the
Unix manual page recv(2) for the meaning of the optional argument
flags; it defaults to zero.
To ensure full data transfer a function like this can be used, which waits for the end of the socket connection (indicated by and empty string returned from recv):
def recv_all(connection):
"""
Function for all data
:param connection: socket connection
:return: received data
"""
data = list()
while True:
data.append(connection.recv(2048))
if not data[-1]:
return b''.join(data)
Another example that might fit your application better could be to wait for a fixed message size (1221 as indicated by your question):
def recv_message(connection):
data = list()
transferred_bytes= 0
while transferred_bytes < 1221:
data.append(connection.recv(min(1221-transferred_bytes, 2048)))
if not data[-1]:
raise RuntimeError("socket connection broken")
transferred_bytes += len(data[-1])
return b''.join(data)
This is only a complement to SimonF's answer. The cause of the problem is indeed that TCP is a stream protocol, so packets can be fragmented or re-assembled at any state: sender TCP/IP stack, network equipments, receiver TCP/IP stack - I include the user layer library in the TCP/IP stack here for simplification.
That is the reason why, you should always use a higher level protocol above TCP to be able to split the stream in sensible messages. Here you could note that the end of a message is '}}}', so you could concatenate the input in a buffer until you find that pattern:
def recv_until(c, guard):
"""Receive data from a socket until guard if found on input"""
guard_sz = len(guard) - 1
data = b''
sz = 0
while True:
buffer = c.recv(1024) # read by chuncks of size 1024 (change value to your needs)
got = len(buffer)
data += buffer # concatenate in buffer
ix = data.find(guard, sz - guard_sz if sz > guard_sz else 0) # is guard found?
if ix != -1:
return (data[:ix + guard_sz + 1], # return the message, and what could be behind it
data[ix + guard_sz + 1:])
sz += got
The trick is to considere guard_sz byte from the last chunk, in the case where the guard could be split in two chunks.
Marco, please use recv_into(buffer[, nbytes[, flags]]) method for the socket.
My example for TCP-microserver:
import socket
import struct
def readReliably(s,n):
buf = bytearray(n)
view = memoryview(buf)
sz = 0
while sz < n:
k = s.recv_into(view[sz:],n-sz)
sz += k
# print 'readReliably()',sz
return sz,buf
def writeReliably(s,buf,n):
sz = 0
while sz < n:
k = s.send(buf[sz:],n-sz)
sz += k
# obj = s.makefile(mode='w')
# obj.flush()
# print 'writeReliably()',sz
return sz
# Client
host = "127.0.0.1"
port = 23456
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(10)
s.connect((host,port))
# Request
buf = struct.pack("4B",*[0x01,0x02,0x03,0x04])
io.writeReliably(s,buf,4)
# Response
sz,buf = io.readReliably(s,4)
a = struct.unpack("4B",buf)
print repr(a)
# Server
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
#s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
#s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
#s.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
s.bind((host,port))
s.listen(10) # unaccepted connections
while True:
sk,skfrom = s.accept()
sz,buf = io.readReliably(sk,4)
a = struct.unpack("4B",buf)
print repr(a)
# ...
io.writeReliably(sk,struct.pack("4B",*[0x01,0x02,0x03,0x04]))
I used the python socket to send a DNS query packet socket and listen to the response. Finally, I got a DNS response packet by the socket.recvfrom(2048) function as expected. But strangely, where I compared the response packet with the packet crawled by Wireshark, I found there exists many difference.
The differences would be found as 3f at the second picture.
The DNS response packet (The highlighted part) crawled by the Wireshark
The DNS response packet got by the socket.recvfrom(2048)
The Creating a Socket Part Codes:
ipv = check_ip(dst)
udp = socket.getprotobyname(Proto.UDP)
if ipv == IPV.ERROR:
return None
elif ipv == IPV.IPV4:
return socket.socket(socket.AF_INET, socket.SOCK_DGRAM, udp)
elif ipv == IPV.IPV6:
return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM, udp)
else:
return None
The Receiving a DNS response packet Part Codes:
remained_time = 0
while True:
remained_time = self.timeout - timeit.default_timer() + sent_time
readable = select.select([sock], [], [], remained_time)[0]
if len(readable) == 0:
return (-1, None)
packet, addr = sock.recvfrom(4096)
Byte 0x3F is the ASCII '?' character. That commonly means the data is being treated as text and is passing through a charset conversion that doesn't support the bytes being converted.
Notice that 0x3F is replacing only the bytes that are > 0x7F (the last byte supported by ASCII). Non-ASCII bytes in the range of 0x80-0xFF are subject to charset interpretation.
That makes sense, as you are using the version of recvfrom() that returns a string, so the received bytes need to be converted to Python's default string encoding.
Since you need raw bytes instead, use recvfrom_into() to fill a pre-allocated bytearray, eg:
packet = bytearray(4096)
remained_time = 0
while True:
remained_time = self.timeout - timeit.default_timer() + sent_time
readable = select.select([sock], [], [], remained_time)[0]
if len(readable) == 0:
return (-1, None)
nbytes, addr = sock.recvfrom_into(packet)
Then you can use packet up to nbytes number of bytes as needed.