We Keep Coding

Retransmission timeout

I am trying to implement UDP stop-and-wait protocol using python socket programming. I have been trying out my code with different retransmission times. And sometimes I get all the files correctly received at the receiver, but sometimes some packets get lost. For example when I ran it with 40ms five times, twice the file was received correctly and three times incorrectly. Why is this variability happening?
Here is my code for the sender and receiver:
senderSocket = socket(AF_INET, SOCK_DGRAM) # Create UDP socket for sender
r = 0 # Number of retransmissions
for i in range(0, len(messages)):
senderSocket.settimeout(retryTimeout) # After the message is sent, set retransmission timeout to listen for acknowledgement
while True:
senderSocket.sendto(messages[i], (hostName, portNumber))
acknowledged = False
while not acknowledged:
try:
ack, receiverAddress = senderSocket.recvfrom(2) # Receive ACK
acknowledged = True
except: # Socket timeout exception occurs when timeout expires but no ACK received
senderSocket.sendto(messages[i], (hostName, portNumber)) # Retransmit the message
r = r + 1 # Increment the number of retransmissions
break # On to the next message
senderSocket.close()
while True:
message, senderAddress = receiverSocket.recvfrom(1027) # Read from UDP socket into message, getting sender's address (sender IP and port)
header = message[:3] # Header is the first 3 bytes (index 0, 1, 2)
data = message[3:] # Rest is the data
first_byte = '{0:08b}'.format(header[0])
second_byte = '{0:08b}'.format(header[1])
seq_num = int(first_byte + second_byte, 2) # Convert bytes to decimal
if seq_num not in seq_nums: # Detect duplicates
seq_nums.append(seq_num)
file_content.extend(data)
ack = header[:2] # ACK is the receipt of the received message (sequence number)
receiverSocket.sendto(ack, senderAddress) # Send ACK
if header[2] == 1: # Sent multiple ACKs at lat message to make sure it receives and the sender closes
receiverSocket.sendto(ack, senderAddress)
receiverSocket.sendto(ack, senderAddress)
receiverSocket.sendto(ack, senderAddress)
break
receiverSocket.close()

Black Hat Python proxy tool no data

I recently bought the book Black Hat Python, 2nd Edition, by Justin Seitz, which seems to be a very good book about networking and all that (i am writing my code on Kali Linux)
I have a problem on the TCP Proxy Tool on chapter 2 :
Here is the code :
import sys
import socket
import threading
HEX_FILTER = ''.join(
[(len(repr(chr(i))) == 3) and chr(i) or '.' for i in range(256)])
def hexdump(src, length = 16, show = True):
# basically translates hexadecimal characters to readable ones
if isinstance(src, bytes):
src = src.decode()
results = list()
for i in range(0, len(src), length):
word = str(src[i:i+length])
printable = word.translate(HEX_FILTER)
hexa = ' '.join(['{ord(c):02X}' for c in word])
hexwidth = length*3
results.append('{i:04x} {hexa:<{hexwidth}} {printable}')
if show :
for line in results :
print(line)
else :
return results
def receive_from(connection):
buffer = b""
connection.settimeout(10)
try :
while True :
data = connection.recvfrom(4096)
if not data :
break
buffer += data
except Exception as e:
pass
return buffer
def request_handler(buffer):
# perform packet modifications
return buffer
def response_handler(buffer):
# perform packet modifications
return buffer
def proxy_handler(client_socket, remote_host, remote_port, receive_first):
remote_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
remote_socket.connect((remote_host, remote_port))
if receive_first :
# Check for any data to receive before
going into the main loop (i guess)
remote_buffer = receive_from(remote_socket)
hexdump(remote_buffer)
remote_buffer = response_handler(remote_buffer)
if len(remote_buffer):
print("[<==] Sending %d bytes to localhost." % len(remote_buffer))
client_socket.send(remote_buffer)
while True : # Start the loop
local_buffer = receive_from(client_socket)
if len(local_buffer):
line = "[==>] Received %d bytes from localhost." % len(local_buffer)
print(line)
hexdump(local_buffer)
local_buffer = request_handler(local_buffer)
remote_socket.send(local_buffer)
print("[==>] Sent to remote.")
remote_buffer = receive_from(remote_socket)
if len(remote_buffer):
print("[==>] Received %d bytes from remote." % len(remote_buffer))
hexdump(remote_buffer)
remote_buffer=response_handler(remote_buffer)
client_socket.send(remote_buffer)
print("[<==] Sent to localhost.")
if not len(local_buffer) or not len(remote_buffer):
# If no data is passed, close the sockets and breaks the loop
client_socket.close()
remote_socket.close()
print("[*] No more data. Closing connections. See you later !")
break
def server_loop(local_host, local_port, remote_host, remote_port, receive_first):
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try :
server.bind((local_host, local_port)) # Bind the local host and the local port
except Exception as e:
print('Problem on bind : %r' %e)
# If an error occurs, prints a
print("[!] Failed to listen on %s:%d" % (local_host, local_port))
print("[!] Check for other listening sockets or correct permissions.")
sys.exit(0)
print("[*] Listening on %s:%d" % (local_host, local_port))
server.listen(5)
while True :
client_socket, addr = server.accept()
# print out the local connection information
line = "> Received incoming connection from %s:%d" % (addr[0], addr[1])
print(line)
# start a thread to talk to the remote host
proxy_thread = threading.Thread(
target = proxy_handler,
args=(client_socket,remote_host,
remote_port, receive_first))
proxy_thread.start()
def main():
if len(sys.argv[1:]) != 5:
print("Usage: ./proxy.py [localhost] [localport]")
print("[remotehost] [remoteport] [receive_first]")
print("Example : ./proxy.py 127.0.0.1 9000 192.168.56.1 9000 True")
sys.exit(0)
loca l_host = sys.argv[1]
local_port = int(sys.argv[2])
remote_host = sys.argv[3]
remote_port = int(sys.argv[4])
receive_first = sys.argv[5]
if "True" in receive_first:
receive_first = True
else :
receive_first = False
server_loop(local_host, local_port,
remote_host, remote_port, receive_first)
if __name__ == '__main__':
main()
(sorry, i had a bit of a trouble formatting it and it's quite long)
Now, normally, i just need to open 2 terminals and run the code with the command line :
sudo python proxy.py 127.0.0.1 21 ftp.dlptest.com 21 True
in one terminal, and :
ftp 127.0.0.1 21
in the other one.
My code seems to be working fine, except that... I receive no data. I tried different ftp servers (notice that i don't use the one quoted in the book), but it still doesn't work. It just says :
[*] Listening on 127.0.0.1
> Received incoming connection from 127.0.0.1:55856
but it doesn't actually displays anything until the connexion times out or that i stop the command with Ctrl + C.
I know this question has already been asked, but they don't resolve my problem.
Please tell me if i forgot a line of code (for example the one that prints the data on the screen lol) or did anything wrong :)

one the hexa variable you need to put and f'{ord(c):02x}' because you just have a string and not using the 'c' variable from the list comprehension. That's a small typo you missed fix that and try the whole process again.
hexa = ' '.join([f'{ord(c):02X}' for c in word])
The f should be here ^

Socket Programming three way handshake (PYTHON) with TCP

So I am working on a project that requires me to simulate a TCP three handshake protocol using UDP. I want to divided the project into two parts: the first is to establish and close connection; the second is to implement the flow control protocol without any package loss. Since the client and server will be interacting with each other, so I need two sockets, which means I also need two ports - 19990 and 19991 - to send and receive data between client and server.
I am having trouble with the three way handshake
Client -> (Send SYN)
Sever -> (Receive SYN and Send SYN ACK)
Client -> Receives SYN ACK and Sends ACK with Data
Server then establishes a connection and says that data has been received
Client -> Server ACK the connection & received data. So send FIN to close the connection
Server -> Receives FIN and closes the connection (with server_socket.close)
output on the client side:
Send: a,1
Receive: b, num:2
Send: c,3
Receive: d, num:4
Client done
output on server side:
Receive: a, num:1
Send: b,2
Receive c, num:3
Send: d,4
Server done.
However, what I am hoping for them to look like is like so:
client.py
Send SYN
Send: ,1
Receive: num,2
Received SYN-ACK. Send ACK & Data
Send: I love ABC University in New York City.,3
Receive: ,num:4
Received ACK for data. Send FIN. Connection closed.
Send: ,4
server.py
Receive: ,num: 1
Received SYN. Send SYN-ACK
Send: ,2
Receive: I love ABC University in New York City.,num:3
Receive ACK. Connection Established. Received Data. Send ACK
Send: ,4
Receive: ,num:4
Connection Closed
Here is the code I have for server.py:
import socket
import sys
import struct
server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_socket.bind(("0.0.0.0", 19990))
def send(c,num):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
ss = struct.pack("!50si",c.encode(),num)
s.sendto(ss,("127.0.0.1",19991))
s.close()
print("Send:%s,%d" % (c,num))
def recv():
global server_socket
data, addr = server_socket.recvfrom(1024)
str,num = struct.unpack("!50si",data)
str = str.decode("utf-8").replace("\0","")
print("Receive:%s,num:%d" % (str,num))
return str,num
while True:
str,num = recv()
num = num + 1
str = chr(ord(str)+1)
send(str,num)
if str == "d":
break
print("Server Done.")
server_socket.close()
Here is the code I have for client.py
import socket
import sys
import struct
server_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_socket.bind(("0.0.0.0", 19991))
def send(c,num):
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
ss = struct.pack("!50si",c.encode(),num)
s.sendto(ss,("127.0.0.1",19990))
s.close()
print("Send:%s,%d" % (c,num))
def recv():
global server_socket
data, addr = server_socket.recvfrom(1024)
str,num = struct.unpack("!50si",data)
str = str.decode("utf-8").replace("\0","")
print("Receive:%s,num:%d" % (str,num))
return str,num
str = 'a'
num = 1
while True:
send(str,num)
str,num = recv()
if str == "d":
break
str = chr(ord(str)+1)
num = num + 1
print("Client done.")
server_socket.close()
#If the number = 1, sender will send and the receiver will receive
#c is the data and num is the sequence number, for the first three msgs it is the flag
#The payload we want to send is seven characters with one sentence
#Window size is 4 with 4 being four characters
#First package is "i lo"
#I is sent as a package, then space as a package, then l as a package, and o as a package as a window
#First byte is i, second is space, l is third, o is forth.
#Send out 4 bytes, receive 4 acknowledgements.
#When the sender sends out the last byte "."
#To do: Change 'a' to past establishment
#Instead of Send: a, 1
#professor wants to see "Send Think message"
#Receive: Think ack message
#Send: Ack msg
#a b c d are just give you some examples

Weird print() output in while() when trying TCP connection

The problem is: print() in client() only output one line, while I expect it to print several lines because the while loop it resides obviously run more than one time.
The problem occurs when I am testing the sample code given in book Foundations of Python Network Programming, 3rd Edition. This sample code basically create a simple TCP server/client that process/send simple text capitalisation request. Following is the code:
#!/usr/bin/env python3
# Foundations of Python Network Programming, Third Edition
# https://github.com/brandon-rhodes/fopnp/blob/m/py3/chapter03/tcp_deadlock.py
# TCP client and server that leave too much data waiting
import argparse, socket, sys
def server(host, port, bytecount):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen(1)
print('Listening at', sock.getsockname())
while True:
sc, sockname = sock.accept()
print('Processing up to 1024 bytes at a time from', sockname)
n = 0
while True:
data = sc.recv(1024)
if not data:
break
output = data.decode('ascii').upper().encode('ascii')
sc.sendall(output) # send it back uppercase
n += len(data)
print('\r %d bytes processed so far' % (n,), end=' ')
sys.stdout.flush()
print()
sc.close()
print(' Socket closed')
def client(host, port, bytecount):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
bytecount = (bytecount + 15) // 15 * 16 # round up to a multiple of 16
message = b'CAP!' # 16-byte message to repeat over and over
print('Sending', bytecount, 'bytes of data, in chunks of 16 bytes')
sock.connect((host, port))
sent = 0
while sent < bytecount:
sock.sendall(message)
sent += len(message)
# print('test')
print('\r %d bytes sent' %(sent,), end=' ') # <-- THIS IS THE PROBLEMATIC PRINT()
sys.stdout.flush()
print()
sock.shutdown(socket.SHUT_WR)
print('Receiving all the data the server sends back')
received = 0
while True:
data = sock.recv(42)
if not received:
print(' The first data received says', repr(data))
if not data:
break
received += len(data)
print('\r %d bytes received' %(received,), end=' ')
print()
sock.close()
if __name__ == '__main__':
choices = {'client': client, 'server': server}
parser = argparse.ArgumentParser(description='Get deadlocked over TCP')
parser.add_argument('role', choices=choices, help='which role to play')
parser.add_argument('host', help='interface the server listens at;'
' host the client sends to')
parser.add_argument('bytecount', type=int, nargs='?', default=16,
help='number of bytes for client to send (default 16)')
parser.add_argument('-p', metavar='PORT', type=int, default=1060,
help='TCP port (default 1060)')
args = parser.parse_args()
function = choices[args.role]
function(args.host, args.p, args.bytecount)
After I have simply created server locally, I started my client with
python listing3-2.py client 0.0.0.0
This is the output I get:
Sending 32 bytes of data, in chunks of 16 bytes
32 bytes sent
Receiving all the data the server sends back The first data received
says b'CAP!CAP!CAP!CAP!CAP!CAP!CAP!CAP!' 32 bytes received
From this output we know that the while() loop has run 8 times(because of 8 'CAP!'s) but print('\r %d bytes sent' %(sent,), end=' ') has run only once. What is more strange to me is that sys.stdout.flush() doesn't work, despite the effort of the author who might have also noticed the problem.
Something even more weird happens when I try to add one line of print('test') just before the problematic print(), see what happens?
python listing3-2.py client 0.0.0.0
Sending 32 bytes of data, in chunks of 16 bytes test
4 bytes sent test
8 bytes sent test
12 bytes sent test
16 bytes sent test
20 bytes sent test
24 bytes sent test
28 bytes sent test
32 bytes sent
Receiving all the data the server sends back The first data received says
b'CAP!CAP!CAP!CAP!CAP!CAP!CAP!CAP!' 32 bytes received
With an additional print() just before, print('\r %d bytes sent' %(sent,), end=' ') run 8 times, though the structure of loop is absolutely unchanged. I am totally confused by the fact that adding a print() could bring such consequence. And I am also confused by the thought that print() method may have some strange characteristics if used in this situation.

It is the \r combined with end=' ' in the print message. That is a carriage return without linefeed, so all the lines print over each other. The extra print adds a linefeed and they stop printing over each other. Change it to \n to fix, or more simply:
print(sent,'bytes sent')
P.S. There is also a math error:
bytecount = (bytecount + 15) // 15 * 16
should be:
bytecount = (bytecount + 15) // 16 * 16

Python UDP program multiple connections with select

I'm making a UDP program that connects multiple clients/neighbors. There is no server everyone is using the same program. I'm trying to test it with localhost so consider all IPs and ports of neighbors work as intended. Using 127.0.0.1 as IP on all and connecting to different ports.
So my question is why do I receive the startup data that I send before the while loop but I cannot send any? Seems that I am doing something wrong with sys.stdin.
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.bind((HOST, PORT))
going = True
input = [s]
while going:
i,o,e = select.select(input,[],[],)
try :
for x in i:
if x == sys.stdin:
data = sys.stdin.read()
for i in neighbors:
addr = (i[1][0], int(i[1][1]))
s.sendto(data, addr)
else:
msg, addr = s.recvfrom(100)
print msg
except socket.error, msg:
print 'Error Code : ' + str(msg[0]) + ' Message ' + msg[1]
sys.exit()
s.close()

input is always [s], so i is always going to be [s], so x == sys.stdin is never going to be True (because x is always s), so only the else clause will ever execute.
Maybe you meant input = [s, sys.stdin]?