We Keep Coding

Python socket hangs on connect/accept

I'm trying to send packets using sockets, and was able to do so just fine until this morning. I'm not sure what's going on. The packets are showing up in tcpdump but the server and the client cannot connect to each other.
netcat.py
import socket
import argparse
import sys
import os
import re
import threading
def convertContent(content: str = "") -> bytes:
byteContent = []
# grab the hex from the content
for i in range(len(content)):
if content[i] == "\\" and content[i+1] == "x":
byteContent.append(f"{content[i+2]}{content[i+3]}")
# grab the non hex from the content, split it on the hex
stringContent = re.split(r"\\x.{2}", content)
byteIndex = 0
newContent = b""
# Re add the non-hex content, and the hex content
for word in stringContent:
newContent += word.encode()
if byteIndex < len(byteContent):
newContent += bytes.fromhex(byteContent[byteIndex])
byteIndex += 1
newContent = newContent.replace(b"\\n", b"\n").replace(b"\\r", b"\r")
return newContent
class Netcat():
'''
Netcat class that can be used to send/receive TCP packets
'''
BUFFER_SIZE = 1024
def __init__(self):
pass
#classmethod
def createSocket(cls):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Address might be in a TIME_WAIT status, ignore this
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Port might be in a TIME_WAIT status, ignore this
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1)
return sock
#classmethod
def send(cls, hostname: str = "127.0.0.1", srcPort: int = 0, destPort: int = 9999, content: str = "", buffer_size: int = 1024):
srcPort = int(srcPort)
destPort = int(destPort)
try:
content = convertContent(content=content)
except:
pass
sock = cls.createSocket()
# Set the source port before sending
sock.connect((hostname, destPort))
sock.sendall(content)
# shutdown might be redundant/unnecessary (tells connected host that we're done sending data)
sock.shutdown(socket.SHUT_WR)
while True:
data = sock.recv(buffer_size)
if len(data) == 0:
break
sock.close()
#classmethod
def receive(cls, port: int = 9999, buffer_size: int = 1024):
if port <= 1024 and os.geteuid() != 0:
print(f"Listening on port {port} requires superuser privileges!")
return
host = ""
sock = cls.createSocket()
sock.bind((host, port))
sock.listen(10)
conn, addr = sock.accept()
while True:
data = conn.recv(buffer_size)
if not data:
break
conn.close()
threading.Thread(target=Netcat.receive,daemon=True).start()
Netcat.send(content="test")
Note: I am sending the packets from one VM to another, rather than sending to myself, but it would be a lot to ask people to spin up a bunch of VMs to reproduce this. The hostname param in the send method should be the actual IP of the receiving machine
I've thrown some print statements, and the server stops on sock.accept(), while the client hangs on sock.connect((hostname, destPort))
I checked the hostname for the server, and it's listening on (0.0.0.0, 8888) (assuming 8888 is the port param), which means its listening on all interfaces on that port, so I dont know why its refusing to connect
I tcpdumped on the server, and its getting the packets, it gets a SYN, then sends out a SYN, ACK, but the rest of the packets are marked as re-transmissions.
I've tried looping the accept & connect lines, thinking maybe some sort of race condition was occurring, but no matter what I do the client can't connect to the server.
Edit: This works on my local machine, but still breaks when I try to send packets over the network. The first 2 steps of the handshake go through SYN & SYN, ACK, but not the third ACK

Don't bind in the client. Working example below, but minor changes to make a standalone script:
import socket
import threading
def receive(port: int = 9999, buffer_size: int = 1024):
host = ""
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Address might be in a TIME_WAIT status, ignore this
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((host, port))
sock.listen()
conn, addr = sock.accept()
while True:
data = conn.recv(buffer_size)
if not data:
break
print(data)
conn.close()
def send(hostname: str = "127.0.0.1", destPort: int = 9999, content: str = b"test", buffer_size: int = 1024):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# Address might be in a TIME_WAIT status, ignore this
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# Removed bind
sock.connect((hostname, destPort))
sock.sendall(content)
# shutdown might be redundant/unnecessary (tells connected host that we're done sending data)
sock.shutdown(socket.SHUT_WR)
while True:
data = sock.recv(buffer_size)
if len(data) == 0:
break
sock.close()
threading.Thread(target=receive,daemon=True).start()
send()
Output:
b'test'

Empty string is appended to the deque

I have a deque on a host. Each String that is received through TCP socket is appended to this deque. When each data is received, I print the data and the deque. Here is the code:
from __future__ import print_function
import commands
import socket
import select
from collections import deque
host = commands.getoutput("hostname -I")
port = 5005
backlog = 5
BUFSIZE = 4096
BUFFER_SIZE = 1024
q = deque()
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
if (data == 'sample.jpg'):
print("start processing")
#processP(q)
else:
print("appended")
q.append(data)
print(q)
conn.close()
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
#print("select.select")
inputready,outputready,exceptready = select.select(input,[],[], 0.1)
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
The problem is that when I print the received data and deque in these lines print("received data:", data) and print(q), the received data is print correctly, but the deque content is printed like these in each step:
deque([''])
deque(['',''])
deque(['','',''])
What's wrong? Here is the sender code which doesn't seem to have any problem:
from __future__ import print_function
import socket
from struct import pack
import commands
import select
#HOST = '10.0.0.2'
PORT = 5005
BUFSIZE = 4096
def tcp_send(s, ip):
TCP_IP = ip
BUFFER_SIZE = 1024
MESSAGE = s
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, PORT))
s.send(MESSAGE)
data = s.recv(BUFFER_SIZE)
s.close()
print ("sent data:", data)
fnames = [
'90.jpg','91.jpg','92.jpg','93.jpg','94.jpg','95.jpg','96.jpg','97.jpg','98.jpg','99.jpg','100.jpg','sample.jpg'
]
def main():
for fname1 in fnames:
tcp_send(fname1,'10.0.0.2')
if __name__ == '__main__':
main()

replace your read_tcp(s) with
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
print(data)
if (data == 'sample.jpg'):
print("start processing")
#processP(q)
else:
print("appended", data)
q.append(data)
print(q)
conn.close()
you are trying to access data outside while that's why it is empty

It looks like indentation problem. You will brake out of the while loop (in read_tcp()) only when data is evaluated False. Only then, already out of the loop you append current value of data to to the deque q. In the loop you print every chunck of data you get. I think the if block needs to be indented one level to be part of the loop.
Also in read_udp() I don't see that you add anything to deque

Sending a message after file transfer completion

I'm new to socket programming. I'm trying to send 4 files from one host to another. Here is the code:
sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
if __name__ == '__main__':
main()
Receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
File transfer is working fine and there is no problem with it. Now I want to send a simple string like "finish" after sending all files, so that receiver will understand that the transfer is completed and it will do some other tasks based on this finish message (however, it still can receive messages at the same time).
I tried to do this by adding another function called sendMessage() to the sender code and a function called recvMessage() to the receiver. Here are the changed codes:
Sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 1024
MESSAGE = "Finish!"
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
#Updated part for sending message
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
sendMessage(MESSAGE)
if __name__ == '__main__':
main()
receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 20
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
#Updated part for receiving message
def recvMessage(conn):
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
recvMessage(conn)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
but after running these codes both sender and receiver freeze after the complete transfer of 4 files and nothing happens. What's wrong and how can I do this?

I suspect you're falling prey to buffering here:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
You perform a send, then immediately try to recv. Except stream connections tend to buffer to avoid excessive packet overhead, so odds are, you don't actually send anything yet, the server doesn't see anything so it doesn't respond, and both sides are blocked waiting for data.
The simplest solution here is to shut down the send side port for writing once you've sent the message, which forces out the last data and lets the receiver know you're done:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.sendall(message) # sendall makes sure the *whole* message is sent
sock.shutdown(socket.SHUT_WR) # We're done writing
data = sock.recv(BUFFER_SIZE)
sock.close()
print("received data:", data)
On the receiver side you have a bigger problem: You close the connection before trying to receive at all:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close() # Closed here!!!
print('saved\n')
recvMessage(conn) # Used again here!!!
So move the close after the recvMessage call, and change recvMessage to use setsockopt to turn on TCP_NODELAY, so buffering isn't occurring (otherwise the echo back may end up buffering indefinitely, though shutting down the sender for write does mean you're likely to detect the sender is done and exit the loop then close the connection, so it may work fine without TCP_NODELAY, as long as the sender isn't expecting to receive data and respond further):
def recvMessage(conn):
# Disable Nagle algorithm so your echoes don't buffer
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.sendall(data) # echo using sendall, again, to ensure it's all really sent

A receiver that can receive both TCP images and UDP texts

I'm new to socket programming . I've implemented 2 separated codes on the same host. One of them is supposed to receive images using TCP protocol and the second one is supposed to receive text messages through UDP protocol. Both of them are working fine separately. Here are the codes:
Image receiver (TCP):
from __future__ import print_function
import socket
from struct import unpack
import Queue
from PIL import Image
HOST = '10.0.0.1'
PORT = 12345
BUFSIZE = 4096
q = Queue.Queue()
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
q.put(name)
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
Text receiver (UDP):
import socket
UDP_IP = "10.0.0.1"
UDP_PORT = 5005
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock.bind((UDP_IP, UDP_PORT))
while True:
data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
print "received message:", data
Now my question is: How can I merge these 2 codes? I don't want to open 2 separate consoles for each of them and I want one code instead of two. Is it possible?
I tried the solution which was provided in the comment and here is the code:
from __future__ import print_function
from select import select
import socket
from struct import unpack
host = '10.0.0.2'
port = 5005
size = 8000
backlog = 5
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
def read_udp(s):
data,addr = s.recvfrom(8000)
print("Recv UDP:'%s'" % data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp.bind((host,port))
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
input = [tcp,udp]
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
if __name__ == '__main__':
run()
but I don't receive any UDP or TCP packet now and it doesn't seem to work for me.

Short answer yes, but you will need to implement multithreading for this. For instance, your program will spawn two threads, one for TCP sockets and other for UDP.

audio over python tcp error

I am writing a simple python tcp code to send over a wav file however I seem to be getting stuck. can someone explain why my code is not working correctly?
Server Code
import socket, time
import scipy.io.wavfile
import numpy as np
def Main():
host = ''
port = 3333
MAX = 65535
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(1)
print "Listening on port..." + str(port)
c, addr = s.accept()
print "Connection from: " + str(addr)
wavFile = np.array([],dtype='int16')
i = 0
while True:
data = c.recvfrom(MAX)
if not data:
break
# print ++i
# wavfile = np.append(wavfile,data)
print data
timestr = time.strftime("%y%m%d-%h%m%s")
print timestr
# wavF = open(timestr + ".wav", "rw+")
scipy.io.wavfile.write(timestr + ".wav",44100, data)
c.close()
if __name__ == '__main__':
Main()
Client Code
host, port = "", 3333
import sys , socket
import scipy.io.wavfile
# create a tcp/ip socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# connect the socket to the port where the server is listening
server_address = (host, port)
print >>sys.stderr, 'connecting to %s port %s' % server_address
input_data = scipy.io.wavfile.read('Voice 005.wav',)
audio = input_data[1]
sock.connect(server_address)
print 'have connected'
try:
# send data
sock.sendall(audio)
print "sent" + str(audio)
sock.close()
except:
print('something failed sending data')
finally:
print >>sys.stderr, 'closing socket'
print "done sending"
sock.close()
Please help someone, I want to send an audio file to my embedded device with tcp since it crucial data to be processed on the embedded device.

Not sure why you go to the trouble of using scipy and numpy for this, since you can just use the array module to create binary arrays that will hold the wave file. Can you adapt and use the simple client/server example below?
(Note: I've copy/pasted a small Windows sound file called 'tada.wav' to the same folder to use with the test scripts.)
Code for the server script:
import socket
HOST = '' # Symbolic name meaning all available interfaces
PORT = 50007 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
print('Listening...')
conn, addr = s.accept()
print('Connected by', addr)
outfile = open("newfile.wav", 'ab')
while True:
data = conn.recv(1024)
if not data: break
outfile.write(data)
conn.close()
outfile.close()
print ("Completed.")
Code for the client:
from array import array
from os import stat
import socket
arr = array('B') # create binary array to hold the wave file
result = stat("tada.wav") # sample file is in the same folder
f = open("tada.wav", 'rb')
arr.fromfile(f, result.st_size) # using file size as the array length
print("Length of data: " + str(len(arr)))
HOST = 'localhost'
PORT = 50007
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.send(arr)
print('Finished sending...')
s.close()
print('done.')
This works for me (though only tested by running both on localhost) and I end up with a second wave file that's an exact copy of the one sent by the client through the socket.