We Keep Coding

Python Socket file transfer without closing sockets [duplicate]

This question already has answers here:
sending multiple files in python
(1 answer)
download multiple subsequent files from one server with one TCP handshake
(1 answer)
Sending multiple images with socket
(1 answer)
Sending multiple files through a TCP socket
(2 answers)
Closed last year.
I've been trying to transfer large files through python sockets, I can do it but it only works well if I close the connection, I want to keep the connection open and keep transfering files after that
Server:
import socket
import sys
from tqdm import tqdm
IP =
PORT =
ADDR = (IP, PORT)
SIZE = 4096
FORMAT = "utf-8"
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(ADDR)
server.listen()
print("[+] Listening...")
conn, addr = server.accept()
print(f"[+] Client connected from {addr[0]}:{addr[1]}")
#last try:
def receiver():
file_name = conn.recv(1024).decode()
file_size = conn.recv(1024).decode()
with open(file_name, "wb") as file:
c = 0
while c <= int(file_size):
data = conn.recv(1024)
if not (data):
break
file.write(data)
c += len(data)
def execute_rem(command):
conn.send(command.encode(FORMAT))
if command[0] == "exit":
conn.close()
server.close()
exit()
def run():
while True:
command = input(">> ")
if len(command) != 0:
conn.send(command.encode(FORMAT))
command = command.split(" ")
if command[0] == "download" and len(command) == 2:
receiver()
else:
result = conn.recv(SIZE)
if result == "1":
continue
else:
print(str(result, FORMAT))
run()
client:
import os
import sys
import socket
import time
import subprocess
from tqdm import tqdm
IP =
PORT =
ADDR = (IP, PORT)
SIZE = 4096
FORMAT = "utf-8"
WAITTIME = 10
client = 0
while True:
try:
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.connect(ADDR)
break
except socket.error:
time.sleep(WAITTIME)
def exec_comnd(command):
cmd = subprocess.Popen(command,shell = True, stderr = subprocess.PIPE, stdout = subprocess.PIPE, stdin = subprocess.PIPE)
byte = cmd.stdout.read()+cmd.stderr.read()
if len(byte) == 0:
byte = "1"
return byte
def f_send(file_name):
file_size = os.path.getsize(file_name)
client.send(file_name.encode())
client.send(str(file_size).encode())
with open(file_name, "rb") as file:
c = 0
while c <= file_size:
data = file.read(1024)
if not (data):
break
client.sendall(data)
c += len(data)
def run():
while True:
command = client.recv(SIZE).decode(FORMAT)
command = command.split(" ")
if command[0] == "exit":
client.close()
exit()
elif command[0] == "cd" and len(command) == 2:
path = command[1]
os.chdir(path)
client.send(("Cambio a directorio " + path).encode(FORMAT))
elif command[0] == "download" and len(command) == 2:
f_send(command[1])
else:
res_comnd = exec_comnd(command)
client.send(res_comnd)
run()
This is my last attempt but I have tried different ways. The file gets sent but the server gets stuck until I ctl+c, after that, based on the output, server gets stuck on "data = conn.recv(1024))" (terminal output stops at "download test.jpg") and client gets stuck on "client.send(res_comnd)". I don't see why, is the only way closing the socket after the file transfer?
server output:
[+] Listening... [+] Client connected from
IP:PORT
>> download test.jpg
^CTraceback (most recent call last): File "/home/xxxx/project/nuev/server.py", line 94, in <module>
run() File "/home/xxxx/project/nuev/server.py", line 84, in run
receiver() File "/home/xxxx/project/nuev/server.py", line 36, in receiver
data = conn.recv(1024) KeyboardInterrupt
client output:
Traceback (most recent call last): File "C:\Users\xxxx\Desktop\client.py", line 108, in <module>
run() File "C:\Users\xxxx\Desktop\client.py", line 105, in run
client.send(res_comnd) ConnectionAbortedError: [WinError 10053] An established connection was aborted by the software in your host machine

Python Sockets multiple file transfers

So I have a python program that basically let's the client connect to the server and send it an excel file which is used as an input for an optimization problem. I then want the server to send the output of this optimization (also an excel file) back to the client. The model itself takes about a minute to solve, and I think this is causing some issues with the client trying to receive the output 'too early'.
The client code:
SEPARATOR = "<SEPARATOR>"
BUFFER_SIZE = 4096
HEADER = 64
HEADERSIZE = 10
port = 1234
FORMAT = 'utf-8'
DISCONNECT_MESSAGE = "!DISCONNECT"
host = "123.45.678.910"
s = socket.socket()
s.connect((host, port))
filename = "input/Model_Input.xlsx"
filesize = os.path.getsize(filename)
s.send(f"{filename}{SEPARATOR}{filesize}".encode())
with open(filename, "rb") as f:
while True:
bytes_read = f.read(BUFFER_SIZE)
if not bytes_read:
break
s.sendall(bytes_read)
out_received = s.recv(BUFFER_SIZE).decode()
out_filename, out_filesize = out_received.split(SEPARATOR)
out_filename = os.path.basename(out_filename)
out_filesize = int(out_filesize)
with open(out_filename, "wb") as h:
while True:
out_bytes_read = s.recv(BUFFER_SIZE)
if not out_bytes_read:
break
h.write(out_bytes_read)
And the server code:
SERVER_PORT = 1234
SERVER_HOST = socket.gethostbyname(socket.gethostname())
BUFFER_SIZE = 4096
SEPARATOR = "<SEPARATOR>"
s = socket.socket()
s.bind((SERVER_HOST, SERVER_PORT))
s.listen(5)
client_socket, address = s.accept()
received = client_socket.recv(BUFFER_SIZE).decode()
filename, filesize = received.split(SEPARATOR)
filename = os.path.basename(filename)
filesize = int(filesize)
with open(filename, "wb") as f:
while True:
bytes_read = client_socket.recv(BUFFER_SIZE)
if not bytes_read:
break
f.write(bytes_read)
##################
## MODEL CODE ##
##################
outfilename = 'Model_Output.xlsx'
outfilesize = os.path.getsize(filename)
client_socket.send(f"{outfilename}{SEPARATOR}{outfilesize}".encode())
with open(outfilename, "rb") as h:
while True:
# read the bytes from the file
bytes_readed = h.read(BUFFER_SIZE)
if not bytes_readed:
break
client_socket.sendall(bytes_readed)
I am able to send the input file to the server and get the model to run, and save the output to storage. However, as soon as I add in the part to try send it back to the client, it gets stuck. It still sends and receives the input file successfully, but then the model never runs. Neither the client nor the server disconnect, they both just seem to be stuck.
Thank you

I can be very difficult for someone (i.e. me) to remotely debug this type of code, so I can't really point to a particular line of code that is necessarily the problem. If, however, your client and server are running on the same machine, then there is a potential problem in the client code that begins:
with open(out_filename, "wb") as h:
while True:
out_bytes_read = s.recv(BUFFER_SIZE)
if not out_bytes_read:
break
h.write(out_bytes_read)
When the open is executed, this sets the file size to 0. The server, meanwhile is reading this file to transmit it to the file and can possibly find that there are now only 0 bytes. But it has already sent a "header" saying that there are N bytes where N is non-zero. But that is a different problem than the one you describe. But it could be happening in the other direction, also. That is, when the client is sending the file and the server is opening the file for output, it is now zeroing out the file that the client is still reading. The code below solves that problem in both directions. Of course, if your client and server are on different computers not accessing the same files concurrently, then what I have described is not an issue. Not yet, anyway.
I can, however, offer a slightly different approach, which does seem to work:
See the Using a Socket section from the Socket Programming HOWTO article in the Python 3 Manual. I have adopted the suggestion to use fixed length messages. It's a little more laborious, perhaps, but also a bit more fullproof. That means that if you want to transmit the filename, you have to first transmit the length of the encoded filename as fixed length length message (3 bytes can handle encoded filenames up to 999 bytes in length) and then you can transmit the encode filename. Similarly, we transmit the length of a file as a 9 byte length (left-padded with zeroes), which can handle file sizes up to 999,999,999 bytes (I set the width of 9 arbitrarily). I have two functions, receive_msg and send_msg that will robustly send and receive complete byte messages and can be used by both the client and server. These are modeled on the MySocket.mysend and MySocket.myreceive methods from the article.
I assumed that the server should be able to handle more than one request before terminating. In fact, it should be able to handle requests concurrently. To that end the server passes a request to a thread pool worker function, process_request, for processing. It wasn't clear what the nature of the so-called "Model Code" was. Assuming the function that performs this computation, process_model, is CPU-intensive, process_requestis passed a multiprocessing pool instance that can be used to perform the process_model processing so that the CPU-intensive portion of processing will not be limited by the Global Interpreter Lock. If there is no real CPU-intensive processing involved, then remove the code that created the multiprocessing pool and then call process_model as a regular function.
Server Code
import socket
from multiprocessing.pool import Pool, ThreadPool
import os.path
BUFFER_SIZE = 4096
SERVER_PORT = 1234
SERVER_HOST = socket.gethostbyname(socket.gethostname())
def receive_msg(sock, msg_length):
chunks = []
bytes_recd = 0
while bytes_recd < msg_length:
chunk = sock.recv(min(msg_length - bytes_recd, BUFFER_SIZE))
if chunk == b'':
raise RuntimeError("socket connection broken")
chunks.append(chunk)
bytes_recd = bytes_recd + len(chunk)
return b''.join(chunks)
def send_msg(sock, msg):
msg_length = len(msg)
totalsent = 0
while totalsent < msg_length:
sent = sock.send(msg[totalsent:])
if sent == 0:
raise RuntimeError("socket connection broken")
totalsent = totalsent + sent
def server():
s = socket.socket()
s.bind((SERVER_HOST, SERVER_PORT))
s.listen(5)
process_pool = Pool(5)
thread_pool = ThreadPool(5)
while True:
client_socket, address = s.accept()
thread_pool.apply_async(process_request, args=(process_pool, client_socket))
def process_request(process_pool, s):
# Fixed length fields:
# width 3 for filename length, followed by filename, width 9 for filesize
filename_size = int(receive_msg(s, 3).decode())
filename = receive_msg(s, filename_size).decode()
filename = os.path.basename(filename)
filesize = int(receive_msg(s, 9).decode())
msg = receive_msg(s, filesize)
with open(filename, "wb") as f:
f.write(msg)
# Assuming processing the model is CPU-intensive,
# we use a process pool for doing that:
out_filename = process_pool.apply(process_model, args=(filename,))
out_filesize = os.path.getsize(out_filename)
encoded_filename = out_filename.encode()
msg1 = b"%03d%s%09dfilesize" % (len(encoded_filename), encoded_filename, out_filesize)
with open(out_filename, "rb") as h:
msg2 = h.read()
send_msg(s, msg1)
send_msg(s, msg2)
def process_model(filename):
...
# Returned filename should probably be a function of the passed filename
return 'Model_Output.xlsx' # name of the output file
if __name__ == '__main__':
server()
Client Code
import socket
import os.path
BUFFER_SIZE = 4096
port = 1234
host = "123.45.678.910"
def receive_msg(sock, msg_length):
chunks = []
bytes_recd = 0
while bytes_recd < msg_length:
chunk = sock.recv(min(msg_length - bytes_recd, BUFFER_SIZE))
if chunk == b'':
raise RuntimeError("socket connection broken")
chunks.append(chunk)
bytes_recd = bytes_recd + len(chunk)
return b''.join(chunks)
def send_msg(sock, msg):
msg_length = len(msg)
totalsent = 0
while totalsent < msg_length:
sent = sock.send(msg[totalsent:])
if sent == 0:
raise RuntimeError("socket connection broken")
totalsent = totalsent + sent
def client():
s = socket.socket()
s.connect((host, port))
filename = "input/Model_Input.xlsx"
filesize = os.path.getsize(filename)
# Fixed length fields:
# width 3 for filename length, followed by filename, width 9 for filesize
encoded_filename = filename.encode()
msg1 = b"%03d%s%09dfilesize" % (len(encoded_filename), encoded_filename, filesize)
with open(filename, "rb") as f:
msg2 = f.read()
send_msg(s, msg1)
send_msg(s, msg2)
out_filename_size = int(receive_msg(s, 3).decode())
out_filename = receive_msg(s, out_filename_size).decode()
out_filename = os.path.basename(out_filename)
out_filesize = int(receive_msg(s, 9).decode())
msg = receive_msg(s, out_filesize)
with open(out_filename, "wb") as h:
h.write(msg)
if __name__ == '__main__':
client()
Update
The entire programming can greatly be simplified by implementing the service as a Remote Procedure Call. The code is based on Python Cookbook, 3rd Edition:
Server
import socket
import pickle
from multiprocessing.connection import Listener
from threading import Thread
from multiprocessing.pool import Pool
import os.path
SERVER_PORT = 1234
SERVER_HOST = socket.gethostbyname(socket.gethostname())
class RPCHandler:
def __init__(self):
self._functions = { }
def register_function(self, func):
self._functions[func.__name__] = func
def handle_connection(self, connection):
try:
while True:
# Receive a message
func_name, args, kwargs = pickle.loads(connection.recv())
# Run the RPC and send a response
try:
r = self._functions[func_name](*args,**kwargs)
connection.send(pickle.dumps(r))
except Exception as e:
connection.send(pickle.dumps(e))
except EOFError:
pass
def server():
global process_pool
handler = RPCHandler()
handler.register_function(process_request)
sock = Listener((SERVER_HOST, SERVER_PORT))
process_pool = Pool(5)
while True:
client = sock.accept()
t = Thread(target=handler.handle_connection, args=(client,))
t.daemon = True
t.start()
def process_request(filename, contents):
filename = os.path.basename(filename)
with open(filename, "wb") as f:
f.write(contents)
# Assuming processing the model is CPU-intensive,
# we use a process pool for doing that:
out_filename = process_pool.apply(process_model, args=(filename,))
with open(out_filename, "rb") as h:
out_contents = h.read()
return (out_filename, out_contents)
def process_model(filename):
...
# Returned filename should probably be a function of the passed filename
return 'Model_Output.xlsx' # name of the output file
if __name__ == '__main__':
server()
Client
import pickle
import socket
import os.path
from multiprocessing.connection import Client
port = 1234
host = "123.45.678.910"
class RPCProxy:
def __init__(self, connection):
self._connection = connection
def __getattr__(self, name):
def do_rpc(*args, **kwargs):
self._connection.send(pickle.dumps((name, args, kwargs)))
result = pickle.loads(self._connection.recv())
if isinstance(result, Exception):
raise result
return result
return do_rpc
def client():
c = Client((host, port))
proxy = RPCProxy(c)
filename = "input/Model_Input.xlsx"
with open(filename, "rb") as f:
contents = f.read()
(out_filename, out_contents) = proxy.process_request(filename, contents)
out_filename = os.path.basename(out_filename)
with open(out_filename, "wb") as h:
h.write(out_contents)
if __name__ == '__main__':
client()

Python Webserver not working

I am building a basic python web server, but I keep having a problem where it is not sending any data (by the way I am accessing the website on the same computer as it is running on and I have the file which the server is trying to access) here is my code:
import socket
HOST, PORT = '', 80
def between(left,right,s):
before,_,a = s.partition(left)
a,_,after = a.partition(right)
return a
filereq = ""
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_socket.bind((HOST, PORT))
lines = []
print("Started!")
listen_socket.listen(1)
print("Listening")
while True:
try:
lines = []
client_connection, client_address = listen_socket.accept()
print("Connected")
request = client_connection.recv(1024)
print("Received Data!")
filereq = between("GET /", " HT", request)
print(filereq)
filereq = open(filereq)
for line in filereq:
lines.append(line)
print(lines)
sendata = ''.join(lines)
print(sendata)
http_response = """\
HTTP/1.1 200 OK
{}
""".format(sendata)
print(http_response)
client_connection.sendall(http_response)
print("Sent the Data!")
client_connection.close()
print("Connection Closed!")
except:
5+5

The problem is that the server is implemented in Python3 but the code mixes bytes and strings, which works in Python2 but not Python3.
This causes an error in the between function, because partition is being called on a bytes object but is being provided with str separator values.
>>> data = b'abc'
>>> data.partition('b')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: a bytes-like object is required, not 'str'
To fix this, decode the data from bytes to str when read from the socket, then encode back to bytes before sending the response (socket.sendall expects bytes as an argument).
Also, print out any exceptions that occur so that you can debug them.
import socket
import sys
import traceback
HOST, PORT = '', 80
def between(left,right,s):
before,_,a = s.partition(left)
a,_,after = a.partition(right)
return a
filereq = ""
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_socket.bind((HOST, PORT))
lines = []
print("Started!")
listen_socket.listen(1)
print("Listening")
while True:
try:
lines = []
client_connection, client_address = listen_socket.accept()
print("Connected")
request = client_connection.recv(1024)
print("Received Data!")
# Decode the data before processing.
decoded = request.decode('utf-8')
filereq = between("GET /", " HT", decoded)
print(filereq)
filereq = open(filereq)
for line in filereq:
lines.append(line)
print(lines)
sendata = ''.join(lines)
print(sendata)
http_response = """\
HTTP/1.1 200 OK
{}
""".format(sendata)
print(http_response)
# Encode the response before sending.
encoded = http_response.encode('utf-8')
client_connection.sendall(encoded)
print("Sent the Data!")
client_connection.close()
print("Connection Closed!")
except Exception:
# Print the traceback if there's an error.
traceback.print_exc(file=sys.stderr)

invalid mode ('wb') or filename: 'torecv.jpg' python

I'm writing a client-server live screenshot streaming. My program works for few seconds. I can see live stream from my client but then my program crushs with this error:
Traceback (most recent call last): File "C:/Users/user-pc/PycharmProjects/TestsFor/Functions‌​/screenshotLive2/ser‌​ver files.py", line 51, in <module> receive_file(filename) File "C:/Users/user-pc/PycharmProjects/TestsFor/Functions‌​/screenshotLive2/ser‌​ver files.py", line 36, in receive_file f = open(file_name, 'wb') IOError: [Errno 22] invalid mode ('wb') or filename: 'torecv.jpg'
This is really weird because its able to open and close it many times before it crushes. Does anyone understand why?
This is my server:
import socket # Import socket module
import cv2
import struct
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # put socket outside of loop?
sock.bind(("0.0.0.0", 8200))
sock.listen(1)
connect, address = sock.accept()
except socket.error:
print "Socket Error"
def recv_msg():
# Read message length and unpack it into an integer
raw_msglen = recvall(4)
if not raw_msglen:
return None
msglen = struct.unpack('L', raw_msglen)[0]
# Read the message data
return recvall(msglen)
def recvall(n):
# Helper function to recv n bytes or return None if EOF is hit
data = ''
while len(data) < n:
packet = connect.recv(n - len(data))
if not packet:
return None
data += packet
return data
def receive_file(file_name):
f = open(file_name, 'wb')
data = recv_msg()
f.writelines(data)
f.close()
while True:
receive_file(r'torecv.jpg')
image = cv2.imread(r'torecv.jpg',1)
cv2.imshow("Live Screen", image)
cv2.waitKey(1)
And this my client:
import ImageGrab
import StringIO
import socket
import struct
try:
connect = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connect.connect(("localhost", 8200))
except socket.error:
print "Error Socket"
def screen_shot():
im = ImageGrab.grab() # take screen shot
saver = StringIO.StringIO() # creating string as 'File'
im.save(saver, "jpeg") # save the file (not on hard drive)
binary_string = saver.getvalue()
saver.close()
msg = struct.pack('L', len(binary_string)) + binary_string
return msg
while True:
connect.sendall(screen_shot())

TypeError in socket, when pickle

I'm trying to write a client-server program where server receives the requests for a database records, or files, and sends it back. Everything was working just fine until I used the pickle function to send data from client to server,
Exception in thread Thread-1:
Traceback (most recent call last):
File "/usr/lib/python2.7/threading.py", line 551, in __bootstrap_inner
self.run()
File "server.py", line 71, in run
data = pickle.loads(data)
File "/usr/lib/python2.7/pickle.py", line 1381, in loads
file = StringIO(str)
TypeError: expected read buffer, NoneType found
When I send data from server to client, there is no problem. I worked like this for a few weeks but when there is about 50 exceptions, the server program closes.
client.py file:
import socket
import sys
import time
import pickle
import struct
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = ''
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data += packet
return data
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def sending(msg):
host = 'localhost'
port = 50000
size = 1024
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
lenght = len(msg)
if lenght>0:
msg = pickle.dumps(msg)
msg = struct.pack('>I', len(msg)) + msg
s.sendall(msg)
print 'sended string lenght: '+str(lenght)
else:
s.send('nothing sended')
data = recv_msg(s)
lenght2 = len(data)
print data
print 'received string lenght: '+str(lenght2)
#sys.stdout.write(data)
s.close()
while 1:
msg = raw_input('Input:')
sending(msg)
server.py:
class Client(threading.Thread):
def __init__(self,(client,address)):
threading.Thread.__init__(self)
self.client = client
self.address = address
self.size = 1024
def run(self):
running = 1
while running:
sock = self.client
data = self.recv_msg(sock)
data = pickle.loads(data)
if data:
msg = struct.pack('>I', len(data)) + data
self.client.sendall(msg)
else:
self.client.close()
running = 0
def recv_msg(self, sock):
# Read message length and unpack it into an integer
raw_msglen = self.recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return self.recvall(sock, msglen)
def recvall(self, sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = ''
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data += packet
return data
In previous version I puted the pickle function from server to client, and it worked fine, no problems, but now I started to write it from the beginning to find the problem, but I didn't.
The recv_msg method was found here:
Python Socket Receive Large Amount of Data

The recv_msg method returns None when EOF is reached, and you pass that None to pickle.loads, which is an error.
To fix the problem, place the call to pickle.loads() after the EOF-check:
data = self.recv_msg(sock)
if data is not None:
data = pickle.loads(data)
msg = struct.pack('>I', len(data)) + data
self.client.sendall(msg)
else:
self.client.close()
running = 0