I have a simple client-server program, in which I try to send the contents of my Documents folder that contains 2 files ["img1.jpg", "img2.jpg"].
Functioning:
The server waits for a client to connect and receives the messages from it, but if the message is text: files then the createExp () function that receives the name of the new folder to be created and the amount of files it goes to start Has receive.
With that data, I start a for cycle that has to be repeated according to the number of files that the user indicated to the server.
Cycle for:
This cycle has the function of receiving the data of each of the files sent by the client and subsequently saved in the indicated route.
Issue:
The server correctly receives a small part of the data, but then throws an error:
Traceback (most recent call last):
File "C:\Users\Dell\Desktop\servidor_recv_archivo.py", line 53, in <module>
if msgp.decode() == "files":
UnicodeDecodeError: 'utf-8' codec can't decode byte 0x89 in position 5: invalid
start byte
server.py
import socket
import os
def bytes_to_int(b):
result = 0
for i in range(4):
result += b[i]<<(i*8)
return result
def makeExp(client):
while True:
FolderName = client.recv(1024).decode()
NumberFiles = client.recv(1024).decode()
print(FolderName,NumberFiles)
if not os.path.exists(FolderName):
os.mkdir(FolderName)
for element in range(int(NumberFiles)):
size = client.recv(4)
size = bytes_to_int(size)
current_size = 0
buffer = b""
while current_size < size:
data = client.recv(1024)
if not data:
break
if len(data) + current_size > size:
data = data[:size-current_size]
buffer += data
current_size += len(data)
with open(str(element),"wb") as f:
f.write(buffer)
break
ip = "192.168.8.8"
port = 5555
data = (ip,port)
listen = 2
server = socket.socket()
server.bind(data)
server.listen(listen)
client,direction = server.accept()
while True:
try:
msgp = client.recv(1024)
print(msgp)
client.sendall("Msg recv".encode())
if msgp.decode() == "files":
makeExp(client)
except ConnectionResetError:
print("{} ".format(direction))
break
client.py
import socket
import os
def convert_to_bytes(length):
result = bytearray()
result.append(length&255)
for i in range(3):
length = length>>8
result.append(length&255)
return result
def makeFolder(client):
rute = "C:/Users/AngelHp/Desktop/Documentos"
FolderName = os.path.basename("C:/Users/AngelHp/Desktop/Documentos")
NumberFiles = str(len(os.listdir("C:/Users/AngelHp/Desktop/Documentos")))
client.sendall(FolderName.encode())
client.sendall(NumberFiles.encode())
for element in (os.listdir(rute)):
length = os.path.getsize(rute+"/"+element)
client.send(convert_to_bytes(length))
with open(rute+"/"+element,"rb") as infile:
d = infile.read(1024)
while d:
client.send(d)
d = infile.read(1024)
ip = "192.168.8.8"
port = 5555
client = socket.socket()
client.connect((ip,port))
while True:
msg = input("> ")
if msg != "files": #Al oprimir el boton guarar en serv, lanzara la funcion crearExpServ
client.sendall(msg.encode())
reply = client.recv(1024).decode()
print(reply)
elif msg == "files":
print("ok")
makeFolder(client)
#mark - edited
import socket
with socket.socket() as s:
s.bind(('',8000))
s.listen(1)
with s.accept()[0] as c:
chunks = []
while True:
chunk = c.recv(4096)
if not chunk: break
chunks.append(chunk)
for i in range(2):
with open('image{}.png'.format(str(i)),'wb') as f:
f.write(b''.join(chunks))
cieent.py
import socket
import os
with socket.socket() as s:
s.connect(('localhost',8000))
for elemento in os.listdir("img"):
print(elemento)
with open("img"+"/"+elemento,'rb') as f:
s.sendall(f.read())
TCP is a streaming protocol with no concept of message boundaries, so if you print msgp you will see it received more than you expected, probably folder name, number of files, and part of the binary file data. Since that data isn't UTF-8 encoded, you get a UnicodeDecodeError.
You have to define a protocol and buffer data from the socket until it satisfies the protocol (read to a newline character, for example). Also see socket.makefile which wraps a socket in a file-like object so you can treat it more like a file. Methods like .readline() and .read(n) exist, so you could define a protocol like:
Send Folder Name + newline
Send number of files + newline
Send filename #1 + newline
send file size + newline
send binary data of exactly “file size” bytes.
Repeat 3-5 for remaining files.
Example implementing the above protocol (no error handling if a client breaks the protocol). Prepare a folder or two to send, then start the server, in another terminal, run client.py <folder> to transmit <folder> to a Downloads folder.
server.py
import socket
import os
s = socket.socket()
s.bind(('', 8000))
s.listen()
while True:
client, address = s.accept()
print(f'{address} connected')
# client socket and makefile wrapper will be closed when with exits.
with client, client.makefile('rb') as clientfile:
while True:
folder = clientfile.readline()
if not folder: # When client closes connection folder == b''
break
folder = folder.strip().decode()
no_files = int(clientfile.readline())
print(f'Receiving folder: {folder} ({no_files} files)')
# put in different directory in case server/client on same system
folderpath = os.path.join('Downloads', folder)
os.makedirs(folderpath, exist_ok=True)
for i in range(no_files):
filename = clientfile.readline().strip().decode()
filesize = int(clientfile.readline())
data = clientfile.read(filesize)
print(f'Receiving file: {filename} ({filesize} bytes)')
with open(os.path.join(folderpath, filename), 'wb') as f:
f.write(data)
client.py
import socket
import sys
import os
def send_string(sock, string):
sock.sendall(string.encode() + b'\n')
def send_int(sock, integer):
sock.sendall(str(integer).encode() + b'\n')
def transmit(sock, folder):
print(f'Sending folder: {folder}')
send_string(sock, folder)
files = os.listdir(folder)
send_int(sock, len(files))
for file in files:
path = os.path.join(folder, file)
filesize = os.path.getsize(path)
print(f'Sending file: {file} ({filesize} bytes)')
send_string(sock, file)
send_int(sock, filesize)
with open(path, 'rb') as f:
sock.sendall(f.read())
s = socket.socket()
s.connect(('localhost', 8000))
with s:
transmit(s, sys.argv[1])
I prepared two folders then ran "client Folder1" and "client Folder2". Client terminal output:
C:\test>client Folder1
Sending folder: Folder1
Sending file: file1 (13 bytes)
Sending file: file2 (13 bytes)
Sending file: file3 (13 bytes)
Sending file: file4 (13 bytes)
C:\test>client Folder2
Sending folder: Folder2
Sending file: file5 (13 bytes)
Sending file: file6 (13 bytes)
Output (server.py):
C:\test>server
('127.0.0.1', 2303) connected
Receiving folder: Folder1 (4 files)
Receiving file: file1 (13 bytes)
Receiving file: file2 (13 bytes)
Receiving file: file3 (13 bytes)
Receiving file: file4 (13 bytes)
('127.0.0.1', 2413) connected
Receiving folder: Folder2 (2 files)
Receiving file: file5 (13 bytes)
Receiving file: file6 (13 bytes)
Other Examples:
Sending Multiple Files Python Using Socket
Example using makefile
The problem is that you can always encode text as bytes, e.g. "🙂".encode(), but can't always decode an arbitrary sequence of bytes as text. This is because not all sequences of bytes are valid UTF-8 text. When you try to decode binary data to check if it's equal to the "files" string, Python will throw an exeption if it detects byte sequences that aren't used by the UTF-8 text standard.
# Will crash if msgp contains byte sequences that aren't defined in the UTF-8 standard.
if msgp.decode() == "files":
I was able to get your code partially working* by first converting the text to bytes, then comparing the bytes themselves**:
if msgp == "files".encode(): # comparing binary won't crash
*You also need to make sure that the client actually sends the message "files" to the server in the elif statement. For testing purposes, I worked around the lack of message boundaries by adding delays after sending each message, but as Mark Tolonen suggested, it would be much better to introduce a message boundary protocol!
**This technique is good for protocols that use a magic number to distingish the message / file, e.g. PDF files start with "%PDF". However, note that some unicode characters have multiple valid binary representations, so comparing unicode at the byte level can lead to issues.
Related
Hi I have these codes to build a client and server under my ip address to transfer a picture. The codes executes file, but the picture is never transferred to my target file. Does anyone know what is off?
This is the codes for server part
`
from socket import socket, SOCK_STREAM, AF_INET
from base64 import b64encode
from json import dumps
from threading import Thread
def main():
class FileTransferHandler(Thread):
def __init__(self, cclient):
super().__init__()
self.cclient = cclient
def run(self):
my_dict = {}
my_dict['filename'] = 'Goz.png'
my_dict['filedata'] = data
json_str = dumps(my_dict)
self.cclient.send(json_str.encode('utf-8'))
self.cclient.close()
server = socket()
server.bind(('127.0.0.1', 5566))
server.listen(512)
print('Server is listening...')
with open('Goz.png', 'rb') as f:
data = b64encode(f.read()).decode('utf-8')
while True:
client, addr = server.accept()
FileTransferHandler(client).start()
if __name__ == '__main__':
main()
This is the codes for the client part
`
from socket import socket
from json import loads
from base64 import b64decode
def main():
client = socket()
client.connect(('127.0.0.1', 5566))
in_data = bytes()
data = client.recv(1024)
while data:
in_data += data
data = client.recv(1024)
my_dict = loads(in_data.decode('utf-8'))
filename = my_dict['filename']
filedata = my_dict['filedata'].encode('utf-8')
with open('/Users/bowenduan/Desktop/Hao' + filename, 'wb') as f:
f.write(b64decode(filedata))
print('Picture saved.')
if __name__ == '__main__':
main()
#at the sending side read the image and convert the dictionary into a string
pic = open('img.jpg','rb')
textcode = pic.read()
mydata = {
'pic' : textcode,
'name' : 'img.jpg'
}
newcode = str(mydata)
##at the recieving side
#size depends on the size of data you are going to recieve
mydata = sock.recv(24000).decode()
newfile = open('new.jpg','wb+') #jpg/jpeg/png/etc
newfile.write(eval(mydata)['pic'])
newfile.close()
This issue appears to have been a filename handling one, not a problem with the actual networking parts of the code. Your client code was expected to save its files into a folder named Hao, but instead your code was using that as the prefix to the filename to be written. So instead of Hao/Goz.png, you were getting HaoGoz.png.
The way to avoid this kind of issue is to use standard library code to handle your paths, rather than doing string manipulation directly. Either the older os.path.join, or the newer pathlib.Path are probably your best bets:
import os.path
folder = '/Users/bowenduan/Desktop/Hao'
# later...
filename = 'Gao.png'
full_filename = os.path.join(folder, filename)
Or:
import pathlib
folder_path = pathlib.Path('/Users/bowenduan/Desktop/Hao')
# later...
filename = 'Gao.png'
full_filename = folder_path / filename
I am trying to learn python socket programming (Networks) and I have a function that sends a file but I am not fully understanding each line of this function (can't get my head around it). Could someone please explain line by line what this is doing. I am also unsure why it needs the length of the data to send the file. Also if you can see any improvements to this function as well, thanks.
def send_file(socket, filename):
with open(filename, "rb") as x:
data = x.read()
length_data = len(data)
try:
socket.sendall(length_data.to_bytes(20, 'big'))
socket.sendall(data)
except Exception as error:
print("Error", error)
This protocol is written to first send the size of the file and then its data. The size is 20 octets (bytes) serialized as "big endian", meaning the most significant byte is sent first. Suppose you had a 1M file,
>>> val = 1000000
>>> val.to_bytes(20, 'big')
b'\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x0fB#'
The program sends those 20 bytes and then the file payload itself. The receiver would read the 20 octets, convert that back into a file size and would know exactly how many more octets to receive for the file. TCP is a streaming protocol. It has no idea of message boundaries. Protocols need some way of telling the other side how much data makes up a message and this is a common way to do it.
As an aside, this code has the serious problem that it reads the entire file in one go. Suppose it was huge, this code would crash.
A receiver would look something like the following. This is a rudimentary.
import io
def recv_exact(skt, count):
buf = io.BytesIO()
while count:
data = skt.recv(count)
if not data:
return b"" # socket closed
buf.write(data)
count -= len(data)
return buf.getvalue()
def recv_file(skt):
data = recv_exact(skt, 20)
if not data:
return b""
file_size = int.from_bytes(data, "big")
file_bytes = recv_exact(skt, file_size)
return file_bytes
I'm having problems with displaying the contents of the file:
def NRecieve_CnD():
host = "localhost"
port = 8080
NServ = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
NServ.bind(("localhost",8080))
NServ.listen(5)
print("Ballot count is waiting for ballots")
conn, addr = NServ.accept()
File = "Ballot1.txt"
f = open(File, 'w+')
data = conn.recv(1024)
print('Ballots recieved initializing information decoding procedures')
while(data):
data1 = data.decode('utf8','strict')
f.write(data1)
data = conn.recv(1024)
print("Ballot saved, Now displaying vote... ")
files = open("Ballot1.txt", "r")
print(files.read())
when the program is run the area where the contents of the file are supposed to be shown is blank.
You're writing to a file, then opening the file again and without flushing your first write either explicitly, or by closing the file handle, you're reading a file. The result is you're reading the file before you finish writing it.
f.write(data1)
f.flush() # Make sure the data is written to disk
data = conn.recv(1024)
print("Ballot saved, Now displaying vote... ")
files = open("Ballot1.txt", "r")
Beyond that, it's probably best if you don't keep the file open for longer than necessary to avoid surprises like this:
with open(File, 'w+') as f:
f.write(data1)
data = conn.recv(1024)
print("Ballot saved, Now displaying vote... ")
with open(File, "r") as f:
print(f.read())
I'm working on a p2p filesharing system in python3 right now and I've come across an issue I don't know how to fix exactly.
I have peers with a server process and a client process where a client process connects to the other nodes, puts it in its own thread, and listens for data over a socket. When downloading from only one other peer, the file is written correctly with no problem, but when it is split up over multiple peers, the file is corrupted. The data is correctly received from both other peers so I'm thinking this would be a file write issue.
When I get the data from a peer, I open the file, seek to the position where the data comes from, and then write it and close the file. Would locks be the solution to this?
This is the code that is in its own thread that is constantly listening
def handleResponse(clientConnection, fileName, fileSize):
# Listen for connections forever
try:
while True:
#fileName = ""
startPos = 0
data = clientConnection.recv(2154)
# If a response, process it
if (len(data) > 0):
split = data.split(b"\r\n\r\n")
#print(split[0])
headers = split[0].replace(b'\r\n', b' ').split(b' ')
# Go through the split headers and grab the startPos and fileName
for i in range(len(headers)):
if (headers[i] == b"Range:"):
startPos = int(headers[i+1])
#fileName = headers[i+2].decode()
break
# Write the file at the seek pos
mode = "ab+"
if (startPos == 0):
mode = "wb+"
with open ("Download/" + fileName, mode) as f:
f.seek(startPos, 0)
f.write(split[1])
f.close()
Answered by Steffen Ullrich.
Solution is to open the file in rb+ instead of ab+, seek to the position and write. Do note that if the file does not exist, it will throw an exception since it is not created in rb+
I have some problems with this code... send not the integer image but some bytes, is there someone than can help me? I want to send all images I find in a folder. Thank you.
CLIENT
import socket
import sys
import os
s = socket.socket()
s.connect(("localhost",9999)) #IP address, port
sb = 'c:\\python27\\invia'
os.chdir(sb) #path
dirs =os.listdir(sb) #list of file
print dirs
for file in dirs:
f=open(file, "rb") #read image
l = f.read()
s.send(file) #send the name of the file
st = os.stat(sb+'\\'+file).st_size
print str(st)
s.send(str(st)) #send the size of the file
s.send(l) #send data of the file
f.close()
s.close()
SERVER
import socket
import sys
import os
s = socket.socket()
s.bind(("localhost",9999))
s.listen(4) #number of people than can connect it
sc, address = s.accept()
print address
sb = 'c:\\python27\\ricevi'
os.chdir(sb)
while True:
fln=sc.recv(5) #read the name of the file
print fln
f = open(fln,'wb') #create the new file
size = sc.recv(7) #receive the size of the file
#size=size[:7]
print size
strng = sc.recv(int(size)) #receive the data of the file
#if strng:
f.write(strng) #write the file
f.close()
sc.close()
s.close()
To transfer a sequence of files over a single socket, you need some way of delineating each file. In effect, you need to run a small protocol on top of the socket which allows to you know the metadata for each file such as its size and name, and of course the image data.
It appears you're attempting to do this, however both sender and receiver must agree on a protocol.
You have the following in your sender:
s.send(file) #send the name of the file
st = os.stat(sb+'\\'+file).st_size
s.send(str(st)) #send the size of the file
s.send(l)
How is the receiver to know how long the file name is? Or, how will the receiver know where the end of the file name is, and where the size of the file starts? You could imagine the receiver obtaining a string like foobar.txt8somedata and having to infer that the name of the file is foobar.txt, the file is 8 bytes long containing the data somedata.
What you need to do is separate the data with some kind of delimeter such as \n to indicate the boundary for each piece of metadata.
You could envisage a packet structure as <filename>\n<file_size>\n<file_contents>. An example stream of data from the transmitter may then look like this:
foobar.txt\n8\nsomedata
The receiver would then decode the incoming stream, looking for \n in the input to determine each field's value such as the file name and size.
Another approach would be to allocate fixed length strings for the file name and size, followed by the file's data.
The parameter to socket.recv only specifies the maximum buffer size for receiving data packages, it doesn't mean exactly that many bytes will be read.
So if you write:
strng = sc.recv(int(size))
you won't necessarily get all the content, specially if size is rather large.
You need to read from the socket in a loop until you have actually read size bytes to make it work.