I am trying to send a file over a tcp socket in a peer to peer chat system coded in python. The receiving socket seems not to know that there is no more file to receive. The only way I can get the receiving socket to not anticipate the data that isn't coming is by closing the sending socket (using socket.shutdown(socket.SHUT_WR)). However, closing the sending socket is not an option because I need that socket to send other messages. I fisrt tried allocating a new port for file sending/receiving and failed. Now, I have tried creating an end of file "signal" but it does not get recognized on the receiving end as a message separate from the tcp segment, so I am stuck.
The sending code looks like this:
def sendFile(self,filePath):
try:
f = open(filePath, 'rb')
print 'file opened'
for soc in self.allClients.keys():
try:
f = open(filePath, 'rb')
except:
print "File does not exist"
print 'Sending File: ' + filePath
l = f.read(1024)
while (l):
print 'Sending...'
soc.send(l)
l = f.read(1024)
soc.send('end')
f.close()
print 'File sent'
except:
print "File does not exist"
the receiving code looks like this:
def receiveFile(self, ext, clientsoc):
f = open('receivedFile' + ext,'wb')
print "Receiving File..."
l = clientsoc.recv(1024)
while(l):
print "Receiving..."
if (l is not 'end'):
f.write(l)
print l + '\n'
l = clientsoc.recv(1024)
else:
break
f.close()
print "Received Fileeeeeooooo"
Even more strange, this code works when I am using it outside of my peer programme. Any help would be greatly appreciated, I have been struggling with this for two days now.
First of all, you must not compare strings with is or is not:
>>> a = "he"
>>> a + "llo" is "hello"
False
Second, TCP is a streaming protocol. recv is getting up to 1024 bytes, but could be less, and if you send two pieces from your server, they can be merged into one recv. So l is not likely to be "end" but "last bytes of the file's end". And if you check for "end", "end" is not allowed to be in the file. Best solution is to send the length of the file first, and then send and recv length bytes.
PS: you probably want to use sendall at the server side.
Sending the file size first worked!!
New code looks like this.
Sending code:
def sendFileOrImage(self,path):
name, ext = os.path.splitext(path)
filesize = str(os.path.getsize(path))
message = filesize + ',' + ext
print 'message'
for client in self.allClients.keys():
client.send(message)
self.sendFile(path)
def sendFile(self,filePath):
try:
f = open(filePath, 'rb')
except:
print "File does not exist"
print 'Sending File: ' + filePathty
for soc in self.allClients.keys():
l = f.read(1024)
while (l):
print 'Sending...'
soc.send(l)
l = f.read(1024)
f.close()
print 'File sent'
Receiving code:
def receiveFile(self,ext, filesize, clientsoc):
total = 0
f = open('receivedFile' + ext,'wb')
print "Receiving File..."
l = clientsoc.recv(1024)
total = len(l)
while(l):
print "Receiving..."
f.write(l)
if (str(total) != filesize):
print 'trying to receive'
l = clientsoc.recv(1024)
total = total + len(l)
else:
break
f.close()
print "Received Fileeeeeooooo"
Thanks everyone for the help!
Your receiving code works but still there is a problem you need to fix real quick. If your filesize is under 1024, your socket will keep listening for ever and in order to fix this problem you need to change your code like below.
def receiveFile(self,ext, filesize, clientsoc):
total = 0
f = open('receivedFile' + ext,'wb')
print("Receiving File...")
l = clientsoc.recv(1024)
total = len(l)
while(l):
print("Receiving...")
f.write(l)
if (str(total) != filesize):
print 'trying to receive'
l = clientsoc.recv(1024)
total = total + len(l)
if(int(total)>= int(filesize)):
break
f.close()
print("Received Fileeeeeoooo")
Related
Client side:
def send_file_to_hashed(data, tcpsock):
time.sleep(1)
f = data
flag = 0
i=0
tcpsock.send(hashlib.sha256(f.read()).hexdigest())
f.seek(0)
time.sleep(1)
l = f.read(BUFFER_SIZE-64)
while True:
while (l):
tcpsock.send(hashlib.sha256(l).hexdigest() + l)
time.sleep(1)
hashok = tcpsock.recv(6)
if hashok == "HASHOK":
l = f.read(BUFFER_SIZE-64)
flag = 1
if hashok == "BROKEN":
flag = 0
if not l:
time.sleep(1)
tcpsock.send("DONE")
break
return (tcpsock,flag)
def upload(filename):
flag = 0
while(flag == 0):
with open(os.getcwd()+'\\data\\'+ filename +'.csv', 'rU') as UL:
tuplol = send_file_to_hashed(UL ,send_to_sock(filename +".csv",send_to("upload",TCP_IP,TCP_PORT)))
(sock,flagn) = tuplol
flag = flagn
time.sleep(2)
sock.close()
Server Side:
elif(message == "upload"):
message = rec_OK(self.sock)
fis = os.getcwd()+'/data/'+ time.strftime("%H:%M_%d_%m_%Y") + "_" + message
f = open(fis , 'w')
latest = open(os.getcwd()+'/data/' + message , 'w')
time.sleep(1)
filehash = rec_OK(self.sock)
print("filehash:" + filehash)
while True:
time.sleep(1)
rawdata = self.sock.recv(BUFFER_SIZE)
log.write("rawdata :" + rawdata + "\n")
data = rawdata[64:]
dhash = rawdata[:64]
log.write("chash: " + dhash + "\n")
log.write("shash: " + hashlib.sha256(data).hexdigest() + "\n")
if dhash == hashlib.sha256(data).hexdigest():
f.write(data)
latest.write(data)
self.sock.send("HASHOK")
log.write("HASHOK\n" )
print"HASHOK"
else:
self.sock.send("HASHNO")
print "HASHNO"
log.write("HASHNO\n")
if rawdata == "DONE":
f.close()
f = open(fis , 'r')
if (hashlib.sha256(f.read()).hexdigest() == filehash):
print "ULDONE"
log.write("ULDONE")
f.close()
latest.close()
break
else:
self.sock.send("BROKEN")
print hashlib.sha256(f.read()).hexdigest()
log.write("BROKEN")
print filehash
print "BROKEN UL"
f.close()
So the data upload is working fine in all tests that i ran from my computer, even worked fine while uploading data over my mobile connection and still sometimes people say it takes a long time and they kill it after a few minutes. the data is there on their computers but not on the server. I don't know what is happening please help!
First of all: this is unrelated to sha.
Streaming over the network is unpredictable. This line
rawdata = self.sock.recv(BUFFER_SIZE)
doesn't guarantee that you read BUFFER_SIZE bytes. You may have read only 1 byte in the worst case scenario. Therefore your server side is completely broken because of the assumption that rawdata contains whole message. It is even worse. If the client sends command and hash fast you may get e.g. rawdata == 'DONEa2daf78c44(...) which is a mixed output.
The "hanging" part just follows from that. Trace your code and see what happens when the server receives partial/broken messages ( I already did that in my imagination :P ).
Streaming over the network is almost never as easy as calling sock.send on one side and sock.recv on the other side. You need some buffering/framing protocol. For example you can implement this simple protocol: always interpret first two bytes as the size of incoming message, like this:
client (pseudocode)
# convert len of msg into two-byte array
# I am assuming the max size of msg is 65536
buf = bytearray([len(msg) & 255, len(msg) >> 8])
sock.sendall(buf)
sock.sendall(msg)
server (pseudocode)
size = to_int(sock.recv(1))
size += to_int(sock.recv(1)) << 8
# You need two calls to recv since recv(2) can return 1 byte.
# (well, you can try recv(2) with `if` here to avoid additional
# syscall, not sure if worth it)
buffer = bytearray()
while size > 0:
tmp = sock.recv(size)
buffer += tmp
size -= len(tmp)
Now you have properly read data in buffer variable which you can work with.
WARNING: the pseudocode for the server is simplified. For example you need to check for empty recv() result everywhere (including where size is calculated). This is the case when the client disconnects.
So unfortunately there's a lot of work in front of you. You have to rewrite whole sending and receving code.
I am trying to write a script that requires files to be sent from server to clients and am having issues with the files being completely sent when the destination is not the local host. So when the server and client are on different machines the client seems to receive the file faster then the server is sending it and stops receiving early. An example is if I try to send a 200 MB file over the client stops receiving it around usually around 180 MB.
Here are the portions of the script that send and receive files.
send file
def t_handle(client, address, s):
file_size = str(os.stat("path to file to send").st_size)
print file_size
client.send(file_size)
work_file = open("path to fil to send", "rb")
print "sending file"
file_size = int(file_size)
while file_size > 0:
buffer = work_file.read(1024)
client.send(buffer)
file_size -= 1024
print file_size
print "done"
Receive file
def main():
server = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
server.settimeout(90)
server.connect((host, port))
file_size = int(server.recv(1024))
print "receiving file"
file_size = int(file_size)
print file_size
f = open("test_output.mkv", "wb+")
while file_size > 0:
work_file = server.recv(1024)
f.write(work_file)
file_size -= 1024
print file_size
f.close()
print "done"
return 0
Any help would be greatly appreciated.
The argument to socket.recv() is the maximum number of bytes that the call will return, and there's no guarantee that you'll actually get that many on any given call:
while file_size > 0:
work_file = server.recv(1024)
f.write(work_file)
file_size -= 1024
print file_size
If you receive fewer than 1024 bytes on each call, yet subtract that many from your target byte count, your code will act as if it's complete before it actually is.
Try modifying it to something like:
while file_size > 0:
work_file = server.recv(1024)
f.write(work_file)
# subtract the actual number of bytes received
file_size -= len(work_file)
print file_size
I'm using python version 2.7.9 and i try to send png file.
But something strange happens..i using sockets and sends a post request(or kind of).
I send the request to the server from the client,then i prints the length of the request received on the server, for example, the length is:1051.
Then I do a regex to take the png file data, and then prints the length, and the length is 2632, that he larger than the response?!
I think the problem is that it's actually write the content, but not the right of representation, I tried different things but they did not work, so I ask here how to solve this problem.
Server source code:
import socket
import re
server = socket.socket()
server.bind(('0.0.0.0',8080))
while True:
server.listen(2)
(client, client_addr) = server.accept()
print 'IP :',client_addr
res = client.recv(0xfffffff)
print len(res)
#get file name
file_name = res.split('&')[0]
file_name = str(file_name.split('=')[1])
print repr(res)
#get the data of the file
raw_img = str(re.findall("&photo_data=(.*)" ,res ,re.DOTALL))
print "File name:" + file_name
print "Size:" + str(len(raw_img))
with open(file_name, 'wb') as f:
f.write(raw_img)
print "Done"
Client source code:
import socket
client = socket.socket()
client.connect(('127.0.0.1',8080))
raw_data = open('test.png', 'rb').read()
save_file_name = raw_input("Enter the file name:")
print len(raw_data)
output = 'POST /upload HTTP/1.1\r\n'
output += 'Content-Length:' + str(len(raw_data)) + str(len(save_file_name)) + '\r\n\r\n'
output += 'file_name=' + save_file_name + '&'
output += 'photo_data=' + raw_data
print len(output)
client.send(output)
client.close()
First, you should use while True to receive the full data:
res = ''
while True:
data = client.recv(1024)
if not data:
break
res += data
print len(res)
Then, re.findall actually returns an array, not a string. So you should do this:
r = re.findall("&photo_data=(.*)" ,res ,re.DOTALL)
raw_img = str(r[0])
Now it works fine.
Why doesn't the code before work? Let's say we have a list:
r = ['\x45']
The data in raw_img part is basically like this. If we brutely convert this list to a str, we have:
print len(str[r])) # ['E'], 5
Actually, what we need is r[0]:
print len(str[r[0])) # 1
That's why the size of the file became larger.
trying out a new way to send files. The client will be run every 10 mins to ask server to send what's new in last 10 mins. What I have so far work 40% of the time. I can't figure out so far why that is the case.
server main loop:
while 1:
conn, addr = s.accept()
last_ai = get_last_sent_ai()
new_ai = get_new_ai(last_ai)
ai_notes = ''
''' send # of file '''
print "sending length ---------"
conn.sendall('{0}'.format(len(new_ai)))
for ai_file in new_ai:
ai_file = ai_file.rstrip()
if not os.path.isfile(ai_file): continue
f = open(ai_file, 'rb')
ai_notes = f.read()
f.close()
print "sending file infor " + '{0:<10d}:{1:>40s}'.format(len(ai_notes), ai_file)
ready_flag = conn.recv(5)
if ready_flag == "Ready":
conn.sendall(ai_notes)
if len(new_ai) > 0:
update_last_sent(new_ai[-1])
else:
print 'nothing to send'
conn.sendall(' ')
conn.close()
s.close()
Client main loop:
if num_f == 0: sys.exit(0)
while num_f > 0:
f_info = ''
f_info = s.recv(50)
f_len,f_name_tmp = f_info.split(':')
f_len = int(f_len)
s.sendall("Ready")
f_name = f_name_tmp
f = open(f_name, 'wb')
recvd = 0
while recvd < f_len:
chunk = s.recv(min(f_len - recvd, 1024))
if chunk == '':
raise RuntimeError("socket connection broken")
f.write(chunk)
recvd = recvd + len(chunk)
f.close()
num_f -= 1
s.close()
update:
the issue seems to be gone after I went back and changed how the send and receive are done. There gotta be some kind of blocking going on when it hangs, so I followed the python doc and modified the code and all the tests so far are working.
I found the code here: Send a file through sockets in Python (the selected answer)
But I will jut post it here again..
server.py
import socket
import sys
s = socket.socket()
s.bind(("localhost",9999))
s.listen(10)
while True:
sc, address = s.accept()
print address
i=1
f = open('file_'+ str(i)+".txt",'wb') #open in binary
i=i+1
while (True):
l = sc.recv(1024)
while (l):
print l #<--- i can see the data here
f.write(l) #<--- here is the issue.. the file is blank
l = sc.recv(1024)
f.close()
sc.close()
s.close()
client.py
import socket
import sys
s = socket.socket()
s.connect(("localhost",9999))
f=open ("test.txt", "rb")
l = f.read(1024)
while (l):
print l
s.send(l)
l = f.read(1024)
s.close()
On server code, the print l line prints the file contents..so that means that content is being transferred..
but then the file is empty??
what am i missing?
thanks
You are probably trying to inspect the file while the program is running. The file is being buffered, so you likely won't see any output in it until the f.close() line is executed, or until a large amount of data is written. Add a call to f.flush() after the f.write(l) line to see output in real time. Note that it will hurt performance somewhat.
Well that server code didn't work anyway, I've modified it to get it working.
The file was empty because it was stuck in the while True and never got around to closing the file.
Also i=1 was inside the loop so it was always writing to the same file.
import socket
import sys
s = socket.socket()
s.bind(("localhost",9999))
s.listen(10)
i=1
while True:
print "WILL accept"
sc, address = s.accept()
print "DID accept"
print address
f = open('file_'+ str(i)+".txt",'wb') #open in binary
i += 1
l = sc.recv(1024)
while (l):
f.write(l) #<--- here is the issue.. the file is blank
l = sc.recv(1024)
f.close()
sc.close()
print "Server DONE"
s.close()