I am trying the multithreaded python program to connect to server by multiple clients at the same time. The program runs successfully but the image I am trying to send has incomplete data until I terminate the program using Control C. After Control-C, the file reloads and complete image is visible.
I am posting my code here :
Server.py
from socket import *
import thread
def handler(clientsocket, clientaddr):
print "Accepted connection from: ", clientaddr
while 1:
data = clientsocket.recv(8192)
if not data:
break
else:
print "The following data was received - ",data
print "Opening file - ",data
fp = open(data,'r')
strng = "hi"
while strng:
strng = fp.read(8192)
clientsocket.send (strng)
clientsocket.close()
if __name__ == "__main__":
host = 'localhost'
port = 55574
buf = 8192
addr = (host, port)
serversocket = socket(AF_INET, SOCK_STREAM)
serversocket.bind(addr)
serversocket.listen(5)
while 1:
print "Server is listening for connections\n"
clientsocket, clientaddr = serversocket.accept()
thread.start_new_thread(handler, (clientsocket, clientaddr))
serversocket.close()
Client.py :
from socket import *
import os
if __name__ == '__main__':
host = 'localhost'
port = 55574
buf = 8192
addr = (host, port)
clientsocket = socket(AF_INET, SOCK_STREAM)
clientsocket.connect(addr)
while 1:
fname = raw_input("Enter the file name that u want>> ")
if not fname:
break
else:
clientsocket.send(fname)
print "\nThe file will be saved and opened- "
fname = '/home/coep/Downloads/'+fname
nf = open(fname,"a")
strng = "hi"
while strng:
strng = clientsocket.recv(8192)
nf.write(strng)
nf.close()
fname = 'viewnior '+ fname
print fname
os.system(fname)
Try changing:
while strng:
strng = clientsocket.recv(8192)
nf.write(strng)
To:
while True:
strng = clientsocket.recv(8192)
if not strng:
break
nf.write(strng)
There's so many things wrong with that code:
1) Both server and client. Sending and receiveing files might be tricky. Have a look at this:
while strng:
strng = clientsocket.recv(8192)
nf.write(strng)
An infinite loop. You have to add
while strng:
strng = clientsocket.recv(8192)
if not strng:
break
nf.write(strng)
to the server. But the client won't know when you've stopped transmiting the file (and that's the source of your problem). Therefore you either have to send some STOP value (which might be tricky if the file contains such string) or send the size of file before sending the content (so the client will know how much data it should read). The second solution is preferred (for example that's how HTTP works).
2) Don't use thread module. It's low level and it is easy to make mistakes. Use threading.
3) The server. You open a file with fp = open(data,'r') but you don't close it anywhere. Instead use with:
with open(data, 'r') as fp:
# the code that uses fp goes here
It will automatically close the file once it leaves the block.
4) Don't use os.system unless you absolutely have to. I understand that this is just for debugging but a good advice anyway.
5) Use socket.sendall instead of socket.send if you don't want to bother with tricky internals of system's send call. Might not matter in your case though.
Related
server.py:
json files from NVD are used here
import socket, json, random, threading, zipfile, requests, re, zipfile
from bs4 import BeautifulSoup
from zipfile import *
def listen_user(user):
for x in range(2018,2021,1):
filename = "nvdcve-1.1-" + str(x) + ".json"
print(filename)
with open(filename, 'rb') as file:
sendfile = file.read()
user.sendall(sendfile)
print('file sent' + str(x))
def start_server():
while True:
user_socket, address = server.accept()
print(f"User <{address[0]}> connected!")
users.append(user_socket)
listen_accepted_user = threading.Thread(
target=listen_user,
args=(user_socket,)
)
listen_accepted_user.start()
if __name__ == '__main__':
users = []
server = socket.socket(
socket.AF_INET,
socket.SOCK_STREAM,
)
server.bind(
("127.0.0.1", 100)
)
server.listen(5)
print('waiting for connection...')
start_server()
client.py
import socket, json, random
from threading import Thread
def start_client(client):
savefilename = str(random.randint(1,10)) + 'new.json'
print(savefilename)
with client,open(savefilename,'wb') as file:
while True:
recvfile = client.recv(4096)
if not recvfile:
print('1 client')
break
file.write(recvfile)
file.close()
print('2 client')
client.close()
if __name__ == '__main__':
client = socket.socket(
socket.AF_INET,
socket.SOCK_STREAM,
)
client.connect(
("127.0.0.1", 100)
)
start_client(client)
when I send files - they are sent almost in full, but the program does not reach the line "print ('1 client')" or "print ('2 client')"
and the *new file contains all lines except a few dozen of the last
please help - how to fix the code?
recvfile = client.recv(4096) is inside the while loop and it is continuously waiting for the next bytes to receive. The client doesn't know the files are sent, so it waits for the next 4096 bytes and doesn't exit the loop.
To let the client know that the file transfer is completed, you can send a message from the server.py which you can validate in the client and break the loop as shown below.
server.py
def listen_user(user):
for x in ["f.json","g.json"]:
filename = x
print(filename)
with open(filename, 'rb') as file:
sendfile = file.read()
user.sendall(sendfile)
print('file sent' + str(x))
user.send(b"Done")
Client.py
def start_client(client):
savefilename = str(random.randint(1,10)) + 'new.json'
print(savefilename)
with client,open(savefilename,'wb') as file:
while True:
recvfile = client.recv(4096)
if recvfile.decode("utf-8") =="Done":
print('1 client')
file.close()
break
file.write(recvfile)
print('2 client')
client.close()
The call client.recv(4096) means that you are waiting for 4096 bytes to be received, then doing something with those bytes. What's likely happening in this case is that you're writing out all of the bytes, minus those that don't quite fill up the buffer at the end. This leaves the client waiting with a buffer with space that is doesn't think it is ready to write out yet.
I'm guessing that you're assuming that client.recv() will return an empty string once you've gotten all the data; this is not the case based on your code. If you want the client to be able to terminate the connection, you're going to need to send some kind of control sequence or try to otherwise assess the bytes received from the server to determined when it's time to close the connection. If you do this, you will probably want to set bufsize when calling client.recv() to 1, and instead use some other method to buffer before you write to a file.
For instance, since you're sending JSON data, you could concatenate the bytes to a variable and then repeatedly try to parse JSON. Once you have managed to successfully parse JSON, you can terminate the connection on the client side (though this would mean you have to open a new connection per file you're sending).
However, that raises the question: why do you need to close from the client side? Usually the server will just close the connection once it is done sending all of the relevant data.
I'm trying to send file from client to server in python. It is sending without any problem but I want to save that received file with same file name. I'm not getting idea how to save that file with same file name as it is sent from Client to Server.The code I've wrote for this is :
Client Code
import socket, os, shutil
from stat import ST_SIZE
HOST=raw_input("Please enter IP-address : ")
PORT=int(raw_input("Please enter PORT Number : "))
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST,PORT))
if s.recv(8)!='READY':
raw_input('Unable to connect \n\n Press any key to exit ...')
s.close()
exit()
path=raw_input("Please enter the complete PATH of your file : ")
f=open(path,'rb')
fsize=os.stat(f.name)[ST_SIZE]
s.sendall(str(fsize).zfill(8))
sfile = s.makefile("wb")
shutil.copyfileobj(f, sfile)
sfile.close()
s.close()
f.close()
Server Code
import socket
import shutil
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
HOST = ''
PORT = 23240
s.bind((HOST, PORT))
s.listen(3)
conn, addr = s.accept()
print 'conn at address',addr
conn.sendall('READY')
i=1
f = open(r'file_'+ str(i)+".txt",'wb')
i=i+1
print 'File size',fsize
sfile = conn.makefile("rb")
shutil.copyfileobj(sfile, f)
sfile.close()
f.write(conn.recv(fsize))
f.close()
conn.close()
s.close()
Your code is not very robust. recv(cnt) delivers up to cnt bytes of data, or less. So it's not sure, you read the whole file. It is even not sure, you get the "READY" in one recv. Instead, you have to use something like that:
def recv_all(sock, bufsize):
result = ''
while bufsize>0:
data = sock.recv(min(bufsize, 4096))
if not data:
raise IOError("Socket closed")
result += data
bufsize -= len(data)
return result
If you want to know the filename at the server, you also have to transfer it to the server, too. By the way, "READY" has 5 characters, not 8.
I am working on a project where images are taken by my android phone and are stored in folders in my SD card. I am working on a python script that needs to periodically move the folders from the SD to a particular folder in my PC. The phone and the PC are connected over the mobile Hotspot.
I wrote a socket program with my PC as client and the mobile as server. But I am facing some problems with it. Though I could not move folders i tried moving images from the folder and i am facing the following problems
the image is copied in the form of an unknown file format.
i am unable to iterate the process at the server side to move all the images present in the folder
at the client I am not able to store it in the location i want. I try to send the folder name and the file name from the server before sending the image but the client is not taking that file name i sent, instead it searches a folder in that name.
I also have a problem with the size of the names sent to the client, how do i randomly change the size at the client side depending on the name sent from the server.
I need someones help to sort this problem.
Here is the client side code
import socket,os
import time
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(("192.168.43.1", 5005))
size = 1024
while True:
fln = client_socket.recv(size) # folder name
fn = client_socket.recv(size) # file name
fname = "E:\\Transfered\\"+fln+"\\"+fn
fp = open(fname,'w')
while True:
strng = client_socket.recv(1024)
if not strng:
break
fp.write(strng)
fp.close()
print "Data Received successfully"
exit()
#data = 'viewnior '+fname
#os.system(data)
My Server side code
import os
import sys,time
import socket
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(("", 5005))
server_socket.listen(5)
client_socket, address = server_socket.accept()
print "Conencted to - ",address,"\n"
sb = '/mnt/sdcard/sb'
while True:
files = os.listdir(sb)
pages = 0;
while (files):
print '\nMaybe, pending work'
for au in files:
if (au.find('d')>-1): # searching for folder with a d
os.chdir(sb+'/'+au)
imgFiles = os.listdir(sb+'/'+au)
images = [img for img in imgFiles if img.endswith('.jpg')]
print '\n%s user done' %au
client_socket.send(au)
pages = 0;
#copies all .img files in the folder from server to client
for imgs in images:
print imgs
client_socket.send(imgs)
file_name = open(imgs,'r')
while True:
strng = file_name.readline(1024)
if not strng:
break
client_socket.send(strng)
file_name.close()
print "Data sent successfully"
os.remove(sb+'/'+au+'/'+imgs)
pages = pages + 1
time.sleep(1)
os.chdir(sb)
os.rmdir(au)
else:
time.sleep(2)
exit()
The problem seems to be using readline() on a binary file at the server side:
file_name = open(imgs,'rb')
while True:
strng = file_name.readline()
readline() reads data from file up to the next '\n' character. Using it on a binary file may result in reading a very long buffer! (Maybe even up to EOF). In that case, using socket.send() may fail to deliver the entire data, and the return value (=bytes transmitted) should be checked. The possibilities for fixing that is:
using socket.sendall() when sending, will send the entire buffer.
or, alternatively (may use both)
using file_name.read(1024) - which will bound the amount of data read each cycle.
I have modified the code enough to solve many of my problems now the only problem i want to solve is the image transfer. I opened the a .jpg file at the client and wrote the data into it. But the final file size is just 1kb less that the original size. I guess my work will be done if I sort that out. Can some one help me with it.
heres the code
server:
import os
import sys,time
import socket
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.bind(("", 5005))
server_socket.listen(5)
client_socket, address = server_socket.accept()
print "Conencted to - ",address,"\n"
sb = '/mnt/sdcard/sb'
while True:
files = os.listdir(sb)
pages = 0;
while (files):
print '\nMaybe, pending work'
for au in files:
if (au.find('d')>-1):
os.chdir(sb+'/'+au)
imgFiles = os.listdir(sb+'/'+au)
images = [img for img in imgFiles if img.endswith('.jpg')]
print '\n%s user done' %au
client_socket.send(au)
#copies all .img files in the folder from server to client
for imgs in images:
client_socket.send(imgs)
file_name = open(imgs,'rb')
while True:
strng = file_name.readline()
if not strng:
break
client_socket.send(strng)
file_name.close()
os.remove(sb+'/'+au+'/'+imgs)
print "Data sent successfully"
time.sleep(1)
os.chdir(sb)
os.rmdir(au)
else:
time.sleep(2)
exit()
Client:
import socket,os
import time
client_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client_socket.connect(("192.168.43.1", 5005))
dst="E:\\Kiosk\\"
while True:
#folder name
fln = client_socket.recv(4)
os.chdir(dst);
dst = "E:\\Kiosk\\"+fln+"\\"
if not os.path.exists(dst): os.makedirs(dst)
fname = client_socket.recv(4)
os.chdir(dst)
fname = fname+'.jpg'
fp = open(fname,'wb')
# image
while True:
strng = client_socket.recv(1024)
if not strng:
break
fp.write(strng)
fp.close()
print "Data Received successfully"
exit()
#time.sleep(10)
#data = 'viewnior '+fname
#os.system(data)
I am trying to program compilation server which compiles a C program sent by client and returns an object file which can then be linked and executed at the client. Here are my client and server programs respectively
client.py:
# Compilation client program
import sys, socket, string
File = raw_input("Enter the file name:")
ssock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssock.connect(('localhost', 5000))
csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
csock.connect(('localhost', 5001))
f = open(File, "rb")
data = f.read()
f.close()
ssock.send(File) #send filename
ssock.send(data) #send file
fd=raw_input("Enter a key to start recieving object file:")
data=csock.recv(1024) #receive status
if data=="sucess\n":
File=File.replace(".c",".o") #objectfile name
print "Object file, "+File+", recieved sucessfully"
else:
print "There are compilation errors in " + File
File="error.txt" #errorfile name
print "Errors are reported in the file error.txt"
fobj=open(File,"wb")
while 1:
data=ssock.recv(1024) # if any error in c sourcefile then error gets
# eported in errorfile "error.txt" else objectfile is
# returned from server
if not data:break
fobj.write(data)
fobj.close()
ssock.close()
csock.close()
server.py
#Compilation Server program
import subprocess
import socket, time, string, sys, urlparse, os
ssock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssock.bind(('', 5000))
ssock.listen(2)
print 'Server Listening on port 5000'
csock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
csock.bind(('', 5001))
csock.listen(2)
print 'Control server listening on port 5001'
client, claddr = ssock.accept()
controlsoc, caddr = csock.accept()
filename=client.recv(1024) #receive filename
print filename
############### This code is not working, i'm not getting the reason #######
############### I want to receive a file more than 1KB from client #######
f = open(filename,"wb") #receive file======
while 1:
data = client.recv(1024)
if not data: break
f.write(data)
f.close()
###############
###############
data="gcc -c " + filename + " 2> error.txt" #shell command to execute c source file
#report errors if any to error.txt
from subprocess import call
call(data,shell=True) #executes the above shell command
fil = filename.replace(".c",".o")
if (os.path.isfile(fil))== True: #test for existence of objectfile
data = "sucess\n" #no objectfile => error in compilation
filename = filename.replace(".c",".o")
else:
data = "unsucessful\n"
print data+"hi"
filename = "error.txt"
controlsoc.send(data)
f = open(filename,"rb")
data=f.read()
f.close()
print data
client.send(data)
client.close()
controlsoc.close()
I'm not able to recieve files of multiple KB. Is there any flaw in my code or how should i modify my code in order to achieve my objective of coding a compilation server.
Please help me with this regard..Thanks in advance
The problem here is you assume that ssock.send(File) will result in filename=client.recv(1024) reading exactly the filename and not more, but in fact the receiving side has no idea where the filename ends and you end up getting the file name and part of the data in the filename variable.
TCP connection is a bi-directional stream of bytes. It doesn't know about boundaries of your messages. One send might correspond to more then one recv on the other side (and the other way around). You need an application-level protocol on top of raw TCP.
The easiest in your case would be to send a text line in the form file-size file-name\n as a header. This way your server would be able to not only separate header from file data (via newline) but also know how many bytes of file content to expect, and reuse same TCP connection for multiple files.
I'm trying to write transfer files or chunks of data over a socket. I feel as if I'm reinventing the wheel, but my searches for a simple solution have failed (everything I find is either too simple or too complex). The server would run on a phone running python 2.5.4. The intended application would be to sync music files between the phone and a host computer.
This is the guts of what I have, which appears to work. I send and receive 'ok' to break up streams.
Is sending 'ok' back and forth essentially as stop bits to break up streams of data a reasonable technique?
Is there a standard way to do this?
Running any sort of library server (ftp, http) on the phone is not a useful solution given the limits of the phone's memory and processing power.
server:
import socket
c = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
c.bind(('', 1234))
c.listen(1)
s,a = c.accept()
while True:
data = s.recv(1024)
cmd = data[:data.find('\n')]
if cmd == 'get':
x, file_name, x = data.split('\n', 2)
s.sendall('ok')
with open(file_name, 'rb') as f:
data = f.read()
s.sendall('%16d' % len(data))
s.sendall(data)
s.recv(2)
if cmd == 'end':
s.close()
c.close()
break
client:
import socket
s = socket.socket()
s.connect(('192.168.1.2', 1234))
def get_file(s, file_name):
cmd = 'get\n%s\n' % (file_name)
s.sendall(cmd)
r = s.recv(2)
size = int(s.recv(16))
recvd = ''
while size > len(recvd):
data = s.recv(1024)
if not data:
break
recvd += data
s.sendall('ok')
return recvd
print get_file(s, 'file1')
print get_file(s, 'file2')
s.sendall('end\n')
Is sending 'ok' back and forth essentially as stop bits to break up
streams of data a reasonable technique?
Most protocols use some terminator or another. Popular alternatives are '\r\n', '\r\n\r\n' or EOF (ctrl+d), but these are just arbitrarily chosen and no worse or better than your 'ok', as long as your client and server know how to handle it.
Your code looks good.
You don't actually need to send across the size of the file. You can use while True, as the check if not data: break will stop the loop.
while True:
data = s.recv(1024)
if not data: print " Done "; break
recvd += data
Also, why are you sending 'ok' is the other side doesn't check for it? You are just skipping 2 bytes at each side.
Don't you need to cater to multiple clients? No need for multi-threading?
Is there a standard way to do this?
Yes. http://www.faqs.org/rfcs/rfc959.html
Describes the standard way to do this.
Here is an implementation: http://docs.python.org/library/ftplib.html
U may look at this implementation. It also take care of if the file is in a sub-directory. Here is the link!
server
import socket
import os
print('Waiting for clinet to connect...')
c = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
c.bind(('', 1234))
c.listen(1)
s, a = c.accept()
print('Connected. Going to receive file.')
s.sendall('getfilename')
filename = s.recv(1024)
if '/' in filename:
dir = os.path.dirname(filename)
try:
os.stat(dir)
except:
print('Directory does not exist. Creating directory.')
os.mkdir(dir)
f = open(filename, 'wb')
print('Filename: ' + filename)
while True:
s.sendall('getfile')
size = int(s.recv(16))
print('Total size: ' + str(size))
recvd = ''
while size > len(recvd):
data = s.recv(1024)
if not data:
break
recvd += data
f.write(data)
#print(len(recvd))
break
s.sendall('end')
print('File received.')
s.close()
c.close()
f.close()
client
import socket
import sys
if len(sys.argv) > 1 :
print('Trying to connect...')
s = socket.socket()
s.connect(('127.0.0.1', 1234))
print('Connected. Wating for command.')
while True:
cmd = s.recv(32)
if cmd == 'getfilename':
print('"getfilename" command received.')
s.sendall(sys.argv[1])
if cmd == 'getfile':
print('"getfile" command received. Going to send file.')
with open(sys.argv[1], 'rb') as f:
data = f.read()
s.sendall('%16d' % len(data))
s.sendall(data)
print('File transmission done.')
if cmd == 'end':
print('"end" command received. Teminate.')
break
rsync is the standard way to sync files between two computers. You could write it in Python like this http://code.activestate.com/recipes/577518-rsync-algorithm/ or you could wrap the C library like this http://freshmeat.net/projects/pysync/ with some tweaks like replacing MD4 with MD5.
Or, if you want to do this at the socket level, you really should be using asynchat with asyncore. Here is an FTP server written with asynchat http://pyftpdlib.googlecode.com/svn-history/r20/trunk/pyftpdlib/FTPServer.py but you should start by reading http://www.doughellmann.com/PyMOTW/asynchat/ Pay attention to the part about Message Terminators point 2. A lot of network protocols do odd stuff like this, i.e. sometimes they send and receive full line commands and responses, and sometimes they send and receive chunks of arbitrary data preceded by the count of how many bytes are in the chunk. You can handle this much more easily with asynchat, and your program will scale much better too.