I've been having a problem with coding the file I/O program using a socket in python. The purpose of my server program is to send a text file to the client program via socket. Even though I confirmed that a client program receives some data from a server program, the client can't write the data to the file. What is the problem??? Please help me..
[server.py]
import socketserver
from time import sleep
class TCPSocketHandler(socketserver.BaseRequestHandler):
def handle(self):
sock = self.request
while True:
with open('imageNames.txt', 'r') as file:
try:
names=file.readlines()
for name in names:
print(name)
sock.send((name.encode('utf-8')))
except Exception as exc:
print(exc)
sleep(60)
def main():
HOST, PORT = "192.168.xx.xx", 7799
server = socketserver.TCPServer((HOST, PORT), TCPSocketHandler)
server.serve_forever()
if __name__ == '__main__':
main()
[client.py]
import socket
def main():
HOST, PORT = "192.168.xx.xx", 7799
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((HOST, PORT))
data = (sock.recv(1024)).decode('utf-8')
with open('imageNames2.txt', 'w') as file:
try:
while data:
print(data)
file.write(data)
data = (sock.recv(1024)).decode('utf-8')
except Exception as e:
print(e)
if __name__ == '__main__':
main()
Big Edit
After some more digging I found that what I wrote before was only partially correct. The real reason is the buffering, both in Python and on the file system level.
If a TextIO object is created, there's a buffer that accumulates input to avoid excessive system calls. Once this buffer fills up, the content will be flushed towards the file system where it might end up in another buffer until this one fills up and the content is finally written to the file.
Now, the content manager exiting, thus internally calling close() on the TextIO implicitly enforces flushing the buffers.
With the loop in the server sleeping for 60 seconds, probably both, you and I have not been patient enough to wait for the buffers to fill up to a level that triggers flushing to see content in the file on disk.
If you want to enforce every chunk that the client receives to show up in the file immediately, you'll have to explicitly flush the TextIO buffer and call os.fsync() to trigger flushing of the file system buffer as well.
So, here's how you could alter the client while keeping the server in its infinite loop over the file content:
import os
import socket
def main():
HOST, PORT = "127.0.0.1", 7799
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
sock.connect((HOST, PORT))
data = (sock.recv(1024)).decode('utf-8')
with open('imageNames2.txt', 'w') as file:
try:
while data:
file.write(data)
file.flush()
os.fsync(file.fileno())
print(data)
data = (sock.recv(1024)).decode('utf-8')
except Exception as e:
print(e)
if __name__ == '__main__':
main()
Please note that in some IDEs, such as PyCharm, the new file might not show up in the project overview until the file has been closed. On the file system level, however, it does exist.
Also, my earlier side note about losing the first line was wrong. Sorry about that.
Related
server.py:
json files from NVD are used here
import socket, json, random, threading, zipfile, requests, re, zipfile
from bs4 import BeautifulSoup
from zipfile import *
def listen_user(user):
for x in range(2018,2021,1):
filename = "nvdcve-1.1-" + str(x) + ".json"
print(filename)
with open(filename, 'rb') as file:
sendfile = file.read()
user.sendall(sendfile)
print('file sent' + str(x))
def start_server():
while True:
user_socket, address = server.accept()
print(f"User <{address[0]}> connected!")
users.append(user_socket)
listen_accepted_user = threading.Thread(
target=listen_user,
args=(user_socket,)
)
listen_accepted_user.start()
if __name__ == '__main__':
users = []
server = socket.socket(
socket.AF_INET,
socket.SOCK_STREAM,
)
server.bind(
("127.0.0.1", 100)
)
server.listen(5)
print('waiting for connection...')
start_server()
client.py
import socket, json, random
from threading import Thread
def start_client(client):
savefilename = str(random.randint(1,10)) + 'new.json'
print(savefilename)
with client,open(savefilename,'wb') as file:
while True:
recvfile = client.recv(4096)
if not recvfile:
print('1 client')
break
file.write(recvfile)
file.close()
print('2 client')
client.close()
if __name__ == '__main__':
client = socket.socket(
socket.AF_INET,
socket.SOCK_STREAM,
)
client.connect(
("127.0.0.1", 100)
)
start_client(client)
when I send files - they are sent almost in full, but the program does not reach the line "print ('1 client')" or "print ('2 client')"
and the *new file contains all lines except a few dozen of the last
please help - how to fix the code?
recvfile = client.recv(4096) is inside the while loop and it is continuously waiting for the next bytes to receive. The client doesn't know the files are sent, so it waits for the next 4096 bytes and doesn't exit the loop.
To let the client know that the file transfer is completed, you can send a message from the server.py which you can validate in the client and break the loop as shown below.
server.py
def listen_user(user):
for x in ["f.json","g.json"]:
filename = x
print(filename)
with open(filename, 'rb') as file:
sendfile = file.read()
user.sendall(sendfile)
print('file sent' + str(x))
user.send(b"Done")
Client.py
def start_client(client):
savefilename = str(random.randint(1,10)) + 'new.json'
print(savefilename)
with client,open(savefilename,'wb') as file:
while True:
recvfile = client.recv(4096)
if recvfile.decode("utf-8") =="Done":
print('1 client')
file.close()
break
file.write(recvfile)
print('2 client')
client.close()
The call client.recv(4096) means that you are waiting for 4096 bytes to be received, then doing something with those bytes. What's likely happening in this case is that you're writing out all of the bytes, minus those that don't quite fill up the buffer at the end. This leaves the client waiting with a buffer with space that is doesn't think it is ready to write out yet.
I'm guessing that you're assuming that client.recv() will return an empty string once you've gotten all the data; this is not the case based on your code. If you want the client to be able to terminate the connection, you're going to need to send some kind of control sequence or try to otherwise assess the bytes received from the server to determined when it's time to close the connection. If you do this, you will probably want to set bufsize when calling client.recv() to 1, and instead use some other method to buffer before you write to a file.
For instance, since you're sending JSON data, you could concatenate the bytes to a variable and then repeatedly try to parse JSON. Once you have managed to successfully parse JSON, you can terminate the connection on the client side (though this would mean you have to open a new connection per file you're sending).
However, that raises the question: why do you need to close from the client side? Usually the server will just close the connection once it is done sending all of the relevant data.
I've written a simple Python script that reads a file and streams the contents over a socket. The application I've connected to currently just reads the data from the socket and writes it to a file. However the data on the receiving end is incomplete. The first ~150 lines of the file do not get received, nor does the last line. I don't see anything glaringly wrong with my Python code, but if someone can point out what I've done wrong I would appreciate it. If there's an alternative method that can accomplish this task that may be helpful as well. Thanks.
EDIT: I'm pretty sure it's an issue with this code and not the receiving side because I have a C++ version of this Python code that the receiving end works fine with. However, I don't know what could be wrong here.
import socket
import sys
import time
__all__=['Stream_File_to_Socket']
def Stream_File_to_Socket(port,input_file):
# create socket
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_address = ('localhost',port)
sock.bind(server_address)
sock.listen(1)
# open file and send data
f = open(input_file,"r")
while True:
#print('waiting for a connection')
connection, client_address = sock.accept()
if connection.fileno() != -1 :
break
#print("no connection!")
return(-1)
time.sleep(0.5)
buffer_size = 1024
while True:
data = f.readline()
if not data:
connection.send(data)
break
connection.send(data)
time.sleep(0.01)
f.close()
return(0)
I'm trying to send a file over a socket in Python 2.7.x . My client and server code is below. It's working, but only after the client connection kills the socket. So I added raw_input('finished') to the end of both for debugging.
So if I start the server, then run the client... It looks like all but the last bit of the file sends, until I forcefully kill the client and then it's all there. So the problem is definitely in the server loop... I just don't know how to fix it. if not data: break isn't being triggered. But, if I do something like if len(data) < 1024: break it won't work for bigger files.
Any help is appreciated!
# client.py
import socket
conn = socket.socket()
conn.connect(('localhost', 1337))
f = open('test.jpg', 'rb')
data = f.read(1024)
while data:
conn.send(data)
data = f.read(1024)
f.close()
raw_input('finished')
# server.py
import socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('localhost', 1337))
s.listen(5)
conn, addr = s.accept()
f = open('test.jpg', 'wb')
while True:
data = conn.recv(1024)
if not data:
break
f.write(data)
f.close()
raw_input('finished')
From your posted code:
while data:
conn.send(data)
data = f.read(1024)
From the Python socket documentation:
socket.send(string[, flags])
[...]
Returns the number of bytes sent. Applications are responsible for checking
that all data has been sent; if only some of the data was transmitted, the
application needs to attempt delivery of the remaining data.
That should tell you what the problem is, but just to be explicit about it: send() may or may not accept all of the bytes you asked it to send before returning, and it's up to you to handle it correctly in the case where it only accepts the first (n) bytes rather than the entire buffer. If you don't check send()'s return value, then you will sometimes drop some of the bytes of your file without knowing it. You need to check send()'s return value and if it is less than len(data), call send() again (as many times as necessary) with the remaining bytes. Alternatively you could call conn.sendall() instead of conn.send(), since sendall() will perform that logic for you.
I run my script on computer "A". Then I connect to computer "A" from computer "B" through my script. I send my message to computer "A" and my script runs it with an exec() instruction.
I want to see the result of executing my message on computer "A", through a socket on computer "B".
I tried to change sys.stdout = socket_response but had a error: "Socket object has no attribute write()"
So, how can I redirect standard output (for print or exec()) from computer "A" to computer "B" through socket connection?
It will be some kind of 'python interpreter' into my script.
SORRY, I CAN'T ANSWER MY OWN QUESTION WITHOUT REPUTATION
Thanks to all!
I use a simple way, which #Torxed advised me of. Here's my pseudo-code (it's just an example, not my real script)
#-*-coding:utf-8-*-
import socket
import sys
class stdout_():
def __init__(self, sock_resp):
self.sock_resp = sock_resp
def write(self, mes):
self.sock_resp.send(mes)
MY_IP = 'localhost'
MY_PORT = 31337
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print("Start server")
old_out = sys.stdout
srv.bind((MY_IP, MY_PORT))
srv.listen(0)
sock_resp, addr_resp = srv.accept()
new_out = stdout_(sock_resp)
sys.stdout = new_out
#sys.stdout = sock_resp ### sock_object has no attribute 'write'
while 1:
try:
a = sock_resp.recv(1024)
exec(a)
except socket.timeout:
#print('server timeout!!' + '\n')
continue
I connected to script with Putty and sent "print 'abc'" and then I received the answer 'abc'.
There is the makefile function in Python's socket class:
socket.makefile(mode='r', buffering=None, *, encoding=None,
errors=None, newline=None)
Return a file object associated with the socket. The exact returned
type depends on the arguments given to makefile(). These arguments are
interpreted the same way as by the built-in open() function.
Closing the file object won’t close the socket unless there are no
remaining references to the socket. The socket must be in blocking
mode; it can have a timeout, but the file object’s internal buffer may
end up in a inconsistent state if a timeout occurs.
You can read how to use it in Mark Lutz's book (chapter 12, "Making Sockets Look Like Files and Streams").
An example from the book (the idea is simple: make a file object from a socket with socket.makefile and link sys.stdout with it):
def redirectOut(port=port, host=host):
"""
connect caller's standard output stream to a socket for GUI to listen
start caller after listener started, else connect fails before accept
"""
sock = socket(AF_INET, SOCK_STREAM)
sock.connect((host, port)) # caller operates in client mode
file = sock.makefile('w') # file interface: text, buffered
sys.stdout = file # make prints go to sock.send
return sock # if caller needs to access it raw
Server side:
from subprocess import Popen, STDOUT, PIPE
from socket import socket
from time import sleep
server_sock = socket()
server_sock.bind(('', 8000))
server_sock.listen(4)
def close_process(p):
p.stdin.close()
p.stdout.close()
while 1:
try:
client, client_address = server_sock.accept()
data = client.recv(8192)
except:
break
# First, we open a handle to the external command to be run.
process = Popen(data.decode('utf-8'), shell=True, stdout=PIPE, stdin=PIPE, stderr=STDOUT)
# Wait for the command to finish
# (.poll() will return the exit code, None if it's still running)
while process.poll() == None:
sleep(0.025)
# Then we send whatever output the command gave us back via the socket
# Python3: sockets never convert data from byte objects to strings,
# so we'll have to do this "manually" in case you're confused from Py2.X
try:
client.send(bytes(process.stdout.read(), 'UTF-8'))
except:
pass
# And finally, close the stdout/stdin of the process,
# otherwise you'll end up with "to many filehandles openened" in your OS.
close_process(process)
client.close()
server_sock.close()
This assumes Python3.
If no one else have a better way of just redirecting output to a socket from a process, this is a solution you could work with.
I want my python application to be able to tell when the socket on the other side has been dropped. Is there a method for this?
Short answer:
use a non-blocking recv(), or a blocking recv() / select() with a very
short timeout.
Long answer:
The way to handle socket connections is to read or write as you need to, and be prepared to handle connection errors.
TCP distinguishes between 3 forms of "dropping" a connection: timeout, reset, close.
Of these, the timeout can not really be detected, TCP might only tell you the time has not expired yet. But even if it told you that, the time might still expire right after.
Also remember that using shutdown() either you or your peer (the other end of the connection) may close only the incoming byte stream, and keep the outgoing byte stream running, or close the outgoing stream and keep the incoming one running.
So strictly speaking, you want to check if the read stream is closed, or if the write stream is closed, or if both are closed.
Even if the connection was "dropped", you should still be able to read any data that is still in the network buffer. Only after the buffer is empty will you receive a disconnect from recv().
Checking if the connection was dropped is like asking "what will I receive after reading all data that is currently buffered ?" To find that out, you just have to read all data that is currently bufferred.
I can see how "reading all buffered data", to get to the end of it, might be a problem for some people, that still think of recv() as a blocking function. With a blocking recv(), "checking" for a read when the buffer is already empty will block, which defeats the purpose of "checking".
In my opinion any function that is documented to potentially block the entire process indefinitely is a design flaw, but I guess it is still there for historical reasons, from when using a socket just like a regular file descriptor was a cool idea.
What you can do is:
set the socket to non-blocking mode, but than you get a system-depended error to indicate the receive buffer is empty, or the send buffer is full
stick to blocking mode but set a very short socket timeout. This will allow you to "ping" or "check" the socket with recv(), pretty much what you want to do
use select() call or asyncore module with a very short timeout. Error reporting is still system-specific.
For the write part of the problem, keeping the read buffers empty pretty much covers it. You will discover a connection "dropped" after a non-blocking read attempt, and you may choose to stop sending anything after a read returns a closed channel.
I guess the only way to be sure your sent data has reached the other end (and is not still in the send buffer) is either:
receive a proper response on the same socket for the exact message that you sent. Basically you are using the higher level protocol to provide confirmation.
perform a successful shutdow() and close() on the socket
The python socket howto says send() will return 0 bytes written if channel is closed. You may use a non-blocking or a timeout socket.send() and if it returns 0 you can no longer send data on that socket. But if it returns non-zero, you have already sent something, good luck with that :)
Also here I have not considered OOB (out-of-band) socket data here as a means to approach your problem, but I think OOB was not what you meant.
It depends on what you mean by "dropped". For TCP sockets, if the other end closes the connection either through
close() or the process terminating, you'll find out by reading an end of file, or getting a read error, usually the errno being set to whatever 'connection reset by peer' is by your operating system. For python, you'll read a zero length string, or a socket.error will be thrown when you try to read or write from the socket.
From the link Jweede posted:
exception socket.timeout:
This exception is raised when a timeout occurs on a socket
which has had timeouts enabled via a prior call to settimeout().
The accompanying value is a string whose value is currently
always “timed out”.
Here are the demo server and client programs for the socket module from the python docs
# Echo server program
import socket
HOST = '' # Symbolic name meaning all available interfaces
PORT = 50007 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print 'Connected by', addr
while 1:
data = conn.recv(1024)
if not data: break
conn.send(data)
conn.close()
And the client:
# Echo client program
import socket
HOST = 'daring.cwi.nl' # The remote host
PORT = 50007 # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.send('Hello, world')
data = s.recv(1024)
s.close()
print 'Received', repr(data)
On the docs example page I pulled these from, there are more complex examples that employ this idea, but here is the simple answer:
Assuming you're writing the client program, just put all your code that uses the socket when it is at risk of being dropped, inside a try block...
try:
s.connect((HOST, PORT))
s.send("Hello, World!")
...
except socket.timeout:
# whatever you need to do when the connection is dropped
If I'm not mistaken this is usually handled via a timeout.
I translated the code sample in this blog post into Python: How to detect when the client closes the connection?, and it works well for me:
from ctypes import (
CDLL, c_int, POINTER, Structure, c_void_p, c_size_t,
c_short, c_ssize_t, c_char, ARRAY
)
__all__ = 'is_remote_alive',
class pollfd(Structure):
_fields_ = (
('fd', c_int),
('events', c_short),
('revents', c_short),
)
MSG_DONTWAIT = 0x40
MSG_PEEK = 0x02
EPOLLIN = 0x001
EPOLLPRI = 0x002
EPOLLRDNORM = 0x040
libc = CDLL('libc.so.6')
recv = libc.recv
recv.restype = c_ssize_t
recv.argtypes = c_int, c_void_p, c_size_t, c_int
poll = libc.poll
poll.restype = c_int
poll.argtypes = POINTER(pollfd), c_int, c_int
class IsRemoteAlive: # not needed, only for debugging
def __init__(self, alive, msg):
self.alive = alive
self.msg = msg
def __str__(self):
return self.msg
def __repr__(self):
return 'IsRemoteAlive(%r,%r)' % (self.alive, self.msg)
def __bool__(self):
return self.alive
def is_remote_alive(fd):
fileno = getattr(fd, 'fileno', None)
if fileno is not None:
if hasattr(fileno, '__call__'):
fd = fileno()
else:
fd = fileno
p = pollfd(fd=fd, events=EPOLLIN|EPOLLPRI|EPOLLRDNORM, revents=0)
result = poll(p, 1, 0)
if not result:
return IsRemoteAlive(True, 'empty')
buf = ARRAY(c_char, 1)()
result = recv(fd, buf, len(buf), MSG_DONTWAIT|MSG_PEEK)
if result > 0:
return IsRemoteAlive(True, 'readable')
elif result == 0:
return IsRemoteAlive(False, 'closed')
else:
return IsRemoteAlive(False, 'errored')
Trying to improve on #kay response. I made a more pythonic version
(Note that it was not yet tested in a "real-life" environment, and only on Linux)
This detects if the remote side closed the connection, without actually consuming the data:
import socket
import errno
def remote_connection_closed(sock: socket.socket) -> bool:
"""
Returns True if the remote side did close the connection
"""
try:
buf = sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT)
if buf == b'':
return True
except BlockingIOError as exc:
if exc.errno != errno.EAGAIN:
# Raise on unknown exception
raise
return False
Here is a simple example from an asyncio echo server:
import asyncio
async def handle_echo(reader, writer):
addr = writer.get_extra_info('peername')
sock = writer.get_extra_info('socket')
print(f'New client: {addr!r}')
# Initial of client command
data = await reader.read(100)
message = data.decode()
print(f"Received {message!r} from {addr!r}")
# Simulate a long async process
for _ in range(10):
if remote_connection_closed(sock):
print('Remote side closed early')
return
await asyncio.sleep(1)
# Write the initial message back
print(f"Send: {message!r}")
writer.write(data)
await writer.drain()
writer.close()
async def main():
server = await asyncio.start_server(
handle_echo, '127.0.0.1', 8888)
addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
print(f'Serving on {addrs}')
async with server:
await server.serve_forever()
if __name__ == '__main__':
asyncio.run(main())