I'm new to socket programming. I'm trying to send 4 files from one host to another. Here is the code:
sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
if __name__ == '__main__':
main()
Receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
File transfer is working fine and there is no problem with it. Now I want to send a simple string like "finish" after sending all files, so that receiver will understand that the transfer is completed and it will do some other tasks based on this finish message (however, it still can receive messages at the same time).
I tried to do this by adding another function called sendMessage() to the sender code and a function called recvMessage() to the receiver. Here are the changed codes:
Sender:
from __future__ import print_function
import socket
from struct import pack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 1024
MESSAGE = "Finish!"
def send(sock, data):
while data:
sent = sock.send(data)
data = data[sent:]
#Updated part for sending message
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
def send_file(fname):
with open(fname, 'rb') as f:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((HOST, PORT))
except socket.error as err:
print(err, HOST, PORT)
sock.close()
return
# Send the file name length & the filename itself in one packet
send(sock, pack('B', len(fname)) + fname.encode())
while True:
data = f.read(BUFSIZE)
if not data:
break
send(sock, data)
sock.close()
fnames = [
'1.jpg',
'2.jpg',
'3.jpg',
'4.jpg',
]
def main():
for fname in fnames:
send_file(fname)
sendMessage(MESSAGE)
if __name__ == '__main__':
main()
receiver:
from __future__ import print_function
import socket
from struct import unpack
HOST = '10.0.0.2'
PORT = 12345
BUFSIZE = 4096
BUFFER_SIZE = 20
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
#Updated part for receiving message
def recvMessage(conn):
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
recvMessage(conn)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
but after running these codes both sender and receiver freeze after the complete transfer of 4 files and nothing happens. What's wrong and how can I do this?
I suspect you're falling prey to buffering here:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.send(message)
data = sock.recv(BUFFER_SIZE)
sock.close()
print ("received data:", data)
You perform a send, then immediately try to recv. Except stream connections tend to buffer to avoid excessive packet overhead, so odds are, you don't actually send anything yet, the server doesn't see anything so it doesn't respond, and both sides are blocked waiting for data.
The simplest solution here is to shut down the send side port for writing once you've sent the message, which forces out the last data and lets the receiver know you're done:
def sendMessage(message):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.connect((HOST, PORT))
sock.sendall(message) # sendall makes sure the *whole* message is sent
sock.shutdown(socket.SHUT_WR) # We're done writing
data = sock.recv(BUFFER_SIZE)
sock.close()
print("received data:", data)
On the receiver side you have a bigger problem: You close the connection before trying to receive at all:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close() # Closed here!!!
print('saved\n')
recvMessage(conn) # Used again here!!!
So move the close after the recvMessage call, and change recvMessage to use setsockopt to turn on TCP_NODELAY, so buffering isn't occurring (otherwise the echo back may end up buffering indefinitely, though shutting down the sender for write does mean you're likely to detect the sender is done and exit the loop then close the connection, so it may work fine without TCP_NODELAY, as long as the sender isn't expecting to receive data and respond further):
def recvMessage(conn):
# Disable Nagle algorithm so your echoes don't buffer
conn.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.sendall(data) # echo using sendall, again, to ensure it's all really sent
Related
everyone.
I send data to the server, it eats them, and then I start waiting for a response from the server by the client, and at this moment both the server and the client hang, but if you send data to the server without waiting for a response from it, it will calmly perform the function without freezes.
Who knows what the problem is?
client
import os
import socket
from time import sleep
import ast
class socker:
def __init__(self):
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.connect(('localhost', 4545))
def send_request(self, request):
# self.sock.settimeout(5)
self.sock.send(request)
def recieve_request(self):
byte = b""
while True:
data = self.sock.recv(1024)
byte += data
if not data:
break
return byte
def close_connection(self):
self.sock.close()
def test():
s = socker()
s.send_request(str({"request": "get_files_list"}).encode())
s.recieve_request()
test()
server
import socket
import ast
import os
def start_server():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(('localhost', 4545))
print("Сервер включён!")
while True:
try:
server.listen(10)
conn, addr = server.accept()
print('Подключён:', addr)
byte = b""
while True:
data = conn.recv(1024)
byte += data
if not data:
break
data = ast.literal_eval(byte.decode())
request = data["request"]
print("Запрос: ", request)
if request == "get_info_about_file":
pass
elif request == "get_files_list":
files = os.walk(saved_dir)
dirpaths = []
file_names = []
for dirpath, dirnames, filenames in files:
dirpaths.append(dirpath)
print(dirpath)
for file in filenames:
file_names.append(os.path.join(dirpath, file))
conn.send(str({"dirs": dirpaths, "files": file_names}).encode())
conn.shutdown(socket.SHUT_RDWR)
except KeyboardInterrupt:
server.close()
except Exception as err:
print(err)
if __name__ == "__main__":
start_server()
I'm trying to send a 2D array across a socket, it needs to be made out of multiple socket.recv() otherwise I will get a _pickle.UnpicklingError: pickle data was truncated error.
I've tried to do this with a while loop that receives packets and appends them to a list until all data is received:
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
Just doing:
def receive(self):
data = self.socket.recv(1024)
data = pickle.loads(data)
return data
Works if I send something smaller than the 2D array e.g. a coordinate pair tuple (2, 3). But not with the 2D array.
I get a OSError: [WinError 10022] when attempting the while loop approach.
I have seen that a OSError: [WinError 10022] could be an issue with not binding the socket but I think I have done that.
I can't figure out where connections are closing and am very confused.
Rest of the code:
Server:
import socket
from _thread import start_new_thread
clients = []
def threaded(client):
while True:
data = client.recv(1024)
if not data:
print('Not Data.')
break
# Send the data received from one client to all the other clients.
for c in clients:
if c != client:
c.send(data)
client.close()
def Main():
host = ""
port = 5555
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
print("Socket binded to port:", port)
s.listen(2)
print("Socket is listening.")
while True:
c, addr = s.accept()
clients.append(c)
print(f"Connected to: {addr[0]}:{addr[1]}")
start_new_thread(threaded, (c,))
s.close()
Client:
import socket
import pickle
class Client:
def __init__(self):
self.host = 'localhost'
self.port = 5555
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((self.host, self.port))
def send(self, data):
message = pickle.dumps(data)
try:
self.socket.send(message)
except socket.error as e:
return str(e)
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
The client-server I want to make will have 2 clients and each client will send data to the other constantly. How can I correct my code and properly implement this?
After doing more research and figuring out what was going wrong I got a solution.
The code never got out of this while loop - because data was constantly being sent and some packet was always incoming.
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
A solution I found was to send a message indicating how big the message being received should be, so I can tell when all the bytes for a particular message have arrived. Links: Sockets Python 3.5: Socket server hangs forever on file receive, Python Socket Receive Large Amount of Data
def send(self, data):
message = pickle.dumps(data)
msg_len = len(message)
try:
# Send what the total length of the message to be sent is in bytes.
self.socket.send(msg_len.to_bytes(4, 'big'))
self.socket.sendall(message)
except socket.error as e:
return str(e)
def receive(self):
remaining = int.from_bytes(self.socket.recv(4), 'big')
chunks = []
while remaining:
# until there are bytes left...
# fetch remaining bytes or 4096 (whatever smaller)
chunk = self.socket.recv(min(remaining, 4096))
remaining -= len(chunk)
# write to file
chunks.append(chunk)
chunks = b"".join(chunks)
data = pickle.loads(chunks)
return data
I have a deque on a host. Each String that is received through TCP socket is appended to this deque. When each data is received, I print the data and the deque. Here is the code:
from __future__ import print_function
import commands
import socket
import select
from collections import deque
host = commands.getoutput("hostname -I")
port = 5005
backlog = 5
BUFSIZE = 4096
BUFFER_SIZE = 1024
q = deque()
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
if (data == 'sample.jpg'):
print("start processing")
#processP(q)
else:
print("appended")
q.append(data)
print(q)
conn.close()
def read_udp(s):
data,addr = s.recvfrom(1024)
print("received message:", data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
tcp.bind((host,port))
except socket.error as err:
print('Bind failed', err)
return
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
print('***Socket now listening at***:', host, port)
input = [tcp,udp]
try:
while True:
#print("select.select")
inputready,outputready,exceptready = select.select(input,[],[], 0.1)
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
raise
tcp.close()
udp.close()
if __name__ == '__main__':
run()
The problem is that when I print the received data and deque in these lines print("received data:", data) and print(q), the received data is print correctly, but the deque content is printed like these in each step:
deque([''])
deque(['',''])
deque(['','',''])
What's wrong? Here is the sender code which doesn't seem to have any problem:
from __future__ import print_function
import socket
from struct import pack
import commands
import select
#HOST = '10.0.0.2'
PORT = 5005
BUFSIZE = 4096
def tcp_send(s, ip):
TCP_IP = ip
BUFFER_SIZE = 1024
MESSAGE = s
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((TCP_IP, PORT))
s.send(MESSAGE)
data = s.recv(BUFFER_SIZE)
s.close()
print ("sent data:", data)
fnames = [
'90.jpg','91.jpg','92.jpg','93.jpg','94.jpg','95.jpg','96.jpg','97.jpg','98.jpg','99.jpg','100.jpg','sample.jpg'
]
def main():
for fname1 in fnames:
tcp_send(fname1,'10.0.0.2')
if __name__ == '__main__':
main()
replace your read_tcp(s) with
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
while 1:
data = conn.recv(BUFFER_SIZE)
if not data: break
print("received data:", data)
conn.send(data) # echo
print(data)
if (data == 'sample.jpg'):
print("start processing")
#processP(q)
else:
print("appended", data)
q.append(data)
print(q)
conn.close()
you are trying to access data outside while that's why it is empty
It looks like indentation problem. You will brake out of the while loop (in read_tcp()) only when data is evaluated False. Only then, already out of the loop you append current value of data to to the deque q. In the loop you print every chunck of data you get. I think the if block needs to be indented one level to be part of the loop.
Also in read_udp() I don't see that you add anything to deque
I'm new to socket programming . I've implemented 2 separated codes on the same host. One of them is supposed to receive images using TCP protocol and the second one is supposed to receive text messages through UDP protocol. Both of them are working fine separately. Here are the codes:
Image receiver (TCP):
from __future__ import print_function
import socket
from struct import unpack
import Queue
from PIL import Image
HOST = '10.0.0.1'
PORT = 12345
BUFSIZE = 4096
q = Queue.Queue()
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
q.put(name)
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
main()
Text receiver (UDP):
import socket
UDP_IP = "10.0.0.1"
UDP_PORT = 5005
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
sock.bind((UDP_IP, UDP_PORT))
while True:
data, addr = sock.recvfrom(1024) # buffer size is 1024 bytes
print "received message:", data
Now my question is: How can I merge these 2 codes? I don't want to open 2 separate consoles for each of them and I want one code instead of two. Is it possible?
I tried the solution which was provided in the comment and here is the code:
from __future__ import print_function
from select import select
import socket
from struct import unpack
host = '10.0.0.2'
port = 5005
size = 8000
backlog = 5
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
def read_tcp(s):
conn, addr = s.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
name = receiver.get(name_size).decode()
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
def read_udp(s):
data,addr = s.recvfrom(8000)
print("Recv UDP:'%s'" % data)
def run():
# create tcp socket
tcp = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
tcp.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
tcp.bind((host,port))
tcp.listen(1)
# create udp socket
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) # UDP
udp.bind((host,port))
input = [tcp,udp]
while True:
inputready,outputready,exceptready = select(input,[],[])
for s in inputready:
if s == tcp:
read_tcp(s)
elif s == udp:
read_udp(s)
else:
print("unknown socket:", s)
if __name__ == '__main__':
run()
but I don't receive any UDP or TCP packet now and it doesn't seem to work for me.
Short answer yes, but you will need to implement multithreading for this. For instance, your program will spawn two threads, one for TCP sockets and other for UDP.
I have a tcp receiver which is listening for incoming images. I also have a foo() def that runs simultaneously and prints the current time every 5 seconds.
Here is the code:
from __future__ import print_function
import socket
from struct import unpack
import Queue
from PIL import Image
HOST = '10.0.0.1'
PORT = 5005
BUFSIZE = 4096
q = Queue.Queue()
class Receiver:
''' Buffer binary data from socket conn '''
def __init__(self, conn):
self.conn = conn
self.buff = bytearray()
def get(self, size):
''' Get size bytes from the buffer, reading
from conn when necessary
'''
while len(self.buff) < size:
data = self.conn.recv(BUFSIZE)
if not data:
break
self.buff.extend(data)
# Extract the desired bytes
result = self.buff[:size]
# and remove them from the buffer
del self.buff[:size]
return bytes(result)
def save(self, fname):
''' Save the remaining bytes to file fname '''
with open(fname, 'wb') as f:
if self.buff:
f.write(bytes(self.buff))
while True:
data = self.conn.recv(BUFSIZE)
if not data:
break
f.write(data)
import time, threading
def foo():
try:
print(time.ctime())
threading.Timer(5, foo).start()
except KeyboardInterrupt:
print('\nClosing')
def main():
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try:
sock.bind((HOST, PORT))
except socket.error as err:
print('Bind failed', err)
return
sock.listen(1)
print('Socket now listening at', HOST, PORT)
try:
while True:
conn, addr = sock.accept()
print('Connected with', *addr)
# Create a buffer for this connection
receiver = Receiver(conn)
# Get the length of the file name
name_size = unpack('B', receiver.get(1))[0]
# Get the file name itself
name = receiver.get(name_size).decode()
q.put(name)
print('name', name)
# Save the file
receiver.save(name)
conn.close()
print('saved\n')
# Hit Break / Ctrl-C to exit
except KeyboardInterrupt:
print('\nClosing')
sock.close()
if __name__ == '__main__':
foo()
main()
The problem is that when I press Ctrl + C buttons in order to terminate the program, the first time it prints "closing" but it isn't terminated and I should press these buttons at least two times.
How can I stop the program the first time I press Ctrl + C? I removed try and except in def foo(), but it didn't change the result.
Just reraise the exception after the print statement:
except KeyboardInterrupt:
print('\nClosing')
raise