I'm trying to send a large amount of JSON (3000 lines or so) to a server from a client.
Sometimes, the server recieves no bytes at all, or it gets some of the bytes and not all.
It works find for small strings, less than a thousand lines of JSON.
Client
def rpiSendTest(request):
host = request.data["IP"]
port = 5098
timeout = 5
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0)
context = ssl.SSLContext(ssl.PROTOCOL_TLS)
context.verify_mode = ssl.CERT_REQUIRED
context.check_hostname = False
context.load_verify_locations(cafile='certs/cert.pem')
ssock = context.wrap_socket(sock, server_hostname=host)
try:
ssock.connect((host, port))
message = json.dumps(request.data)
success = ssock.sendall(message.encode())
return response.Response(status=status.HTTP_200_OK, data={"status": "testing"})
except Exception as e:
return response.Response(status=status.HTTP_400_BAD_REQUEST)
Server
while True:
# Once you grab a connection, start recieving data
try:
conn.settimeout(20)
data = conn.recv(90000)
if not data:
print("No more data")
# If there's no more data chunks
break
# If the test suite is big, it'll be sent in chunks of bytes, so we take 10000 sized chunks at a time and append to array
print("appending")
print(data)
fragments.append(data)
except Exception as e:
print(e)
data = b''.join(fragments)
# The array is in bytes, so we append to a big byte string
print("loading {}".format(data))
data = json.loads(data)
It would error on json.loads(data) as not all the bytes were sent. Or it wouldn't recieve any data at all.
I'm on a company network, would there be any reasons why this could be causing issues.
Related
I am developing a python socket server. The client send each message to start with a STX (\x02) and end with ETX (\x03). My code can receive message successfully but I can't implement receiving full string using STX and ETX condition. Need help in resolving this issue. Below I have sharing my code for better understanding.
import socket
import time
# Start New RnD
# Global Veriable
enq = chr(5)
ack = chr(6)
stx = chr(2)
etx = chr(3)
# Connect to the server with `telnet $HOSTNAME 5000`.
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setblocking(False)
server.bind(('0.0.0.0', 5000))
server.listen(1)
connections = []
while True:
try:
connection, address = server.accept()
connection.setblocking(False)
connections.append(connection)
except BlockingIOError:
pass
# Incoming Data Processing
for connection in connections:
try:
full_message = ''
data = ""
while True:
try:
received = connection.recv(1)
if received == enq.encode('utf-8'):
print("Received <ENQ>, Sending <ACK>")
connection.sendall(ack.encode('utf-8'))
if not received:
raise RuntimeError("unexpected end-of-message", data)
data += received.decode('utf-8')
#print("Received: {!r}".format(data))
if "\x03" in received.decode("utf-8") :
break
except BlockingIOError:
pass
print("Full Received: {!r}".format(data))
print("Data Received, Sending <ACK>")
connection.sendall(ack.encode('utf-8'))
except BlockingIOError:
continue
While trying to send a list to a client process, I pickle it, but when I receive this on the client side it always gives me the error pickle data was truncated, and I don't know how to fix it.
sv
def handle_client(connection):
connection.send(str.encode('welcome to sv'))
stock = random.sample(output, 1)
order = pickle.dumps(stock)
while True:
data = connection.recv(2048)
if not data:
break
if data.decode('utf-8') == 'wanna trade!':
print("trade order received")
tcp.send(order)
reply = connection.recv(2048)
if reply.decode('utf-8') == 'Y':
tcp.send(order)
output.remove(order)
elif reply.decode('utf-8') == 'N':
print("doesn't wish to buy.")
connection.close()
client
while True:
Cliente.send(str.encode('wanna trade!'))
recv = Cliente.recv(2048)
if not recv:
break
if recv:
Response = pickle.loads(recv)
print(Response)
Check = input('Y/N: ')
Cliente.send(str.encode(Check))
recv2 = Cliente.recv(2048)
if recv2:
final = pickle.load(recv2)
purchases.append(final.decode('utf-8'))
print(purchases)
Cliente.close()
I can't test it but data can be longer than 2048 bytes and when you get from socket only 2048 bytes then you get truncated data.
Socket doesn't know how many data to get in client so you have to send size before data. And size should have always the same length (so client will know if it get full size) so sending it as string may not work (or you would have to read size char after char until you get some spcial char - ie. new line which server would have to send after string with size.)
Server should first send data size (as ie. 4 bytes converted with struct.pack()) and later send data.
And client should first read 4 bytes and convert it to integer with struct.unpack() and later use this value to read all data.
Server:
import struct
stock = ...
data = pickle.dumps(stock)
size = len(data)
size_in_4_bytes = struct.pack('I', size)
print(size, size_in_4_bytes)
tcp.send(size_in_4_bytes)
tcp.send(data)
Client:
import struct
size_in_4_bytes = Cliente.recv(4) # get only 4 bytes
size = struct.unpack('I', size_in_4_bytes)
size = size[0]
print(size, size_in_4_bytes)
data = Cliente.recv(size)
stock = pickle.loads(data)
EDIT:
If you put code in functions then you could use it many times in simply way. You could use it also to send different object: pickle, normal string, data as JSON string, image, any file.
import struct
def send_data(conn, data):
size = len(data)
size_in_4_bytes = struct.pack('I', size)
conn.send(size_in_4_bytes)
conn.send(data)
def recv_data(conn):
size_in_4_bytes = conn.recv(4)
size = struct.unpack('I', size_in_4_bytes)
size = size[0]
data = conn.recv(size)
return data
# -------------------------------------
# --- pickle ---
# send pickle
data = pickle.dumps(stock)
send_data(Client, data)
# recv pickle
data = recv_data(Client)
stock = pickle.loads(data)
# --- text ---
# send normal string
data = text.encode()
send_data(Client, data)
# recv normal string
data = recv_data(Client)
text = data.decode()
# --- JSON ---
# send data as JSON
stock = {'open': 12, 'close': 15}
text = json.dumps(stock)
data = text.encode()
send_data(Client, data)
# recv data as JSON
data = recv_data(Client)
text = data.decode()
stock = json.loads(text)
print(stock) # {'open': 12, 'close': 15}
# --- image (or any other file) ---
# send image
with open('image.jpg', 'rb') as image
data = image.read()
send_data(Client, data)
# recv image
with open('image.jpg', 'wb') as image
data = recv_data(Client)
image.write(data)
EDIT:
Full working example.
Client first sends text and receives text, next it sends directory converted to JSON, and it receives JSON with other directory.
Server uses threads to run with many clients at the same time. There is sleep() to have time to start another client.
I use my code from answer for question:
How to handle multithreading with sockets in Python?
Server:
import socket
import threading
import time
import struct
import json
# --- functions ---
def send_data(conn, data):
size = len(data)
size_in_4_bytes = struct.pack('I', size)
conn.send(size_in_4_bytes)
conn.send(data)
def recv_data(conn):
size_in_4_bytes = conn.recv(4)
size = struct.unpack('I', size_in_4_bytes)
size = size[0]
data = conn.recv(size)
return data
def handle_client(conn, addr):
print("[thread] starting")
# ---
# recv message
data = recv_data(conn)
text = data.decode()
print("[thread] client:", addr, 'recv:', text)
# simulate longer work - to start next client at the same time
time.sleep(5)
# send message
text = "Bye!"
print("[thread] client:", addr, 'send:', text)
data = text.encode()
send_data(conn, data)
# ---
# recv JSON
data = recv_data(conn)
text = data.decode()
stock = json.loads(text)
print("[thread] client:", addr, 'recv:', stock)
# send JSON
stock = {'diff': stock['close'] - stock['open']}
print("[thread] client:", addr, 'send:', stock)
text = json.dumps(stock)
data = text.encode()
send_data(conn, data)
# ---
conn.close()
print("[thread] ending")
# --- main ---
host = '0.0.0.0'
port = 8080
s = socket.socket()
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) # solution for "[Error 89] Address already in use". Use before bind()
s.bind((host, port))
s.listen(1)
all_threads = []
try:
while True:
print("Waiting for client")
conn, addr = s.accept()
print("Client:", addr)
t = threading.Thread(target=handle_client, args=(conn, addr))
t.start()
all_threads.append(t)
except KeyboardInterrupt:
print("Stopped by Ctrl+C")
finally:
if s:
s.close()
for t in all_threads:
t.join()
Client:
import socket
import struct
import json
# --- functions ---
def send_data(conn, data):
size = len(data)
size_in_4_bytes = struct.pack('I', size)
conn.send(size_in_4_bytes)
conn.send(data)
def recv_data(conn):
size_in_4_bytes = conn.recv(4)
size = struct.unpack('I', size_in_4_bytes)
size = size[0]
data = conn.recv(size)
return data
# --- main ---
host = '0.0.0.0'
port = 8080
s = socket.socket()
s.connect((host, port))
print("Connected to the server")
# ---
# send message
text = "Hello"
print('send:', text)
data = text.encode()
send_data(s, data)
# recv message
data = recv_data(s)
text = data.decode()
print('recv:', text)
# ---
# send JSON
stock = {'open': 12, 'close': 15}
print('send:', stock)
text = json.dumps(stock)
data = text.encode()
send_data(s, data)
# recv JSON
data = recv_data(s)
text = data.decode()
stock = json.loads(text)
print('recv:', stock)
# ---
s.close()
Similar way client could send filename and server could send back image data. But for files it may need receiving in chunks because socket has limited buffer. It may need also to send extra iformation if server found image or not.
I'm using a remote linux server and I want to send an array via sockets from client with python so I used this code :
message = pickle.dumps(faceBlob)
message_header = bytes(f"{len(message):<{HEADER_LENGTH}}", "utf-8")
client_socket.send(message_header + message)
to receive it in the server I used a while loop to catch all the message since it is > 4096 :
def receive_blop(client_socket):
# Receive our "header" containing message length, it's size is defined and constant
message_header = client_socket.recv(HEADER_LENGTH)
# If we received no data, client gracefully closed a connection, for example using socket.close() or socket.shutdown(socket.SHUT_RDWR)
if not len(message_header):
return False
# Convert header to int value
message_length = int(message_header.decode('utf-8').strip())
fragments = []
print(message_length)
while True:
# this loop is infinite
print("I arrived her")
chunk = client_socket.recv(4096)
if not chunk:
break
fragments.append(chunk)
data_arr = b"".join(fragments)
# Return an object of message header and message data
return {'header': message_header, 'data': data_arr}
the server still printing the 'I arrived here' but receive the message until the connection is ended from the client
Your loop will continue until the client closes the connection. If they don't close until they get a response from you, you've got a deadlock.
Since you know the message length, you can stop the loop when you've received that many bytes.
received_length = 0
while received_len < message_length:
print("I arrived her")
chunk = client_socket.recv(message_length - received_length)
if not chunk:
break
fragments.append(chunk)
received_length += len(chunk)
I'm trying to send a 2D array across a socket, it needs to be made out of multiple socket.recv() otherwise I will get a _pickle.UnpicklingError: pickle data was truncated error.
I've tried to do this with a while loop that receives packets and appends them to a list until all data is received:
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
Just doing:
def receive(self):
data = self.socket.recv(1024)
data = pickle.loads(data)
return data
Works if I send something smaller than the 2D array e.g. a coordinate pair tuple (2, 3). But not with the 2D array.
I get a OSError: [WinError 10022] when attempting the while loop approach.
I have seen that a OSError: [WinError 10022] could be an issue with not binding the socket but I think I have done that.
I can't figure out where connections are closing and am very confused.
Rest of the code:
Server:
import socket
from _thread import start_new_thread
clients = []
def threaded(client):
while True:
data = client.recv(1024)
if not data:
print('Not Data.')
break
# Send the data received from one client to all the other clients.
for c in clients:
if c != client:
c.send(data)
client.close()
def Main():
host = ""
port = 5555
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
print("Socket binded to port:", port)
s.listen(2)
print("Socket is listening.")
while True:
c, addr = s.accept()
clients.append(c)
print(f"Connected to: {addr[0]}:{addr[1]}")
start_new_thread(threaded, (c,))
s.close()
Client:
import socket
import pickle
class Client:
def __init__(self):
self.host = 'localhost'
self.port = 5555
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((self.host, self.port))
def send(self, data):
message = pickle.dumps(data)
try:
self.socket.send(message)
except socket.error as e:
return str(e)
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
The client-server I want to make will have 2 clients and each client will send data to the other constantly. How can I correct my code and properly implement this?
After doing more research and figuring out what was going wrong I got a solution.
The code never got out of this while loop - because data was constantly being sent and some packet was always incoming.
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
A solution I found was to send a message indicating how big the message being received should be, so I can tell when all the bytes for a particular message have arrived. Links: Sockets Python 3.5: Socket server hangs forever on file receive, Python Socket Receive Large Amount of Data
def send(self, data):
message = pickle.dumps(data)
msg_len = len(message)
try:
# Send what the total length of the message to be sent is in bytes.
self.socket.send(msg_len.to_bytes(4, 'big'))
self.socket.sendall(message)
except socket.error as e:
return str(e)
def receive(self):
remaining = int.from_bytes(self.socket.recv(4), 'big')
chunks = []
while remaining:
# until there are bytes left...
# fetch remaining bytes or 4096 (whatever smaller)
chunk = self.socket.recv(min(remaining, 4096))
remaining -= len(chunk)
# write to file
chunks.append(chunk)
chunks = b"".join(chunks)
data = pickle.loads(chunks)
return data
I wrote a simplistic socket client for reading data in Python 3.4
The problem I'm having is that when the server sends a small amount of data (around 1000) bytes, it will read it perfectly but when a large chunk of data is being handled around (9500 bytes) it will only give me a small chunk of data (like 1100-ish chunks). I can't seem to figure out why its behaving so erratically when handling the huge amount of data. I know that my data is not larger than ssize_t maximum of 32767.
It works perfectly when handling small data and completely turns 180 and behaves differently when handling a huge amount of data. I know that this is not a problem in the TCP server because I tested it with a PHP TCP client and it worked perfectly when handling the huge amount of data.
Any help is greatly appreciated.
import socket
import json
# Written in Python 3.4.
class SocketClient:
def __init__(self, host, port, format, timeout = None):
# Constructor
self.host = host
self.port = port
self.format = format
self.timeout = timeout
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def send(self, firstname, lastname, parameters = [], format = 'json'):
if self.socket is not None:
self.socket.connect((self.host, self.port))
data = {}
data['firstname'] = firstname
data['lastname'] = lastname
data['parameters'] = parameters
data['format'] = format
self.socket.send(bytes(json.dumps(data), "utf-8"))
result = self.socket.recv(32767)
result = result.decode()
return result
def shutdown(self):
if socket is not None:
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
if __name__ == __main__:
client = SocketClient("127.0.0.1", 8080, 'json')
response = client.send('foo', 'bar', ['foobar'])
print(response)
client.shutdown()
TCP is a streaming protocol. Data is delivered in junks of bytes, where the length is determined by many factors. One is, that internal buffers are limited to some thousand bytes. You never can read 32767 bytes at once.
The only guarantee with recv is, that you get at least 1 byte and at most the number of bytes you say. Your code have to cope with this, that means, you have to do more than one recv call until you have the amount of bytes you need. That means on the other side, a protocol that don't have end-of-message indicators or a length encoded is badly broken. In your case: you have to parse the json byte stream until a valid json expression is sent. But what about 1245.6? Is it finished after receiving 1 or 12 or ...?
To repair your protocol, simply send some length information with your json data.
For sending you should use sendall instead of send.
You can use recv_into(buffer[, nbytes[, flags]]) method:
def readReliably(s,n):
buf = bytearray(n)
view = memoryview(buf)
sz = 0
while sz < n:
k = s.recv_into(view[sz:],n-sz)
sz += k
# print 'readReliably()',sz
return sz,buf
def writeReliably(s,buf,n):
sz = 0
while sz < n:
k = s.send(buf[sz:],n-sz)
sz += k
# obj = s.makefile(mode='w')
# obj.flush()
# print 'writeReliably()',sz
return sz
See the full example at: https://stackoverflow.com/a/55446223/966789
while True:
sk,skfrom = s.accept()
sz,buf = io.readReliably(sk,4)
a = struct.unpack("4B",buf)
print repr(a)
# ...
io.writeReliably(sk,struct.pack("4B",*[0x01,0x02,0x03,0x04]))
That is how recv works in more than one language... https://docs.python.org/3.4/library/socket.html#socket.socket.recv