We Keep Coding

Server/Client app and JSONDecodeError: Unterminated string python

I have async socket server file and client file.
When i send something like that "download filename.ex" to the client, this client's code hande my request:
try:
content = read(sp_data[-1]).decode('utf-8')
print(content)
msg = json.dumps({'file': sp_data[-1], 'command': data, 'content': content,
'msg': f'[+] File {sp_data[-1]} has been successfully downloaded.'}).encode('utf-8')
except FileNotFoundError:
msg = json.dumps({'msg': f'[-] File {sp_data[-1]} not found', 'command': data}).encode('utf-8')
s.send(msg)
When client send some data to the socketserver, this server's code handle received message:
def recv_message(client_socket):
global messages
data = json.loads(client_socket.recv(4096).decode('utf-8').strip()) ##Important here i got this error json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 67 (char 66)
raddr = get_raddr(str(client_socket))
raddr = f'{raddr[0]}:{raddr[1]}'
message = f'From: {raddr}\nCommand: {data["command"]}\nOutput: \n\n{data["msg"]}'
try:
d = messages[raddr]
d.append(message)
messages[raddr] = d
except KeyError:
messages[raddr] = [message]
except AttributeError:
print(message, messages)
if 'content' in data.keys(): ##Important
print(data['content'])
threading.Thread(target=create_file, args=(data['file'], data['content'],), daemon=False).start()
Error:
data = json.loads(client_socket.recv(4096).decode('utf-8').strip())
json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 67 (char 66)
But server's code above give me this error when it receive message from the first code(when i send something like that "download file.ex" to the client, client detect my command as its special command, execute the first code, send json file to the server. But if i send "dir" command to the client, it will detect my command like shell command, will run command through subprocess, will send result to the server back and i won't get any errors.)
Note: I also reduced socketserver's code. Therefore, something in my code can work worse. The main goal of this post - make download feature works. I also understand that my code is big. I left "##Important" comments in my files. U can watch only code that located by these comments.
Server:
import selectors
import socket
import threading
import json
import base64
import shlex
selector = selectors.DefaultSelector()
connections = {}
def accept_conn(server_socket):
sock, addr = server_socket.accept()
connections[len(connections) + 1] = [sock, f'{addr[0]}:{addr[-1]}']
selector.register(fileobj=sock, events=selectors.EVENT_READ, data=recv_message)
s = socket.socket()
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind(('localhost', 4444))
s.listen()
selector.register(fileobj=s, events=selectors.EVENT_READ, data=accept_conn)
messages = {}
##Important
def create_file(file, content): #content - base64 string
print(content)
with open(file, 'wb') as f:
f.write(base64.b64decode(content.encode('utf-8')))
def recv_message(client_socket):
global messages
data = json.loads(client_socket.recv(4096).decode('utf-8').strip()) ##Important here i got this error json.decoder.JSONDecodeError: Unterminated string starting at: line 1 column 67 (char 66)
raddr = get_raddr(str(client_socket))
raddr = f'{raddr[0]}:{raddr[1]}'
message = f'From: {raddr}\nCommand: {data["command"]}\nOutput: \n\n{data["msg"]}'
try:
d = messages[raddr]
d.append(message)
messages[raddr] = d
except KeyError:
messages[raddr] = [message]
except AttributeError:
print(message, messages)
if 'content' in data.keys(): ##Important
print(data['content'])
threading.Thread(target=create_file, args=(data['file'], data['content'],), daemon=False).start()
def get_raddr(string):
'''Get raddr parameter from client socket'''
raddr = string.replace('>', '')
return eval(raddr[raddr.find('raddr')::].replace('raddr=', ''))
def is_manage_string(sub, string):
tokens = shlex.split(string)
try:
if len(tokens) == 2 and tokens[0] == sub and str(int(tokens[-1])):
return True, int(tokens[-1])
except Exception as e:
print(e)
return False
manage_process = False
def manage():
global manage_process
while True:
manage_process = False
command = input('>>> ').strip()
if command == 'list':
try:
for i in range(1, len(connections) + 1):
print(f'{i}\t{connections[i][-1]}')
except KeyError:
pass
if len(connections) == 0:
print('[-] There are not any connections')
elif 'manage' in command:
index = is_manage_string('manage', command)
if index:
index = index[-1]
else:
print('[-] Invalid command\nUse manage "number_of_connection"\nEx: manage 1')
continue
if index >= 1 and index <= len(connections):
sock, addr = connections[index]
print(addr)
print(f'{addr} is used')
while True: ##Important here i launch loop which send data to socket
manage_process = True
command = input('>>> ').strip()
if command == 'messages':
try:
if messages[addr] == list():
print()
continue
except KeyError:
pass
try:
print('\n\n'.join(messages[addr]))
except KeyError:
print()
elif command == 'message':
try:
print(messages[addr][-1])
except:
print()
elif command == 'clear_messages':
try:
if messages[addr]:
messages[addr] = []
except KeyError:
print('[-] There are not any messages for cleaning up')
elif command == 'leave':
print(f'Leaving connection {addr}')
break
elif command: ##Important if command hasn't been detected as my special command(leave, messages), it will be executed like shell command
try:
sock.send(command.encode('utf-8'))
print(
'Your input has not been detected as special command and will execute like shell command or like client special command(ex: download; see client file)')
except ConnectionResetError:
print("Connection has been lost, therefore shell commands can't be used")
else:
continue
else:
print('[-] Invalid number of connection')
elif command:
print('[-] Invalid command\nType "help" to see avalible commands')
##Important
def event_loop():
while True:
data = selector.select()
for key, _ in data:
try:
key.data(key.fileobj)
except ConnectionResetError:
selector.unregister(key.fileobj)
##Important
threading.Thread(target=manage, daemon=True).start()
event_loop()
Client:
import socket
import subprocess
import shlex
import threading
import json
import base64
s = socket.socket()
s.connect(('localhost', 4444))
##Important
def read(file):
with open(file, 'rb') as f:
return base64.b64encode(f.read())
def runner(data):
sp_data = shlex.split(data)
try:
print(sp_data)
if len(sp_data) == 2 and sp_data[0] == 'download': ###Important here we create json object which will be send to socketserver
try:
content = read(sp_data[-1]).decode('utf-8')
print(content)
msg = json.dumps({'file': sp_data[-1], 'command': data, 'content': content,
'msg': f'[+] File {sp_data[-1]} has been successfully downloaded.'}).encode('utf-8')
except FileNotFoundError:
msg = json.dumps({'msg': f'[-] File {sp_data[-1]} not found', 'command': data}).encode('utf-8')
s.send(msg)
return ''
except Exception as e:
print(e)
command = subprocess.run(data, shell=True, encoding='cp866', text=True, capture_output=True)
command = command.stderr if command.stderr else command.stdout
command = json.dumps({'msg': command, 'command': data})
s.send(command.encode('utf-8'))
while True:##Important
data = s.recv(4096).decode('utf-8').strip()
threading.Thread(target=runner, args=(data,)).start()

import socket
import struct
class Socket(socket.socket):
def __init__(self):
self.sock = socket.socket()
super().__init__(socket.AF_INET, socket.SOCK_STREAM)
def send_msg(self, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
self.sock.sendall(msg)
def recv_msg(self):
# Read message length and unpack it into an integer
raw_msglen = self.recv_all(4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return self.recv_all(msglen)
def recv_all(self, n):
data = bytearray()
while len(data) < n:
packet = self.sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data
I reshaped ur code into Socket class.

Solution - usage these features:
def send_msg(sock, msg):
# Prefix each message with a 4-byte length (network byte order)
msg = struct.pack('>I', len(msg)) + msg
sock.sendall(msg)
def recv_msg(sock):
# Read message length and unpack it into an integer
raw_msglen = recvall(sock, 4)
if not raw_msglen:
return None
msglen = struct.unpack('>I', raw_msglen)[0]
# Read the message data
return recvall(sock, msglen)
def recvall(sock, n):
# Helper function to recv n bytes or return None if EOF is hit
data = bytearray()
while len(data) < n:
packet = sock.recv(n - len(data))
if not packet:
return None
data.extend(packet)
return data

Can't figure out how to send a 2D array over a socket - OSError: [WinError 10022]

I'm trying to send a 2D array across a socket, it needs to be made out of multiple socket.recv() otherwise I will get a _pickle.UnpicklingError: pickle data was truncated error.
I've tried to do this with a while loop that receives packets and appends them to a list until all data is received:
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
Just doing:
def receive(self):
data = self.socket.recv(1024)
data = pickle.loads(data)
return data
Works if I send something smaller than the 2D array e.g. a coordinate pair tuple (2, 3). But not with the 2D array.
I get a OSError: [WinError 10022] when attempting the while loop approach.
I have seen that a OSError: [WinError 10022] could be an issue with not binding the socket but I think I have done that.
I can't figure out where connections are closing and am very confused.
Rest of the code:
Server:
import socket
from _thread import start_new_thread
clients = []
def threaded(client):
while True:
data = client.recv(1024)
if not data:
print('Not Data.')
break
# Send the data received from one client to all the other clients.
for c in clients:
if c != client:
c.send(data)
client.close()
def Main():
host = ""
port = 5555
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
print("Socket binded to port:", port)
s.listen(2)
print("Socket is listening.")
while True:
c, addr = s.accept()
clients.append(c)
print(f"Connected to: {addr[0]}:{addr[1]}")
start_new_thread(threaded, (c,))
s.close()
Client:
import socket
import pickle
class Client:
def __init__(self):
self.host = 'localhost'
self.port = 5555
self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.socket.connect((self.host, self.port))
def send(self, data):
message = pickle.dumps(data)
try:
self.socket.send(message)
except socket.error as e:
return str(e)
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
data = b"".join(packets)
data = pickle.loads(data)
return data
The client-server I want to make will have 2 clients and each client will send data to the other constantly. How can I correct my code and properly implement this?

After doing more research and figuring out what was going wrong I got a solution.
The code never got out of this while loop - because data was constantly being sent and some packet was always incoming.
def receive(self):
packets = []
while True:
packet = self.socket.recv(1024)
if not packet:
break
packets.append(packet)
A solution I found was to send a message indicating how big the message being received should be, so I can tell when all the bytes for a particular message have arrived. Links: Sockets Python 3.5: Socket server hangs forever on file receive, Python Socket Receive Large Amount of Data
def send(self, data):
message = pickle.dumps(data)
msg_len = len(message)
try:
# Send what the total length of the message to be sent is in bytes.
self.socket.send(msg_len.to_bytes(4, 'big'))
self.socket.sendall(message)
except socket.error as e:
return str(e)
def receive(self):
remaining = int.from_bytes(self.socket.recv(4), 'big')
chunks = []
while remaining:
# until there are bytes left...
# fetch remaining bytes or 4096 (whatever smaller)
chunk = self.socket.recv(min(remaining, 4096))
remaining -= len(chunk)
# write to file
chunks.append(chunk)
chunks = b"".join(chunks)
data = pickle.loads(chunks)
return data

Python Webserver not working

I am building a basic python web server, but I keep having a problem where it is not sending any data (by the way I am accessing the website on the same computer as it is running on and I have the file which the server is trying to access) here is my code:
import socket
HOST, PORT = '', 80
def between(left,right,s):
before,_,a = s.partition(left)
a,_,after = a.partition(right)
return a
filereq = ""
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_socket.bind((HOST, PORT))
lines = []
print("Started!")
listen_socket.listen(1)
print("Listening")
while True:
try:
lines = []
client_connection, client_address = listen_socket.accept()
print("Connected")
request = client_connection.recv(1024)
print("Received Data!")
filereq = between("GET /", " HT", request)
print(filereq)
filereq = open(filereq)
for line in filereq:
lines.append(line)
print(lines)
sendata = ''.join(lines)
print(sendata)
http_response = """\
HTTP/1.1 200 OK
{}
""".format(sendata)
print(http_response)
client_connection.sendall(http_response)
print("Sent the Data!")
client_connection.close()
print("Connection Closed!")
except:
5+5

The problem is that the server is implemented in Python3 but the code mixes bytes and strings, which works in Python2 but not Python3.
This causes an error in the between function, because partition is being called on a bytes object but is being provided with str separator values.
>>> data = b'abc'
>>> data.partition('b')
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
TypeError: a bytes-like object is required, not 'str'
To fix this, decode the data from bytes to str when read from the socket, then encode back to bytes before sending the response (socket.sendall expects bytes as an argument).
Also, print out any exceptions that occur so that you can debug them.
import socket
import sys
import traceback
HOST, PORT = '', 80
def between(left,right,s):
before,_,a = s.partition(left)
a,_,after = a.partition(right)
return a
filereq = ""
listen_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
listen_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
listen_socket.bind((HOST, PORT))
lines = []
print("Started!")
listen_socket.listen(1)
print("Listening")
while True:
try:
lines = []
client_connection, client_address = listen_socket.accept()
print("Connected")
request = client_connection.recv(1024)
print("Received Data!")
# Decode the data before processing.
decoded = request.decode('utf-8')
filereq = between("GET /", " HT", decoded)
print(filereq)
filereq = open(filereq)
for line in filereq:
lines.append(line)
print(lines)
sendata = ''.join(lines)
print(sendata)
http_response = """\
HTTP/1.1 200 OK
{}
""".format(sendata)
print(http_response)
# Encode the response before sending.
encoded = http_response.encode('utf-8')
client_connection.sendall(encoded)
print("Sent the Data!")
client_connection.close()
print("Connection Closed!")
except Exception:
# Print the traceback if there's an error.
traceback.print_exc(file=sys.stderr)

invalid mode ('wb') or filename: 'torecv.jpg' python

I'm writing a client-server live screenshot streaming. My program works for few seconds. I can see live stream from my client but then my program crushs with this error:
Traceback (most recent call last): File "C:/Users/user-pc/PycharmProjects/TestsFor/Functions‌​/screenshotLive2/ser‌​ver files.py", line 51, in <module> receive_file(filename) File "C:/Users/user-pc/PycharmProjects/TestsFor/Functions‌​/screenshotLive2/ser‌​ver files.py", line 36, in receive_file f = open(file_name, 'wb') IOError: [Errno 22] invalid mode ('wb') or filename: 'torecv.jpg'
This is really weird because its able to open and close it many times before it crushes. Does anyone understand why?
This is my server:
import socket # Import socket module
import cv2
import struct
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # put socket outside of loop?
sock.bind(("0.0.0.0", 8200))
sock.listen(1)
connect, address = sock.accept()
except socket.error:
print "Socket Error"
def recv_msg():
# Read message length and unpack it into an integer
raw_msglen = recvall(4)
if not raw_msglen:
return None
msglen = struct.unpack('L', raw_msglen)[0]
# Read the message data
return recvall(msglen)
def recvall(n):
# Helper function to recv n bytes or return None if EOF is hit
data = ''
while len(data) < n:
packet = connect.recv(n - len(data))
if not packet:
return None
data += packet
return data
def receive_file(file_name):
f = open(file_name, 'wb')
data = recv_msg()
f.writelines(data)
f.close()
while True:
receive_file(r'torecv.jpg')
image = cv2.imread(r'torecv.jpg',1)
cv2.imshow("Live Screen", image)
cv2.waitKey(1)
And this my client:
import ImageGrab
import StringIO
import socket
import struct
try:
connect = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
connect.connect(("localhost", 8200))
except socket.error:
print "Error Socket"
def screen_shot():
im = ImageGrab.grab() # take screen shot
saver = StringIO.StringIO() # creating string as 'File'
im.save(saver, "jpeg") # save the file (not on hard drive)
binary_string = saver.getvalue()
saver.close()
msg = struct.pack('L', len(binary_string)) + binary_string
return msg
while True:
connect.sendall(screen_shot())

PyAudio Over Network crashes

Hello I am facing an issue with PyAudio I can not solve. When I use it over the network (after a while) it crashes. The error I get is
Exception in thread Thread-1:
Traceback (most recent call last):
File "C:\Python27\lib\threading.py", line 552, in __bootstrap_inner
self.run()
File "C:\Python27\lib\threading.py", line 505, in run
self.__target(*self.__args, **self.__kwargs)
File "C:\Users\maboroshi\Desktop\myChat\chat.py", line 71, in server
frames_per_buffer = chunk)
File "C:\Python27\lib\site-packages\pyaudio.py", line 714, in open
stream = Stream(self, *args, **kwargs)
File "C:\Python27\lib\site-packages\pyaudio.py", line 396, in __init__
self._stream = pa.open(**arguments)
IOError: [Errno Device unavailable] -9985
My code is below (well most of it :-P
This is for decrypting the data and managing connections
def decrypt_my_message(msg):
iv = "1234567812345678"
key = your_friends_key
if len(key) not in (16, 24, 32):
raise ValueError("Key must be 16, 24, or 32 bytes")
if (len(msg) % 16) != 0:
raise ValueError("Message must be a multiple of 16 bytes")
if len(iv) != 16:
raise ValueError("IV must be 16 bytes")
cipher = AES.new(key, AES.MODE_CBC, iv)
plaintext = cipher.decrypt(msg)
return plaintext
### Server function ###
def server():
HOST = ''
PORT = 9001
### Initialize socket
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(5)
### Start recieve loop
read_list = [server_socket]
while True:
readable, writable, errored = select.select(read_list, [], [])
for s in readable:
if s is server_socket:
conn, addr = s.accept()
read_list.append(conn)
print "Connection from ", addr
else:
msg = conn.recv(2024)
if msg:
cmd, msg = ord(msg[0]),msg[1:]
if cmd == CMD_MSG:
listb1.insert(END, decrypt_my_message(msg.strip()) + "\n")
listb1.yview(END)
elif cmd == CMD_AUDIO:
#d = speex.Decoder()
#d.initialize(speex.SPEEX_MODEID_WB)
p = pyaudio.PyAudio()
stream = p.open(format = FORMAT,
channels = CHANNELS,
rate = RATE,
input = True,
output = True,
frames_per_buffer = chunk)
stream.write(decrypt_my_message(msg), chunk) #Write the data back out to the speakers
else:
s.close()
read_list.remove(s)
And this is for connecting and sending encrypted audio
def encrypt_my_audio_message(msg):
key = your_key
iv = '1234567812345678'
aes = AES.new(key, AES.MODE_CBC, iv)
encoder = PKCS7Encoder()
pad_text = encoder.encode(msg)
msg = aes.encrypt(pad_text)
return msg
def connectToServer():
CLIENT_PORT = 9001
CLIENT_HOST = str(entryhost.get())
global s
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((CLIENT_HOST, CLIENT_PORT))
print "Connected\n"
except:
print "Could not connect"
### Client Function ###
def client(cmd, msg):
try:
s.send(cmd + msg)
except:
print "You are not connected"
def sendAudio():
p = pyaudio.PyAudio()
stream = p.open(format = FORMAT,
channels = CHANNELS,
rate = RATE,
input = True,
output = True,
frames_per_buffer = chunk)
data = stream.read(chunk)
return encrypt_my_audio_message(data)
## stream.stop_stream()
## stream.close()
## p.terminate()
def keypress(event):
if event.keysym == 'Escape':
root.destroy()
#x = event.char
if event.keysym == 'Control_L':
#print("Sending Data...")
client(chr(CMD_AUDIO), sendAudio())
#print("Data Sent!")
Any ideas why this happens would be most helpful

The thing that jumps out at me is that you're mixing a thread that blocks with select, and the loop looks a bit weird. You've got a lot of stuff going on in both directions - try and focus on making the flow of packets in and out work like gears meshing together, or ensure that there's no way that somebody is going to be empty or full when they're expected to be neither.
You might find some inspiration from:
this portaudio/speex python wrapper
asyncore
queue

My guess is that you're running into problems because you are creating multiple PyAudio objects and opening multiple streams without closing any of them.
You should create one PyAudio object, and one stream, and re-use those objects.