We Keep Coding

Python JSON parse from tcp socket

I'm new to python and i'm stuck with a program that should receive json from tcp socket and forward it to http with simple http server. With the code below i am getting following error Other error occurred: 'length' I am assuming it has something to do with dictionary or key in it since i am trying to set tcp buffer size from received json(it has 'length' before actual data). Example {"length": 203} {"agent_version":3.09,"build_version":"Agent/3.09 (ubuntu; x86_64; conntrack; netlink; dns-cache; plugins; tcmalloc; inotify; regex) no/2.9.0 JSON/1.90","json_version":1.9,"type":"agent_hello"} {"length": 484}
Can someone point me in the right direction?
Thanks.
#!/usr/bin/python3
import http.server
from logging import exception
import socketserver
import json
import socket
PORT = 8000
buff_size = 219
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('10.10.10.10', 2100))
class MyHandler(http.server.SimpleHTTPRequestHandler):
global do_rcv, data, qdata
def do_rcv():
global buff_size, data, qdata
qdata = []
while True:
try:
data = s.recv(buff_size).decode('utf-8').split('\n')
data += data
for i in data:
i = str('[' + str(i) + ']')
i = str(i).replace('}{', '},{')
i = str(i).replace('\n', '')
qdata = json.loads(i)
buff_size = qdata[-1]['length']
buff_size = buff_size + 16
if not data:
s.close()
return
#except KeyError:
#pass
except Exception as err:
print(buff_size)
print(qdata)
print(f'Other error occurred: {err}')
s.close()
return
def do_GET(self):
do_rcv()
output_data = []
# - request -
content_length = int(self.headers['Content-Length'])
#print('content_length:', content_length)
if content_length:
input_json = self.rfile.read(content_length)
input_data = json.loads(input_json)
else:
input_data = None
print(input_data)
# - response -
self.send_response(200)
self.send_header('Content-type', 'text/json')
self.end_headers()
output_data += qdata
output_json = json.dumps(output_data)
self.wfile.write(output_json.encode('utf-8'))
Handler = MyHandler
try:
with socketserver.TCPServer(("", PORT), Handler) as httpd:
print(f"Starting http://0.0.0.0:{PORT}")
httpd.allow_reuse_address = True
httpd.serve_forever()
except KeyboardInterrupt:
print("Stopping by Ctrl+C")
httpd.server_close() # to resolve problem `OSError: [Errno 98] Address already in use`
s.close()
except Exception:
print("Stopping by other exception")
httpd.server_close()

browser doesn't response well to https proxy

I'v built my own https proxy and when ever I send some data to a browser the browser responses with nothing and also after a lot of time.
basically all the proxy should do is just forward the message to browser, get the response and forward back to the client
the code of the proxy:
import socket
import select
serverSock = socket.socket()
serverSock.bind(('0.0.0.0', 8080))
serverSock.listen(3)
waiting_clients = {} # client : browser
users_dict = {}
open_clients = {}
browsers_clients = {} # browser : client
threading.Thread(target=browserCom).start()
while True:
try:
rlist, wlist, xlist = select.select(list(users_dict.keys()) + [serverSock], [], [], 0.3)
except:
pass
else:
for current_socket in rlist:
if current_socket is serverSock:
# new client
client, address = serverSock.accept()
print(f'{address} - connected to proxy')
# add to dictionary
users_dict[client] = address
open_clients[address] = client
else:
# receive info
receiving = True
msg = bytearray()
while receiving:
try:
data = current_socket.recv(1024)
except Exception as e:
print(e, 3)
if current_socket in users_dict.keys():
disconnect(users_dict[current_socket])
else:
current_socket.close()
break
else:
msg.extend(data)
# got the full msg
if len(data) < 1024:
receiving = False
if len(msg) == 0:
if current_socket in users_dict.keys():
disconnect(users_dict[current_socket])
else:
print("GOT FROM CLIENT", msg)
if current_socket in waiting_clients.keys():
# sending the data from client to browser
waiting_clients[current_socket].send(msg)
else:
msg = msg.decode()
msgSplit = msg.split()
address = msgSplit[1]
if address.split(':')[1].isnumeric():
if msg.startswith('CONNECT'):
browserLink, browserPort = address.split(':')
browserPort = int(browserPort)
browserIP = socket.gethostbyname(browserLink)
address = (browserIP, browserPort)
# connect to the site
browserSocket = socket.socket()
print(address)
browserSocket.connect((browserIP, browserPort))
waiting_clients[current_socket] = browserSocket
browsers_clients[browserSocket] = current_socket
msg_ret = "HTTP/1.1 200 Connection established\r\n\r\n"
sendMsg(users_dict[current_socket], msg_ret)
m
The proxy is able to make the connection after the CONNECT and notify to the client but after I send to the browser I got from the data with a function running in the background:
def browserCom():
while True:
try:
rlist, wlist, xlist = select.select(list(browsers_clients.keys()), [], [], 0.3)
except:
pass
else:
for current_browser in rlist:
# receive data from the browser
receiving = True
resp_msg = bytearray()
while receiving:
try:
data = current_browser.recv(1024)
except Exception as e:
print(e)
del waiting_clients[browsers_clients[current_browser]]
current_browser.close()
browsers_clients[current_browser].close()
del browsers_clients[current_browser]
else:
resp_msg.extend(data)
# got the full msg
if len(data) < 1024:
receiving = False
print("RESPONSE FROM BROWSER", resp_msg)
# sending the msg to the client
sendMsg(users_dict[browsers_clients[current_browser]], resp_msg)
I need to wait a lot of time for the response and most of the responses come empty the responses are mostly bytearray(b'') and even when I get the response even though I sent the response back to the client:
# sending the msg to the client
sendMsg(users_dict[browsers_clients[current_browser]],resp_msg)
using this
def sendMsg(address, msg):
"""
:param ip: ip to send to
:param msg: msg to send
:return: sends the msg to the ip
"""
if address in open_clients.keys():
sock = open_clients[address]
if type(msg) == str:
msg = msg.encode()
try:
sock.send(msg)
except Exception as e:
print(e, 4)
disconnect(address)
I hope you are abled to understand my code, please if something is unclear ask me in the comments and I will try to help you understand as soon as possible
this is the best that I can do to keep the code minimal for this problem without removing crucial parts

My mistake was that I didn't understand that while tunneling both the browser and the client exchange messages, adding another select to all the browser helped me check all the data from all the browsers and now it works.
The code above is updated and works
basically what I added is :
def browserCom():
while True:
try:
rlist, wlist, xlist = select.select(list(browsers_clients.keys()), [], [], 0.3)
except:
pass
else:
for current_browser in rlist:
# receive data from the browser
receiving = True
resp_msg = bytearray()
while receiving:
try:
data = current_browser.recv(1024)
except Exception as e:
print(e)
del waiting_clients[browsers_clients[current_browser]]
current_browser.close()
browsers_clients[current_browser].close()
del browsers_clients[current_browser]
else:
resp_msg.extend(data)
# got the full msg
if len(data) < 1024:
receiving = False
# disconnecting browser
if resp_msg == bytearray(b''):
del waiting_clients[browsers_clients[current_browser]]
current_browser.close()
browsers_clients[current_browser].close()
del browsers_clients[current_browser]
print("RESPONSE FROM BROWSER", resp_msg)
# sending the msg to the client
if current_browser in browsers_clients and browsers_clients[current_browser] in users_dict:
sendMsg(users_dict[browsers_clients[current_browser]], resp_msg)

UDP Tracker only gives me my ip as answer to announce request

I've recently been trying to create a torrent client in python, and have just got the UDP announce protocol to work.
The tracker accepts my connect request just fine but only returns my IP and port as the peer list when I announce to it...
I've tried to look at the same torrents in other torrent clients and they have multiple working peers while my request only shows me my computer (I've tried this on many torrents, all return just my IP and port)
Here's the code for the sending function itself:
async def announce_udp(self, try_num = 1):
self.sock.settimeout(15)
answer = {}
inner_while = False
while try_num < 4:
while try_num < 4:
try:
print("trying to send")
sended = self.send(1, self.announce_payload())
print("sending the following packet: {0}".format(sended))
print(self.url)
inner_while = True
break
except Exception:
print("problem in sending")
try_num += 1
if not inner_while:
break
try:
answer = self.interpret(15)
break
except Exception:
print("problem in receiving")
try_num += 1
print("announce answer is: {0}".format(answer))
return answer
here's the code for the make payload function:
def announce_payload(self, downloaded = 0, left = 0, uploaded = 0, event = 0, key = get_transaction_id()):
payload = [self.torrent.get_torrent_info_hash_decoded(), get_peer_id().encode(), downloaded,
self.torrent.get_torrent_size(), uploaded, event, 0, key, -1, 6988]
p_tosend = None
try:
p_tosend = struct.pack('!20s20sqqqiIIiH', *payload)
except Exception as e:
print("there was an error: {0}".format(e))
return p_tosend
here's the code for the interpret + process function:
def interpret(self, timeout=10):
self.sock.settimeout(timeout)
print("got to interpret")
try:
response = self.sock.recv(10240)
print("answer recieved")
except socket.timeout:
print("no answer, try again")
raise TrackerResponseException("no answer", 0)
headers = response[:8]
payload = response[8:]
action, trans_id = struct.unpack('!ll', headers)
try:
trans = self.transactions[trans_id]
except KeyError:
raise TrackerResponseException("InvalidTransaction: id not found", trans_id)
try:
trans['response'] = self.process(action, payload, trans)
except Exception as e:
trans['response'] = None
print("error occured: {0}".format(e))
trans['completed'] = True
del self.transactions[trans_id]
#print(trans)
return trans
def process_announce(self, payload, trans):
response = {}
info = payload[:struct.calcsize("!lll")]
interval, leechers, seeders = struct.unpack("!lll", info)
print(interval, leechers, seeders, "noamsssssss")
peer_data = payload[struct.calcsize("!lll"):]
peer_size = struct.calcsize("!lH")
num_of_peers = int(len(peer_data) / peer_size)
print("the number of peers is: {0} and the peer data is: {1}".format(num_of_peers, peer_data))
print()
peers = []
for peer_offset in range(num_of_peers):
off = peer_size * peer_offset
peer = peer_data[off:off + peer_size]
addr, port = struct.unpack("!lH", peer)
peers.append({
'addr': socket.inet_ntoa(struct.pack('!L', addr)),
'port': port,
})
print(payload)
return dict(interval=interval, leechers=leechers, seeders=seeders, peers=peers)
I'm sorry if any of this is irrelevant, but I want to give you all of the code incase it tells you something.
(get_peer_id() returns a random peer id per the tracker protocol specification, and the get_transaction_id() returns random.randint(0, 1 << 32 - 1))
EDIT:
Alright, I've found the problem and now I'm feeling pretty dumb...
turns out even in the udp tracker whenever you send the info hash it has to be SHA1 encoded.
Hopefully this can help someone if they are stuck in the same problem :)

How to receive all data after TCP connection has been closed by the peer?

Running some production code I faced the following problem:
When sending HTTP requests to some server, server immediately closes the connection after sending response, which, for some reason, results in data loss.
Analyzing TCP dumps i can see that conversation goes as this:
client request
server ack
server push
server fin, ack (sent after ~0.000020 secs after previous push)
As the result my code can't get data sent by the server, (i'm guessing because of the small delay after push POLLHUP event might go before POLLIN?)
Trying to mimic the problem I've written the following code:
(It mimics the client behaviour on my side)
client:
import time
import socket
from errno import EAGAIN
from select import poll, POLLIN, POLLPRI, POLLERR, POLLHUP, POLLNVAL
def main(buf=""):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.setblocking(False)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
polling_object = poll()
polling_object.register(client, POLLPRI | POLLIN | POLLERR | POLLHUP)
in_buf = ""
sock_closed = False
try:
client.connect(("127.0.0.1", 8877))
except socket.error, e:
pass
while True and not sock_closed:
events = polling_object.poll(0)
for _, e in events:
if e & (POLLIN | POLLPRI):
while True:
try:
data = client.recv(1024)
if data:
in_buf += data
elif data == "":
client.close()
sock_closed = True
break
except socket.error, e:
if e.args[0] == EAGAIN:
break
else:
raise
elif e & (POLLERR|POLLHUP|POLLNVAL):
client.close()
sock_closed = True
if buf and not sock_closed:
try:
b_sent = client.send(buf)
if b_sent == len(buf):
buf = ""
else:
buf = buf[b_sent:]
except socket.error, e:
if e.args[0] != EAGAIN:
client.close()
sock_closed = True
time.sleep(0.5)
if sock_closed:
return in_buf
if __name__ == '__main__':
import sys
if len(sys.argv) > 1:
buf = sys.argv[1]
else:
buf = 'hello'
print main(buf)
server
import datetime
import time
import socket
def main():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", 8877))
server.listen(0)
client, _ = server.accept()
t1 = time.time()
data = ""
while not data:
data += client.recv(1024)
print "recv data %s" % data
client.sendall('{"ok": 1}')
t2 = time.time()
client.close()
t3 = time.time()
server.close()
return t1, t2, t3
if __name__ == '__main__':
c_r, d_s, c_c = main()
print "Connection received at ", datetime.datetime.fromtimestamp(c_r)
print "All Data sent after %.12f secs" % (d_s - c_r)
print "Connection closed after %.12f secs" % (c_c - d_s)
Running this code won't help me reproduce the problem because my client still can get data from socket buffer, which is kind of obviously by just following the code. The only difference is that in tcp dump it goes like this:
client request
server ack
server push
client ack
server fin, ack
I'm wondering is there a way to send fin, ack right after push without "letting" client to sent ack? Can it be done using python?

Sending numerous packets instead of 1 SSL Python

I have set up an experiment where I pass Modbus traffic over a SSL tunnel (this being the first thing I've ever done in python). I am able to send and receive data but when I send one request numerous requests are actually sent (see screenshot)
I've tried numerous configurations including (in both client and server):
send()--no change
sendall() --no change
setblocking(1)
setblocking(0)--doesn't read all the data
On the server side:
if data == Read_Coils_Answer-- I don't think I'm converting the big endian properly for comparison and this didn't work
while data: --the while loop seems to be the only way to prevent either side from stopping short with a "Broken Pipe" error. So this is what I'm using.
I eventually plan to use a for loop (now commented out and set to 4).
My Server code:
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def do_something(connstream, data):
readCoilsReq = str('\x01\x01\x00')
answer = str(ModbusPDU01_Read_Coils_Answer)
while data:
print ("Request Recevied from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
return False
def deal_with_client(connstream):
data = connstream.recv(64)
while data:
if not do_something(connstream, data):
break
data = connstream.recv(64)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Recieved\n")
sys.exit(1)
My Client Side code:
from ModLib import *
from time import sleep
import socket, ssl, pprint
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_sock = ssl.wrap_socket(s,
ca_certs="server.crt",
cert_reqs=ssl.CERT_REQUIRED)
ssl_sock.connect(('localhost', 502))
ssl_sock.setblocking(1)
readCoils = ModbusPDU01_Read_Coils()
#for i in range(4):
sleep(2)
ssl_sock.sendall(str(readCoils))
print ("Request for Read Coils Sent")
#start receive
data = ssl_sock.recv(64)
print ("Response from Server:")
print pprint.pformat(data)
if False: #from the python docs
ssl_sock.write("""GET / HTTP/1.0\r
Host: www.verisign.com\n\n""")
data = ssl_sock.read()
ssl_sock.close()

The do_something() loop was not necessary, as the deal_with_client() loop was doing the same thing. I removed do_something() and put the code in deal_with_client() which allows me to keep the connection open (see below)
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def deal_with_client(connstream):
data = connstream.recv(1120)
answer = str(ModbusPDU01_Read_Coils_Answer())
while data:
print ("Request Received from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
data = connstream.recv(1120)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Received\n")
sys.exit(1)