We Keep Coding

Redirect instead of HTTP403

I'm trying to make a proxy with a Blocklist. I initially used 403 when a user goes to a blocked page. However, it doesn't work with HTTPS and returns ERR_TUNNEL_CONNECTION_FAILED as explained in Respond with 403 in an HTTPS proxy
Thus, I want to redirect the user to a html page like this
This is my code:
import socket
import threading
import signal
import sys
import fnmatch
import errno
import time
import pdb
import re
from time import gmtime, strftime, localtime
import logging
import config
import rule
import tag as tag_store
from Ignite import flame
core = flame()
p=re.compile('(http:\/\/)?([\w\.-]*)(\:(\d*))?(\/.*)?')
thread_logger = logging.getLogger('thread')
access_logger = logging.getLogger('access')
csv_logger = logging.getLogger('csv')
def proxy(browser_conn, client_addr):
print("hi")
def ishostAllowed(host):
print("\n\nHost:")
print(str(host))
access_logger.info(str(host))
if host.split('.')[-1].isdigit():
thread_logger.warn("Invalid host:".format(host),extra=req)
return core.check_allow(host)
#pdb.set_trace()
tags=tag_store.get(host)
if not tags:
thread_logger.warn("{0} isn't allowed: empty tags".format(host),extra=req)
return core.check_allow(host)
for tag in tag_store.get(host):
if not rule.isTagAllowed(tag):
thread_logger.warn("{0}:{1} isn't allowed".format(host,tag),extra=req)
return core.check_allow(host)
return core.check(host)
def proxy_http(request):
try:
# create a socket to connect to the web server
#pdb.set_trace()
server_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_conn.settimeout(config.connection_timeout)
server_conn.connect((request['host'], request['port']))
server_conn.sendall(request['raw_data']) # send request to webserver
while 1:
data = server_conn.recv(config.max_request_len) # receive data from web server
if (len(data) > 0):
browser_conn.send(data) # send to browser
else:
break
except socket.error as error_msg:
thread_logger.error(str(error_msg)+":"+str(request),extra=req);
finally:
if server_conn:
server_conn.close()
if browser_conn:
browser_conn.close()
return
def response(status,message):
reply = "HTTP/1.1 {0} {1}\r\n"
reply += "Proxy-agent: Sinktrap\r\n"
reply += "\r\n"
reply = reply.format(status,message);
#pdb.set_trace()
browser_conn.sendall( reply.encode() )
def proxy_https(request):
#pdb.set_trace()
try:
server_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# If successful, send 200 code response
server_conn.connect((req['host'], req['port']))
response(200,'Connection established')
except socket.error as err:
# If the connection could not be established, exit
# Should properly handle the exit with http error code here
thread_logger.error("Cannot establish https connection:"+err,extra=req);
if server_conn:
server_conn.close()
if browser_conn:
browser_conn.close()
return
# Indiscriminately forward bytes
browser_conn.setblocking(0)
server_conn.setblocking(0)
timeout=time.time()+60 # 1 minute
while timeout-time.time()>0:
request_done=False
replied_done=False
try:
request =browser_conn.recv(config.max_request_len) # receive data from browser
if (len(request) > 0):
server_conn.sendall(request) # send to web server
else:
request_done=True
#hread_logger.info("REQUEST len: " + str(len(request)),extra=req);
except socket.error as e:
if e.errno==errno.EWOULDBLOCK:
time.sleep(0.1)
pass
else:
thread_logger.error("pipe error:"+str(e),extra=req);
break
try:
reply = server_conn.recv(config.max_request_len) # receive data from web server
if (len(reply) > 0):
browser_conn.sendall(reply) # send to browser
else:
replied_done=True
#thread_logger.info("reply len: " + str(len(reply)),extra=req);
except socket.error as e:
if e.errno==errno.EWOULDBLOCK:
time.sleep(0.1)
pass
else:
thread_logger.error("pipe error:"+str(e),extra=req);
break
if request_done and replied_done:
break
server_conn.close()
browser_conn.close()
raw_data = browser_conn.recv(config.max_request_len) # get the request from browser
req={'raw_data':raw_data,
'tname' : threading.currentThread().getName(),
'client_ip' : client_addr[0],
'client_port' : client_addr[1]
}
thread_logger.info("REQUEST: {0}".format(raw_data),extra=req);
#pdb.set_trace()
try:
# request_line is the first one. https://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html
msg_body_pos=len(raw_data)
for i in range(4,len(raw_data)):
if raw_data[i-4:i].decode()=='\r\n\r\n':
msg_body_pos=i
break
lines=raw_data[:msg_body_pos-4].decode('utf-8').split('\r\n')
if len(lines[0])<16:
thread_logger.warn("INVALU REQUEST:{0}".format(raw_data),extra=req);
return
headers = {k:v for k,v in (x.split(':',1) for x in lines[1:]) }
if 'Referer' in headers:
req['Referer']=headers['Referer']
else:
req['Referer']=''
if 'User-Agent' in headers:
req['User-Agent']=headers['User-Agent']
else:
req['User-Agent']=''
req['request_line'] =lines[0]
req['method'],req['request_uri'],req['http_version']=lines[0].split(' ')
#check if the first line is valid request. request_line might be empty
if not req['method'] or not req['request_uri'] or not req['http_version']:
thread_logger.warn("INVALU REQUEST:{0}".format(raw_data),extra=req);
return
except Exception as e:
thread_logger.error("INVALU REQUEST:{0} {1}".format(e, raw_data),extra=req);
logging.exception("INVALU REQUEST")
return
access_logger.info("",extra=req)
#pdb.set_trace()
m=p.match(req['request_uri'])
req['host']=m.group(2)
req['port']=int(m.group(4)) if m.group(4) else 80
# Check if request is allowed or not
if not ishostAllowed(req['host']):
csv_logger.info("blocked",extra=req);
thread_logger.warn("Block REQUEST:{0}".format(raw_data),extra=req);
response(403,"The website has been blocked by Ignite's proxy.")
#Breq = req
#Breq['host'] = "azlancoding.github.io/Blocked"
#proxy_https(Breq)
#response(307,"https://azlancoding.github.io/BLOCKED")
return
csv_logger.info("allowed",extra=req);
#pdb.set_trace()
if req['method']=='CONNECT':
proxy_https(req)
else:
proxy_http(req)
The original proxy is pcxy
See my github project here

browser doesn't response well to https proxy

I'v built my own https proxy and when ever I send some data to a browser the browser responses with nothing and also after a lot of time.
basically all the proxy should do is just forward the message to browser, get the response and forward back to the client
the code of the proxy:
import socket
import select
serverSock = socket.socket()
serverSock.bind(('0.0.0.0', 8080))
serverSock.listen(3)
waiting_clients = {} # client : browser
users_dict = {}
open_clients = {}
browsers_clients = {} # browser : client
threading.Thread(target=browserCom).start()
while True:
try:
rlist, wlist, xlist = select.select(list(users_dict.keys()) + [serverSock], [], [], 0.3)
except:
pass
else:
for current_socket in rlist:
if current_socket is serverSock:
# new client
client, address = serverSock.accept()
print(f'{address} - connected to proxy')
# add to dictionary
users_dict[client] = address
open_clients[address] = client
else:
# receive info
receiving = True
msg = bytearray()
while receiving:
try:
data = current_socket.recv(1024)
except Exception as e:
print(e, 3)
if current_socket in users_dict.keys():
disconnect(users_dict[current_socket])
else:
current_socket.close()
break
else:
msg.extend(data)
# got the full msg
if len(data) < 1024:
receiving = False
if len(msg) == 0:
if current_socket in users_dict.keys():
disconnect(users_dict[current_socket])
else:
print("GOT FROM CLIENT", msg)
if current_socket in waiting_clients.keys():
# sending the data from client to browser
waiting_clients[current_socket].send(msg)
else:
msg = msg.decode()
msgSplit = msg.split()
address = msgSplit[1]
if address.split(':')[1].isnumeric():
if msg.startswith('CONNECT'):
browserLink, browserPort = address.split(':')
browserPort = int(browserPort)
browserIP = socket.gethostbyname(browserLink)
address = (browserIP, browserPort)
# connect to the site
browserSocket = socket.socket()
print(address)
browserSocket.connect((browserIP, browserPort))
waiting_clients[current_socket] = browserSocket
browsers_clients[browserSocket] = current_socket
msg_ret = "HTTP/1.1 200 Connection established\r\n\r\n"
sendMsg(users_dict[current_socket], msg_ret)
m
The proxy is able to make the connection after the CONNECT and notify to the client but after I send to the browser I got from the data with a function running in the background:
def browserCom():
while True:
try:
rlist, wlist, xlist = select.select(list(browsers_clients.keys()), [], [], 0.3)
except:
pass
else:
for current_browser in rlist:
# receive data from the browser
receiving = True
resp_msg = bytearray()
while receiving:
try:
data = current_browser.recv(1024)
except Exception as e:
print(e)
del waiting_clients[browsers_clients[current_browser]]
current_browser.close()
browsers_clients[current_browser].close()
del browsers_clients[current_browser]
else:
resp_msg.extend(data)
# got the full msg
if len(data) < 1024:
receiving = False
print("RESPONSE FROM BROWSER", resp_msg)
# sending the msg to the client
sendMsg(users_dict[browsers_clients[current_browser]], resp_msg)
I need to wait a lot of time for the response and most of the responses come empty the responses are mostly bytearray(b'') and even when I get the response even though I sent the response back to the client:
# sending the msg to the client
sendMsg(users_dict[browsers_clients[current_browser]],resp_msg)
using this
def sendMsg(address, msg):
"""
:param ip: ip to send to
:param msg: msg to send
:return: sends the msg to the ip
"""
if address in open_clients.keys():
sock = open_clients[address]
if type(msg) == str:
msg = msg.encode()
try:
sock.send(msg)
except Exception as e:
print(e, 4)
disconnect(address)
I hope you are abled to understand my code, please if something is unclear ask me in the comments and I will try to help you understand as soon as possible
this is the best that I can do to keep the code minimal for this problem without removing crucial parts

My mistake was that I didn't understand that while tunneling both the browser and the client exchange messages, adding another select to all the browser helped me check all the data from all the browsers and now it works.
The code above is updated and works
basically what I added is :
def browserCom():
while True:
try:
rlist, wlist, xlist = select.select(list(browsers_clients.keys()), [], [], 0.3)
except:
pass
else:
for current_browser in rlist:
# receive data from the browser
receiving = True
resp_msg = bytearray()
while receiving:
try:
data = current_browser.recv(1024)
except Exception as e:
print(e)
del waiting_clients[browsers_clients[current_browser]]
current_browser.close()
browsers_clients[current_browser].close()
del browsers_clients[current_browser]
else:
resp_msg.extend(data)
# got the full msg
if len(data) < 1024:
receiving = False
# disconnecting browser
if resp_msg == bytearray(b''):
del waiting_clients[browsers_clients[current_browser]]
current_browser.close()
browsers_clients[current_browser].close()
del browsers_clients[current_browser]
print("RESPONSE FROM BROWSER", resp_msg)
# sending the msg to the client
if current_browser in browsers_clients and browsers_clients[current_browser] in users_dict:
sendMsg(users_dict[browsers_clients[current_browser]], resp_msg)

How to fix: "TypeError: 'bool' object is not subscriptable"

I am currently working with a basic client/server application and implementing a simple RSA / public-key authentication system. I have ran into this error and can not, for the life of me, figure it out.
I am using the latest version of python.
server.py
def getUserData(username):
global privateKeysFilename
filename = privateKeysFilename
with open(filename, "r") as keysFile:
for line in keysFile:
line = [token.rstrip("\n") for token in line.split(",")]
if(username == line[0]):
if DEBUG:
print("\n=== DEBUG\nUser data = %s\n===\n" %(line))
return line
return False
def running(self):
global BUFFER, DEBUG, start, final
while 1:
print('Waiting for a connection')
connection, client_address = self.server_socket.accept()
connection.send("Successful connection!".encode())
x = randint(start, final)
self.fx = function(x)
connection.send(str(x).encode())
try:
# Output that a client has connected
print('connection from', client_address)
write_connection()
# Set the time that the client connected
start_time = datetime.datetime.now()
# Loop until the client disconnects from the server
while 1:
# Receive information from the client
userData = connection.recv(BUFFER)
#data = connection.recv(1024).decode()
if(userData != "0"):
#define split character
ch = ","
userData = userData.split(ch.encode())
username = userData[0]
r = int(userData[1])
userData = getUserData(username)
e, n = int(userData[1]), int(userData[2])
y = modularPower(r, e, n)
if DEBUG:
print("=== DEBUG\ne = %d\nn = %d\nr = %d\ny = %d\n===\n" %(e, n, r, y))
if(self.fx == y):
#if authentication passed
connection.send("Welcome!!!".encode())
else:
connection.send("Failure!!!".encode())
if (userData != 'quit') and (userData != 'close'):
print('received "%s" ' % userData)
connection.send('Your request was successfully received!'.encode())
write_data(userData)
# Check the dictionary for the requested artist name
# If it exists, get all their songs and return them to the user
if userData in self.song_dictionary:
songs = ''
for i in range(len(self.song_dictionary.get(userData))):
songs += self.song_dictionary.get(userData)[i] + ', '
songs = songs[:-2]
print('sending data back to the client')
connection.send(songs.encode())
print("Sent", songs)
# If it doesn't exist return 'error' which tells the client that the artist does not exist
else:
print('sending data back to the client')
connection.send('error'.encode())
else:
# Exit the while loop
break
# Write how long the client was connected for
write_disconnection(start_time)
except socket.error:
# Catch any errors and safely close the connection with the client
print("There was an error with the connection, and it was forcibly closed.")
write_disconnection(start_time)
connection.close()
data = ''
finally:
if data == 'close':
print('Closing the connection and the server')
# Close the connection
connection.close()
# Exit the main While loop, so the server does not listen for a new client
break
else:
print('Closing the connection')
# Close the connection
connection.close()
# The server continues to listen for a new client due to the While loop
and here is the output with error:
Traceback <most recent call last>:
File "server.py", line 165, in running
e, n = int(userData[1]), int(userData[2])
TypeError: 'bool' object is not subscriptable
Any help would be greatly appreciated! :)

By using userData[n] you are trying to access the nth element in a subscriptable object.
This can be a list, dict, tuple or even a string.
The error you see means that your object userData is neither of the previous mentioned types, and it's a bool ( True or False )
Since it's the result of calling the function getUserData(), I recommend you check the return type of this function and make sure it's of the mentioned types and revise your code logic.
[Update]
By checking the function getUserData() it looks it only returns line if the username is included, if not it returns False which is not handled in the main code.
I suggest this edit to the code to inlclude success status to the return value as follows.
def getUserData(username):
global privateKeysFilename
filename = privateKeysFilename
with open(filename, "r") as keysFile:
for line in keysFile:
line = [token.rstrip("\n") for token in line.split(",")]
if(username == line[0]):
if DEBUG:
print("\n=== DEBUG\nUser data = %s\n===\n" %(line))
return True, line
return False, None
And then in your code when you call getUserData() you check for the success first before parsing data like this
userData = getUserData(username)
if userData [0]:
e, n = int(userData[1]), int(userData[2])
y = modularPower(r, e, n)
else:
# Your failure condition

Why h.getresponse() is needed in python logging HTTPHandler?

I overrided the method emit of python logging httphandler to adapt it to my needs, and I noticed the line
h.getresponse() #can't do anything with the result
Why is this line necessary?
I noticed that removing this line has no effect when using unsecure logging, but makes the logs fail when using secure connection.
def emit(self, record):
"""
Emit a record.
Send the record to the Web server as a percent-encoded dictionary
"""
try:
import http.client, urllib.parse
host = self.host
if self.secure:
h = http.client.HTTPSConnection(host, context=self.context)
else:
h = http.client.HTTPConnection(host)
url = self.url
data = urllib.parse.urlencode(self.mapLogRecord(record))
if self.method == "GET":
if (url.find('?') >= 0):
sep = '&'
else:
sep = '?'
url = url + "%c%s" % (sep, data)
h.putrequest(self.method, url)
# support multiple hosts on one IP address...
# need to strip optional :port from host, if present
i = host.find(":")
if i >= 0:
host = host[:i]
# See issue #30904: putrequest call above already adds this header
# on Python 3.x.
# h.putheader("Host", host)
if self.method == "POST":
h.putheader("Content-type",
"application/x-www-form-urlencoded")
h.putheader("Content-length", str(len(data)))
if self.credentials:
import base64
s = ('%s:%s' % self.credentials).encode('utf-8')
s = 'Basic ' + base64.b64encode(s).strip().decode('ascii')
h.putheader('Authorization', s)
h.endheaders()
if self.method == "POST":
h.send(data.encode('utf-8'))
h.getresponse() #can't do anything with the result
except Exception:
self.handleError(record)

The getresponse() call guarantees that the request is actually sent to the server by getting the response to the request.

Partial read from Socket.File.Read

Im coding a python script that connects to a remote server, and parses the returned response. For some odd reason, 9 out of 10 times, Once the header is read, the script continues and returns before getting the body of the response. Im no expert at python, but im certain that my code is correct on the python side of things. Here is my code:
class miniclient:
"Client support class for simple Internet protocols."
def __init__(self, host, port):
"Connect to an Internet server."
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.settimeout(30)
try:
self.sock.connect((host, port))
self.file = self.sock.makefile("rb")
except socket.error, e:
#if e[0] == 111:
# print "Connection refused by server %s on port %d" % (host,port)
raise
def writeline(self, line):
"Send a line to the server."
try:
# Updated to sendall to resolve partial data transfer errors
self.sock.sendall(line + CRLF) # unbuffered write
except socket.error, e:
if e[0] == 32 : #broken pipe
self.sock.close() # mutual close
self.sock = None
raise e
except socket.timeout:
self.sock.close() # mutual close
self.sock = None
raise
def readline(self):
"Read a line from the server. Strip trailing CR and/or LF."
s = self.file.readline()
if not s:
raise EOFError
if s[-2:] == CRLF:
s = s[:-2]
elif s[-1:] in CRLF:
s = s[:-1]
return s
def read(self, maxbytes = None):
"Read data from server."
if maxbytes is None:
return self.file.read()
else:
return self.file.read(maxbytes)
def shutdown(self):
if self.sock:
self.sock.shutdown(1)
def close(self):
if self.sock:
self.sock.close()
self.sock = None
I use the ReadLine() method to read through the headers until i reach the empty line (Delimiter between headers and body). From there, my objects just call the "Read()" method to read the body. As stated before, 9 of 10 times, read returns nothing, or just partial data.
Example use:
try:
http = miniclient(host, port)
except Exception, e:
if e[0] == 111:
print "Connection refused by server %s on port %d" % (host,port)
raise
http.writeline("GET %s HTTP/1.1" % str(document))
http.writeline("Host: %s" % host)
http.writeline("Connection: close") # do not keep-alive
http.writeline("")
http.shutdown() # be nice, tell the http server we're done sending the request
# Determine Status
statusCode = 0
status = string.split(http.readline())
if status[0] != "HTTP/1.1":
print "MiniClient: Unknown status response (%s)" % str(status[0])
try:
statusCode = string.atoi(status[1])
except ValueError:
print "MiniClient: Non-numeric status code (%s)" % str(status[1])
#Extract Headers
headers = []
while 1:
line = http.readline()
if not line:
break
headers.append(line)
http.close() # all done
#Check we got a valid HTTP response
if statusCode == 200:
return http.read()
else:
return "E\nH\terr\nD\tHTTP Error %s \"%s\"\n$\tERR\t$" % (str(statusCode), str(status[2]))

You call http.close() before you call http.read(). Delay the call to http.close() until after you have read all of the data.