We Keep Coding

https over http proxy socket connection in python

I'm trying to write a simple 'https over http tunnel' server in python.
Every other thing works out fine except the connection between the client and the server persist and ends up blocking( forever ).
I'm pretty sure they carry out the SLL handshake because they both send and receive a couple of times before it hangs.
here's the server code:
import socket
import threading
class SocketWrapper:
def __init__(self,sock = None):
if sock is None:
self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
else:
self.socket = sock
def connect(self,host,port):
try:
self.socket.connect((host,int(port)))
return True
except socket.error:
return False
def close(self):
# close the socket connection
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
def send(self,data):
bytes_sent = 0
msg_len = len(data)
while bytes_sent < msg_len:
sent = self.socket.send(data[bytes_sent:])
bytes_sent += sent
def receive(self):
chunks = []
while True:
try:
self.socket.settimeout(0.5)
chunk = self.socket.recv(4096)
chunks.append(chunk)
except socket.error:
self.socket.settimeout(0)
break;
return b''.join(chunks)
class HttpTunnel:
def __init__(self,host='localhost',port=3000):
# create the server socket,bind and listen
self.host,self.port = host,port
self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
self.socket.bind((self.host,self.port))
self.socket.listen(3)
print("listening on port",self.port)
self.running = True
def handleClientRequest(self,connection,address):
print("Connected to",address)
clientSocket = SocketWrapper(connection)
meta = clientSocket.receive().decode().split('\r\n')[0]
# getting method,uri,version from 'CONNECT host:port HTTP/1.1'
method,uri,version = meta.split(' ')
host,port = uri.split(':')
serverSocket = SocketWrapper()
# if connection to the remote server is created successfuly
if(serverSocket.connect(host,port)):
print("Connected to remote server")
# send connection success message to the client
clientSocket.send(b'HTTP/1.1 200 OK\r\n\r\n');
while True:
try:
clientResponse = clientSocket.receive()
serverSocket.send(clientResponse)
print("Sent client - server")
serverResponse = serverSocket.receive()
clientSocket.send(serverResponse)
print("Sent server - client")
except socket.error:
break;
else:
# send someking of error. In this case 404
serverSocket.send(b'HTTP/1.1 404 Not Found\r\n\r\n')
# close the connection
clientSocket.close()
serverSocket.close()
def mainloop(self):
while self.running:
# accept client connection
connection,address = self.socket.accept()
self.handleClientRequest(connection,address)
proxy = HttpTunnel()
proxy.mainloop()
the client code:
import urllib
import urllib.request as request
proxy = request.ProxyHandler({
'https':'https://127.0.0.1:3000'
})
opener = request.build_opener(proxy)
request.install_opener(opener)
try:
resp = request.urlopen('https://google.com')
print(resp.read())
except Exception as e:
print(e)
the client did not get the response from the server and therefore prints nothing.
here's the server output:
listening on port 3000
Connected to ('127.0.0.1', 54888)
Connected to remote server
Sent client - server
Sent server - client
Sent client - server
Sent server - client
Sent client - server

There are several problems here:
The main problem is that you don't handle the case when recv returns 0 since the socket gets closed. Instead you run into an endless loop where no data get read and no data get send. Some simple print statements which actually show how much data are read would have helped to track this problem down.
Apart from that the idea of polling each file handle after each other using settimeout is a bad approach. Instead check the file handles in parallel and then read from the one which has data - see select.
And finally you are assuming that socket.send will send all data given. This is not the case, it might send less. Check the return code or just use socket.sendall

Error with python socket programming, winerror10057

This is my python code:
class SimpleWebServer():
def __init__(self, port):
threading.Thread.__init__(self)
self.port = port
self.BUFFER_SIZE = 8192
#Create a server socket
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)#YOUR CODE
try:
#Bind to a port
#YOUR CODE; 1 line
self.server.bind(('localhost', self.port))
except socket.error:
print('Bind failed %s' % (socket.error))
sys.exit()
#Listen to the server socket
#YOUR CODE; 1 line
self.server.listen(5)
def run_thread(self, conn, addr):
# connection timeout after 60-second inactivity
conn.settimeout(60.0)
print('Client connected with ' + addr[0] + ':' + str(addr[1]))
while True:
try:
# Receives the request message from the client
message = self.server.recv(1024)
# .decode("utf-8") #YOUR CODE
if not message: break
# Extract the path of the requested object from the message
# The path is the second part of HTTP header, identified by [1]
filename = message.split()[1].decode()
print('Client request ' + filename)
# Extract the file extention to determine file type
filename_extension = os.path.splitext(filename)[1]
# Open the local file f specified in filename for reading
# Because the extracted path of the HTTP request includes
# a character '\', we read the path from the second character
f = open(filename)#YOUR CODE
# Store the entire content of the requested file in a temporary buffer
msg = f #YOUR CODE
# Send the HTTP response headers to the connection socket
# 1. HTTP version and status code
# YOUR CODE 1-line
#conn.send_response(200, addr)
conn.sendall(bytearray("HTTP/1.1 200 OK"))
# 2. Keep-Alive field
# YOUR CODE 1-line
#conn.send_head("Timeout = 5", "Max - 30", addr)
conn.sendall(bytearray("Timeout = 5", "Max - 30"))
# 3. Content length field
# YOUR CODE 1 - 3 lines
#conn.send_header("Content-length", len(msg), addr)
conn.sendall(bytearray("Content-length", len(msg)))
# 4. Content type field (based on the file type)
# YOUR CODE
#conn.send_header("Content-type", "text/html", addr)
#conn.end_headers()
conn.sendall(bytearray("Content-type", "text/html"))
# Send the HTTP response body
for i in range(0, len(msg), self.BUFFER_SIZE):
end = min(i + self.BUFFER_SIZE, len(msg))
conn.send(msg[i: end])
# Exception handling
except FileNotFoundError:
# Send HTTP response message for file not found
# YOUR CODE 1 - 3 lines
self.send_response(404)
except socket.timeout:
#Socket timeout
print("Conn socket timeout!")
break
except socket.error as e:
#Other socket exceptions
print("Socket error: %s" % e)
break
conn.close() # Close socket
def run(self):
print('Waiting for connections on port %s' % (self.port))
while True:
#Accept a new connection
try:
#Waiting for connection request
(conn, addr) = self.server.accept()
#Start a new thread to handle HTTP request/response
threading.Thread(target=self.run_thread, args=(conn, addr)).start()
except:
break
# Close the server socket
self.exit()
def exit(self):
"""Close server socket"""
self.server.close()
I call the class here:
if __name__ == '__main__':
parser = optparse.OptionParser(usage="%prog ServerPort")
options, args = parser.parse_args()
if len(args) < 1:
parser.error("No ServerPort")
else:
if validate_port(args[0]):
server_port = int(args[0])
else:
parser.error("ServerPort invalid!")
#Create and start the server
server = SimpleWebServer(server_port)
server.run()
I call the script via powershell and I get this error:
Client connected with 127.0.0.1:2654
Socket error: [WinError 10057] A request to send or receive data was disallowed because the socket is not connected and (when sending on a datagram socket using a sendto call) no address was supplied
I'm running this command in my browser to try and access this socket:
http://localhost:8888/index.html
I have no clue what I should be doing to fix this. There is a lot of documentation for python socket programming and I've tried many of the different methods but nothing has worked so far.

Socket data not getting received when the request is made from a raspberry pi

I am trying to send and receive data from a server. When I run the below python program from eclipse or python IDLE I get the response from the server. But, when I run the program from Raspberry Pi I am getting an error. Below is the simple code and error.
import socket,time
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ip = '127.0.0.1'
port = 1883
address = (ip,port)
client.connect(address)
print("connected")
def close():
global client
client.close()
client=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
client.connect(address)
while 1:
client.send(b'\x01\x04\x00\x00\xxxxxx')
print("sent")
try:
raw_data = client.recv(1024)
except:
close()
raw_data = client.recv(1024)
received_data = " ".join("{:02x}".format(byte) for byte in
(raw_data))
print(received_data)
time.sleep(1)
When I run this code in raspberry pi,I get BrokenPipe error. why it is happening in raspberry pi?
Error:
connected
sent
received
sent
Traceback (most recent call last):
File "exdu.py", line 18, in <module>
client.send(b'\x01\x04\x00\x00\xxxxxxxxx')
socket.error: [Errno 32] Broken pipe

A broken pipe is caused when a pipe (in this case a TCP socket) is closed unexpectedly by the other side and recv gets back an empty response or send tries to write to a closed connection.
There are quite a couple pieces of not so best practice code may be causing a broken pipe error in your code.
I'll try and point out a few things that stand out to me using comments:
import socket,time
# it would be nice to have a function wrapping this but as it is this is not the issue.
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ip = '127.0.0.1'
port = 1883
address = (ip,port)
client.connect(address)
print("connected")
def close():
global client
# If you read the python socket documentation https://docs.python.org/2/howto/sockets.html
# you will see that they advise sending a shutdown packet to the other side of the connection.
# In my experience this is actually extremely important when properly handling TCP connections and may be causing this issue.
# you must do this from the side that closes the connection, this includes when the server closes a connection.
client.shutdown(SHUT_RDWR) # this is what you want to do here.
client.close()
# connecting to a new socket here makes no sense and may also cause an issue.
client=socket.socket(socket.AF_INET,socket.SOCK_STREAM)
client.connect(address)
raw_data=client.recv(1024)
while 1:
client.send(b'\x01\x04\x00\x00\xxxxxx')
print("sent")
try:
raw_data = client.recv(1024)
except:
# this except statement may cause an exception itself.
# Not only that but you are probably getting your broken pipe error
# here from the recv in close().
close()
raw_data = client.recv(1024)
received_data = " ".join("{:02x}".format(byte) for byte in
(raw_data))
print(received_data)
time.sleep(1)
Apologies for the code comments, but i find they're useful when the segment is copy pasted when you are trying to experiment with a solution.
As an extension i would write the code you have to be much more like the python docs example:
class mysocket:
def __init__(self, sock=None):
if sock is None:
self.sock = socket.socket(
socket.AF_INET, socket.SOCK_STREAM)
else:
self.sock = sock
def connect(self, host, port):
self.sock.connect((host, port))
# important bit of missing code
def disconnect(self):
self.sock.shutdown(SHUT_RDWR)
self.sock.close()
def mysend(self, msg):
totalsent = 0
while totalsent < MSGLEN:
sent = self.sock.send(msg[totalsent:])
if sent == 0:
raise RuntimeError("socket connection broken")
totalsent = totalsent + sent
def myreceive(self):
chunks = []
bytes_recd = 0
while bytes_recd < MSGLEN:
chunk = self.sock.recv(min(MSGLEN - bytes_recd, 2048))
if chunk == '':
raise RuntimeError("socket connection broken")
chunks.append(chunk)
bytes_recd = bytes_recd + len(chunk)
return ''.join(chunks)
You can rewrite the send and recieve functions to do your bidding and then call it like:
socket = mysocket()
socket.connect('127.0.0.1', 1883)
try:
socket.mysend(b'\x01\x04\x00\x00\xxxxxx')
raw_data = socket.myrecieve()
# do more stuff
except:
socket.disconnect()
This is example code but may set you on the right path.

Broken pipe implies something isn't going through exactly as you want it to. Are you sure the server is running correctly on the raspberry pi on localhost?
Otherwise you might want to either run the server seperately(maybe heroku or digitalocean is relevant) or check your pi's firewall(I highly doubt this is the problem for localhost -> localhost connections)

create multi-thread tcp server python 3

Hello I tried to make a simple server that accept multiple clients simultaneously I'm new to python and I have a difficult to understand it....I try to change my code in multi-thread applications but without positive result...here is the code:
import socket, threading
def message():
while 1:
data = connection.recv(1024)
if not data: break
#connection.sendall(b'-- Message Received --\n')
print(data.decode('utf-8'))
connection.close()
def connection():
address = input("Insert server ip")
port = 44444
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((address, port))
s.listen(1)
print("Server started! Waiting for connections...")
def accept connection():
connection, address = s.accept()
print('Client connected with address:', address)
t=thread.Threading(target=message,args=(connection))
t.run()
I know that there are many errors but I'm new in python sorry :(
The original non-threaded code is:
import socket
address = input("Insert server ip:")
port = 44444
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((address, port))
s.listen(1)
print("Server started! Waiting for connections...")
connection, address = s.accept()
print('Client connected with address:', address)
while 1:
data = connection.recv(1024)
if not data: break
#connection.sendall(b'-- Message Received --\n')
print(data.decode('utf-8'))
connection.close()

Your basic design is close, but you've got a whole lot of little problems making it hard to move forward.
First, you have a function name with a space in it, which isn't allowed. And you have an IndentationError because you didn't indent its contents.
Next, inside that accept_connection function, you're using threading wrong.
thread.Threading doesn't exist; you probably meant threading.Thread.
args has to be a sequence (tuple, list, etc.) of values. You probably expected (connection) to be a tuple of one value, but it's not; tuples are defined by commas, not parentheses, and what you have is just the value connection with superfluous parentheses around it. You wanted (connection,) here.
Also, calling run on a thread object just runs the thread's code in the current thread. You want to call start, which will start a new thread and call the run method on that thread.
Meanwhile, you're never actually calling this function anywhere, so of course it can't do anything. Think about where you want to call it. After creating the listener socket, you want to loop around accept, kicking off a new client thread for each accepted connection, right? So, you want to call it in a loop, either inside connection, or at the top level (in which case connection has to return s).
And finally, your accept_connection function can't access local variables from some other function; if you want it to use a socket named s, you have to pass it as a parameter.
So:
def connection():
address = input("Insert server ip")
port = 44444
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((address, port))
s.listen(1)
print("Server started! Waiting for connections...")
while True:
accept_connection(s)
def accept_connection(s):
connection, address = s.accept()
print('Client connected with address:', address)
t=thread.Threading(target=message, args=(connection,))
t.start()
As a side note, be careful with using sock.recv(1024) and assuming you're going to get the whole message that the other side sent with send(msg). You might get that, or you might get half the message, or the whole message plus half of another message the client sent later. Sockets are just streams of bytes, like files, not streams of separate messages; you need some kind of protocol to separate messages.
The simplest possible protocol is to send each message on its own line. Then you can just do socket.makefile() and for line in f:, just like you would for a real file. Of course this doesn't work if your messages can have newlines, but you can, e.g., backslash-escape them on one side and unescape them on the other.

This is a pretty old post but there's a nice way to do what you're talking about. Here's a link to an example I posted a little while back:
https://bitbucket.org/matthewwachter/tcp_threadedserver/src/master/
And the script:
from datetime import datetime
from json import loads, dumps
from pprint import pprint
import socket
from threading import Thread
class ThreadedServer(Thread):
def __init__(self, host, port, timeout=60, debug=False):
self.host = host
self.port = port
self.timeout = timeout
self.debug = debug
Thread.__init__(self)
# run by the Thread object
def run(self):
if self.debug:
print(datetime.now())
print('SERVER Starting...', '\n')
self.listen()
def listen(self):
# create an instance of socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# bind the socket to its host and port
self.sock.bind((self.host, self.port))
if self.debug:
print(datetime.now())
print('SERVER Socket Bound', self.host, self.port, '\n')
# start listening for a client
self.sock.listen(5)
if self.debug:
print(datetime.now())
print('SERVER Listening...', '\n')
while True:
# get the client object and address
client, address = self.sock.accept()
# set a timeout
client.settimeout(self.timeout)
if self.debug:
print(datetime.now())
print('CLIENT Connected:', client, '\n')
# start a thread to listen to the client
Thread(target = self.listenToClient,args = (client,address)).start()
# send the client a connection message
# res = {
# 'cmd': 'connected',
# }
# response = dumps(res)
# client.send(response.encode('utf-8'))
def listenToClient(self, client, address):
# set a buffer size ( could be 2048 or 4096 / power of 2 )
size = 1024
while True:
try:
# try to receive data from the client
data = client.recv(size).decode('utf-8')
if data:
data = loads(data.rstrip('\0'))
if self.debug:
print(datetime.now())
print('CLIENT Data Received', client)
print('Data:')
pprint(data, width=1)
print('\n')
#send a response back to the client
res = {
'cmd': data['cmd'],
'data': data['data']
}
response = dumps(res)
client.send(response.encode('utf-8'))
else:
raise error('Client disconnected')
except:
if self.debug:
print(datetime.now())
print('CLIENT Disconnected:', client, '\n')
client.close()
return False
if __name__ == "__main__":
ThreadedServer('127.0.0.1', 8008, timeout=86400, debug=True).start()

Here is some example code I have showing a threaded socket connection.
def sock_connection( sock, host ):
"Handle socket"
pass
while 1:
try:
newsock = sock.accept()
thread = Thread( target=sock_connection, args=newsock )
thread.start()
except Exception, e:
print "error on socket connection: " % e)

Python Socket Listening

All of the below mentioned is on windows machines using python 2.7
Hello,
I am currently attempting to listen on a socket for data send by a remote program. This data is then printed to the screen and user input is requested that is then returned to remote program. In testing I have been able to have the remote program send me a menu of command line programs (cmd, ipconfig, whoami, ftp) and then my program returns with a number as a selection of the menu option.
The remote program receives my response and sends the output of the selected command. ipconfig and whoami work perfectly, but cmd and ftp only returns the output of the terminal once. (I.E. I can enter one command into the FTP program and send that too the remote program before I never hear back)
The part of my code that fails is that
if ready[0]: never becomes ready a second time after the first conversation.
I know the remote program is functioning correctly as I can use netcat to act in lieu of my code and operate the cmd terminal indefinitely.
How do I go about properly implementing a python socket listener that can account for this type of connection?
My "program" in its entirety:
import socket, sys, struct, time, select
host = ''
port = 50000
connectionSevered=0
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
except socket.error:
print 'Failed to create socket'
sys.exit()
print '[+] Listening for connections on port '+str(port)+'.'
s.bind((host,port))
s.listen(5)
def recvall(the_socket,timeout=2):
global connectionSevered
data=''; # Data found by recv
total_data=[]; # Finally list of everything
s.setblocking(0) #make socket non blocking
begin=time.time() #beginning time
while 1:
ready = select.select([client], [], [], .2)
if time.time()-begin > timeout:
print 'Timeout reached'
#Leave loop, timer has reached its threshold
break
if ready[0]:
print 'In ready loop!'
try:
data = client.recv(4096) #attempt to fetch data
if data:
begin=time.time() #reset timeout timer
total_data.append(data)
data='';
except socket.error:
print '[+] Lost connection to client. Printing buffer...'
connectionSevered=1 # Let main loop know connection has errored
pass
time.sleep(1)
#join all parts to make final string
return ''.join(total_data)
client, address = s.accept()
print '[+] Client connected!'
while (connectionSevered==0): # While connection hasn't errored
print "connectionSevered="+str(connectionSevered) # DEBUG
recvall(s)
response = raw_input() #take user input
client.sendto(response) #send input
client.close(0)
Please let me know if you need more information, any help would be greatly appreciated, I am very new to this and eager to learn.

Playing around with this for a while finally got it working nice with a telnet session locally using python 2.7.
What it does is it sets up a thread that runs when the client connects listening for client stuff.
When the client sends a return ("\r\n" might have to change that if your interacting with a Linux system?) the message gets printed to the server, while this is happening if there is a raw input at the server side this will get sent to the client:
import socket
import threading
host = ''
port = 50000
connectionSevered=0
class client(threading.Thread):
def __init__(self, conn):
super(client, self).__init__()
self.conn = conn
self.data = ""
def run(self):
while True:
self.data = self.data + self.conn.recv(1024)
if self.data.endswith(u"\r\n"):
print self.data
self.data = ""
def send_msg(self,msg):
self.conn.send(msg)
def close(self):
self.conn.close()
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host,port))
s.listen(5)
except socket.error:
print 'Failed to create socket'
sys.exit()
print '[+] Listening for connections on port: {0}'.format(port)
conn, address = s.accept()
c = client(conn)
c.start()
print '[+] Client connected: {0}'.format(address[0])
c.send_msg(u"\r\n")
print "connectionSevered:{0}".format(connectionSevered)
while (connectionSevered==0):
try:
response = raw_input()
c.send_msg(response + u"\r\n")
except:
c.close()
The above answer will not work for more than a single connection. I have updated it by adding another thread for taking connections. It it now possible to have more than a single user connect.
import socket
import threading
import sys
host = ''
port = 50000
class client(threading.Thread):
def __init__(self, conn):
super(client, self).__init__()
self.conn = conn
self.data = ""
def run(self):
while True:
self.data = self.data + self.conn.recv(1024)
if self.data.endswith(u"\r\n"):
print self.data
self.data = ""
def send_msg(self,msg):
self.conn.send(msg)
def close(self):
self.conn.close()
class connectionThread(threading.Thread):
def __init__(self, host, port):
super(connectionThread, self).__init__()
try:
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.bind((host,port))
self.s.listen(5)
except socket.error:
print 'Failed to create socket'
sys.exit()
self.clients = []
def run(self):
while True:
conn, address = self.s.accept()
c = client(conn)
c.start()
c.send_msg(u"\r\n")
self.clients.append(c)
print '[+] Client connected: {0}'.format(address[0])
def main():
get_conns = connectionThread(host, port)
get_conns.start()
while True:
try:
response = raw_input()
for c in get_conns.clients:
c.send_msg(response + u"\r\n")
except KeyboardInterrupt:
sys.exit()
if __name__ == '__main__':
main()
Clients are not able to see what other clients say, messages from the server will be sent to all clients. I will leave that as an exercise for the reader.

If you're in Python 3 by now and still wondering about sockets, here's a basic way of using them:
server.py
import time
import socket
# creating a socket object
s = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
# get local Host machine name
host = socket.gethostname() # or just use (host == '')
port = 9999
# bind to pot
s.bind((host, port))
# Que up to 5 requests
s.listen(5)
while True:
# establish connection
clientSocket, addr = s.accept()
print("got a connection from %s" % str(addr))
currentTime = time.ctime(time.time()) + "\r\n"
clientSocket.send(currentTime.encode('ascii'))
clientSocket.close()
client.py
import socket
# creates socket object
s = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
host = socket.gethostname() # or just use (host = '')
port = 9999
s.connect((host, port))
tm = s.recv(1024) # msg can only be 1024 bytes long
s.close()
print("the time we got from the server is %s" % tm.decode('ascii'))
Run server.py first, then run client.py.
This is just send and receive the currentTime.
What's new in Python 3.4 sockets?
A major difference between python 2.7 sockets and python 3.4 sockets is the sending messages. you have to .encode() (usually using 'ascii' or blank as parameters/arguments)
and then using .decode()
For example use .encode() to send, and use .decode() to receive.
Extra info: client/server socket tutorial