We Keep Coding

https over http proxy socket connection in python

I'm trying to write a simple 'https over http tunnel' server in python.
Every other thing works out fine except the connection between the client and the server persist and ends up blocking( forever ).
I'm pretty sure they carry out the SLL handshake because they both send and receive a couple of times before it hangs.
here's the server code:
import socket
import threading
class SocketWrapper:
def __init__(self,sock = None):
if sock is None:
self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
else:
self.socket = sock
def connect(self,host,port):
try:
self.socket.connect((host,int(port)))
return True
except socket.error:
return False
def close(self):
# close the socket connection
self.socket.shutdown(socket.SHUT_RDWR)
self.socket.close()
def send(self,data):
bytes_sent = 0
msg_len = len(data)
while bytes_sent < msg_len:
sent = self.socket.send(data[bytes_sent:])
bytes_sent += sent
def receive(self):
chunks = []
while True:
try:
self.socket.settimeout(0.5)
chunk = self.socket.recv(4096)
chunks.append(chunk)
except socket.error:
self.socket.settimeout(0)
break;
return b''.join(chunks)
class HttpTunnel:
def __init__(self,host='localhost',port=3000):
# create the server socket,bind and listen
self.host,self.port = host,port
self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
self.socket.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1)
self.socket.bind((self.host,self.port))
self.socket.listen(3)
print("listening on port",self.port)
self.running = True
def handleClientRequest(self,connection,address):
print("Connected to",address)
clientSocket = SocketWrapper(connection)
meta = clientSocket.receive().decode().split('\r\n')[0]
# getting method,uri,version from 'CONNECT host:port HTTP/1.1'
method,uri,version = meta.split(' ')
host,port = uri.split(':')
serverSocket = SocketWrapper()
# if connection to the remote server is created successfuly
if(serverSocket.connect(host,port)):
print("Connected to remote server")
# send connection success message to the client
clientSocket.send(b'HTTP/1.1 200 OK\r\n\r\n');
while True:
try:
clientResponse = clientSocket.receive()
serverSocket.send(clientResponse)
print("Sent client - server")
serverResponse = serverSocket.receive()
clientSocket.send(serverResponse)
print("Sent server - client")
except socket.error:
break;
else:
# send someking of error. In this case 404
serverSocket.send(b'HTTP/1.1 404 Not Found\r\n\r\n')
# close the connection
clientSocket.close()
serverSocket.close()
def mainloop(self):
while self.running:
# accept client connection
connection,address = self.socket.accept()
self.handleClientRequest(connection,address)
proxy = HttpTunnel()
proxy.mainloop()
the client code:
import urllib
import urllib.request as request
proxy = request.ProxyHandler({
'https':'https://127.0.0.1:3000'
})
opener = request.build_opener(proxy)
request.install_opener(opener)
try:
resp = request.urlopen('https://google.com')
print(resp.read())
except Exception as e:
print(e)
the client did not get the response from the server and therefore prints nothing.
here's the server output:
listening on port 3000
Connected to ('127.0.0.1', 54888)
Connected to remote server
Sent client - server
Sent server - client
Sent client - server
Sent server - client
Sent client - server

There are several problems here:
The main problem is that you don't handle the case when recv returns 0 since the socket gets closed. Instead you run into an endless loop where no data get read and no data get send. Some simple print statements which actually show how much data are read would have helped to track this problem down.
Apart from that the idea of polling each file handle after each other using settimeout is a bad approach. Instead check the file handles in parallel and then read from the one which has data - see select.
And finally you are assuming that socket.send will send all data given. This is not the case, it might send less. Check the return code or just use socket.sendall

I want to implement retransmission. Through the timeout function of the recv method [duplicate]

I need to set timeout on python's socket recv method. How to do it?

The typical approach is to use select() to wait until data is available or until the timeout occurs. Only call recv() when data is actually available. To be safe, we also set the socket to non-blocking mode to guarantee that recv() will never block indefinitely. select() can also be used to wait on more than one socket at a time.
import select
mysocket.setblocking(0)
ready = select.select([mysocket], [], [], timeout_in_seconds)
if ready[0]:
data = mysocket.recv(4096)
If you have a lot of open file descriptors, poll() is a more efficient alternative to select().
Another option is to set a timeout for all operations on the socket using socket.settimeout(), but I see that you've explicitly rejected that solution in another answer.

there's socket.settimeout()

As mentioned both select.select() and socket.settimeout() will work.
Note you might need to call settimeout twice for your needs, e.g.
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.bind(("",0))
sock.listen(1)
# accept can throw socket.timeout
sock.settimeout(5.0)
conn, addr = sock.accept()
# recv can throw socket.timeout
conn.settimeout(5.0)
conn.recv(1024)

You could set timeout before receiving the response and after having received the response set it back to None:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5.0)
data = sock.recv(1024)
sock.settimeout(None)

The timeout that you are looking for is the connection socket's timeout not the primary socket's, if you implement the server side. In other words, there is another timeout for the connection socket object, which is the output of socket.accept() method. Therefore:
sock.listen(1)
connection, client_address = sock.accept()
connection.settimeout(5) # This is the one that affects recv() method.
connection.gettimeout() # This should result 5
sock.gettimeout() # This outputs None when not set previously, if I remember correctly.
If you implement the client side, it would be simple.
sock.connect(server_address)
sock.settimeout(3)

Got a bit confused from the top answers so I've wrote a small gist with examples for better understanding.
Option #1 - socket.settimeout()
Will raise an exception in case the sock.recv() waits for more than the defined timeout.
import socket
sock = socket.create_connection(('neverssl.com', 80))
timeout_seconds = 2
sock.settimeout(timeout_seconds)
sock.send(b'GET / HTTP/1.1\r\nHost: neverssl.com\r\n\r\n')
data = sock.recv(4096)
data = sock.recv(4096) # <- will raise a socket.timeout exception here
Option #2 - select.select()
Waits until data is sent until the timeout is reached. I've tweaked Daniel's answer so it will raise an exception
import select
import socket
def recv_timeout(sock, bytes_to_read, timeout_seconds):
sock.setblocking(0)
ready = select.select([sock], [], [], timeout_seconds)
if ready[0]:
return sock.recv(bytes_to_read)
raise socket.timeout()
sock = socket.create_connection(('neverssl.com', 80))
timeout_seconds = 2
sock.send(b'GET / HTTP/1.1\r\nHost: neverssl.com\r\n\r\n')
data = recv_timeout(sock, 4096, timeout_seconds)
data = recv_timeout(sock, 4096, timeout_seconds) # <- will raise a socket.timeout exception here

You can use socket.settimeout() which accepts a integer argument representing number of seconds. For example, socket.settimeout(1) will set the timeout to 1 second

try this it uses the underlying C.
timeval = struct.pack('ll', 2, 100)
s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVTIMEO, timeval)

As mentioned in previous replies, you can use something like: .settimeout()
For example:
import socket
s = socket.socket()
s.settimeout(1) # Sets the socket to timeout after 1 second of no activity
host, port = "somehost", 4444
s.connect((host, port))
s.send("Hello World!\r\n")
try:
rec = s.recv(100) # try to receive 100 bytes
except socket.timeout: # fail after 1 second of no activity
print("Didn't receive data! [Timeout]")
finally:
s.close()
I hope this helps!!

#! /usr/bin/python3.6
# -*- coding: utf-8 -*-
import socket
import time
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
s.settimeout(5)
PORT = 10801
s.bind(('', PORT))
print('Listening for broadcast at ', s.getsockname())
BUFFER_SIZE = 4096
while True:
try:
data, address = s.recvfrom(BUFFER_SIZE)
except socket.timeout:
print("Didn't receive data! [Timeout 5s]")
continue

Shout out to: https://boltons.readthedocs.io/en/latest/socketutils.html
It provides a buffered socket, this provides a lot of very useful functionality such as:
.recv_until() #recv until occurrence of bytes
.recv_closed() #recv until close
.peek() #peek at buffer but don't pop values
.settimeout() #configure timeout (including recv timeout)

trying to receive data from remote device using udp socket

I'm trying to send a command to a remote device: E5071C ENA Vector Network Analyzer
These are my problems:
When I try to send and receive data from the socket, it "hangs".
I am not sure which type of socket I should use.
For the other commands, I used s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM), but I only had to send data in those cases. Here I need to send and receive data. I have tried to use a while Trueloop like this:
while True:
s.settimeout(1)
print(sys.stderr, 'waiting for a connection')
connection, client_address = s.accept()
try:
print(sys.stderr, 'client connected:', client_address)
while True:
data = connection.recv(1024)
print(sys.stderr, 'received "%s"') % data
if data:
connection.sendall(data)
else:
break
finally:
connection.close()
I could not get the result I wanted with the while loop either, so this is what I have so far instead:
## import modules
## for loop iterations
## commands sent to remote device using UDP socket
def is_hanging(host, port):
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,port))
min_msg = ':CALCulate:SELected:MARKer:FUNCtion:TYPE MINimum'
s.send(min_msg.encode())
s.settimeout(1)
try:
s.recv(1024)
print("Received data!")
return True
except socket.timeout as e:
print(e)
return False
finally:
s.close()
if is_hanging('10.5.33.16',5025) is True:
pass
else:
raise Exception("Could not receive data.")
I am not entirely sure how s.recv works, but I what I am hoping/expecting is that data I send to the remote device generates a response which sent back to me.
Right now it is just hanging.

socket.accept() is only relevant in the context of a stream-based socket that's been configured as a listening socket. It doesn't make sense to use on a datagram socket, especially one that's already associated with a remote host.
(Are you sure SOCK_DGRAM -- i.e. UDP -- is correct here? Most SCPI devices I've worked with use TCP, i.e. SOCK_STREAM.)
If you're sure that you're using the right protocol, remove the call to s.accept(), and call .recv() on the existing socket s.

python3 - broken pipe error when using socket.send()

I am programming a client-server instant message program. I created a similar program in Python 2, and am trying to program it in Python 3. The problem is when the server takes the message and tries to send it to the other client, it gives me "[Errno 32] Broken Pipe" and exits.
I have done some research, and found that this occurs when the client disconnects, so I did some more testing but could not find when the client disconnects. (I am using Ubuntu 14.04 and Python 3.4)
Here is the server code:
import socket, select, sys
def broadcast(sock, messaged):
for socket in connection_list:
if socket != s and socket != sock:
# Here is where it gives me the broken pipe error
try:
s.send(messaged.encode("utf-8"))
except BrokenPipeError as e:
print(e)
sys.exit()
connection_list = []
host = ''
port = 5558
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.bind((host,port))
s.listen(5)
connection_list.append(s)
read_sockets,write_sockets,error_sockets = select.select(connection_list,[],[])
while True:
for sock in read_sockets:
if sock == s:
conn, addr = s.accept()
connection_list.append(conn)
client = "Client (%s,%s) connected" % addr
print(client)
broadcast(sock,client)
else:
try:
data = sock.recv(2048)
decodeddata = data.decode("utf-8")
if data:
broadcast(sock, decodeddata)
except:
offline = "Client " + addr + "is offline"
broadcast(sock, offline)
print(offline)
connection_list.remove(sock)
sock.close()
continue
And the client code:
import socket, select, string, sys, time
def prompt(data) :
print("<You> " + data)
def Person(data) :
print("<Receiver> " + data)
if __name__ == "__main__":
host = "localhost"
port = 5558
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(2)
try:
s.connect((host,port))
except:
print('Unable to connect')
sys.exit()
print('Connected.')
socket_list = [s]
read_sockets,write_sockets,error_sockets = select.select(socket_list,[],[])
while 1:
for sock in read_sockets:
if sock == s:
try:
time.sleep(1)
data = sock.recv(1024)
Person(data.decode("utf-8"))
except:
msg = input("Send a message: ")
try:
s.send(str.encode(msg))
except:
print("Server is offline")
sys.exit()
else:
print("Server is offline")
sys.exit()

There are two problems that you have to fix to make this work.
First, on both the client side and the server side, you have to put the select inside the loop, not outside. Otherwise, if there was something to read before you got to the loop, you'll recv over and over, and if there wasn't, you'll never recv. Once you fix this, you can get rid of the time.sleep(1). (You should never need a sleep to solve a problem like this; at best it masks the problem, and usually introduces new ones.)
Meanwhile, on the server side, inside broadcast, you're doing s.send. But s is your listener socket, not a connected client socket. You want socket.send here, because socket is each socket in connection_list.
There are a number of unrelated problems in your code as well. For example:
I'm not sure what the except: in the client is supposed to be catching. What it mainly seems to catch is that, about 50% of the time, hitting ^C to end the program triggers the send prompt. But of course, like any bare except:, it also masks any other problems with your code.
There's no way to send any data back and forth other than the "connected" message except for that except: clause.
addr is a tuple of host and port, so when someone goes offline, the server raises a TypeError from trying to format the offline message.
addr is always the last client who connected, not the one who's disconnecting.
You're not setting your sockets to nonblocking mode.
You're not checking for EOF on the recv. This means that you don't actually detect that a client has gone offline until you get an error. Which normally happens only after you try to send them a message (e.g., because someone else has connected or disconnected).

How to tell if a connection is dead in python

I want my python application to be able to tell when the socket on the other side has been dropped. Is there a method for this?

Short answer:
use a non-blocking recv(), or a blocking recv() / select() with a very
short timeout.
Long answer:
The way to handle socket connections is to read or write as you need to, and be prepared to handle connection errors.
TCP distinguishes between 3 forms of "dropping" a connection: timeout, reset, close.
Of these, the timeout can not really be detected, TCP might only tell you the time has not expired yet. But even if it told you that, the time might still expire right after.
Also remember that using shutdown() either you or your peer (the other end of the connection) may close only the incoming byte stream, and keep the outgoing byte stream running, or close the outgoing stream and keep the incoming one running.
So strictly speaking, you want to check if the read stream is closed, or if the write stream is closed, or if both are closed.
Even if the connection was "dropped", you should still be able to read any data that is still in the network buffer. Only after the buffer is empty will you receive a disconnect from recv().
Checking if the connection was dropped is like asking "what will I receive after reading all data that is currently buffered ?" To find that out, you just have to read all data that is currently bufferred.
I can see how "reading all buffered data", to get to the end of it, might be a problem for some people, that still think of recv() as a blocking function. With a blocking recv(), "checking" for a read when the buffer is already empty will block, which defeats the purpose of "checking".
In my opinion any function that is documented to potentially block the entire process indefinitely is a design flaw, but I guess it is still there for historical reasons, from when using a socket just like a regular file descriptor was a cool idea.
What you can do is:
set the socket to non-blocking mode, but than you get a system-depended error to indicate the receive buffer is empty, or the send buffer is full
stick to blocking mode but set a very short socket timeout. This will allow you to "ping" or "check" the socket with recv(), pretty much what you want to do
use select() call or asyncore module with a very short timeout. Error reporting is still system-specific.
For the write part of the problem, keeping the read buffers empty pretty much covers it. You will discover a connection "dropped" after a non-blocking read attempt, and you may choose to stop sending anything after a read returns a closed channel.
I guess the only way to be sure your sent data has reached the other end (and is not still in the send buffer) is either:
receive a proper response on the same socket for the exact message that you sent. Basically you are using the higher level protocol to provide confirmation.
perform a successful shutdow() and close() on the socket
The python socket howto says send() will return 0 bytes written if channel is closed. You may use a non-blocking or a timeout socket.send() and if it returns 0 you can no longer send data on that socket. But if it returns non-zero, you have already sent something, good luck with that :)
Also here I have not considered OOB (out-of-band) socket data here as a means to approach your problem, but I think OOB was not what you meant.

It depends on what you mean by "dropped". For TCP sockets, if the other end closes the connection either through
close() or the process terminating, you'll find out by reading an end of file, or getting a read error, usually the errno being set to whatever 'connection reset by peer' is by your operating system. For python, you'll read a zero length string, or a socket.error will be thrown when you try to read or write from the socket.

From the link Jweede posted:
exception socket.timeout:
This exception is raised when a timeout occurs on a socket
which has had timeouts enabled via a prior call to settimeout().
The accompanying value is a string whose value is currently
always “timed out”.
Here are the demo server and client programs for the socket module from the python docs
# Echo server program
import socket
HOST = '' # Symbolic name meaning all available interfaces
PORT = 50007 # Arbitrary non-privileged port
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((HOST, PORT))
s.listen(1)
conn, addr = s.accept()
print 'Connected by', addr
while 1:
data = conn.recv(1024)
if not data: break
conn.send(data)
conn.close()
And the client:
# Echo client program
import socket
HOST = 'daring.cwi.nl' # The remote host
PORT = 50007 # The same port as used by the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((HOST, PORT))
s.send('Hello, world')
data = s.recv(1024)
s.close()
print 'Received', repr(data)
On the docs example page I pulled these from, there are more complex examples that employ this idea, but here is the simple answer:
Assuming you're writing the client program, just put all your code that uses the socket when it is at risk of being dropped, inside a try block...
try:
s.connect((HOST, PORT))
s.send("Hello, World!")
...
except socket.timeout:
# whatever you need to do when the connection is dropped

If I'm not mistaken this is usually handled via a timeout.

I translated the code sample in this blog post into Python: How to detect when the client closes the connection?, and it works well for me:
from ctypes import (
CDLL, c_int, POINTER, Structure, c_void_p, c_size_t,
c_short, c_ssize_t, c_char, ARRAY
)
__all__ = 'is_remote_alive',
class pollfd(Structure):
_fields_ = (
('fd', c_int),
('events', c_short),
('revents', c_short),
)
MSG_DONTWAIT = 0x40
MSG_PEEK = 0x02
EPOLLIN = 0x001
EPOLLPRI = 0x002
EPOLLRDNORM = 0x040
libc = CDLL('libc.so.6')
recv = libc.recv
recv.restype = c_ssize_t
recv.argtypes = c_int, c_void_p, c_size_t, c_int
poll = libc.poll
poll.restype = c_int
poll.argtypes = POINTER(pollfd), c_int, c_int
class IsRemoteAlive: # not needed, only for debugging
def __init__(self, alive, msg):
self.alive = alive
self.msg = msg
def __str__(self):
return self.msg
def __repr__(self):
return 'IsRemoteAlive(%r,%r)' % (self.alive, self.msg)
def __bool__(self):
return self.alive
def is_remote_alive(fd):
fileno = getattr(fd, 'fileno', None)
if fileno is not None:
if hasattr(fileno, '__call__'):
fd = fileno()
else:
fd = fileno
p = pollfd(fd=fd, events=EPOLLIN|EPOLLPRI|EPOLLRDNORM, revents=0)
result = poll(p, 1, 0)
if not result:
return IsRemoteAlive(True, 'empty')
buf = ARRAY(c_char, 1)()
result = recv(fd, buf, len(buf), MSG_DONTWAIT|MSG_PEEK)
if result > 0:
return IsRemoteAlive(True, 'readable')
elif result == 0:
return IsRemoteAlive(False, 'closed')
else:
return IsRemoteAlive(False, 'errored')

Trying to improve on #kay response. I made a more pythonic version
(Note that it was not yet tested in a "real-life" environment, and only on Linux)
This detects if the remote side closed the connection, without actually consuming the data:
import socket
import errno
def remote_connection_closed(sock: socket.socket) -> bool:
"""
Returns True if the remote side did close the connection
"""
try:
buf = sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT)
if buf == b'':
return True
except BlockingIOError as exc:
if exc.errno != errno.EAGAIN:
# Raise on unknown exception
raise
return False
Here is a simple example from an asyncio echo server:
import asyncio
async def handle_echo(reader, writer):
addr = writer.get_extra_info('peername')
sock = writer.get_extra_info('socket')
print(f'New client: {addr!r}')
# Initial of client command
data = await reader.read(100)
message = data.decode()
print(f"Received {message!r} from {addr!r}")
# Simulate a long async process
for _ in range(10):
if remote_connection_closed(sock):
print('Remote side closed early')
return
await asyncio.sleep(1)
# Write the initial message back
print(f"Send: {message!r}")
writer.write(data)
await writer.drain()
writer.close()
async def main():
server = await asyncio.start_server(
handle_echo, '127.0.0.1', 8888)
addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
print(f'Serving on {addrs}')
async with server:
await server.serve_forever()
if __name__ == '__main__':
asyncio.run(main())