After long hours of research and testing I finally ask here.
My script has to handle multiple client connections and in the same time has to get and send a stream from another socket.
Finally I've been able to make it work but only for one user. That user connects to the socket, the script connects to the other socket, then return the stream to the client.
The script works pretty well but has a some hard limitations :
- it send the stream to the client but,
- even if the socket is in non-blocking mode I think that calling a socket inside another one is the main reason why it reacts like it was in blocking mode (because one ot these is continuously sending datas ?)
By the way I think that the select() method could allow me to do what I want, but I don't clearly understand how.
Here is the server code taht works for one client, but is blocking
#!/usr/bin/env python
# coding: utf-8
from __future__ import print_function
import sys, time, base64, socket
server_ip = 'XX.XX.XX.XX'
def caster_connect(connected_client, address):
username = 'XXXXXXX'
password = 'XXXXXXXXX'
host = 'XX.XX.XX.XX'
port = 2102
pwd = base64.b64encode("{}:{}".format(username, password).encode('ascii'))
pwd = pwd.decode('ascii')
u_message = ''
stream_type = 'CMRp'
header = \
"GET /" + str(stream_type) + " HTTP/1.1\r\n" +\
"Host " + str(host) + "\r\n" +\
"Ntrip-Version: Ntrip/1.0\r\n" +\
"User-Agent: my_script.py/0.1\r\n" +\
"Accept: */*\r\n" +\
"Authorization: Basic {}\r\n\r\n".format(pwd) +\
"Connection: close\r\n"
print("Connecting to caster...\n")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,int(port)))
s.send(header.encode('ascii'))
print("Waiting answer from caster...\n")
while True:
try:
data = s.recv(2048)
connected_client.send(data)
print("Sending data from caster at %s" % time.time())
sys.stdout.flush()
# On any error, close sockets
except socket.error, e:
print("No data received from caster : %s" % e)
print("Close client connection at %s" % format(address))
s.close()
break
return
#----------------
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((server_ip, 5680))
sock.settimeout(3)
try:
while True:
try:
sock.listen(5)
client, address = sock.accept()
print ("%s connected" % format(address) )
msg = client.recv(4096)
except socket.timeout, e:
err = e.args[0]
if err == 'timed out':
print("Timed out, retry later")
continue
else:
print(socket.error)
sock.close()
except socket.error:
print(socket.error)
sock.close()
else:
if len(msg) == 0:
print("Shutdown on client end")
sock.close()
else:
print(msg)
caster_response = caster_connect(client, address)
sys.stdout.flush()
print("Close")
client.close()
sock.close()`enter code here`
except KeyboardInterrupt:
print("W: Keyboard interrupt, closing socket")
finally:
sock.close()
And this is the code I found to handle select()
#!/usr/bin/env python
# coding: utf-8
import select, socket, sys, Queue
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setblocking(0)
server.bind(('XX.XX.XX.XX', 64000))
server.listen(5)
inputs = [server]
outputs = []
message_queues = {}
while inputs:
readable, writable, exceptional = select.select(
inputs, outputs, inputs)
for s in readable:
if s is server:
connection, client_address = s.accept()
print("New connection from %s" % client_address)
connection.setblocking(0)
inputs.append(connection)
message_queues[connection] = Queue.Queue()
else:
data = s.recv(1024)
print("Data received : %s" % data)
if data:
message_queues[s].put(data)
if s not in outputs:
outputs.append(s)
else:
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
del message_queues[s]
for s in writable:
try:
next_msg = message_queues[s].get_nowait()
print("Next msg : %s" % next_msg)
except Queue.Empty:
outputs.remove(s)
else:
s.send(next_msg)
for s in exceptional:
inputs.remove(s)
if s in outputs:
outputs.remove(s)
s.close()
del message_queues[s]
In this code (found at this page) I didn't make changes as I don't know how to handle this.
Maybe by creating another server script that would only handle the stream part, so the main script would act as a server for clients, but as client for the stream part ?
Related
I have written a Python async socket server, basically listens on port and responds to it. However, sometimes I observe that the request is sent partially e.g. the Content-Length indicates that the length is X but actual request received is kinda trimmed halfway. What could go wrong here?
SERVER_HOST = '127.0.0.1'
SERVER_PORT = 12345
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.setblocking(False)
server_socket.bind((SERVER_HOST, SERVER_PORT))
server_socket.listen(5)
inputs = [server_socket]
outputs = []
messages = {}
while True:
for i in inputs:
if i.fileno() < 0:
inputs.remove(i)
for i in outputs:
if i.fileno() < 0:
outputs.remove(i)
readable, writable, exceptional = select.select(inputs, outputs, inputs, 1)
if not (readable or writable or exceptional):
continue
for s in readable:
if s is server_socket:
connection, client_address = s.accept()
connection.setblocking(0)
inputs.append(connection)
messages[connection] = queue.Queue()
else:
data = s.recv(self.params["buffer_size"])
if data:
messages[s].put(data)
if s not in outputs:
outputs.append(s)
else:
# Interpret empty result as closed connection
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
del messages[s]
# Handle outputs
for s in writable:
try:
next_msg = messages[s].get_nowait()
# handle next_msg here
handle(next_msg)
except Exception as ex:
print('Error: ', s.getpeername(), ': ', ex)
outputs.remove(s)
else:
s.sendall(response.encode("utf-8"))
outputs.remove(s)
s.shutdown(socket.SHUT_WR)
s.close()
# close socket
server_socket.close()
And based on the comments, the TCP is stream based, I changed a little bit on recv e.g. read all until no more.
for s in readable:
if s is server_socket:
# A "readable" server socket is ready to accept a connection
connection, client_address = s.accept()
print('new connection from', client_address)
connection.setblocking(0)
inputs.append(connection)
# Give the connection a queue for data we want to send
messages[connection] = queue.Queue()
else:
_data = s.recv(self.params["buffer_size"])
if _data:
data = _data
while _data:
print('received "%s" from %s' % (_data, s.getpeername()))
try:
_data = s.recv(self.params["buffer_size"])
if _data:
data += _data
except:
break
# Add output channel for response
if s not in outputs:
outputs.append(s)
messages[s].put(data)
else:
# Interpret empty result as closed connection
print('closing', client_address, 'after reading no data')
# Stop listening for input on the connection
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
# Remove message queue
del messages[s]
But, still, there is a chance this will not work.
Running some production code I faced the following problem:
When sending HTTP requests to some server, server immediately closes the connection after sending response, which, for some reason, results in data loss.
Analyzing TCP dumps i can see that conversation goes as this:
client request
server ack
server push
server fin, ack (sent after ~0.000020 secs after previous push)
As the result my code can't get data sent by the server, (i'm guessing because of the small delay after push POLLHUP event might go before POLLIN?)
Trying to mimic the problem I've written the following code:
(It mimics the client behaviour on my side)
client:
import time
import socket
from errno import EAGAIN
from select import poll, POLLIN, POLLPRI, POLLERR, POLLHUP, POLLNVAL
def main(buf=""):
client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
client.setblocking(False)
client.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
client.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
polling_object = poll()
polling_object.register(client, POLLPRI | POLLIN | POLLERR | POLLHUP)
in_buf = ""
sock_closed = False
try:
client.connect(("127.0.0.1", 8877))
except socket.error, e:
pass
while True and not sock_closed:
events = polling_object.poll(0)
for _, e in events:
if e & (POLLIN | POLLPRI):
while True:
try:
data = client.recv(1024)
if data:
in_buf += data
elif data == "":
client.close()
sock_closed = True
break
except socket.error, e:
if e.args[0] == EAGAIN:
break
else:
raise
elif e & (POLLERR|POLLHUP|POLLNVAL):
client.close()
sock_closed = True
if buf and not sock_closed:
try:
b_sent = client.send(buf)
if b_sent == len(buf):
buf = ""
else:
buf = buf[b_sent:]
except socket.error, e:
if e.args[0] != EAGAIN:
client.close()
sock_closed = True
time.sleep(0.5)
if sock_closed:
return in_buf
if __name__ == '__main__':
import sys
if len(sys.argv) > 1:
buf = sys.argv[1]
else:
buf = 'hello'
print main(buf)
server
import datetime
import time
import socket
def main():
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server.bind(("127.0.0.1", 8877))
server.listen(0)
client, _ = server.accept()
t1 = time.time()
data = ""
while not data:
data += client.recv(1024)
print "recv data %s" % data
client.sendall('{"ok": 1}')
t2 = time.time()
client.close()
t3 = time.time()
server.close()
return t1, t2, t3
if __name__ == '__main__':
c_r, d_s, c_c = main()
print "Connection received at ", datetime.datetime.fromtimestamp(c_r)
print "All Data sent after %.12f secs" % (d_s - c_r)
print "Connection closed after %.12f secs" % (c_c - d_s)
Running this code won't help me reproduce the problem because my client still can get data from socket buffer, which is kind of obviously by just following the code. The only difference is that in tcp dump it goes like this:
client request
server ack
server push
client ack
server fin, ack
I'm wondering is there a way to send fin, ack right after push without "letting" client to sent ack? Can it be done using python?
I am writing a script for handling HTTP request through socket programming. My Python Script just reads each HTTP response, search for few keywords and increment the counters.
Only starting the script takes CPU upto 90-99% when there is no incoming messages. How should i handle this?
HOST = ''
SOCKET_LIST = []
RECV_BUFFER = 40966
PORT=int(sys.argv[1])
serviceInitiatedEvent=0
deliveredEvent=0
EstablishedEvent=0
ConnectionClearedEvent=0
def chat_server():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
SOCKET_LIST.append(server_socket)
print "Chat server started on port " + str(PORT)
try:
while 1:
ready_to_read,ready_to_write,in_error = select.select(SOCKET_LIST,[],[],0)
for sock in ready_to_read:
if sock == server_socket:
sockfd, addr = server_socket.accept()
SOCKET_LIST.append(sockfd)
else:
try:
data = sock.recv(RECV_BUFFER)
if data:
if re.search('serviceInitiatedEvent></SOAP-ENV',data):
global serviceInitiatedEvent
serviceInitiatedEvent+=1
if re.search('deliveredEvent></SOAP-ENV',data):
global deliveredEvent
deliveredEvent+=1
else:
if sock in SOCKET_LIST:
SOCKET_LIST.remove(sock)
except:
broadcast(server_socket, sock, "Client (%s, %s) is offline\n" % addr)
continue
except KeyboardInterrupt:
print "service Initiated Event:%s" % (serviceInitiatedEvent)
print "delivered Event: %s" % (deliveredEvent)
server_socket.close()
if __name__ == "__main__":
sys.exit(chat_server())
If you have code with while 1 loop utilizing 100%, that's probably the culprit. It's called busy waiting.
select function has timeout parameter that specifies how long it should wait for events. In your code, you set it to 0, so that when there is no data available in sockets, control returns immediately, causing busy waiting loop.
Specify some larger timeout, based on your needs, so that your code won't spin when there's nothing to do:
ready_to_read,ready_to_write,in_error = select.select(SOCKET_LIST,[],[], 1)
# ^^^ here
I have been writing a transparent proxy server in python to log where the request is going. Most pages load e.g. google.co.uk, however, pages such as google.com get stuck loading and some pages such as a local IP get the "Connection reset" error in the browser.
Any help would be greatly appreciated.
#!/usr/bin/env python
import socket, optparse, thread
def proxy(url, port, connection, address, data):
try:
get = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
get.connect((url, port))
get.send(data)
while True:
reply = get.recv(BUFFER)
if len(reply) > 0:
connection.send(reply)
info = float(len(reply))
info = float(info / 1024)
info = "%.3s" %(str(info))
info = "%s KB" %(info)
print("[*] Request Complete: %s => %s <=" %(str(address[0]), str(info)))
else:
break
get.close()
connection.close()
except Exception as e:
get.close()
connection.close()
def handle(connection, address, data):
first = data.split("\n")[0]
url = first.split(" ")[1]
protocolPosition = url.find("://")
if protocolPosition == -1:
# No protocol so default
temp = url
else:
temp = url[(protocolPosition + 3):]
if ":" in temp:
# Port other than 80 has been specified
port = temp.split(":")[-1].strip("/")
webserver = temp.split(":")[:-1]
try:
# Incase there is ':' in the URL
webserver = "".join(webserver)
except:
pass
else:
port = 80
webserver = temp.strip("/")
print("[*] '%s' => '%s'" %(address[0], webserver))
proxy(webserver, port, connection, address, data)
receive = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
receive.bind(("0.0.0.0", PORT))
except socket.error as e:
print("Failed to bind to 0.0.0.0:%d" %(PORT))
print("Error: " + str(e))
raise SystemExit
receive.listen(MAXCONNECTIONS)
print("Listening on 0.0.0.0:%d" %(PORT))
while True:
try:
connection, address = receive.accept()
data = connection.recv(BUFFER)
thread.start_new_thread(handle, (connection, address, data,))
except KeyboardInterrupt:
break
print("\nReleasing socket")
receive.close()
Edit: After some digging around and error handling I narrowed the error down to
[Errno -2] Name or service not known
When a client connects to the pipe, and sends data I can receive this fine and I can keep receiving the data. Trouble comes when the client disconnects and the while loop is still active, connection.recv() doesn't block and therefore keeps looping frantically! So I need a way to detect if a client is still connected.
I have the following code:
pipe = './pipes/uds_defzone-lrecv'
try:
os.unlink(pipe)
except OSError:
if os.path.exists(pipe):
raise
self.logger.debug('Created UDS pipe: ' + pipe)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(pipe)
sock.listen(1)
self.logger.debug('Waiting for connection: ' + pipe)
connection, client_address = sock.accept()
self.logger.debug('Connection from: ' + client_address)
while True:
self.logger.debug('Waiting for data')
data = connection.recv(4096)
self.logger.debug('Received: ' + str(data))
For reference, the sender.py code:
# Create a UDS socket
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Connect the socket to the port where the server is listening
pipe = './pipes/uds_defzone-lrecv'
logger.debug('connecting to: ' + pipe)
try:
sock.connect(pipe)
except socket.error, msg:
logger.debug(msg)
sys.exit(1)
try:
message = 'THIS IS A TEST'
logger.debug('sending: ' + message)
sock.sendall(message)
time.sleep(2)
finally:
logger.debug('closing socket')
sock.close()
TIA!
UPDATE
I can slow it down with the following code I suppose, but not exactly what I want.
while True:
try:
self.logger.debug('Waiting for data')
data_present = select.select([sock], [], [], 30)
if data_present[0]:
data = connection.recv(4096)
self.logger.debug('Received: ' + data)
except select.timeout:
pass
UPDATE 2
For reference this is the code I came up with:
while True:
logger.debug('Waiting for data')
data = connection.recv(4096)
if not data == '':
logger.debug('Received: ' + data)
else:
logger.debug('Nothing received')
break
A hack I came up with in the process... Might be usable where it is legitimate that a client might send empty data, for signalling perhaps?
while True:
try:
logger.debug('Waiting for data')
data = connection.recv(4096)
# *** This throws an exception when client has disconnected
x = connection.getpeername()
logger.debug('Received: ' + data)
except:
logger.debug('Client disconnected')
break
connection.recv() doesn't block and therefore keeps looping frantically! So I need a way to detect if a client is still connected.
If the peer disconnects recv data will return empty data (''). You need to check this and exit the loop.