this is my first entry #stackoverflow;-))
I need a Python3 TCP server, which sends all inputs of the clients to the other participants. The server works, but unfortunately the message is only returned to the sending client. As soon as the second client sends something, it gets all entries.The entries are therefore available in the output queue.
Why are the entries not recognized by select ()?
Thx for the support.
`
import select
import socket
import sys
import queue
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setblocking(0)
server_address = ('192.168.100.28', 8888)
print ("starting up on %s port %s" % server_address)
server.bind(server_address)
server.listen(5)
inputs = [ server ]
outputs = [ ]
message_queues = {}
while inputs:
print ("\nwaiting for the next event")
readable, writable, exceptional = select.select(inputs, outputs, inputs)
Handle inputs
for s in readable:
if s is server:
# A "readable" server socket is ready to accept a connection
connection, client_address = s.accept()
print ("new connection from", client_address)
connection.setblocking(0)
inputs.append(connection)
# Give the connection a queue for data we want to send
message_queues[connection] = queue.Queue()
else:
data = s.recv(1024)
if data:
# A readable client socket has data
print ('received "%s" from %s' % (data, s.getpeername()))
# Add output channel for response
#message_queues[s].put(data)
if s not in outputs:
outputs.append(s)
for aQueue in message_queues:
message_queues[aQueue].put(data)
# AQueue.send ("Test".encode())
else:
# Interpret empty result as closed connection
print ("closing", client_address, "after reading no data")
# Stop listening for input on the connection
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
# Remove message queue
del message_queues[s]
Handle outputs
for s in writable:
try:
next_msg = message_queues[s].get_nowait()
except queue.Empty:
# No messages waiting so stop checking for writability.
print ("output queue for", s.getpeername(), "is empty")
outputs.remove(s)
else:
print ('sending "%s" to %s' % (next_msg, s.getpeername()))
s.send(next_msg)
Handle "exceptional conditions"
for s in exceptional:
print ("handling exceptional condition for", s.getpeername())
# Stop listening for input on the connection
inputs.remove(s)
if s in outputs:
outputs.remove(s)
s.close()
# Remove message queue
del message_queues[s]
`
Your problem is that you only add a peer to the output list when you read something from it. You should instead do it when you write somthing to its message queue.
The receiving part should become:
...
data = s.recv(1024)
if data:
# A readable client socket has data
print ('received "%s" from %s' % (data, s.getpeername()))
# Add output channel for response
#message_queues[s].put(data)
# if s not in outputs:
# outputs.append(s)
for aQueue in message_queues:
message_queues[aQueue].put(data)
if aQueue not in outputs: # enqueue the msg
outputs.append(aQueue) # and ask select to warn when it can be sent
# AQueue.send ("Test".encode())
else:
...
According to your own requirements, you should only enqueue messages for other participants:
for aQueue in message_queues:
if aQueue != s: # only send to other participants
...
Finally, you often test existence of a socket in outputs. That let think that a set would be more appropriate than a list.
Related
I have written a Python async socket server, basically listens on port and responds to it. However, sometimes I observe that the request is sent partially e.g. the Content-Length indicates that the length is X but actual request received is kinda trimmed halfway. What could go wrong here?
SERVER_HOST = '127.0.0.1'
SERVER_PORT = 12345
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.setblocking(False)
server_socket.bind((SERVER_HOST, SERVER_PORT))
server_socket.listen(5)
inputs = [server_socket]
outputs = []
messages = {}
while True:
for i in inputs:
if i.fileno() < 0:
inputs.remove(i)
for i in outputs:
if i.fileno() < 0:
outputs.remove(i)
readable, writable, exceptional = select.select(inputs, outputs, inputs, 1)
if not (readable or writable or exceptional):
continue
for s in readable:
if s is server_socket:
connection, client_address = s.accept()
connection.setblocking(0)
inputs.append(connection)
messages[connection] = queue.Queue()
else:
data = s.recv(self.params["buffer_size"])
if data:
messages[s].put(data)
if s not in outputs:
outputs.append(s)
else:
# Interpret empty result as closed connection
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
del messages[s]
# Handle outputs
for s in writable:
try:
next_msg = messages[s].get_nowait()
# handle next_msg here
handle(next_msg)
except Exception as ex:
print('Error: ', s.getpeername(), ': ', ex)
outputs.remove(s)
else:
s.sendall(response.encode("utf-8"))
outputs.remove(s)
s.shutdown(socket.SHUT_WR)
s.close()
# close socket
server_socket.close()
And based on the comments, the TCP is stream based, I changed a little bit on recv e.g. read all until no more.
for s in readable:
if s is server_socket:
# A "readable" server socket is ready to accept a connection
connection, client_address = s.accept()
print('new connection from', client_address)
connection.setblocking(0)
inputs.append(connection)
# Give the connection a queue for data we want to send
messages[connection] = queue.Queue()
else:
_data = s.recv(self.params["buffer_size"])
if _data:
data = _data
while _data:
print('received "%s" from %s' % (_data, s.getpeername()))
try:
_data = s.recv(self.params["buffer_size"])
if _data:
data += _data
except:
break
# Add output channel for response
if s not in outputs:
outputs.append(s)
messages[s].put(data)
else:
# Interpret empty result as closed connection
print('closing', client_address, 'after reading no data')
# Stop listening for input on the connection
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
# Remove message queue
del messages[s]
But, still, there is a chance this will not work.
I am writing a script for handling HTTP request through socket programming. My Python Script just reads each HTTP response, search for few keywords and increment the counters.
Only starting the script takes CPU upto 90-99% when there is no incoming messages. How should i handle this?
HOST = ''
SOCKET_LIST = []
RECV_BUFFER = 40966
PORT=int(sys.argv[1])
serviceInitiatedEvent=0
deliveredEvent=0
EstablishedEvent=0
ConnectionClearedEvent=0
def chat_server():
server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
server_socket.bind((HOST, PORT))
server_socket.listen(10)
SOCKET_LIST.append(server_socket)
print "Chat server started on port " + str(PORT)
try:
while 1:
ready_to_read,ready_to_write,in_error = select.select(SOCKET_LIST,[],[],0)
for sock in ready_to_read:
if sock == server_socket:
sockfd, addr = server_socket.accept()
SOCKET_LIST.append(sockfd)
else:
try:
data = sock.recv(RECV_BUFFER)
if data:
if re.search('serviceInitiatedEvent></SOAP-ENV',data):
global serviceInitiatedEvent
serviceInitiatedEvent+=1
if re.search('deliveredEvent></SOAP-ENV',data):
global deliveredEvent
deliveredEvent+=1
else:
if sock in SOCKET_LIST:
SOCKET_LIST.remove(sock)
except:
broadcast(server_socket, sock, "Client (%s, %s) is offline\n" % addr)
continue
except KeyboardInterrupt:
print "service Initiated Event:%s" % (serviceInitiatedEvent)
print "delivered Event: %s" % (deliveredEvent)
server_socket.close()
if __name__ == "__main__":
sys.exit(chat_server())
If you have code with while 1 loop utilizing 100%, that's probably the culprit. It's called busy waiting.
select function has timeout parameter that specifies how long it should wait for events. In your code, you set it to 0, so that when there is no data available in sockets, control returns immediately, causing busy waiting loop.
Specify some larger timeout, based on your needs, so that your code won't spin when there's nothing to do:
ready_to_read,ready_to_write,in_error = select.select(SOCKET_LIST,[],[], 1)
# ^^^ here
After long hours of research and testing I finally ask here.
My script has to handle multiple client connections and in the same time has to get and send a stream from another socket.
Finally I've been able to make it work but only for one user. That user connects to the socket, the script connects to the other socket, then return the stream to the client.
The script works pretty well but has a some hard limitations :
- it send the stream to the client but,
- even if the socket is in non-blocking mode I think that calling a socket inside another one is the main reason why it reacts like it was in blocking mode (because one ot these is continuously sending datas ?)
By the way I think that the select() method could allow me to do what I want, but I don't clearly understand how.
Here is the server code taht works for one client, but is blocking
#!/usr/bin/env python
# coding: utf-8
from __future__ import print_function
import sys, time, base64, socket
server_ip = 'XX.XX.XX.XX'
def caster_connect(connected_client, address):
username = 'XXXXXXX'
password = 'XXXXXXXXX'
host = 'XX.XX.XX.XX'
port = 2102
pwd = base64.b64encode("{}:{}".format(username, password).encode('ascii'))
pwd = pwd.decode('ascii')
u_message = ''
stream_type = 'CMRp'
header = \
"GET /" + str(stream_type) + " HTTP/1.1\r\n" +\
"Host " + str(host) + "\r\n" +\
"Ntrip-Version: Ntrip/1.0\r\n" +\
"User-Agent: my_script.py/0.1\r\n" +\
"Accept: */*\r\n" +\
"Authorization: Basic {}\r\n\r\n".format(pwd) +\
"Connection: close\r\n"
print("Connecting to caster...\n")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host,int(port)))
s.send(header.encode('ascii'))
print("Waiting answer from caster...\n")
while True:
try:
data = s.recv(2048)
connected_client.send(data)
print("Sending data from caster at %s" % time.time())
sys.stdout.flush()
# On any error, close sockets
except socket.error, e:
print("No data received from caster : %s" % e)
print("Close client connection at %s" % format(address))
s.close()
break
return
#----------------
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((server_ip, 5680))
sock.settimeout(3)
try:
while True:
try:
sock.listen(5)
client, address = sock.accept()
print ("%s connected" % format(address) )
msg = client.recv(4096)
except socket.timeout, e:
err = e.args[0]
if err == 'timed out':
print("Timed out, retry later")
continue
else:
print(socket.error)
sock.close()
except socket.error:
print(socket.error)
sock.close()
else:
if len(msg) == 0:
print("Shutdown on client end")
sock.close()
else:
print(msg)
caster_response = caster_connect(client, address)
sys.stdout.flush()
print("Close")
client.close()
sock.close()`enter code here`
except KeyboardInterrupt:
print("W: Keyboard interrupt, closing socket")
finally:
sock.close()
And this is the code I found to handle select()
#!/usr/bin/env python
# coding: utf-8
import select, socket, sys, Queue
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.setblocking(0)
server.bind(('XX.XX.XX.XX', 64000))
server.listen(5)
inputs = [server]
outputs = []
message_queues = {}
while inputs:
readable, writable, exceptional = select.select(
inputs, outputs, inputs)
for s in readable:
if s is server:
connection, client_address = s.accept()
print("New connection from %s" % client_address)
connection.setblocking(0)
inputs.append(connection)
message_queues[connection] = Queue.Queue()
else:
data = s.recv(1024)
print("Data received : %s" % data)
if data:
message_queues[s].put(data)
if s not in outputs:
outputs.append(s)
else:
if s in outputs:
outputs.remove(s)
inputs.remove(s)
s.close()
del message_queues[s]
for s in writable:
try:
next_msg = message_queues[s].get_nowait()
print("Next msg : %s" % next_msg)
except Queue.Empty:
outputs.remove(s)
else:
s.send(next_msg)
for s in exceptional:
inputs.remove(s)
if s in outputs:
outputs.remove(s)
s.close()
del message_queues[s]
In this code (found at this page) I didn't make changes as I don't know how to handle this.
Maybe by creating another server script that would only handle the stream part, so the main script would act as a server for clients, but as client for the stream part ?
When a client connects to the pipe, and sends data I can receive this fine and I can keep receiving the data. Trouble comes when the client disconnects and the while loop is still active, connection.recv() doesn't block and therefore keeps looping frantically! So I need a way to detect if a client is still connected.
I have the following code:
pipe = './pipes/uds_defzone-lrecv'
try:
os.unlink(pipe)
except OSError:
if os.path.exists(pipe):
raise
self.logger.debug('Created UDS pipe: ' + pipe)
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(pipe)
sock.listen(1)
self.logger.debug('Waiting for connection: ' + pipe)
connection, client_address = sock.accept()
self.logger.debug('Connection from: ' + client_address)
while True:
self.logger.debug('Waiting for data')
data = connection.recv(4096)
self.logger.debug('Received: ' + str(data))
For reference, the sender.py code:
# Create a UDS socket
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Connect the socket to the port where the server is listening
pipe = './pipes/uds_defzone-lrecv'
logger.debug('connecting to: ' + pipe)
try:
sock.connect(pipe)
except socket.error, msg:
logger.debug(msg)
sys.exit(1)
try:
message = 'THIS IS A TEST'
logger.debug('sending: ' + message)
sock.sendall(message)
time.sleep(2)
finally:
logger.debug('closing socket')
sock.close()
TIA!
UPDATE
I can slow it down with the following code I suppose, but not exactly what I want.
while True:
try:
self.logger.debug('Waiting for data')
data_present = select.select([sock], [], [], 30)
if data_present[0]:
data = connection.recv(4096)
self.logger.debug('Received: ' + data)
except select.timeout:
pass
UPDATE 2
For reference this is the code I came up with:
while True:
logger.debug('Waiting for data')
data = connection.recv(4096)
if not data == '':
logger.debug('Received: ' + data)
else:
logger.debug('Nothing received')
break
A hack I came up with in the process... Might be usable where it is legitimate that a client might send empty data, for signalling perhaps?
while True:
try:
logger.debug('Waiting for data')
data = connection.recv(4096)
# *** This throws an exception when client has disconnected
x = connection.getpeername()
logger.debug('Received: ' + data)
except:
logger.debug('Client disconnected')
break
connection.recv() doesn't block and therefore keeps looping frantically! So I need a way to detect if a client is still connected.
If the peer disconnects recv data will return empty data (''). You need to check this and exit the loop.
I have set up an experiment where I pass Modbus traffic over a SSL tunnel (this being the first thing I've ever done in python). I am able to send and receive data but when I send one request numerous requests are actually sent (see screenshot)
I've tried numerous configurations including (in both client and server):
send()--no change
sendall() --no change
setblocking(1)
setblocking(0)--doesn't read all the data
On the server side:
if data == Read_Coils_Answer-- I don't think I'm converting the big endian properly for comparison and this didn't work
while data: --the while loop seems to be the only way to prevent either side from stopping short with a "Broken Pipe" error. So this is what I'm using.
I eventually plan to use a for loop (now commented out and set to 4).
My Server code:
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def do_something(connstream, data):
readCoilsReq = str('\x01\x01\x00')
answer = str(ModbusPDU01_Read_Coils_Answer)
while data:
print ("Request Recevied from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
return False
def deal_with_client(connstream):
data = connstream.recv(64)
while data:
if not do_something(connstream, data):
break
data = connstream.recv(64)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Recieved\n")
sys.exit(1)
My Client Side code:
from ModLib import *
from time import sleep
import socket, ssl, pprint
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_sock = ssl.wrap_socket(s,
ca_certs="server.crt",
cert_reqs=ssl.CERT_REQUIRED)
ssl_sock.connect(('localhost', 502))
ssl_sock.setblocking(1)
readCoils = ModbusPDU01_Read_Coils()
#for i in range(4):
sleep(2)
ssl_sock.sendall(str(readCoils))
print ("Request for Read Coils Sent")
#start receive
data = ssl_sock.recv(64)
print ("Response from Server:")
print pprint.pformat(data)
if False: #from the python docs
ssl_sock.write("""GET / HTTP/1.0\r
Host: www.verisign.com\n\n""")
data = ssl_sock.read()
ssl_sock.close()
The do_something() loop was not necessary, as the deal_with_client() loop was doing the same thing. I removed do_something() and put the code in deal_with_client() which allows me to keep the connection open (see below)
from ModLib import *
import socket, ssl, sys, pprint
try:
bindsocket = socket.socket()
bindsocket.bind(('', 502))
bindsocket.listen(5)
bindsocket.setblocking(1)
def deal_with_client(connstream):
data = connstream.recv(1120)
answer = str(ModbusPDU01_Read_Coils_Answer())
while data:
print ("Request Received from Client:")
print pprint.pformat(data)
connstream.send(answer)
print ("Answer Sent to Client")
print pprint.pformat(answer)
data = connstream.recv(1120)
while True:
newsocket, fromaddr = bindsocket.accept()
connstream = ssl.wrap_socket(newsocket,
server_side=True,
certfile="server.crt",
keyfile="server.key",
ssl_version=ssl.PROTOCOL_TLSv1)
try:
deal_with_client(connstream)
finally:
connstream.shutdown(socket.SHUT_RDWR)
connstream.close()
except KeyboardInterrupt:
print ("\nTerminating Session at User Request")
print ("No More Data Will be Sent/Received\n")
sys.exit(1)