I use python's socket and epoll to make a web server.
My operating system is Linux CentOS 6,
My python version is python 2.7.8.
My source code is:
# -*-coding:utf-8-*-
import socket
import select
import time
EOL1 = b'\n\n'
EOL2 = b'\n\r\n'
response = b'HTTP/1.0 200 OK\r\nDate: Mon, 1 Jan 1996 01:01:01 GMT\r\n'
response += b'Content-Type: text/plain\r\nContent-Length: 13\r\n\r\n'
response += b'<html><head><title>title</title></head><body><p>Hello, world!</p></body></html>'
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
serversocket.bind(('0.0.0.0', 8080))
serversocket.listen(1) # the number of client that connect to server
serversocket.setblocking(0) # set 0 not block other block
serversocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
epoll = select.epoll()
epoll.register(serversocket.fileno(), select.EPOLLIN)
try:
connections = {}
requests = {}
responses = {}
while True:
events = epoll.poll(1)
for fileno, event in events:
if fileno == serversocket.fileno(): # if request come
connection, address = serversocket.accept() # waiting income connection
connection.setblocking(0) # none block
epoll.register(connection.fileno(), select.EPOLLIN) # register socket read event to epoll
connections[connection.fileno()] = connection # add connection to connections dict
requests[connection.fileno()] = b''
responses[connection.fileno()] = response # write data to responses dict
elif event & select.EPOLLIN: # when data in os's read buffer area
requests[fileno] += connections[fileno].recv(1024) # read data from connections
if EOL1 in requests[fileno] or EOL2 in requests[fileno]: # if http message
print('-' * 40 + '\n' + requests[fileno].decode()[:-2])
responses[fileno] += str(time.time())
epoll.modify(fileno, select.EPOLLOUT) # change file number to epoll out mode
elif event & select.EPOLLOUT: # if out mode
byteswritten = connections[fileno].send(responses[fileno]) # write data to os's write buffer
responses[fileno] = responses[fileno][byteswritten:] # get http response message
if len(responses[fileno]) == 0: # if file sent
epoll.modify(fileno, 0) # change file number to hup mode
connections[fileno].shutdown(socket.SHUT_RDWR) # set socket read and write mode shutdown
elif event & select.EPOLLHUP: # if message sent and file number in epoll is hup
epoll.unregister(fileno) # remove file number from epoll
connections[fileno].close() # close connection
del connections[fileno] # delete connection from connections dict
finally:
epoll.unregister(serversocket.fileno())
epoll.close()
serversocket.close()
But when I open web browser and visit "http://localhost:8080/", I get some data like these <html><head><,it is not full data, it just a part of my data.What's the matter in my project.
view more info please look this picture.
You have "Content-Length: 13" in your code and hence only first 13 characters are showing up!
Related
I have an industrial robot connected over TCP which recieves a string in bytes. I have successfully done some simple comms between them but I now need to add another client, where the server can send out messages to either client. I'm using selectors and as its a little abstract I'm getting confused.
This is the server code - the client is written in SPEL+ so i won't show it, but is very very simple - Print [port number] [message]
import socket
import selectors
import types
import time
import queue
sel = selectors.DefaultSelector()
# # Any data received by this queue will be sent
# send_queue = queue.Queue()
def accept_wrapper(sock):
conn, addr = sock.accept() # Should be ready to read
fd = sock.fileno()
print(f"Accepted connection from {addr}, FD# {fd}")
conn.setblocking(False)
data = types.SimpleNamespace(addr=addr, inb=b"", outb=b"")
events = selectors.EVENT_READ | selectors.EVENT_WRITE
sel.register(conn, events, data=data)
def service_connection(key, mask, pending_messages):
sock = key.fileobj
data = key.data
print("Key.fd~~~~~~")
print(key.fd)
print("Key.fileobj.fileno()~~~~~~")
print(key.fileobj.fileno())
if mask & selectors.EVENT_READ:
recv_data = sock.recv(1024) # Should be ready to read
if recv_data:
data.outb += recv_data
else:
print(f"Closing connection to {data.addr}")
sel.unregister(sock)
sock.close()
if mask & selectors.EVENT_WRITE:
if len(pending_messages) > 0 and pending_messages[0][0] == key.fd:
print(pending_messages[0][0])
data.outb = pending_messages[0][1]
pending_messages.pop(0)
print(pending_messages)
if data.outb:
print(f"Echoing {data.outb!r} to {data.addr}")
sent = sock.send(data.outb) # Should be ready to write
data.outb = data.outb[sent:]
time.sleep(1)
def main(host, port):
#host, port = sys.argv[1], int(sys.argv[2])
lsock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
lsock.bind((host, port))
lsock.listen()
print(f"Listening on {(host, port)}")
lsock.setblocking(False)
sel.register(lsock, selectors.EVENT_READ, data=None)
pending_messages = []
try:
while True:
events = sel.select(timeout=None)
for key, mask in events:
if key.data is None:
fd, addr = accept_wrapper(key.fileobj)
else:
pending_messages = [(fd, b"helloWorld\r")]
service_connection(key, mask, pending_messages)
except KeyboardInterrupt:
print("Caught keyboard interrupt, exiting")
finally:
sel.close()
if __name__ == '__main__':
main("",2000)
I thought fd was the interger assigned to the connection within the selector, Also (using many print statements) im not sure how the selector knows that there is data to write.
The code might be familiar as its from RealPython, but like most tutorials im finding they all stop at the server being able to accept a message from other code to then send to a client.
At the moment, I cannot get multi socket server to accept a message to then send it to the client. The client and server do communicate but this is just a basic echo.
I am trying to implement UDP stop-and-wait protocol using python socket programming. I have been trying out my code with different retransmission times. And sometimes I get all the files correctly received at the receiver, but sometimes some packets get lost. For example when I ran it with 40ms five times, twice the file was received correctly and three times incorrectly. Why is this variability happening?
Here is my code for the sender and receiver:
senderSocket = socket(AF_INET, SOCK_DGRAM) # Create UDP socket for sender
r = 0 # Number of retransmissions
for i in range(0, len(messages)):
senderSocket.settimeout(retryTimeout) # After the message is sent, set retransmission timeout to listen for acknowledgement
while True:
senderSocket.sendto(messages[i], (hostName, portNumber))
acknowledged = False
while not acknowledged:
try:
ack, receiverAddress = senderSocket.recvfrom(2) # Receive ACK
acknowledged = True
except: # Socket timeout exception occurs when timeout expires but no ACK received
senderSocket.sendto(messages[i], (hostName, portNumber)) # Retransmit the message
r = r + 1 # Increment the number of retransmissions
break # On to the next message
senderSocket.close()
while True:
message, senderAddress = receiverSocket.recvfrom(1027) # Read from UDP socket into message, getting sender's address (sender IP and port)
header = message[:3] # Header is the first 3 bytes (index 0, 1, 2)
data = message[3:] # Rest is the data
first_byte = '{0:08b}'.format(header[0])
second_byte = '{0:08b}'.format(header[1])
seq_num = int(first_byte + second_byte, 2) # Convert bytes to decimal
if seq_num not in seq_nums: # Detect duplicates
seq_nums.append(seq_num)
file_content.extend(data)
ack = header[:2] # ACK is the receipt of the received message (sequence number)
receiverSocket.sendto(ack, senderAddress) # Send ACK
if header[2] == 1: # Sent multiple ACKs at lat message to make sure it receives and the sender closes
receiverSocket.sendto(ack, senderAddress)
receiverSocket.sendto(ack, senderAddress)
receiverSocket.sendto(ack, senderAddress)
break
receiverSocket.close()
I'm trying to send random size int array from multiple-clients to server which will keep adding the newly received int array to a global array and return accumulated sorted array to client. My client code is able to send and receive int array to/from server. But server is not able to read the int array and sort and send back to client (My server can just read and send back original int array to client, but it's not what I want).
In my server code, commented part is not working. I am very new in python and socket programming.
Client.py
# Import socket module
import socket, pickle
import random
def Main():
# local host IP '127.0.0.1'
host = '127.0.0.1'
# Define the port on which you want to connect
port = 12345
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# connect to server on local computer
s.connect((host, port))
while True:
# Generate Random array to be sent to server
arr = []
# Generate random size between 10 and 20
# random_size = random.randint(10, 20)
random_size = random.randint(1, 3)
for i in range(0, random_size):
arr.append(random.randint(0, 10))
print('Array = ' + str(arr))
# Serialise the array to byte stream before sending to server
data_stream = pickle.dumps(arr)
#Array byte stream sent to server
s.send(data_stream)
# messaga received from server
data = s.recv(1024)
#deserialise the byte stream into array after receiving from server
data_arr = pickle.loads(data)
# print the received message
#print('Received', repr(data_arr))
print('Received from Server: ', data_arr)
# ask the client whether he wants to continue
ans = input('\nDo you want to continue(y/n) :')
if ans == 'y':
continue
else:
break
# close the connection
s.close()
if __name__ == '__main__':
Main()
Server.py
# import socket programming library
import socket, pickle
# import thread module
from _thread import *
import threading
from sortedcontainers import SortedList
import bisect
#Container to store accumulated int array from multiple clients
sl = SortedList()
# To protect
print_lock = threading.Lock()
# thread fuction
def threaded(c):
while True:
# data received from client
data = c.recv(1024)
# Data from client can't be printed =============== why?
print(data)
if not data:
print('No data received from client - Bye')
# lock released on exit
print_lock.release()
break
c.send(data) # ===> It works but I want to add received int array into global sl before sending back to client
'''
////////////////////// Code in this comment section is not working //////////////////
#Deserialise Byte stream array from client into array list
data_arr = pickle.loads(data)
#Add received int array from client to global sortedList sl in sorted order
for i in data_arr:
bisect.insort(sl, i)
sl.add(i)
print(sl)
#Serialise sorted sl into Byte stream before sending to client
data_stream = pickle.dumps(sl)
# send back sorted integer list to client
c.send(data_stream)
'''
# connection will never be closed, server will run always
#c.close()
def Main():
host = ""
# We can use a port on our specific computer
# But in this case it is 12345 (it can be anything)
port = 12345
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.bind((host, port))
print("socket binded to post", port)
# put the socket into listening mode
s.listen(5)
print("socket is listening")
# a forever loop until client wants to exit
while True:
# establish connection with client
c, addr = s.accept()
# lock acquired by client
print_lock.acquire()
print('Connected to :', addr[0], ':', addr[1])
# Start a new thread and return its identifier
start_new_thread(threaded, (c,))
s.close()
if __name__ == '__main__':
Main()
I run it in terminal and I see error
NotImplementedError: use ``sl.add(value)`` instead
but it seems to be incomplete message.
After removing
bisect.insort(sl, i)
it starts working.
Probably there was: use ``sl.add(value)`` instead of ``bisect.insort(sl, i)``
Server Side (server prints the first line of information sent from the client then it JUST STAYS open and doesn't seem to continue listening it just stays open. Is there a way to get the server to listen to the client more and print?)
import time
import socket
import signal
from datetime import datetime
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('localhost', 8089))
serversocket.listen(1024) # become a server socket, maximum 5 connectionn
def clientsocketentry():
while True:
connection, addr = serversocket.accept()
buf = connection.recv(64)
if not buf:
break
elif buf == 'killsrv':
connection.close()
sys.exit()
else:
print (buf)
buf = buf.decode("utf-8")
buf = buf.split(',')
serverLong = buf[0]
print('Longitude:' + '' + serverLong)
serverLat = buf[1]
print('Lattitude:' + '' + serverLat)
serverAlt = buf[2]
print('Altitude:' + '' + serverAlt)
serverTime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('Time of Entry:' + ' ' + serverTime)
connection.close()
clientsocketentry()
Client Side (I am only able to send one of the strings of information then the server stays open ut does not take more information from the client)
import socket
import time
clientsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
clientsocket.connect(('localhost', 8089))
a = '39.163100,-76.899428,0'
clientsocket.send(a.encode('utf-8'))
time.sleep(5)
a = '4.2,2.2415,0'
clientsocket.send(a.encode('utf-8'))
time.sleep(5)
a = '43454,354354,35435'
clientsocket.send(a.encode('utf-8'))
time.sleep(5)
a = '435742.,35.452,52434'
clientsocket.send(a.encode('utf-8'))
time.sleep(5)
clientsocket.close()
If you accept one single connection at a time (no need for a 1024 backlog then...) you can simply nest 2 loops: the outer one waiting for new connections the inner one processing input from the only one established connection. If you need to process more than one connection, you will have to use select or threads.
Here is an example for one single connection:
def clientsocketentry():
serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
serversocket.bind(('localhost', 8089))
serversocket.listen(5) # become a server socket, maximum 5 connectionn
cont = True
while cont:
connection, addr = serversocket.accept()
while True:
buf = connection.recv(64)
if len(buf) == 0: # end of connection
connection.close()
break
elif buf == b'killsrv': # request for closing server (beware of the byte string)
connection.close()
serversocket.close()
cont = False
break
else:
print (buf)
buf = buf.decode("utf-8")
buf = buf.split(',')
serverLong = buf[0]
print('Longitude:' + '' + serverLong)
serverLat = buf[1]
print('Lattitude:' + '' + serverLat)
serverAlt = buf[2]
print('Altitude:' + '' + serverAlt)
serverTime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('Time of Entry:' + ' ' + serverTime)
# connection.close() # wait for client to close
You are closing the socket at the end of your print logic in the ClientSocketEntry function.
serverTime = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('Time of Entry:' + ' ' + serverTime)
connection.close()
Instead of closing the connection there only close it when the user sends killsrv
Because every time you close the connection on the socket it is saying that you are expecting another client to connect to the server. So maybe before going into the while statement accept the connection and then pass it into the while statement, because the way you have it structured at the moment is expecting multiple connections from different clients.
I'm running into issues transferring data over TCP with a remote client and server written in Python. The server is located in a pretty remote region with relatively slow internet connection (<2Mb/sec). When the client is run on the LAN with the server the complete string is transferred (2350 bytes); however, when I run the client outside of the LAN sometimes the string is truncated (1485 bytes) and sometimes the full string comes through (2350 bytes). The size of the truncated string always seems to be 1485 bytes. The full size of the string is well below the set buffer size for the client and server.
I've copied abbreviated versions of the client and server code below, where I have tried to edit out all extraneous details:
Client
import socket
from time import sleep
class FTIRdataClient():
def __init__(self,TCP_IP="xxx.xxx.xxx.xxx",TCP_Port=xxx,BufferSize=4096):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def writeTCP(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.send(message)
incomming = sock.recv(self.RECV_BUFFER)
sock.close()
except:
print "Unable to connect to data server!!"
incomming = False
return incomming
if __name__ == "__main__":
#----------------------------------
# Initiate remote data client class
#----------------------------------
dataClass = FTIRdataClient(TCP_IP=dataServer_IP,TCP_Port=portNum,BufferSize=4096)
#--------------------------------
# Ask database for all parameters
#--------------------------------
allParms = dataClass.writeTCP("LISTALL")
Server
import os
import sys
import socket
import select
import smtplib
import datetime as dt
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
#------------------------------------------------
# Three types of call to server:
# 1) set -- sets the value of a data parameter
# 2) get -- gets the value of a data parameter
# 3) write -- write data to a file
#------------------------------------------------
splitVals = data.strip().split()
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#---------------------------------------------------
# Remove client from socket list after disconnection
#---------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
def closeServer(self):
''' Close the TCP data server '''
self.server_socket.close()
Your help is greatly appreciated!!!
For anyone who is interested I found the solution to this problem. John Nielsen has a pretty good explanation here. Basically, TCP stream only guarantees that bytes will not arrive out of order or be duplicated; however, it does not guarantee how many groups the data will be sent in. So one needs to continually read (socket.recv) until all the data is sent. The previous code work on the LAN because the server was sending the entire string in one group. Over a remote connection the string was split into several groups.
I modified the client to continually loop on socket.recv() until the socket is closed and I modified the server to immediately close the socket after sending the data. There are several other ways to do this mentioned in the above link. The new code looks like:
Client
class FTIRdataClient(object):
def __init__(self,TCP_IP="xxx.xxx.xx.xxx",TCP_Port=xxxx,BufferSize=4024):
#-----------------------------------
# Configuration parameters of server
#-----------------------------------
self.TCP_IP = TCP_IP
self.TCP_Port = int(TCP_Port)
self.RECV_BUFFER = int(BufferSize)
def setParam(self,message):
try:
sock = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
sock.connect((self.TCP_IP,self.TCP_Port))
sock.sendall("set "+message)
#-------------------------
# Loop to recieve all data
#-------------------------
incommingTotal = ""
while True:
incommingPart = sock.recv(self.RECV_BUFFER)
if not incommingPart: break
incommingTotal += incommingPart
sock.close()
except:
print "Unable to connect to data server!!"
incommingTotal = False
return incommingTotal
Server
class FTIRdataServer(object):
def __init__(self,ctlFvars):
...
def runServer(self):
self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server_socket.bind((self.TCP_IP,self.TCP_Port))
#self.server_socket.setsockopt(socket.IPPROTO_TCP,socket.TCP_NODELAY,1)
self.server_socket.listen(10)
self.connection_list.append(self.server_socket)
#-------------------------------------
# Start loop to listen for connections
#-------------------------------------
while True:
#--------------------
# Get list of sockets
#--------------------
read_sockets,write_sockets,error_sockets = select.select(self.connection_list,[],[],5)
for sock in read_sockets:
#-----------------------
# Handle new connections
#-----------------------
if sock == self.server_socket:
#----------------------------------------------
# New connection recieved through server_socket
#----------------------------------------------
sockfd, addr = self.server_socket.accept()
self.connection_list.append(sockfd)
print "Client (%s, %s) connected" % addr
#-------------------------------------
# Handle incomming request from client
#-------------------------------------
else:
#------------------------
# Handle data from client
#------------------------
try:
data = sock.recv(self.RECV_BUFFER)
...
elif splitVals[0].upper() == 'LISTALL':
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParams:
msgLst.append("{0:}={1:}".format(k," ".join(self.dataParams[k])))
msg = ";".join(msgLst)
sock.sendall(msg)
elif splitVals[0].upper() == 'LISTALLTS': # List all time stamps
msgLst = []
#----------------------------
# Create a string of all keys
# and values to send back
#----------------------------
for k in self.dataParamTS:
msgLst.append("{0:}={1:}".format(k,self.dataParamTS[k]))
msg = ";".join(msgLst)
sock.sendall(msg)
...
else:
pass
#------------------------
# Close socket connection
#------------------------
sock.close()
self.connection_list.remove(sock)
#------------------------------------------------------
# Remove client from socket list if client discconnects
#------------------------------------------------------
except:
sock.close()
self.connection_list.remove(sock)
continue
#-------------
# Close server
#-------------
self.closeServer()
Whatever. This is probably common knowledge and I'm just a little slow.