We Keep Coding

How to keep listening to the connected TCP sockets by "select"

All my clients sockets do the same thing: send a package every second(22 bytes)
Server code as below:
import select
import socket
import datetime
SList = []
class Tserver:
def __init__(self, portNum):
host = '127.0.0.1'
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server.bind((host, portNum))
self.server.listen(1)
def GETPACK():
# function for CRC check
def CRC(DATA_STR):
return 1
# generate 100 sockets to listen
for x in range(100):
SList.append(Tserver(x+10000))
inputs = []
# put in inputs
for x in range(100):
inputs.append(SList[x].server)
while(True):
ready_socks, _, _ = select.select(inputs, [], [])
for sock in ready_socks:
c, addr = sock.accept()
while(True):
data = c.recv(22)
if len(data) == 22: # To make sure the data-length is 22
# Turn the pack string into bytearray
data_bytes = bytearray()
data_bytes.extend(data)
if CRC(data_bytes) == 1:
print "Connected from client IP Address:" + str(addr)
# ID
ID = 256*data_bytes[1] + data_bytes[2]
print "ID: ", ID
now = datetime.datetime.now()
print "now: ", str(now)
if __name__ == "__main__":
GETPACK()
My server can only print the packages sent by the first connected socket.
And my question is how to print out all message from each ports whenever a package is sent to the server.

See this PyMOTW entry for a detailed explanation of how to use the select module to write a select-based server.
The main differences between that example and your code are:
You just create one listening socket - server. There is no need to listen on multiple ports.
The variable inputs will be a list consisting of server and any other open socket connections to clients.
Your service loop will look like:
while true:
readable, _, _ = select.select(inputs, [], [])
for r in readable:
if r is server:
# handle a new incoming connection
# this will add an entry to the variable inputs
else:
# read some data from socket r and process it
When you attempt to read from a client socket and get an EOF condition, you can close that socket and remove it from the inputs variable.

#ErikR Thanks for your help, i changed my code, and it worked fine.
The reason that my code doesn't work was because of two things:
1.I only create one connection to recv data from my clients.
2.The same connection can't be accepted again for recv, if the clients does't reconnect.(my code doesn't check the exception when clients shutdown)
Code as below:
import select, socket, datetime
SList = []
SconnList = []
class Tserver:
def __init__(self, portNum):
host = '127.0.0.1'
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.server.bind((host,portNum))
self.server.listen(1)
print "Server ports: "+str(portNum)
class Sconn:
def __init__(self, sock):
self.conn, self.addr = sock.accept()
def GETPACK():
# function for CRC check
def CRC(DATA_STR):
return 1
# generate 100 sockets to listen
for x in range(100):
SList.append(Tserver(x+10000))
inputs = []
# put in inputs
for x in range(100):
inputs.append(SList[x].server)
while(True):
ready_socks,_,_ = select.select(inputs, [], [])
for sock in ready_socks:
try:
SconnList.append(Sconn(sock))
SconnList.reverse()
inputs.append(SconnList[0].conn)
except:
data = sock.recv(22)
if len(data) == 22: # To make sure the data-length is 22
#Turn the pack string into bytearray
data_bytes = bytearray()
data_bytes.extend(data)
if CRC(data_bytes) == 1:
print "IP Address:" + str(sock.getsockname())
#ID
ID = 256*data_bytes[1] + data_bytes[2]
print "ID: ",ID
now = datetime.datetime.now()
print "now: ",str(now)
print ""
print ""
if __name__ == "__main__":
GETPACK()

How to finish a socket file transfer in Python?

I have a Client and a Server and I need to transfer some files using sockets. I can send small messages, but when I try to send a File, the problems begins...
client.py:
from socket import *
from threading import Thread
import sys
import hashlib
class Client(object):
ASK_LIST_FILES = "#001" # 001 is the requisition code to list
# all the files
ASK_SPECIFIC_FILE = "#002" # 002 is the requisition code to a
# specific file
SEND_FILE = "#003" # 003 is the requisition code to send one
# file
AUTHENTICATION = "#004" # 004 is the requisition code to user
# authentication
listOfFiles = []
def __init__(self):
try:
self.clientSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.clientSocket.connect(addr)
except (error):
print("Failed to connect.")
sys.exit()
print(self.clientSocket.recv(1024).decode())
def closeConnection(self):
self.clientSocket.close()
def _askFileList(self):
try:
data = Client.ASK_LIST_FILES
self.clientSocket.sendall(data.encode())
# self._recvFileList()
except (error):
print("Failed asking for the list of files.")
self.closeConnection()
sys.exit()
thread = Thread(target = self._recvFileList)
thread.start()
def _recvFileList(self):
print("Waiting for the list...")
self.listOfFiles = []
while len(self.listOfFiles) == 0:
data = self.clientSocket.recv(1024).decode()
if (data):
self.listOfFiles = data.split(',')
if(len(self.listOfFiles) > 0):
print (self.listOfFiles)
def _askForFile(self, fileIndex):
fileIndex = fileIndex - 1
try:
data = Client.ASK_SPECIFIC_FILE + "#" + str(fileIndex)
self.clientSocket.sendall(data.encode())
except(error):
print("Failed to ask for an specific file.")
self.closeConnection()
sys.exit()
self._downloadFile(fileIndex)
def _downloadFile(self, fileIndex):
print("Starting receiving file")
f = open("_" + self.listOfFiles[fileIndex], "wb+")
read = self.clientSocket.recv(1024)
# print(read)
# f.close
while len(read) > 0:
print(read)
f.write(read)
f.flush()
read = self.clientSocket.recv(1024)
f.flush()
f.close()
self.closeConnection()
server.py
from socket import *
from threading import Thread
import sys
import glob
class Server(object):
def __init__(self):
try:
self.serverSocket = socket(AF_INET, SOCK_STREAM)
except (error):
print("Failed to create a Socket.")
sys.exit()
def connect(self, addr):
try:
self.serverSocket.bind(addr)
except (error):
print ("Failed on binding.")
sys.exit()
def closeConnection(self):
self.serverSocket.close()
def waitClients(self, num):
while True:
print("Waiting for clients...")
self.serverSocket.listen(num)
conn, addr = self.serverSocket.accept()
print("New client found...")
thread = Thread(target = self.clientThread, args = (conn,))
thread.start()
def clientThread(self, conn):
WELCOME_MSG = "Welcome to the server"
conn.send(WELCOME_MSG.encode())
while True:
data = conn.recv(2024).decode()
if(data):
# print(data)
# reply = 'OK: ' + data
# conn.sendall(reply.encode())
if(data == "#001"):
listOfFiles = self.getFileList()
strListOfFiles = ','.join(listOfFiles)
self._sendFileList(strListOfFiles, conn)
else:
dataCode = data.split('#')
print(dataCode)
if(dataCode[1] == "002"):
print("Asking for file")
self._sendFile(int(dataCode[2]), conn)
if(dataCode[1] == "003"):
print("Pedido de login")
if self._authentication(dataCode[2]):
conn.send("OK".encode())
# self._recvFile(conn)
else:
conn.send("FAILED".encode())
def _sendFile(self, fileIndex, conn):
listOfFiles = self.getFileList()
print(fileIndex)
print(listOfFiles[fileIndex])
f = open(listOfFiles[fileIndex], "rb")
read = f.read(1024)
while len(read) > 0:
conn.send(read)
read = f.read(1024)
f.close()
def _sendFileList(self, strList, conn):
try:
conn.sendall(strList.encode())
except (error):
print("Failed to send list of files.")
def getFileList(self):
return glob.glob("files/*")
When I try to get a file from my server, I can transfer everything but the connection never ends. What is going on with my code?

First, you are doing here the most common error using TCP: assume all data sent in a single send() will be got identically in a single recv(). This is untrue for TCP, because it is an octet stream, not a message stream. Your code will work only under ideal (lab) conditions and could mysteriously fail in a real world usage. You should either explicitly invent message boundaries in TCP streams, or switch e.g. to SCTP. The latter is available now almost everywhere and keeps message boundaries across a network connection.
The second your error is directly connected to the first one. When sending file, you don't provide any explicit mark that file has been finished. So, clients waits forever. You might try to close server connection to show that file is finished, but in that case client won't be able to distinguish real file end and connection loss; moreover, the connection won't be reusable for further commands. You would select one of the following ways:
Prefix a file contents with its length. In this case, client will know how many bytes shall be received for the file.
Send file contents as a chunk sequence, prefixing each chunk with its length (only for TCP) and with mark whether this chunk is last (for both transports). Alternatively, a special mark "EOF" can be sent without data.
Similarly, control messages and their responses shall be provided with either length prefix or a terminator which can't appear inside such message.
When you finish developing this, you would look at FTP and HTTP; both addresses all issues I described here but in principally different ways.

Sending DHCP Discover using python scapy

I am new to python and learning some network programming, I wish to send an DHCP Packet through my tap interface to my DHCP server and expecting some response from it. I tried with several packet building techniques such a structs and ctypes and ended up with using scapy. Here I am able to send DHCP Packet but unable to get any response from the DHCP server(Analyzed using wireshark and tcpdump)..My packet looked like same as original DHCP packet but failed to get response. Here is my code
import socket
from scapy.all import *
def main():
if len(sys.argv)<3:
print " fewer arguments."
sys.exit(1)
else:
tap_interface = sys.argv[1]
src_mac_address = sys.argv[2]
ethernet = Ether(dst='ff:ff:ff:ff:ff:ff',src=src_mac_address,type=0x800)
ip = IP(src ='0.0.0.0',dst='255.255.255.255')
udp =UDP (sport=68,dport=67)
fam,hw = get_if_raw_hwaddr(tap_interface)
bootp = BOOTP(chaddr = hw, ciaddr = '0.0.0.0',xid = 0x01020304,flags= 1)
dhcp = DHCP(options=[("message-type","discover"),"end"])
packet = ethernet / ip / udp / bootp / dhcp
fd = open('/dev/net/tun','r+')
TUNSETIFF = 0x400454ca
IFF_TAP = 0x0002
IFF_NO_PI = 0x1000
mode = IFF_TAP | IFF_NO_PI
ifr = struct.pack('16sH', tap_interface, IFF_TAP | IFF_NO_PI)
fcntl.ioctl(fd,TUNSETIFF,ifr)
while True:
sendp(packet, iface = tap_interface)
time.sleep(10)
if __name__ == '__main__':
main()
Is there any other ways of achieving this? If so please do mention them as well.
Thanks in Advance.

Solved ! I had the same problem,
The problem I think was on the srp() function, it can't receive packets on port 68, but I've created a new function with a new thread that sniffs BOOTP messages and displays the packet fields.
you can simulate it :
sniff(iface=myiface, filter="port 68 and port 67")
then send the packet using srp() or sendp() func :)
NOTE:
I have used multithreading mechanism cause my program sends messages and sniffs if a rogue DHCP Server is on the network

I am not sure if this would qualify as an answer, but we use scapy to simulate DHCP server/client exchange, and the following does the job for us:
discover = Ether(dst='ff:ff:ff:ff:ff:ff', src=cliMAC, type=0x0800) / IP(src='0.0.0.0', dst='255.255.255.255') / UDP(dport=67,sport=68) / BOOTP(op=1, chaddr=cliMACchaddr) / DHCP(options=[('message-type','discover'), ('end')])
The main difference between my code and yours seem to be how the BOOTP header is defined. Maybe you could try my packet definition and see if it works?

Here is an example that I did that gets a dhcp address and assigns it to an ip interface:
My rough POC, while creating code for my project:
#!/usr/bin/python
from scapy.all import Ether,IP,UDP,DHCP,BOOTP,get_if_raw_hwaddr,get_if_hwaddr,conf,sniff,sendp
from pyroute2 import IPDB
from Queue import Empty
from multiprocessing import Process, Queue, Manager
from wpa_supplicant.core import WpaSupplicantDriver
from twisted.internet.selectreactor import SelectReactor
import threading
import time
import errno
import sys
import types
import netifaces
import dbus
import json
import re
class PythonDHCPScanner:
def change_ip(self,ipObject,netInterface):
ipdb = IPDB()
ips= ipdb.interfaces[self.get_interface(netInterface)]
ipAddrs = ips.ipaddr.ipv4[0]
ips.del_ip(ipAddrs['address'],ipAddrs['prefixlen'])
ips.add_ip(ipObject['ipAddr'],24)
ipdb.commit()
ipdb.routes.add(dst="default",gateway=ipObject['router'])
ipdb.commit()
def queue_get_all(self):
items = []
maxItems = 50
for numOfItemsRetrieved in range(0, maxItems):
try:
items.append(self.q.get_nowait())
except Empty, e:
break
return items
def __init__(self):
self.net_iface = netifaces.interfaces()
def dhcp_print(self,pkt):
self.q.put(str(pkt))
def get_interface(self,number):
return str(self.net_iface[number].decode())
def get_interfaces(self):
return self.net_iface
def get_dhcp_object(self,interfaceNumber):
self.q = Manager().Queue()
c = Process(target=self.callSniffer,args=(interfaceNumber,)).start()
time.sleep(0.1)
p = Process(target=self.callPacket(interfaceNumber)).start()
time.sleep(5)
if c is not None:
c.join()
dhcp = {}
for strPkt in self.queue_get_all():
try:
pkt = Ether(strPkt)
pkt.show()
if pkt[Ether].dst == get_if_hwaddr(self.get_interface(interfaceNumber)):
if pkt[DHCP]:
if pkt.getlayer(DHCP).fields['options'][0][1] == 2:
if pkt[IP]:
dhcp['ipAddr'] = pkt[IP].dst
for option in pkt.getlayer(DHCP).fields['options']:
if option == 'end':
break
dhcp[option[0]] = option[1]
print dhcp['router']
print dhcp['subnet_mask']
break
except:
continue
return dhcp
def callSniffer(self,interfaceNumber):
inter = self.get_interface(interfaceNumber)
conf.iface = inter
print inter
sniff(iface=inter,filter="udp",prn=self.dhcp_print, timeout=10)
def callPacket(self,interfaceNumber):
inter = self.get_interface(interfaceNumber)
print inter
fam,hw = get_if_raw_hwaddr(inter)
macaddress= get_if_hwaddr(inter)
conf.iface = inter
ethernet = Ether(dst="ff:ff:ff:ff:ff:ff",src=macaddress,type=0x800)
ip = IP(src="0.0.0.0",dst="255.255.255.255")
udp = UDP(sport=68,dport=67)
bootp = BOOTP(chaddr =hw,xid=0x10000000)
dhcp = DHCP(options=[("message-type","discover"),("end")])
packet=ethernet/ip/udp/bootp/dhcp
sendp(packet,iface=inter)
# get dhcp object
dave = PythonDHCPScanner()
dhcpObject = dave.get_dhcp_object(3)
# Pick interface number 3 on my box
time.sleep(1)
for dhcpKey in dhcpObject.keys():
print str(dhcpKey) + ":" + str(dhcpObject[dhcpKey])
time.sleep(1)
dave.change_ip(dhcpObject,3)

telnet Connection closed when I use yield in diesel

hi I want to create chat server with diesel . I use this code for running simple chat server :
from diesel import Application, Service, until_eol, fire, wait
def chat_server(addr):
my_nick = (yield until_eol()).strip()
while True:
my_message, other_message = yield (until_eol(), wait('chat_message'))
if my_message:
yield fire('chat_message', (my_nick, my_message.strip()))
else:
nick, message = other_message
yield "<%s> %s\r\n" % (nick, message)
app = Application()
app.add_service(Service(chat_server, 8000))
app.run()
but when I try to telnet this server , telnet Connection closed by foreign host.
[nima#ca005 Desktop]$ telnet localhost 8000
Trying 127.0.0.1...
Connected to localhost.
Escape character is '^]'.
Connection closed by foreign host.
when I remove yield from code I have no problem with connectiong to server.
def chat_server(addr):
my_nick = until_eol().strip()
while True:
message = diesel.until_eol()
shouted_message = my_nick + ":" + message
diesel.send(shouted_message)
my_message = until_eol()
other_message = wait('chat_message')
if my_message:
fire('chat_message', (my_nick, my_message.strip()))
what is wrong with this code?

Maybe something like this. It mightn't work straight away as I am unfamiliar with how diesel works.
import StringIO
import socket
import threading
from diesel import Application, Service, until_eol, fire, wait
class socket_thread(threading.Thread):
def __init__(self, line_filter = None):
threading.Thread.__init__(self)
self.daemon = True
self.lock = threading.Lock()
self.event = threading.Event()
self.event.clear()
self.buffer = StringIO.StringIO()
if(line_filter == None):
self.line_filter = lambda x: x
else:
self.line_filter = line_filter
def run(self):
message = True
while message:
message = diesel.until_eol()
self.lock.acquire()
self.buffer.write(message)
self.lock.release()
self.event.set()
def readlines(self):
self.lock.acquire()
self.buffer.seek(0)
raw_lines = self.buffer.readlines()
self.buffer.truncate(0)
self.lock.release()
lines = map(self.line_filter, raw_lines)
return lines
def chat_server(addr):
server_recv = socket_thread()
my_nick = until_eol().strip()
data = []
server_recv.start()
while True:
server_recv.event.wait()
data = server_recv.readlines()
if(data):
shouted_message = my_nick + ":" + data
diesel.send(shouted_message)
server_recv.event.clear()
my_message = until_eol()
other_message = wait('chat_message')
if my_message:
fire('chat_message', (my_nick, my_message.strip()))
app = Application()
app.add_service(Service(chat_server, 8000))
app.run()

I found the right code here :
# vim:ts=4:sw=4:expandtab
'''Simple chat server.
telnet, type your name, hit enter, then chat. Invite
a friend to do the same.
'''
from diesel import Application, Service, until_eol, fire, first, send
def chat_server(addr):
my_nick = until_eol().strip()
while True:
evt, data = first(until_eol=True, waits=['chat_message'])
if evt == 'until_eol':
fire('chat_message', (my_nick, data.strip()))
else:
nick, message = data
send("<%s> %s\r\n" % (nick, message))
app = Application()
app.add_service(Service(chat_server, 8000))
app.run()

Passing from one socket to another

I am trying to write a program that works as an intermedium. (M)
I can only use telnet to connect :
A needs to connect to M, B connects to M.
A sends data to M on a socket, M needs to pass it to B
B sends data to M on another socket
I tried this by starting four threads with a shared list
The problem is it seems it is not writing to the other socket, or even accepting writing.
Does anyone know a better way to implement this and pass it through to another socket
My code :
import sys
import arduinoReadThread
import arduinoWriteThread
import socket
class ControllerClass(object):
'''
classdocs
'''
bolt = 0
socketArray=list()
def __init__(self):
self.readAndParseArgv()
self.createThreads()
def readAndParseArgv(self):
array = sys.argv
print sys.argv
if len(array) != 3:
print "Too few arguments : ./script host:port host:port"
else:
for line in array:
if ":" in line:
splitted = line.split(':')
HOST = splitted[0]
print HOST
PORT = int(splitted[1])
print PORT
s=socket.socket(socket.AF_INET, socket.SOCK_STREAM ) #create an INET, STREAMing socket
s.bind((HOST,PORT)) #bind to that port
print "test"
s.listen(1) #listen for user input and accept 1 connection at a time.
self.socketArray.append(s)
def createThreads(self):
print "Creating Threads"
sharedArray1 = list()
sharedArray2 = list()
s1 = self.socketArray.pop()
s2 = self.socketArray.pop()
sT1 = arduinoWriteThread.writeThread().run(self.bolt,sharedArray1,s2)
sT2 = arduinoReadThread.readThread().run(self.bolt,sharedArray1,s1)
sT3 = arduinoReadThread.readThread().run(self.bolt,sharedArray2,s2)
sT4 = arduinoWriteThread.writeThread().run(self.bolt,sharedArray2,s1)
sT1.start()
sT2.start()
sT3.start()
sT4.start()
x = ControllerClass()
x
Two Threads :
Write Thread :
import threading
class writeThread ( threading.Thread ):
def run ( self,bolt,writeList,sockeToWriteTo ):
s = sockeToWriteTo
while(bolt == 0):
conn, addr = s.accept()
if len(writeList) > 0:
socket.send(writeList.pop(0))
Read Thread
import threading
class readThread ( threading.Thread ):
def run ( self,bolt,writeList,socketToReadFrom ):
s = socketToReadFrom
while(bolt == 0):
conn, addr = s.accept()
f = conn.rcv()
print f
writeList.append(f)

You don't really need threads for this...
When a new connection is accepted, add it to a list. When receiving anything from one of the connection in the list, send to all connections except the one you got the message from.
Use select to see which connections have send data to you.
Edit
Example using select:
# serversocket: One server socket listening on some port, has to be non-blocking
# all_sockets : List containing all connected client sockets
while True:
readset = [serversocket]
readset += all_sockets
# Wait for sockets to be ready, with a 0.1 second timeout
read_ready = select.select(readset, None, None, 0.1)
# If the listening socket can be read, it means it has a new connection
if serversocket in read_ready:
new_connection = serversocket.accept()
new_connection.setblocking(0); # Make socket non-blocking
all_sockets += [new_connection]
read_ready.remove(serversocket) # To not loop over it below
for socket in read_ready:
# Read data from socket
data = socket.recv(2048)
for s in all_sockets:
# Do not send to self
if s != socket:
s.send(data)
Disclaimer I have never really used the Python socket functions, the code above was made from reading the manual pages just now. The code is probably not optimal or very Pythonic either.