We Keep Coding

multi threading http server in python

I am trying to build a HTTP server in python,
that sniffs packets and sends them to an other interface.
the server can get routing paths through a POST http request.
So that I need that the server will parallely sniff pakets and listen to http requests.
this is my code:
from scapy.all import *
from scapy.layers.inet import IP, UDP
from http.server import BaseHTTPRequestHandler, HTTPServer
import json
from socketserver import ThreadingMixIn
import threading
ROUTING_LIST = []
INTERFACE_TO_SNIFF = 'vEthernet'
PORT = 80
class Route:
def __init__(self):
self.first_IP_src = ""
self.first_port_src = ""
self.first_IP_dst = ""
self.first_port_dst = ""
self.first_iface = ""
self.second_IP_src = ""
self.second_port_src = ""
self.second_IP_dst = ""
self.second_port_dst = ""
self.second_iface = ""
class Server(BaseHTTPRequestHandler):
# POST echoes the message adding a JSON field
def do_POST(self):
# read the message and convert it into a python dictionary
length = int(self.headers['Content-length'])
message = self.rfile.read(length)
routing_dict = json.loads(message, strict=False)
if add_routing_http(routing_dict) is True:
print("New Routing received:")
print("{" + "\n".join("{!r}: {!r},".format(k, v) for k, v in routing_dict.items()) + "}")
self.send_response(200)
self.send_header('Content-type', 'text/html')
self.end_headers()
self.wfile.write(
("POST routing request received! now we have " + str(len(ROUTING_LIST)) + " routes").encode("utf8"))
def run_server():
global PORT
server_address = ('', PORT)
httpd = HTTPServer(server_address, Server)
print('Starting httpd on port %d...' % PORT)
httpd.serve_forever()
def add_routing_local(first_IP_src, first_port_src, first_IP_dst, first_port_dst, first_iface,
second_IP_src, second_port_src, second_IP_dst, second_port_dst, second_iface):
global ROUTING_LIST
temp = Route()
temp.first_IP_src = first_IP_src
temp.first_port_src = first_port_src
temp.first_IP_dst = first_IP_dst
temp.first_port_dst = first_port_dst
temp.first_iface = first_iface
temp.second_IP_src = second_IP_src
temp.second_port_src = second_port_src
temp.second_IP_dst = second_IP_dst
temp.second_port_dst = second_port_dst
temp.second_iface = second_iface
ROUTING_LIST.append(temp)
def add_routing_http(routing_dict):
global ROUTING_LIST
temp = Route()
temp.first_IP_src = routing_dict.get('firstIpSrc')
temp.first_port_src = routing_dict.get('firstPortSrc')
temp.first_IP_dst = routing_dict.get('firstIpDst')
temp.first_port_dst = routing_dict.get('firstPortDst')
temp.first_iface = routing_dict.get('firstIface')
temp.second_IP_src = routing_dict.get('secondIpSrc')
temp.second_port_src = routing_dict.get('secondPortSrc')
temp.second_IP_dst = routing_dict.get('secondIpDst')
temp.second_port_dst = routing_dict.get('secondPortDst')
temp.second_iface = routing_dict.get('secondIface')
ROUTING_LIST.append(temp)
return True
def packets_filter(packet):
return IP in packet and UDP in packet and Raw in packet
def match_packet(packet, routing):
match = True
if routing.first_IP_src != '' and packet[IP].src != routing.first_IP_src:
return False
if routing.first_IP_dst != '' and packet[IP].dst != routing.first_IP_dst:
return False
if routing.first_port_src != '' and packet[UDP].sport != routing.first_port_src:
return False
if routing.first_port_dst != '' and packet[UDP].dport != routing.first_port_dst:
return False
if routing.first_iface != '' and packet.sniffed_on is not None and routing.first_iface != packet.sniffed_on:
return False
return True
def handle_packet(packet):
global ROUTING_LIST
for routing in ROUTING_LIST:
if match_packet(packet, routing) is True:
new_packet = packet.copy()
new_packet[IP].src = routing.second_IP_src
new_packet[IP].dst = routing.second_IP_dst
new_packet[UDP].sport = routing.second_port_src
new_packet[UDP].dport = routing.second_port_dst
new_packet.show()
sendp(new_packet) # sendp(new_packet, iface=routing.second_iface)iface='eth0'
return
def main():
daemon = threading.Thread(name='daemon_server', target=run_server, args=())
daemon.setDaemon(True) # Set as a daemon so it will be killed once the main thread is dead.
daemon.start()
print("start sniffing")
sniff(lfilter=packets_filter, prn=handle_packet) # sniff(lfilter=packets_filter, prn=handle_packet, iface=INTERFACE_TO_SNIFF)
if __name__ == "__main__":
main()
In short - I wantthe main function to run in parallel both of functions: run_server, sniff. if I try to run inly one of them - both work great.
In this code only the run_server works but not the sniffing.
What is wrong?
thank you

You have created Thread only for the run_server method. In order to run the sniff function on multithreaded, you will have to create a thread for the sniff function too.
You can learn about basic multithreading from this document:
https://www.geeksforgeeks.org/multithreading-python-set-1/

Why am I getting a timeout error while trying to send TCP packets over the internet?

I wanted to demonstrate Asynchronous Non-Blocking threads through an EventLoop using a chatroom that I coded out in Python.
The chatroom is working fine when I simulate a server and clients on my desktop, but whenever I want to send packets over the internet to my friends who stay at a geographically distant location, I receive Timeout errors.
Obviously, I change the IP_ADDR accordingly while doing so. In fact, I have tried both IPv4 and IPv6. The firewall is off, and there are no anti-viruses installed.
I tried setting the timeout options as well, but the problem still exists.
Over a small geographical distance, the connection works.
I have also checked whether I am even able to send packets over the Internet at all to the target computer using the tracert command, and it seems like I can.
Server.py
import sys
import select
import msvcrt
import socket
IP_ADDR = socket.gethostbyname(socket.gethostname())
PORT = 5555
HEADER_SIZE = 10
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind((IP_ADDR, PORT))
def add_header(username, msg):
username = f'{len(username) :< {HEADER_SIZE}}' + username
msg_len = len(msg)
msg = username + f'{msg_len :< {HEADER_SIZE}}' + msg
return msg.encode("utf-8")
sock_list = [server]
sock_dict = {server : 'Server'}
def broadcast_message(client, broadcast_msg):
try: #EAFP
client.send(broadcast_msg)
except:
username = sock_dict[client]
del sock_dict[client]
sock_list.remove(client)
broadcast_msg = add_header(sock_dict[server], f"{username} has left the group!!")
for clients in sock_list:
if clients is server:
print(f"{username} has left the group!!")
else:
broadcast_message(clients, broadcast_msg)
server.listen()
while True:
readers, _, err_sockets = select.select(sock_list, [], [], 1)
if(msvcrt.kbhit()):
msg = input("[:] >> ")
#msg = sys.stdin.readline()[:-1]
msg = add_header(sock_dict[server], msg)
for client in sock_list:
if client is server:
continue
else:
broadcast_message(client, msg)
for reader in readers:
if reader is server:
client_socc, client_addr = server.accept()
try:
client_username = client_socc.recv(1024).decode("utf-8")
if not len(client_username):
continue
else:
print(f"Connection accepted from {client_username[HEADER_SIZE : ].title()} : {client_addr[0]} : {client_addr[1]}")
sock_dict[client_socc] = client_username[HEADER_SIZE : ].title()
sock_list.append(client_socc)
broadcast_msg = add_header(sock_dict[server], f"{sock_dict[client_socc]} has joined the group!!")
for client in sock_list:
if client is server or client is client_socc:
continue
else:
broadcast_message(client, broadcast_msg)
except:
continue
else:
try:
client_msg = reader.recv(1024).decode("utf-8")
if not len(client_msg):
del sock_dict[reader]
sock_list.remove(reader)
else:
while len(client_msg):
broadcast_msg = add_header(sock_dict[reader], client_msg[HEADER_SIZE : HEADER_SIZE + int(client_msg[:HEADER_SIZE])])
print(f"{sock_dict[reader]} >> {client_msg[HEADER_SIZE : HEADER_SIZE + int(client_msg[:HEADER_SIZE])]}")
client_msg = client_msg[HEADER_SIZE + int(client_msg[:HEADER_SIZE]) : ]
for client in sock_list:
if client is server or client is reader:
continue
else:
broadcast_message(client, broadcast_msg)
except:
continue
Client.py
import sys
import select
import socket
import msvcrt
IP_ADDR = socket.gethostbyname(socket.gethostname())
PORT = 5555
HEADER_SIZE = 10
class Connection():
def __init__(self, default = (IP_ADDR, PORT)):
self.client_conn = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
print(f"Trying to connect to {default[0]} : {default[1]}")
self.client_conn.connect(default)
print("Connection succesful!")
username = input("Enter your username : ")
username = f'{len(username) :< {HEADER_SIZE}}' + username
self.client_conn.send(username.encode("utf-8"))
def fileno(self):
return self.client_conn.fileno()
def on_read(self):
msg = self.client_conn.recv(1024).decode("utf-8")
self.decode_message(msg)
def decode_message(self, msg):
while len(msg):
username = msg[HEADER_SIZE : HEADER_SIZE + int(msg[: HEADER_SIZE])]
msg = msg[HEADER_SIZE + int(msg[: HEADER_SIZE]) : ]
user_msg = msg[HEADER_SIZE : HEADER_SIZE + int(msg[: HEADER_SIZE])]
msg = msg[HEADER_SIZE + int(msg[: HEADER_SIZE]) : ]
print(f"{username} >> {user_msg}")
class Input():
def __init__(self, client):
self.client = client.client_conn
def fileno(self):
return sys.stdin.fileno()
def on_read(self):
#msg = sys.stdin.readline()[:-1]
msg = input("[:] >> ")
msg_len = len(msg)
msg = f'{msg_len :< {HEADER_SIZE}}' + msg
self.client.send(msg.encode("utf-8"))
connection = Connection()
read_input = Input(connection)
while True:
readers, _, _ = select.select([connection], [], [], 1)
if(msvcrt.kbhit()):
readers.append(read_input)
for reader in readers:
reader.on_read()

How to get the modbus_tk master's IP?

I write a slave.py(server) base on modbus_tk(https://github.com/ljean/modbus-tk).
Then,I use the client tools to connect the slave，It's OK.
And now,I want to get which master(client) access to me,I want to get source IP and port.
I find the key point 'client, address = self._sock.accept()' in function _do_run(self) in Class TcpServer(server) ,modbus_tcp.py.
I tried some methods, but it didn't work.
slave.py
import modbus_tk
import modbus_tk.modbus_tcp as modbus_tcp
import modbus_tk.defines as mdef
import threading
import sys
from lxml import etree
logger = modbus_tk.utils.create_logger(name='console', record_format='%(message)s')
server = modbus_tcp.TcpServer(port=502,address='0.0.0.0')
class Modbus_server(object):
def main(self):
try:
logger.info("running...")
logger.info("enter 'quit' for closing the server")
server.start()
self.config_slaves()
while True:
cmd = sys.stdin.readline()
if cmd.find('quit')==0:
#when input"quit",the program exit!
sys.stdout.write('bye-bye\r\n')
sys.exit(0)
finally:
server.stop()
#get slave configuration
def config_slaves(self):
dom = etree.parse('modbus.xml')
slaves = dom.xpath('//modbus/slaves/*')
try:
for s in slaves:
slave_id = int(s.attrib['id'])
slave = server.add_slave(slave_id)
logger.debug('Added slave with id %s.', slave_id)
for b in s.xpath('./blocks/*'):
name = b.attrib['name']
request_type = eval('mdef.' + b.xpath('./type/text()')[0])
start_addr = int(b.xpath('./starting_address/text()')[0])
size = int(b.xpath('./size/text()')[0])
slave.add_block(name, request_type, start_addr, size)
logger.debug(
'Added block %s to slave %s. '
'(type=%s, start=%s, size=%s)',
name, slave_id, request_type, start_addr, size)
logger.info('modbus initialized')
except (Exception) as e:
logger.info(e)
modbus=Modbus_server()
modbus.main()`
modbus_tcp.py
''''''
......
def _do_run(self):
"""called in a almost-for-ever loop by the server"""
#check the status of every socket
inputready = select.select(self._sockets, [], [], 1.0)[0]
#handle data on each a socket
for sock in inputready:
try:
if sock == self._sock:
# handle the server socket
client, address = self._sock.accept()
client.setblocking(0)
LOGGER.info("%s is connected with socket %d...", str(address), client.fileno())
self._sockets.append(client)
call_hooks("modbus_tcp.TcpServer.on_connect", (self, client, address))
''''''
''''''

Thread a server to accept more than one client doesn't work properly

Well, I'm trying to make a simple Network TCP chatting program to dive deeper in python, threading and networking. The program worked but with just one user, I looked this up, I found that I need threading to make the server accept more than one user. I threaded the server but now when you connect the second user it disconnect the first one. Source code may not be that good..
#!/usr/bin/python
import socket, sys, threading
from time import sleep
# Global Stuff
localhost = socket.gethostbyname(socket.gethostname())
#localhost = '192.168.x.x'
serverPort = 5003
buffer = 1024 #Bytes
backlog = 5
userThread= []
count = 0
class server(object):
''' Constructor to Establish Bind server once an object made'''
def __init__(self, localhost, serverPort): # Connect Tcp
global backlog, count
self.servSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.servSock.bind((localhost, serverPort))# bind((host,Port))
self.servSock.listen(backlog)
print count
except Exception, e:
print "[Bind ]", e
sys.exit()
def accept(self):
global userThread, conn, addr, count
"""
- PROBLEM IS IN HERE SOMEWHERE SERVER DOESN'T ADD THE OTHER CLIENT EXCEPT ONCE.
- THREAD DOEN'T KEEP THE CLIENT.
- THE SECOND CLIENT FREEZES WHILE SENDING THE VERY FIRST MESSAGE TILL THE FIRST
CLIENT SEND A MESSAGE THEN IT CAN SEND MESSAGES AND THE FIRST CLIENT CAN'T SEND SHIT.
"""
count+=1
while True:
print count
self.conn, self.addr = self.servSock.accept()
conn = self.conn
print("This is a connection: ", conn)
#acceptThread = threading.start_new_thread(target=serverObj.accept, args=(conn))
#addr = self.addr
print "[Listening..]"
if(self.addr not in userThread):
userThread.append(self.addr)
print "Client's added Successfully"
else:
pass
def redirect(self):
global buffer, userThread, conn, count
count+=1
while True:
try:
print "Redirecting " + str(count)
self.data = conn.recv(buffer)
if self.data:
for user in userThread:
#conn.send(b'Recieved by server!\n')
conn.sendto("Sent!\n"+self.data+"\n", user)
print "Server: Data sent[" +self.data+"] to ["+str(user)+"]"
else:
self.data = conn.recv(buffer)
print "No dataa found"
except Exception, e:
print "[Redirect ] ",e
sleep(7)
print "OUT"# testing if it's getting out this infinite loop.
def exit(self):
self.server.close()
def main():
global localhost, serverPort, conn
try:
serverObj = server(localhost, serverPort)
print("[+] Server is UP!")
except Exception, e:
print "[Main ] ",e
exit()
acceptThread = threading.Thread(name = "Accepting Connections", target=serverObj.accept)
redirThread = threading.Thread(name = "Redirecting Data", target=serverObj.redirect)
acceptThread.start()
redirThread.start()
print userThread
main()
######################################### Client ##########################################
#!/usr/bin/python
# This is client file
"""
http://eli.thegreenplace.net/2011/05/18/code-sample-socket-client-thread-in-python
https://docs.python.org/2/library/threading.html
"""
import socket
import threading
from time import sleep
# Client Info
#clientSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
#localhost = '192.168.x.x'
# Global Stuff
serverIP = socket.gethostbyname(socket.gethostname())
#serverIP = '192.168.x.x'
serverPort, MsgSendError, MsgSendSucc, clientPort, data, buffer =\
5003, False, True, 12345, '',1024 #Bytes
class client(object):
global MsgSendError, MsgSendSucc, buffer, data
''' Constructor to Establish Connection once client is up'''
def __init__(self, serverIP, serverPort): # Connect Tcp
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.sock.connect((serverIP, serverPort))
except Exception, e:
return "[Connecting to Server]", e
def send(self, data):
try:
self.sock.send(data) # covnert it from string into byte streams to be in proper format.
#print str(data)
return MsgSendSucc
except:
return MsgSendError
def receive2(self):
try:
data = self.sock.recv(buffer)
#print "Function: Receive2."# testing
#print(str(data))# testing
#print "Received!"# testing
return str(data)
except Exception, e:
return "[In receive2]", e
def main():
global serverIP, serverPort, data#, sock
clientObj = client(serverIP, serverPort)
alias = raw_input("Your Name USER! ")
sentData = ''
while sentData is not 'Quit':
sentData = raw_input("Data>> ")
data = alias + ": "+sentData
if clientObj.send(data) == MsgSendSucc:
#print "Sent!"
#print "Fetching..\n"# testing
print(clientObj.receive2())
# testing
main()

Connecting admin client only recieves once

When I run the admin client after connecting some clients, my admin returns the ip addresses and port numbers fine. If i close the admin and rerun it nothing happens. This has me baffled. I am unsure why it is doing this
#Admin Client
from functools import partial
import ssl
import socket
s = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
ts = ssl.wrap_socket(s, certfile="100298750.crt",
keyfile="100298750.key",
ca_certs="5cc515-root-ca.cer")
ts.connect(('192.168.0.5', 4001))
ts.send("Hello\r\n".encode())
if ts.recv(80).decode() == "Admin-Greetings\r\n":
print("The players currently online are:\n")
ts.send("Who\r\n".encode())
for data in iter(partial(ts.recv, 1000), b''):
print(data.decode())
ts.close()
Server
import threading
import socket
import math
import random
import ssl
addressList = []
def within(guess,goal,n):
absValue = abs(guess - goal)
if absValue <= n:
return True
else:
return False
def HandleAdmin(adminSocket,):
while True:
global addressList
(c,a) = adminSocket.accept()
ts = ssl.wrap_socket(c, certfile="5cc515_server.crt",
keyfile="5cc515_server.key",
server_side=True,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs="5cc515-root-ca.cer")
if ts.recv(80).decode() == 'Hello\r\n':
ts.send('Admin-Greetings\r\n'.encode())
if ts.recv(80).decode() == 'Who\r\n':
for i in addressList:
ts.send(i.encode())
ts.close()
return
def HandleClient(c,a):
global addressList
address, port = a
address = str(address) + ' ' + str(port) + '\r\n'
addressList.append(address)
scoreCount = 0
guess = 0
if(c.recv(80).decode()) == 'Hello\r\n':
c.send('Greetings\r\n'.encode())
goal = random.randrange(1,21)
while guess!= goal:
guess =c.recv(80).decode()
guess = int(guess[7:len(guess)-2])
if guess == goal:
c.send('Correct\r\n'.encode())
addressList.remove(address)
c.close()
elif within(guess, goal, 2) == True:
c.send('Close\r\n'.encode())
else:
c.send('Far\r\n'.encode())
else:
c.close()
return
clientSocket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
clientSocket.bind(("192.168.0.5",4000))
clientSocket.listen(5)
adminSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
adminSocket.bind(("192.168.0.5",4001))
adminSocket.listen(5)
handleAdminThread = threading.Thread(target = HandleAdmin,
args = (adminSocket,))
handleAdminThread.start()
while True:
(c,a) = clientSocket.accept()
clientThread = threading.Thread(target = HandleClient, args = (c,a))
clientThread.start()

If i close the admin and rerun it nothing happens. This has me
baffled. I am unsure why it is doing this
It is simply doing this because the HandleAdmin() server thread code has the line
return
at the end of its loop and thus exits after one run. Drop it, and it is alright.