We Keep Coding

wanted to improve python socket processing

The requirement is to receive all traffic from port1(p1p1) and then encapsulate those packets to udp port xxx and send it out on port2 on 5 different source ips.
import socket
import struct
import sys
import time
import re
import random
from collections import namedtuple
def get_socket_connection():
global sw_list
udp_port = int('xxx')
for i in range(0, 5):
IP = '10.10.1.%d'%(1+i)
try:
#socket connections to send out the encapsulated traffic
globals()[f"send_sock_{i}"] = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
globals()[f"send_sock_{i}"].bind((IP, 0))
except socket.error as msg:
print('Socket could not be created for IP %s. Error Code : %s ' %(IP, str(msg)))
sys.exit()
try:
#socket connection to receive raw/native traffic
s = socket.socket(socket.AF_PACKET, socket.SOCK_RAW, socket.ntohs(0x0003))
except socket.error as msg:
print('Socket could not be created to read the traffic. Error Code : %s ' %(str(msg)))
sys.exit()
# receive a packet
while True:
packet = s.recvfrom(65565)
#receive traffic ONLY from port p1p1 otherwise DO NOT PROCESS
if packet[1][0] != 'p1p1':
continue
packet = packet[0]
num = random.randint(0, 4)
globals()[f"send_sock_{num}"].sendto(header_get(sw_list[num].port, sw_list[num].dpid, packet), (str('1.1.1.1'), int(udp_port)) )
def mac_to_int(mac):
res = re.match('^((?:(?:[0-9a-f]{2}):){5}[0-9a-f]{2})$', mac.lower())
if res is None:
raise ValueError('invalid mac address')
#print(int(res.group(0).replace(':', ''), 16))
return int(res.group(0).replace(':', ''), 16)
def header_get(port, dpid, data_packet):
global iteration
pkt=struct.pack("!2I2Q", 2, port, dpid, round(time.clock_gettime(time.CLOCK_REALTIME)))
iteration += 1
print(iteration)
return (pkt+data_packet)
def main():
global proto_dic, iteration, sw_list
iteration = int(0)
sw = namedtuple("sw", ["num", "sw_name", "dpid", "port"])
sw_list = []
count = int(0)
sw_list.append(sw(count+1, "s1", mac_to_int('3c:2c:30:5c:61:80'), int(1)))
sw_list.append(sw(count+2, "s2", mac_to_int('98:5d:82:c1:a0:80'), int(1)))
sw_list.append(sw(count+3, "s3", mac_to_int('20:04:0f:2c:cf:42'), int(1)))
sw_list.append(sw(count+4, "s4", mac_to_int('e4:f0:04:32:94:1a'), int(1)))
sw_list.append(sw(count+5, "s5", mac_to_int('3c:2c:99:69:2e:2c'), int(1)))
sw_list.append(sw(count+6, "s5", mac_to_int('3c:2c:99:69:2e:2c'), int(2)))
get_socket_connection()
if __name__ == '__main__':
main()
As such the above code is working fine however, it is unable to process more than 20k pps guaranteed rate. I'm looking for atleast 75K guaranteed rate so I would like to know what optimizations can to be done to get such performances? This server has 4 CPU core and 16G ram.
Appreciate any help.

Black Hat Python proxy tool no data

I recently bought the book Black Hat Python, 2nd Edition, by Justin Seitz, which seems to be a very good book about networking and all that (i am writing my code on Kali Linux)
I have a problem on the TCP Proxy Tool on chapter 2 :
Here is the code :
import sys
import socket
import threading
HEX_FILTER = ''.join(
[(len(repr(chr(i))) == 3) and chr(i) or '.' for i in range(256)])
def hexdump(src, length = 16, show = True):
# basically translates hexadecimal characters to readable ones
if isinstance(src, bytes):
src = src.decode()
results = list()
for i in range(0, len(src), length):
word = str(src[i:i+length])
printable = word.translate(HEX_FILTER)
hexa = ' '.join(['{ord(c):02X}' for c in word])
hexwidth = length*3
results.append('{i:04x} {hexa:<{hexwidth}} {printable}')
if show :
for line in results :
print(line)
else :
return results
def receive_from(connection):
buffer = b""
connection.settimeout(10)
try :
while True :
data = connection.recvfrom(4096)
if not data :
break
buffer += data
except Exception as e:
pass
return buffer
def request_handler(buffer):
# perform packet modifications
return buffer
def response_handler(buffer):
# perform packet modifications
return buffer
def proxy_handler(client_socket, remote_host, remote_port, receive_first):
remote_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
remote_socket.connect((remote_host, remote_port))
if receive_first :
# Check for any data to receive before
going into the main loop (i guess)
remote_buffer = receive_from(remote_socket)
hexdump(remote_buffer)
remote_buffer = response_handler(remote_buffer)
if len(remote_buffer):
print("[<==] Sending %d bytes to localhost." % len(remote_buffer))
client_socket.send(remote_buffer)
while True : # Start the loop
local_buffer = receive_from(client_socket)
if len(local_buffer):
line = "[==>] Received %d bytes from localhost." % len(local_buffer)
print(line)
hexdump(local_buffer)
local_buffer = request_handler(local_buffer)
remote_socket.send(local_buffer)
print("[==>] Sent to remote.")
remote_buffer = receive_from(remote_socket)
if len(remote_buffer):
print("[==>] Received %d bytes from remote." % len(remote_buffer))
hexdump(remote_buffer)
remote_buffer=response_handler(remote_buffer)
client_socket.send(remote_buffer)
print("[<==] Sent to localhost.")
if not len(local_buffer) or not len(remote_buffer):
# If no data is passed, close the sockets and breaks the loop
client_socket.close()
remote_socket.close()
print("[*] No more data. Closing connections. See you later !")
break
def server_loop(local_host, local_port, remote_host, remote_port, receive_first):
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try :
server.bind((local_host, local_port)) # Bind the local host and the local port
except Exception as e:
print('Problem on bind : %r' %e)
# If an error occurs, prints a
print("[!] Failed to listen on %s:%d" % (local_host, local_port))
print("[!] Check for other listening sockets or correct permissions.")
sys.exit(0)
print("[*] Listening on %s:%d" % (local_host, local_port))
server.listen(5)
while True :
client_socket, addr = server.accept()
# print out the local connection information
line = "> Received incoming connection from %s:%d" % (addr[0], addr[1])
print(line)
# start a thread to talk to the remote host
proxy_thread = threading.Thread(
target = proxy_handler,
args=(client_socket,remote_host,
remote_port, receive_first))
proxy_thread.start()
def main():
if len(sys.argv[1:]) != 5:
print("Usage: ./proxy.py [localhost] [localport]")
print("[remotehost] [remoteport] [receive_first]")
print("Example : ./proxy.py 127.0.0.1 9000 192.168.56.1 9000 True")
sys.exit(0)
loca l_host = sys.argv[1]
local_port = int(sys.argv[2])
remote_host = sys.argv[3]
remote_port = int(sys.argv[4])
receive_first = sys.argv[5]
if "True" in receive_first:
receive_first = True
else :
receive_first = False
server_loop(local_host, local_port,
remote_host, remote_port, receive_first)
if __name__ == '__main__':
main()
(sorry, i had a bit of a trouble formatting it and it's quite long)
Now, normally, i just need to open 2 terminals and run the code with the command line :
sudo python proxy.py 127.0.0.1 21 ftp.dlptest.com 21 True
in one terminal, and :
ftp 127.0.0.1 21
in the other one.
My code seems to be working fine, except that... I receive no data. I tried different ftp servers (notice that i don't use the one quoted in the book), but it still doesn't work. It just says :
[*] Listening on 127.0.0.1
> Received incoming connection from 127.0.0.1:55856
but it doesn't actually displays anything until the connexion times out or that i stop the command with Ctrl + C.
I know this question has already been asked, but they don't resolve my problem.
Please tell me if i forgot a line of code (for example the one that prints the data on the screen lol) or did anything wrong :)

one the hexa variable you need to put and f'{ord(c):02x}' because you just have a string and not using the 'c' variable from the list comprehension. That's a small typo you missed fix that and try the whole process again.
hexa = ' '.join([f'{ord(c):02X}' for c in word])
The f should be here ^

Why is my script telling me that my Minecraft server on port 25565 is unreachable?

My Python script is not working properly. It says that kandicraft.finlaydag33k.nl on port 25565 is down, whilst, it's responding to pings (and I can connect to the game itself)
I know it should be a bug somewhere in the code, but I can't find it as I started python like half an hour ago.
The output that I get is: 24-02-2016 16:05:30] kandicraft.finlaydag33k.nl on port 25565 seems to be unreachable!
I've editted the question as the port 80 with google now works, but the main purpose of this script (pinging minecraft servers) later on doesn't.
the error I get from the exception is an integer is required (so port 25565 doesn't seem to be an integer???)
import os
import RPi.GPIO as gpio
import time
import socket
## set variables for the machine to ping and pin for the LED
hostname = ['kandicraft.finlaydag33k.nl:25565','google.com:80']
led_pin = 37
## prepare
led_status = gpio.LOW
gpio.setmode(gpio.BOARD)
gpio.setup(led_pin, gpio.OUT, gpio.PUD_OFF, led_status)
## PING FUNCTION GALORE!!
def check_ping(host,port):
captive_dns_addr = ""
host_addr = ""
try:
host_addr = socket.gethostbyname(host)
if (captive_dns_addr == host_addr):
return False
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(1)
s.connect((host,port))
s.close()
except:
return False
return True
## Run the script itself infinitely
while True:
host_up = ""
for host in hostname:
if ":" in host:
temphost, tempport = host.split(":")
pingstatus = check_ping(temphost, tempport)
if pingstatus == False:
print('[' + time.strftime("%d-%m-%Y %H:%M:%S") + '] ' + temphost + ' on port ' + tempport + ' seems to be unreachable!')
host_up = "False"
if host_up == "False":
led_status = gpio.HIGH
else:
led_status = gpio.LOW
gpio.output(led_pin,led_status)
time.sleep(1)

I managed to solve all issues that I found by using check_ping(temphost,int(tempport))
thanks all for helping me solve it!

To debug your program just replace
except:
return False
by:
except Exception as exc:
print exc
return False
in check_ping() function

How to keep listening to the connected TCP sockets by "select"

All my clients sockets do the same thing: send a package every second(22 bytes)
Server code as below:
import select
import socket
import datetime
SList = []
class Tserver:
def __init__(self, portNum):
host = '127.0.0.1'
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.server.bind((host, portNum))
self.server.listen(1)
def GETPACK():
# function for CRC check
def CRC(DATA_STR):
return 1
# generate 100 sockets to listen
for x in range(100):
SList.append(Tserver(x+10000))
inputs = []
# put in inputs
for x in range(100):
inputs.append(SList[x].server)
while(True):
ready_socks, _, _ = select.select(inputs, [], [])
for sock in ready_socks:
c, addr = sock.accept()
while(True):
data = c.recv(22)
if len(data) == 22: # To make sure the data-length is 22
# Turn the pack string into bytearray
data_bytes = bytearray()
data_bytes.extend(data)
if CRC(data_bytes) == 1:
print "Connected from client IP Address:" + str(addr)
# ID
ID = 256*data_bytes[1] + data_bytes[2]
print "ID: ", ID
now = datetime.datetime.now()
print "now: ", str(now)
if __name__ == "__main__":
GETPACK()
My server can only print the packages sent by the first connected socket.
And my question is how to print out all message from each ports whenever a package is sent to the server.

See this PyMOTW entry for a detailed explanation of how to use the select module to write a select-based server.
The main differences between that example and your code are:
You just create one listening socket - server. There is no need to listen on multiple ports.
The variable inputs will be a list consisting of server and any other open socket connections to clients.
Your service loop will look like:
while true:
readable, _, _ = select.select(inputs, [], [])
for r in readable:
if r is server:
# handle a new incoming connection
# this will add an entry to the variable inputs
else:
# read some data from socket r and process it
When you attempt to read from a client socket and get an EOF condition, you can close that socket and remove it from the inputs variable.

#ErikR Thanks for your help, i changed my code, and it worked fine.
The reason that my code doesn't work was because of two things:
1.I only create one connection to recv data from my clients.
2.The same connection can't be accepted again for recv, if the clients does't reconnect.(my code doesn't check the exception when clients shutdown)
Code as below:
import select, socket, datetime
SList = []
SconnList = []
class Tserver:
def __init__(self, portNum):
host = '127.0.0.1'
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR,1)
self.server.bind((host,portNum))
self.server.listen(1)
print "Server ports: "+str(portNum)
class Sconn:
def __init__(self, sock):
self.conn, self.addr = sock.accept()
def GETPACK():
# function for CRC check
def CRC(DATA_STR):
return 1
# generate 100 sockets to listen
for x in range(100):
SList.append(Tserver(x+10000))
inputs = []
# put in inputs
for x in range(100):
inputs.append(SList[x].server)
while(True):
ready_socks,_,_ = select.select(inputs, [], [])
for sock in ready_socks:
try:
SconnList.append(Sconn(sock))
SconnList.reverse()
inputs.append(SconnList[0].conn)
except:
data = sock.recv(22)
if len(data) == 22: # To make sure the data-length is 22
#Turn the pack string into bytearray
data_bytes = bytearray()
data_bytes.extend(data)
if CRC(data_bytes) == 1:
print "IP Address:" + str(sock.getsockname())
#ID
ID = 256*data_bytes[1] + data_bytes[2]
print "ID: ",ID
now = datetime.datetime.now()
print "now: ",str(now)
print ""
print ""
if __name__ == "__main__":
GETPACK()

List of IP addresses/hostnames from local network in Python

How can I get a list of the IP addresses or host names from a local network easily in Python?
It would be best if it was multi-platform, but it needs to work on Mac OS X first, then others follow.
Edit: By local I mean all active addresses within a local network, such as 192.168.xxx.xxx.
So, if the IP address of my computer (within the local network) is 192.168.1.1, and I have three other connected computers, I would want it to return the IP addresses 192.168.1.2, 192.168.1.3, 192.168.1.4, and possibly their hostnames.

If by "local" you mean on the same network segment, then you have to perform the following steps:
Determine your own IP address
Determine your own netmask
Determine the network range
Scan all the addresses (except the lowest, which is your network address and the highest, which is your broadcast address).
Use your DNS's reverse lookup to determine the hostname for IP addresses which respond to your scan.
Or you can just let Python execute nmap externally and pipe the results back into your program.

Update: The script is now located on github.
I wrote a small python script, that leverages scapy's arping().

If you know the names of your computers you can use:
import socket
IP1 = socket.gethostbyname(socket.gethostname()) # local IP adress of your computer
IP2 = socket.gethostbyname('name_of_your_computer') # IP adress of remote computer
Otherwise you will have to scan for all the IP addresses that follow the same mask as your local computer (IP1), as stated in another answer.

I have collected the following functionality from some other threads and it works for me in Ubuntu.
import os
import socket
import multiprocessing
import subprocess
def pinger(job_q, results_q):
"""
Do Ping
:param job_q:
:param results_q:
:return:
"""
DEVNULL = open(os.devnull, 'w')
while True:
ip = job_q.get()
if ip is None:
break
try:
subprocess.check_call(['ping', '-c1', ip],
stdout=DEVNULL)
results_q.put(ip)
except:
pass
def get_my_ip():
"""
Find my IP address
:return:
"""
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(("8.8.8.8", 80))
ip = s.getsockname()[0]
s.close()
return ip
def map_network(pool_size=255):
"""
Maps the network
:param pool_size: amount of parallel ping processes
:return: list of valid ip addresses
"""
ip_list = list()
# get my IP and compose a base like 192.168.1.xxx
ip_parts = get_my_ip().split('.')
base_ip = ip_parts[0] + '.' + ip_parts[1] + '.' + ip_parts[2] + '.'
# prepare the jobs queue
jobs = multiprocessing.Queue()
results = multiprocessing.Queue()
pool = [multiprocessing.Process(target=pinger, args=(jobs, results)) for i in range(pool_size)]
for p in pool:
p.start()
# cue hte ping processes
for i in range(1, 255):
jobs.put(base_ip + '{0}'.format(i))
for p in pool:
jobs.put(None)
for p in pool:
p.join()
# collect he results
while not results.empty():
ip = results.get()
ip_list.append(ip)
return ip_list
if __name__ == '__main__':
print('Mapping...')
lst = map_network()
print(lst)

For OSX (and Linux), a simple solution is to use either os.popen or os.system and run the arp -a command.
For example:
import os
devices = []
for device in os.popen('arp -a'): devices.append(device)
This will give you a list of the devices on your local network.

I found this network scanner in python article and wrote this short code. It does what you want! You do however need to know accessible ports for your devices. Port 22 is ssh standard and what I am using. I suppose you could loop over all ports. Some defaults are:
linux: [20, 21, 22, 23, 25, 80, 111, 443, 445, 631, 993, 995]
windows: [135, 137, 138, 139, 445]
mac: [22, 445, 548, 631]
import socket
def connect(hostname, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.setdefaulttimeout(1)
result = sock.connect_ex((hostname, port))
sock.close()
return result == 0
for i in range(0,255):
res = connect("192.168.1."+str(i), 22)
if res:
print("Device found at: ", "192.168.1."+str(i) + ":"+str(22))
EDIT by TheLizzard:
Using the code above and adding threading:
from threading import Thread, Lock
from time import perf_counter
from sys import stderr
from time import sleep
import socket
# I changed this from "192.168.1.%i" to "192.168.0.%i"
BASE_IP = "192.168.0.%i"
PORT = 80
class Threader:
"""
This is a class that calls a list of functions in a limited number of
threads. It uses locks to make sure the data is thread safe.
Usage:
from time import sleep
def function(i):
sleep(2)
with threader.print_lock:
print(i)
threader = Threader(10) # The maximum number of threads = 10
for i in range(20):
threader.append(function, i)
threader.start()
threader.join()
This class also provides a lock called: `<Threader>.print_lock`
"""
def __init__(self, threads=30):
self.thread_lock = Lock()
self.functions_lock = Lock()
self.functions = []
self.threads = []
self.nthreads = threads
self.running = True
self.print_lock = Lock()
def stop(self) -> None:
# Signal all worker threads to stop
self.running = False
def append(self, function, *args) -> None:
# Add the function to a list of functions to be run
self.functions.append((function, args))
def start(self) -> None:
# Create a limited number of threads
for i in range(self.nthreads):
thread = Thread(target=self.worker, daemon=True)
# We need to pass in `thread` as a parameter so we
# have to use `<threading.Thread>._args` like this:
thread._args = (thread, )
self.threads.append(thread)
thread.start()
def join(self) -> None:
# Joins the threads one by one until all of them are done.
for thread in self.threads:
thread.join()
def worker(self, thread:Thread) -> None:
# While we are running and there are functions to call:
while self.running and (len(self.functions) > 0):
# Get a function
with self.functions_lock:
function, args = self.functions.pop(0)
# Call that function
function(*args)
# Remove the thread from the list of threads.
# This may cause issues if the user calls `<Threader>.join()`
# But I haven't seen this problem while testing/using it.
with self.thread_lock:
self.threads.remove(thread)
start = perf_counter()
# I didn't need a timeout of 1 so I used 0.1
socket.setdefaulttimeout(0.1)
def connect(hostname, port):
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
result = sock.connect_ex((hostname, port))
with threader.print_lock:
if result == 0:
stderr.write(f"[{perf_counter() - start:.5f}] Found {hostname}\n")
threader = Threader(10)
for i in range(255):
threader.append(connect, BASE_IP%i, PORT)
threader.start()
threader.join()
print(f"[{perf_counter() - start:.5f}] Done searching")
input("Press enter to exit.\n? ")

Try:
import socket
print ([ip for ip in socket.gethostbyname_ex(socket.gethostname())[2] if not ip.startswith("127.")][:1])

I have done following code to get the IP of MAC known device. This can be modified accordingly to obtain all IPs with some string manipulation. Hope this will help you.
#running windows cmd line statement and put output into a string
cmd_out = os.popen("arp -a").read()
line_arr = cmd_out.split('\n')
line_count = len(line_arr)
#search in all lines for ip
for i in range(0, line_count):
y = line_arr[i]
z = y.find(mac_address)
#if mac address is found then get the ip using regex matching
if z > 0:
ip_out= re.search('[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+', y, re.M | re.I)

I have just had the problem. I solved it like this:
import kthread #pip install kthread
from time import sleep
import subprocess
def getips():
ipadressen = {}
def ping(ipadresse):
try:
outputcap = subprocess.run([f'ping', ipadresse, '-n', '1'], capture_output=True) #sends only one package, faster
ipadressen[ipadresse] = outputcap
except Exception as Fehler:
print(Fehler)
t = [kthread.KThread(target = ping, name = f"ipgetter{ipend}", args=(f'192.168.0.{ipend}',)) for ipend in range(255)] #prepares threads
[kk.start() for kk in t] #starts 255 threads
while len(ipadressen) < 255:
print('Searching network')
sleep(.3)
alldevices = []
for key, item in ipadressen.items():
if not 'unreachable' in item.stdout.decode('utf-8') and 'failure' not in item.stdout.decode('utf-8'): #checks if there wasn't neither general failure nor 'unrechable host'
alldevices.append(key)
return alldevices
allips = getips() #takes 1.5 seconds on my pc

One of the answers in this question might help you. There seems to be a platform agnostic version for python, but I haven't tried it yet.

Here is a small tool scanip that will help you to get all ip addresses and their corresponding mac addresses in the network (Works on Linux).
https://github.com/vivkv/scanip