Here's the ScanUtility.py file that the BeaconScanner.py file uses to find and list the ble beacons.
#This is a working prototype. DO NOT USE IT IN LIVE PROJECTS
import sys
import struct
import bluetooth._bluetooth as bluez
OGF_LE_CTL=0x08
OCF_LE_SET_SCAN_ENABLE=0x000C
def hci_enable_le_scan(sock):
hci_toggle_le_scan(sock, 0x01)
def hci_disable_le_scan(sock):
hci_toggle_le_scan(sock, 0x00)
def hci_toggle_le_scan(sock, enable):
cmd_pkt = struct.pack("<BB", enable, 0x00)
bluez.hci_send_cmd(sock, OGF_LE_CTL, OCF_LE_SET_SCAN_ENABLE, cmd_pkt)
def packetToString(packet):
"""
Returns the string representation of a raw HCI packet.
"""
if sys.version_info > (3, 0):
return ''.join('%02x' % struct.unpack("B", bytes([x]))[0] for x in packet)
else:
return ''.join('%02x' % struct.unpack("B", x)[0] for x in packet)
def parse_events(sock, loop_count=100):
old_filter = sock.getsockopt( bluez.SOL_HCI, bluez.HCI_FILTER, 14)
flt = bluez.hci_filter_new()
bluez.hci_filter_all_events(flt)
bluez.hci_filter_set_ptype(flt, bluez.HCI_EVENT_PKT)
sock.setsockopt( bluez.SOL_HCI, bluez.HCI_FILTER, flt )
results = []
for i in range(0, loop_count):
packet = sock.recv(255)
ptype, event, plen = struct.unpack("BBB", packet[:3])
packetOffset = 0
dataString = packetToString(packet)
"""
If the bluetooth device is an beacon then show the beacon.
"""
#print (dataString)
if dataString[34:50] == '0303aafe1516aafe' or '0303AAFE1116AAFE':
"""
Selects parts of the bluetooth packets.
"""
broadcastType = dataString[50:52]
if broadcastType == '00' :
type = "Eddystone UID"
namespace = dataString[54:74].upper()
instance = dataString[74:86].upper()
resultsArray = [
{"type": type, "namespace": namespace, "instance": instance}]
return resultsArray
elif broadcastType == '10':
type = "Eddystone URL"
urlprefix = dataString[54:56]
if urlprefix == '00':
prefix = 'http://www.'
elif urlprefix == '01':
prefix = 'https://www.'
elif urlprefix == '02':
prefix = 'http://'
elif urlprefix == '03':
prefix = 'https://'
hexUrl = dataString[56:][:-2]
url = prefix + hexUrl.decode("hex")
rssi, = struct.unpack("b", packet[packetOffset -1])
resultsArray = [{"type": type, "url": url}]
return resultsArray
elif broadcastType == '20':
type = "Eddystone TLM"
resultsArray = [{"type": type}]
return resultsArray
elif broadcastType == '30':
type = "Eddystone EID"
resultsArray = [{"type": type}]
return resultsArray
elif broadcastType == '40':
type = "Eddystone RESERVED"
resultsArray = [{"type": type}]
return resultsArray
if dataString[38:46] == '4c000215':
"""
Selects parts of the bluetooth packets.
"""
type = "iBeacon"
uuid = dataString[46:54] + "-" + dataString[54:58] + "-" + dataString[58:62] + "-" + dataString[62:66] + "-" + dataString[66:78]
major = dataString[78:82]
minor = dataString[82:86]
majorVal = int("".join(major.split()[::-1]), 16)
minorVal = int("".join(minor.split()[::-1]), 16)
"""
Organises Mac Address to display properly
"""
scrambledAddress = dataString[14:26]
fixStructure = iter("".join(reversed([scrambledAddress[i:i+2] for i in range(0, len(scrambledAddress), 2)])))
macAddress = ':'.join(a+b for a,b in zip(fixStructure, fixStructure))
rssi, = struct.unpack("b", packet[packetOffset -1])
resultsArray = [{"type": type, "uuid": uuid, "major": majorVal, "minor": minorVal, "rssi": rssi, "macAddress": macAddress}]
return resultsArray
return results
The orginal Beaconscanner.py file works as it should by listing the beacons.
import ScanUtility
import bluetooth._bluetooth as bluez
#Set bluetooth device. Default 0.
dev_id = 0
try:
sock = bluez.hci_open_dev(dev_id)
print ("\n *** Looking for BLE Beacons ***\n")
print ("\n *** CTRL-C to Cancel ***\n")
except:
print ("Error accessing bluetooth")
ScanUtility.hci_enable_le_scan(sock)
#Scans for iBeacons
try:
while True:
returnedList = ScanUtility.parse_events(sock, 10)
for item in returnedList:
print(item)
print("")
except KeyboardInterrupt:
pass
Here's the modified BeaconScanner.py file which should print "Works" if the scanner finds the wanted beacon by it's mac address.
import ScanUtility
import bluetooth._bluetooth as bluez
#Set bluetooth device. Default 0.
dev_id = 0
try:
sock = bluez.hci_open_dev(dev_id)
print ("\n *** Looking for BLE Beacons ***\n")
print ("\n *** CTRL-C to Cancel ***\n")
except:
print ("Error accessing bluetooth")
ScanUtility.hci_enable_le_scan(sock)
#Scans for iBeacons
try:
while True:
returnedList = ScanUtility.parse_events(sock, 10)
for macAddress in returnedList:
if macAddress == "e2:e3:23:d1:b0:54":
print("Works")
else:
print("Nope")
except KeyboardInterrupt:
pass
The modified file however always prints "Nope". I think the "macAddress" part in the if statement can't be used to identify the beacons. What have to be changed in the code so the beacon can be identified by it's mac address in the if statement?
According to the source of ScanUtility.py, it seems like the function returns a list of one dict which is a bit odd. You should query the dict as follow:
for item in returnedList:
try:
if item['macAddress'] == "e2:e3:23:d1:b0:54":
print("Works")
else:
print("Nope")
except KeyError:
print('MAC Address is missing')
Note that I have added a try/except statement to deal with cases where the macAddress key is not present in your dict. This works only if your dict is not a subclass of defaultdict.
Searching for a beacon by its mac address is not always a good solution as it is possible a beacon is using a random private addresses.
Are you sure that the mac address you are looking for is ever broadcast?
Your code also only seems to return the mac address for iBeacons.
The other thing I noticed about this code is that it bypasses the bluetoothd running on your system by doing direct calls to the hci socket. This is generally not a good idea and requires the python script to be run with root priveliages.
A way to avoid this is use the BlueZ D-Bus API. An example of this is included below and prints out beacon data plus a message when it sees the mac address of interest. This code requires pydbus and gi.repository
import argparse
from gi.repository import GLib
from pydbus import SystemBus
import uuid
DEVICE_INTERFACE = 'org.bluez.Device1'
remove_list = set()
def stop_scan():
"""Stop device discovery and quit event loop"""
adapter.StopDiscovery()
mainloop.quit()
def clean_beacons():
"""
BlueZ D-Bus API does not show duplicates. This is a
workaround that removes devices that have been found
during discovery
"""
not_found = set()
for rm_dev in remove_list:
try:
adapter.RemoveDevice(rm_dev)
except GLib.Error as err:
not_found.add(rm_dev)
for lost in not_found:
remove_list.remove(lost)
def process_eddystone(data):
"""Print Eddystone data in human readable format"""
_url_prefix_scheme = ['http://www.', 'https://www.',
'http://', 'https://', ]
_url_encoding = ['.com/', '.org/', '.edu/', '.net/', '.info/',
'.biz/', '.gov/', '.com', '.org', '.edu',
'.net', '.info', '.biz', '.gov']
tx_pwr = int.from_bytes([data[1]], 'big', signed=True)
# Eddystone UID Beacon format
if data[0] == 0x00:
namespace_id = int.from_bytes(data[2:12], 'big')
instance_id = int.from_bytes(data[12:18], 'big')
print(f'\t\tEddystone UID: {namespace_id} - {instance_id} \u2197 {tx_pwr}')
# Eddystone URL beacon format
elif data[0] == 0x10:
prefix = data[2]
encoded_url = data[3:]
full_url = _url_prefix_scheme[prefix]
for letter in encoded_url:
if letter < len(_url_encoding):
full_url += _url_encoding[letter]
else:
full_url += chr(letter)
print(f'\t\tEddystone URL: {full_url} \u2197 {tx_pwr}')
def process_ibeacon(data, beacon_type='iBeacon'):
"""Print iBeacon data in human readable format"""
beacon_uuid = uuid.UUID(bytes=bytes(data[2:18]))
major = int.from_bytes(bytearray(data[18:20]), 'big', signed=False)
minor = int.from_bytes(bytearray(data[20:22]), 'big', signed=False)
tx_pwr = int.from_bytes([data[22]], 'big', signed=True)
print(f'\t\t{beacon_type}: {beacon_uuid} - {major} - {minor} \u2197 {tx_pwr}')
def ble_16bit_match(uuid_16, srv_data):
"""Expand 16 bit UUID to full 128 bit UUID"""
uuid_128 = f'0000{uuid_16}-0000-1000-8000-00805f9b34fb'
return uuid_128 == list(srv_data.keys())[0]
def on_iface_added(owner, path, iface, signal, interfaces_and_properties):
"""
Event handler for D-Bus interface added.
Test to see if it is a new Bluetooth device
"""
iface_path, iface_props = interfaces_and_properties
if DEVICE_INTERFACE in iface_props:
on_device_found(iface_path, iface_props[DEVICE_INTERFACE])
def on_device_found(device_path, device_props):
"""
Handle new Bluetooth device being discover.
If it is a beacon of type iBeacon, Eddystone, AltBeacon
then process it
"""
address = device_props.get('Address')
address_type = device_props.get('AddressType')
name = device_props.get('Name')
alias = device_props.get('Alias')
paired = device_props.get('Paired')
trusted = device_props.get('Trusted')
rssi = device_props.get('RSSI')
service_data = device_props.get('ServiceData')
manufacturer_data = device_props.get('ManufacturerData')
if address.casefold() == 'e2:e3:23:d1:b0:54':
print('Found mac address of interest')
if service_data and ble_16bit_match('feaa', service_data):
process_eddystone(service_data['0000feaa-0000-1000-8000-00805f9b34fb'])
remove_list.add(device_path)
elif manufacturer_data:
for mfg_id in manufacturer_data:
# iBeacon 0x004c
if mfg_id == 0x004c and manufacturer_data[mfg_id][0] == 0x02:
process_ibeacon(manufacturer_data[mfg_id])
remove_list.add(device_path)
# AltBeacon 0xacbe
elif mfg_id == 0xffff and manufacturer_data[mfg_id][0:2] == [0xbe, 0xac]:
process_ibeacon(manufacturer_data[mfg_id], beacon_type='AltBeacon')
remove_list.add(device_path)
clean_beacons()
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('-d', '--duration', type=int, default=0,
help='Duration of scan [0 for continuous]')
args = parser.parse_args()
bus = SystemBus()
adapter = bus.get('org.bluez', '/org/bluez/hci0')
bus.subscribe(iface='org.freedesktop.DBus.ObjectManager',
signal='InterfacesAdded',
signal_fired=on_iface_added)
mainloop = GLib.MainLoop()
if args.duration > 0:
GLib.timeout_add_seconds(args.duration, stop_scan)
adapter.SetDiscoveryFilter({'DuplicateData': GLib.Variant.new_boolean(True)})
adapter.StartDiscovery()
try:
print('\n\tUse CTRL-C to stop discovery\n')
mainloop.run()
except KeyboardInterrupt:
stop_scan()
Example invocation and output:
$ python3 beacon_scanner.py
Use CTRL-C to stop discovery
iBeacon: 1e9fdc8c-96e0-4d68-b34a-3b635cec0489 - 5555 - 99 ↗ -65
Eddystone URL: http://www.bluetooth.com/ ↗ -69
Found mac address of interest
Related
Need some help to set the configuration for sasl.mechanism PLAIN (API) and GSSAPI (Kerberos) authentication.
We are using confluent Kafka here, there are two scripts, one a python script and the second one is a bash script which calls the python one. You can find the script below.
Thanks for the help in advance!
import json
import os
import string
import random
import socket
import uuid
import re
from datetime import datetime
import time
import hashlib
import math
import sys
from functools import cache
from confluent_kafka import Producer, KafkaError, KafkaException
topic_name = os.environ['TOPIC_NAME']
partition_count = int(os.environ['PARTITION_COUNT'])
message_key_template = json.loads(os.environ['KEY_TEMPLATE'])
message_value_template = json.loads(os.environ['VALUE_TEMPLATE'])
message_header_template = json.loads(os.environ['HEADER_TEMPLATE'])
bootstrap_servers = os.environ['BOOTSTRAP_SERVERS']
perf_counter_batch_size = int(os.environ.get('PERF_COUNTER_BATCH_SIZE', 100))
messages_per_aggregate = int(os.environ.get('MESSAGES_PER_AGGREGATE', 1))
max_message_count = int(os.environ.get('MAX_MESSAGE_COUNT', sys.maxsize))
def error_cb(err):
""" The error callback is used for generic client errors. These
errors are generally to be considered informational as the client will
automatically try to recover from all errors, and no extra action
is typically required by the application.
For this example however, we terminate the application if the client
is unable to connect to any broker (_ALL_BROKERS_DOWN) and on
authentication errors (_AUTHENTICATION). """
print("Client error: {}".format(err))
if err.code() == KafkaError._ALL_BROKERS_DOWN or \
err.code() == KafkaError._AUTHENTICATION:
# Any exception raised from this callback will be re-raised from the
# triggering flush() or poll() call.
raise KafkaException(err)
def acked(err, msg):
if err is not None:
print("Failed to send message: %s: %s" % (str(msg), str(err)))
producer_configs = {
'bootstrap.servers': bootstrap_servers,
'client.id': socket.gethostname(),
'error_cb': error_cb
}
# TODO: Need to support sasl.mechanism PLAIN (API) and GSSAPI (Kerberos) authentication.
# TODO: Need to support truststores for connecting to private DCs.
producer = Producer(producer_configs)
# generates a random value if it is not cached in the template_values dictionary
def get_templated_value(term, template_values):
if not term in template_values:
template_values[term] = str(uuid.uuid4())
return template_values[term]
def fill_template_value(value, template_values):
str_value = str(value)
template_regex = '{{(.+?)}}'
templated_terms = re.findall(template_regex, str_value)
for term in templated_terms:
str_value = str_value.replace(f"{{{{{term}}}}}", get_templated_value(term, template_values))
return str_value
def fill_template(template, templated_terms):
# TODO: Need to address metadata field, as it's treated as a string instead of a nested object.
return {field: fill_template_value(value, templated_terms) for field, value in template.items()}
#cache
def get_partition(lock_id):
bits = 128
bucket_size = 2**bits / partition_count
partition = (int(hashlib.md5(lock_id.encode('utf-8')).hexdigest(), 16) / bucket_size)
return math.floor(partition)
sequence_number = int(time.time() * 1000)
sequence_number = 0
message_count = 0
producing = True
start_time = time.perf_counter()
aggregate_message_counter = 0
# cache for templated term values so that they match across the different templates
templated_values = {}
try:
while producing:
sequence_number += 1
aggregate_message_counter += 1
message_count += 1
if aggregate_message_counter % messages_per_aggregate == 0:
# reset templated values
templated_values = {}
else:
for term in list(templated_values):
if term not in ['aggregateId', 'tenantId']:
del(templated_values[term])
# Fill in templated field values
message_key = fill_template(message_key_template, templated_values)
message_value = fill_template(message_value_template, templated_values)
message_header = fill_template(message_header_template, templated_values)
ts = datetime.utcnow().isoformat()[:-3]+'Z'
message_header['timestamp'] = ts
message_header['sequence_number'] = str(sequence_number)
message_value['timestamp'] = ts
message_value['sequenceNumber'] = sequence_number
lock_id = message_header['lock_id']
partition = get_partition(lock_id) # partition by lock_id, since key could be random, but a given aggregate_id should ALWAYS resolve to the same partition, regardless of key.
# Send message
producer.produce(topic_name, partition=partition, key=json.dumps(message_key), value=json.dumps(message_value), headers=message_header, callback=acked)
if sequence_number % perf_counter_batch_size == 0:
producer.flush()
end_time = time.perf_counter()
total_duration = end_time - start_time
messages_per_second=(perf_counter_batch_size/total_duration)
print(f'{messages_per_second} messages/second')
# reset start time
start_time = time.perf_counter()
if message_count >= max_message_count:
break
except Exception as e:
print(f'ERROR: %s' % e)
sys.exit(1)
finally:
producer.flush()
In Python, I am trying to use the J1939 filtering as mentionned in the linux kernel docs: https://www.kernel.org/doc/html/latest/networking/j1939.html
The following code fails at the setsockopt() line (setting up filters):
import socket
import struct
def pack_J1939_filters(can_filters):
can_filter_fmt = "=" + "2Q2B2I" * len(can_filters)
filter_data = []
for can_filter in can_filters:
name = can_filter['name']
name_mask = can_filter['name_mask']
addr = can_filter['addr']
addr_mask = can_filter['addr_mask']
pgn = can_filter['pgn']
pgn_mask = can_filter['pgn_mask']
filter_data.append(name)
filter_data.append(name_mask)
filter_data.append(addr)
filter_data.append(addr_mask)
filter_data.append(pgn)
filter_data.append(pgn_mask)
return struct.pack(can_filter_fmt, *filter_data)
s = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939)
interface = "vcan0"
src_name = socket.J1939_NO_NAME
src_pgn = socket.J1939_NO_PGN
src_addr = 0x81
src_sck_addr = (interface, src_name, src_pgn, src_addr)
s.bind(src_sck_addr)
filters = [{"name": 0, "name_mask":0, "addr":0, "addr_mask":0, "pgn": 0, "pgn_mask": 0}]
packed_filters = pack_J1939_filters(filters)
# socket.SOL_CAN_J1939 does not seem to exist
SOL_CAN_BASE = 100
CAN_J1939 = 7
SOL_CAN_J1939 = SOL_CAN_BASE + CAN_J1939
s.setsockopt(SOL_CAN_J1939, socket.SO_J1939_FILTER , packed_filters)
s.recvfrom(128)
s.close()
First, the kernel documentation mentions to use SOL_CAN_J1939 as the first argument. However socket.SOL_CAN_J1939 does not exist in the socket package. So looking at the code at this location I was able to understand that this int value should be 107: http://socket-can.996257.n3.nabble.com/RFC-v3-0-6-CAN-add-SAE-J1939-protocol-td7571.html
As for the setsockopt() third argument, I packed the filters to match the j1939_filter structure (26 bytes as described in the code from the previous link). This is similar to what is done in can.interfaces.socketcan.utils for raw CAN.
What am I doing wrong to cause setsockopt() to fail?
The first issue was with the struct.pack format (can_filter_fmt) being wrong. I first assumed that the kernel j1939_filter structure size was the sum of the members. This is wrong since the compiler adds padding. This can be added to the struct.pack format as x such as 2Q2I2B6x. Please see Why isn't sizeof for a struct equal to the sum of sizeof of each member?
The second issue was that can_filter_fmt is not packed as 2Q2B2I but as 2Q2I2B6x (the addr member is in the middle).
As for SOL_CAN_J1939 I was correct and needs to be created in file because it is not yet in the package.
The final code is the following:
#!/usr/bin/env python3
import socket
import struct
def pack_J1939_filters(can_filters=None):
if can_filters is None:
# Pass all messages
can_filters = [{}]
can_filter_fmt = "=" + "2Q2I2B6x" * len(can_filters)
filter_data = []
for can_filter in can_filters:
if 'name' in can_filter:
name = can_filter['name']
else:
name = 0
if 'name_mask' in can_filter:
name_mask = can_filter['name_mask']
else:
name_mask = 0
if 'pgn' in can_filter:
pgn = can_filter['pgn']
else:
pgn = 0
if 'pgn_mask' in can_filter:
pgn_mask = can_filter['pgn_mask']
else:
pgn_mask = 0
if 'addr' in can_filter:
addr = can_filter['addr']
else:
addr = 0
if 'addr_mask' in can_filter:
addr_mask = can_filter['addr_mask']
else:
addr_mask = 0
filter_data.append(name)
filter_data.append(name_mask)
filter_data.append(pgn)
filter_data.append(pgn_mask)
filter_data.append(addr)
filter_data.append(addr_mask)
return struct.pack(can_filter_fmt, *filter_data)
def print_msg(data, sck_addr):
print(f"SA:{hex(sck_addr[3])} PGN:{hex(sck_addr[2])}")
for j in range(len(data)):
if j % 8 == 0 and j != 0:
print()
if j % 8 == 0:
print(f"bytes {j} to {j+7}: ", end="")
print(f"{hex(data[j])} ", end="")
print()
print()
def main():
s = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939)
# allows to receive broadcast messages
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
interface = "vcan0"
src_name = socket.J1939_NO_NAME
src_pgn = socket.J1939_NO_PGN # always no PGN for source, unless filtering is needed
src_addr = 0x81 # recvfrom() will not return destination specific messages for other addresses
src_sck_addr = (interface, src_name, src_pgn, src_addr)
s.bind(src_sck_addr)
packed_filters = pack_J1939_filters()
SOL_CAN_BASE = 100
CAN_J1939 = 7
SOL_CAN_J1939 = SOL_CAN_BASE + CAN_J1939
s.setsockopt(SOL_CAN_J1939, socket.SO_J1939_FILTER , packed_filters)
(recv_data, recv_sck_addr) = s.recvfrom(128)
print_msg(recv_data, recv_sck_addr)
s.close()
if __name__ == "__main__":
main()
Thank you.
For J1939 to work with SocketCAN you need two things:
kernel 5.4+
can-j1939 kernel module enabled
Testing for can-1939:
If you install can-utils and after sudo modprobe can-j1939 all you get is fatal error, or if you start testj1939 from can-utils and you get error that protocol is not supported, then it means that can-j1939 was not enabled in your kernel and you need to compile it manually.
Here are my instructions for enabling can-j1939 in Debian 10 kernel:
https://github.com/linux-can/can-utils/blob/master/can-j1939-install-kernel-module.md
I have a list of emails i want to validate from same domain. Firstly, I check if the domain allows verfication or is a catchall (accepts all email user as valid). E.g Assuming [a#domain.com, b#domain.com] is the list, I will check if foo-fake#domain.com would be valid, if it is return from function.
If not use multiple thread to verify all emails in list - if all fail return the last item. If one is valid and other threads are running stop them and return the valid one.
from Queue import Queue
import threading
import smtplib
class MailWorker(threading.Thread):
kill = False
def __init__(self, in_que, out_que):
super(MailWorker, self).__init__()
self.in_que = in_que
self.out_que = out_que
def run(self):
while True:
email, host = self.in_que.get()
self.test_email(email, host)
self.in_que.task_done()
def test_email(self, email, host, retry=0):
status = "Unknown"
code = None
rand = "info#example.com"
try:
server = smtplib.SMTP(timeout=20)
server.connect(host, 25)
server.ehlo_or_helo_if_needed()
code, response = server.docmd('mail from:', "<{}>".format(rand))
code, response = server.docmd('rcpt to:', "<{}>".format(email))
if code == 421:
while retry < 3:
if retry >= 3:
server.quit()
self.out_que.put((email, "Service not available", code))
server.quit()
return self.test_email(email, host, retry=retry)
if code == 250:
status = 'valid'
self.out_que.put((email, status, code,))
except smtplib.SMTPServerDisconnected:
while retry < 3:
retry += 1
status = "(SMTP Disconnected Unexpectedly) Retry # {}".format(retry)
code = -2
time.sleep(2)
if retry >= 3:
self.out_que.put((email, "SMTP Disconnected", code))
else:
return self.test_email(email, host, retry=retry)
self.out_que.put((email, status, code,))
def check_email(emails, domain, index=0, is_main=False):
email = status = code = None
in_que = Queue(maxsize=10)
out_que = Queue(maxsize=10)
if 'brighthouse' in domain:
host = 'brighthouse-co-uk.mail.protection.outlook.com'
else:
host = 'eu-smtp-inbound-2.mimecast.com'
# is it a catchall? if it is i want to return from the function ---
# If the email is valid then it is not a catchall all so execute line 91
# till return checking multipe emails in threads but exit when one valid is found else do all email in list
if not is_main: # check if the email is main
in_que.put(('JUNK_EMAIL_CANT_BE_REAL_fewfwewefew#' + domain, host)) # put rubbish email in thread if valid it is a catchall, then exit
for i in range(1):
mw = MailWorker(in_que, out_que)
print mw
print "*" * 20
mw.daemon = True
mw.start()
print mw.getName()
print in_que.qsize(), " in queue size"
print out_que.qsize(), " out queue size"
print "*" * 20
email, status, code = out_que.get()
print "Contet = ", email, status, code
if code == 250:
print 'Domain is a Catch-All. email: %s host: %s' % (emails[0], host)
MailWorker.kill = True
return emails, "catchall", -99, index
elif code == -1:
return email, status, code, index
# in_que.join()
# out_que.join()
for index, email_address in enumerate(emails):
in_que.put((email_address, host,))
for i in range(10):
mw = MailWorker(in_que, out_que)
mw.daemon = True
mw.start()
while not out_que.empty():
email, status, code, index = out_que.get()
if code == 250:
MailWorker.kill = True
return email, status, code, index
in_que.join()
out_que.join()
return email, status, code, index
emails_list = [
['fred#brighthouse.co.uk', 'joe.fred#brighthouse.co.uk', 'joe#brighthouse.co.uk'],
['fred#cqs.com', 'joe.fred#cqs.com', 'joe#cqs.com']
]
for emails in emails_list:
domain = emails[0].split('#')[1]
print(check_email(emails, domain))
My expectation is for the next list item to run.
The result with the above code is:
# CODE FREEZES and DOESN'T RUN SECOND item IN EMAILS_LIST -- SEE output
"""
<MailWorker(Thread-1, initial)>
********************
Thread-1
1 in queue size
0 out queue size
********************
Contet = rubbish_fQ94hEAi93#brighthouse.co.uk valid 250
Domain is a Catch-All. email: fred#brighthouse.co.uk host: brighthouse-co-uk.mail.protection.outlook.com
(['fred#brighthouse.co.uk', 'joe.fred#brighthouse.co.uk', 'joe#brighthouse.co.uk'], 'catchall', -99, 0)
<MailWorker(Thread-2, initial)>
********************
Thread-2
0 in queue size
0 out queue size
******************** <---- code blocks here
"""
I'm trying to understand a Python script that analyzes beacon frames. But I'm stuck at something called .payload. Looking at the Python documentation and doing research didn't help me out. I found out that the payload is the data carried by the frame.
def insert_ap(pkt):
## Done in the lfilter param
# if Dot11Beacon not in pkt and Dot11ProbeResp not in pkt:
# return
bssid = pkt[Dot11].addr3
if bssid in aps:
return
p = pkt[Dot11Elt]
cap = pkt.sprintf("{Dot11Beacon:%Dot11Beacon.cap%}"
"{Dot11ProbeResp:%Dot11ProbeResp.cap%}").split('+')
ssid, channel = None, None
crypto = set()
while isinstance(p, Dot11Elt):
if p.ID == 0:
ssid = p.info
elif p.ID == 3:
channel = ord(p.info)
elif p.ID == 48:
crypto.add("WPA2")
elif p.ID == 221 and p.info.startswith('\x00P\xf2\x01\x01\x00'):
crypto.add("WPA")
p = p.payload # HERE IT IS
if not crypto:
if 'privacy' in cap:
crypto.add("WEP")
else:
crypto.add("OPN")
print "NEW AP: %r [%s], channed %d, %s" % (ssid, bssid, channel,
' / '.join(crypto))
aps[bssid] = (ssid, channel, crypto)
aps = {}
sniff(iface='mon0', prn=insert_ap, store=False,
lfilter=lambda p: (Dot11Beacon in p or Dot11ProbeResp in p))
The payload function is written in a while loop. The loop is active as long as the packet is an instance of Dot11Elt. But what does .payload do that it's no longer Dot11Elt (?)
Thank you!
packet.payload is just a pointer to the next layer.
Take a look at the Scapy documentation.
For example, if pkt were constructed as such:
pkt = Dot11()/foo()/IP()/TCP()
In your example, p is initially set to pkt[Dot11]. Therefore, p.payload is pkt[Dot11].payload, which points to the foo object. At the end of the loop, p is advanced to p.payload. As long as p is of type Dot11Elt, the loop will keep running.
If we assume that both Dot11, and foo are of type Dot11Elt, then the loop will run until p is pointing to the IP layer.
I'm trying to transmit TCP/IP over a radio that is connected to my computer (specifically, the USRP). Right now, it's done very simply using Tun/Tap to set up a new network interface. Here's the code:
from gnuradio import gr, gru, modulation_utils
from gnuradio import usrp
from gnuradio import eng_notation
from gnuradio.eng_option import eng_option
from optparse import OptionParser
import random
import time
import struct
import sys
import os
# from current dir
from transmit_path import transmit_path
from receive_path import receive_path
import fusb_options
#print os.getpid()
#raw_input('Attach and press enter')
# Linux specific...
# TUNSETIFF ifr flags from <linux/tun_if.h>
IFF_TUN = 0x0001 # tunnel IP packets
IFF_TAP = 0x0002 # tunnel ethernet frames
IFF_NO_PI = 0x1000 # don't pass extra packet info
IFF_ONE_QUEUE = 0x2000 # beats me ;)
def open_tun_interface(tun_device_filename):
from fcntl import ioctl
mode = IFF_TAP | IFF_NO_PI
TUNSETIFF = 0x400454ca
tun = os.open(tun_device_filename, os.O_RDWR)
ifs = ioctl(tun, TUNSETIFF, struct.pack("16sH", "gr%d", mode))
ifname = ifs[:16].strip("\x00")
return (tun, ifname)
# /////////////////////////////////////////////////////////////////////////////
# the flow graph
# /////////////////////////////////////////////////////////////////////////////
class my_top_block(gr.top_block):
def __init__(self, mod_class, demod_class,
rx_callback, options):
gr.top_block.__init__(self)
self.txpath = transmit_path(mod_class, options)
self.rxpath = receive_path(demod_class, rx_callback, options)
self.connect(self.txpath);
self.connect(self.rxpath);
def send_pkt(self, payload='', eof=False):
return self.txpath.send_pkt(payload, eof)
def carrier_sensed(self):
"""
Return True if the receive path thinks there's carrier
"""
return self.rxpath.carrier_sensed()
# /////////////////////////////////////////////////////////////////////////////
# Carrier Sense MAC
# /////////////////////////////////////////////////////////////////////////////
class cs_mac(object):
"""
Prototype carrier sense MAC
Reads packets from the TUN/TAP interface, and sends them to the PHY.
Receives packets from the PHY via phy_rx_callback, and sends them
into the TUN/TAP interface.
Of course, we're not restricted to getting packets via TUN/TAP, this
is just an example.
"""
def __init__(self, tun_fd, verbose=False):
self.tun_fd = tun_fd # file descriptor for TUN/TAP interface
self.verbose = verbose
self.tb = None # top block (access to PHY)
def set_top_block(self, tb):
self.tb = tb
def phy_rx_callback(self, ok, payload):
"""
Invoked by thread associated with PHY to pass received packet up.
#param ok: bool indicating whether payload CRC was OK
#param payload: contents of the packet (string)
"""
if self.verbose:
print "Rx: ok = %r len(payload) = %4d" % (ok, len(payload))
if ok:
os.write(self.tun_fd, payload)
def main_loop(self):
"""
Main loop for MAC.
Only returns if we get an error reading from TUN.
FIXME: may want to check for EINTR and EAGAIN and reissue read
"""
min_delay = 0.001 # seconds
while 1:
payload = os.read(self.tun_fd, 10*1024)
if not payload:
self.tb.send_pkt(eof=True)
break
if self.verbose:
print "Tx: len(payload) = %4d" % (len(payload),)
delay = min_delay
while self.tb.carrier_sensed():
sys.stderr.write('B')
time.sleep(delay)
if delay < 0.050:
delay = delay * 2 # exponential back-off
self.tb.send_pkt(payload)
# /////////////////////////////////////////////////////////////////////////////
# main
# /////////////////////////////////////////////////////////////////////////////
def main():
mods = modulation_utils.type_1_mods()
demods = modulation_utils.type_1_demods()
parser = OptionParser (option_class=eng_option, conflict_handler="resolve")
expert_grp = parser.add_option_group("Expert")
parser.add_option("-m", "--modulation", type="choice", choices=mods.keys(),
default='gmsk',
help="Select modulation from: %s [default=%%default]"
% (', '.join(mods.keys()),))
parser.add_option("-v","--verbose", action="store_true", default=False)
expert_grp.add_option("-c", "--carrier-threshold", type="eng_float", default=30,
help="set carrier detect threshold (dB) [default=%default]")
expert_grp.add_option("","--tun-device-filename", default="/dev/net/tun",
help="path to tun device file [default=%default]")
transmit_path.add_options(parser, expert_grp)
receive_path.add_options(parser, expert_grp)
for mod in mods.values():
mod.add_options(expert_grp)
for demod in demods.values():
demod.add_options(expert_grp)
fusb_options.add_options(expert_grp)
(options, args) = parser.parse_args ()
if len(args) != 0:
parser.print_help(sys.stderr)
sys.exit(1)
if options.rx_freq is None or options.tx_freq is None:
sys.stderr.write("You must specify -f FREQ or --freq FREQ\n")
parser.print_help(sys.stderr)
sys.exit(1)
# open the TUN/TAP interface
(tun_fd, tun_ifname) = open_tun_interface(options.tun_device_filename)
# Attempt to enable realtime scheduling
r = gr.enable_realtime_scheduling()
if r == gr.RT_OK:
realtime = True
else:
realtime = False
print "Note: failed to enable realtime scheduling"
# If the user hasn't set the fusb_* parameters on the command line,
# pick some values that will reduce latency.
if options.fusb_block_size == 0 and options.fusb_nblocks == 0:
if realtime: # be more aggressive
options.fusb_block_size = gr.prefs().get_long('fusb', 'rt_block_size', 1024)
options.fusb_nblocks = gr.prefs().get_long('fusb', 'rt_nblocks', 16)
else:
options.fusb_block_size = gr.prefs().get_long('fusb', 'block_size', 4096)
options.fusb_nblocks = gr.prefs().get_long('fusb', 'nblocks', 16)
#print "fusb_block_size =", options.fusb_block_size
#print "fusb_nblocks =", options.fusb_nblocks
# instantiate the MAC
mac = cs_mac(tun_fd, verbose=True)
# build the graph (PHY)
tb = my_top_block(mods[options.modulation],
demods[options.modulation],
mac.phy_rx_callback,
options)
mac.set_top_block(tb) # give the MAC a handle for the PHY
if tb.txpath.bitrate() != tb.rxpath.bitrate():
print "WARNING: Transmit bitrate = %sb/sec, Receive bitrate = %sb/sec" % (
eng_notation.num_to_str(tb.txpath.bitrate()),
eng_notation.num_to_str(tb.rxpath.bitrate()))
print "modulation: %s" % (options.modulation,)
print "freq: %s" % (eng_notation.num_to_str(options.tx_freq))
print "bitrate: %sb/sec" % (eng_notation.num_to_str(tb.txpath.bitrate()),)
print "samples/symbol: %3d" % (tb.txpath.samples_per_symbol(),)
#print "interp: %3d" % (tb.txpath.interp(),)
#print "decim: %3d" % (tb.rxpath.decim(),)
tb.rxpath.set_carrier_threshold(options.carrier_threshold)
print "Carrier sense threshold:", options.carrier_threshold, "dB"
print
print "Allocated virtual ethernet interface: %s" % (tun_ifname,)
print "You must now use ifconfig to set its IP address. E.g.,"
print
print " $ sudo ifconfig %s 192.168.200.1" % (tun_ifname,)
print
print "Be sure to use a different address in the same subnet for each machine."
print
tb.start() # Start executing the flow graph (runs in separate threads)
mac.main_loop() # don't expect this to return...
tb.stop() # but if it does, tell flow graph to stop.
tb.wait() # wait for it to finish
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
pass
(Anyone familiar with GNU Radio will recognize this as tunnel.py)
My question is, is there a better way to move packets to and from the kernel than tun/tap? I've been looking at ipip or maybe using sockets, but I'm pretty sure those won't be very fast. Speed is what I'm most concerned with.
Remember that tunnel.py is a really, really rough example, and hasn't been updated in a while. It's not really meant to be a basis for other code, so be careful of how much you rely on the code.
Also, remember that TCP over unreliable radio links has significant issues:
http://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_over_wireless_networks