I'm trying to transmit TCP/IP over a radio that is connected to my computer (specifically, the USRP). Right now, it's done very simply using Tun/Tap to set up a new network interface. Here's the code:
from gnuradio import gr, gru, modulation_utils
from gnuradio import usrp
from gnuradio import eng_notation
from gnuradio.eng_option import eng_option
from optparse import OptionParser
import random
import time
import struct
import sys
import os
# from current dir
from transmit_path import transmit_path
from receive_path import receive_path
import fusb_options
#print os.getpid()
#raw_input('Attach and press enter')
# Linux specific...
# TUNSETIFF ifr flags from <linux/tun_if.h>
IFF_TUN = 0x0001 # tunnel IP packets
IFF_TAP = 0x0002 # tunnel ethernet frames
IFF_NO_PI = 0x1000 # don't pass extra packet info
IFF_ONE_QUEUE = 0x2000 # beats me ;)
def open_tun_interface(tun_device_filename):
from fcntl import ioctl
mode = IFF_TAP | IFF_NO_PI
TUNSETIFF = 0x400454ca
tun = os.open(tun_device_filename, os.O_RDWR)
ifs = ioctl(tun, TUNSETIFF, struct.pack("16sH", "gr%d", mode))
ifname = ifs[:16].strip("\x00")
return (tun, ifname)
# /////////////////////////////////////////////////////////////////////////////
# the flow graph
# /////////////////////////////////////////////////////////////////////////////
class my_top_block(gr.top_block):
def __init__(self, mod_class, demod_class,
rx_callback, options):
gr.top_block.__init__(self)
self.txpath = transmit_path(mod_class, options)
self.rxpath = receive_path(demod_class, rx_callback, options)
self.connect(self.txpath);
self.connect(self.rxpath);
def send_pkt(self, payload='', eof=False):
return self.txpath.send_pkt(payload, eof)
def carrier_sensed(self):
"""
Return True if the receive path thinks there's carrier
"""
return self.rxpath.carrier_sensed()
# /////////////////////////////////////////////////////////////////////////////
# Carrier Sense MAC
# /////////////////////////////////////////////////////////////////////////////
class cs_mac(object):
"""
Prototype carrier sense MAC
Reads packets from the TUN/TAP interface, and sends them to the PHY.
Receives packets from the PHY via phy_rx_callback, and sends them
into the TUN/TAP interface.
Of course, we're not restricted to getting packets via TUN/TAP, this
is just an example.
"""
def __init__(self, tun_fd, verbose=False):
self.tun_fd = tun_fd # file descriptor for TUN/TAP interface
self.verbose = verbose
self.tb = None # top block (access to PHY)
def set_top_block(self, tb):
self.tb = tb
def phy_rx_callback(self, ok, payload):
"""
Invoked by thread associated with PHY to pass received packet up.
#param ok: bool indicating whether payload CRC was OK
#param payload: contents of the packet (string)
"""
if self.verbose:
print "Rx: ok = %r len(payload) = %4d" % (ok, len(payload))
if ok:
os.write(self.tun_fd, payload)
def main_loop(self):
"""
Main loop for MAC.
Only returns if we get an error reading from TUN.
FIXME: may want to check for EINTR and EAGAIN and reissue read
"""
min_delay = 0.001 # seconds
while 1:
payload = os.read(self.tun_fd, 10*1024)
if not payload:
self.tb.send_pkt(eof=True)
break
if self.verbose:
print "Tx: len(payload) = %4d" % (len(payload),)
delay = min_delay
while self.tb.carrier_sensed():
sys.stderr.write('B')
time.sleep(delay)
if delay < 0.050:
delay = delay * 2 # exponential back-off
self.tb.send_pkt(payload)
# /////////////////////////////////////////////////////////////////////////////
# main
# /////////////////////////////////////////////////////////////////////////////
def main():
mods = modulation_utils.type_1_mods()
demods = modulation_utils.type_1_demods()
parser = OptionParser (option_class=eng_option, conflict_handler="resolve")
expert_grp = parser.add_option_group("Expert")
parser.add_option("-m", "--modulation", type="choice", choices=mods.keys(),
default='gmsk',
help="Select modulation from: %s [default=%%default]"
% (', '.join(mods.keys()),))
parser.add_option("-v","--verbose", action="store_true", default=False)
expert_grp.add_option("-c", "--carrier-threshold", type="eng_float", default=30,
help="set carrier detect threshold (dB) [default=%default]")
expert_grp.add_option("","--tun-device-filename", default="/dev/net/tun",
help="path to tun device file [default=%default]")
transmit_path.add_options(parser, expert_grp)
receive_path.add_options(parser, expert_grp)
for mod in mods.values():
mod.add_options(expert_grp)
for demod in demods.values():
demod.add_options(expert_grp)
fusb_options.add_options(expert_grp)
(options, args) = parser.parse_args ()
if len(args) != 0:
parser.print_help(sys.stderr)
sys.exit(1)
if options.rx_freq is None or options.tx_freq is None:
sys.stderr.write("You must specify -f FREQ or --freq FREQ\n")
parser.print_help(sys.stderr)
sys.exit(1)
# open the TUN/TAP interface
(tun_fd, tun_ifname) = open_tun_interface(options.tun_device_filename)
# Attempt to enable realtime scheduling
r = gr.enable_realtime_scheduling()
if r == gr.RT_OK:
realtime = True
else:
realtime = False
print "Note: failed to enable realtime scheduling"
# If the user hasn't set the fusb_* parameters on the command line,
# pick some values that will reduce latency.
if options.fusb_block_size == 0 and options.fusb_nblocks == 0:
if realtime: # be more aggressive
options.fusb_block_size = gr.prefs().get_long('fusb', 'rt_block_size', 1024)
options.fusb_nblocks = gr.prefs().get_long('fusb', 'rt_nblocks', 16)
else:
options.fusb_block_size = gr.prefs().get_long('fusb', 'block_size', 4096)
options.fusb_nblocks = gr.prefs().get_long('fusb', 'nblocks', 16)
#print "fusb_block_size =", options.fusb_block_size
#print "fusb_nblocks =", options.fusb_nblocks
# instantiate the MAC
mac = cs_mac(tun_fd, verbose=True)
# build the graph (PHY)
tb = my_top_block(mods[options.modulation],
demods[options.modulation],
mac.phy_rx_callback,
options)
mac.set_top_block(tb) # give the MAC a handle for the PHY
if tb.txpath.bitrate() != tb.rxpath.bitrate():
print "WARNING: Transmit bitrate = %sb/sec, Receive bitrate = %sb/sec" % (
eng_notation.num_to_str(tb.txpath.bitrate()),
eng_notation.num_to_str(tb.rxpath.bitrate()))
print "modulation: %s" % (options.modulation,)
print "freq: %s" % (eng_notation.num_to_str(options.tx_freq))
print "bitrate: %sb/sec" % (eng_notation.num_to_str(tb.txpath.bitrate()),)
print "samples/symbol: %3d" % (tb.txpath.samples_per_symbol(),)
#print "interp: %3d" % (tb.txpath.interp(),)
#print "decim: %3d" % (tb.rxpath.decim(),)
tb.rxpath.set_carrier_threshold(options.carrier_threshold)
print "Carrier sense threshold:", options.carrier_threshold, "dB"
print
print "Allocated virtual ethernet interface: %s" % (tun_ifname,)
print "You must now use ifconfig to set its IP address. E.g.,"
print
print " $ sudo ifconfig %s 192.168.200.1" % (tun_ifname,)
print
print "Be sure to use a different address in the same subnet for each machine."
print
tb.start() # Start executing the flow graph (runs in separate threads)
mac.main_loop() # don't expect this to return...
tb.stop() # but if it does, tell flow graph to stop.
tb.wait() # wait for it to finish
if __name__ == '__main__':
try:
main()
except KeyboardInterrupt:
pass
(Anyone familiar with GNU Radio will recognize this as tunnel.py)
My question is, is there a better way to move packets to and from the kernel than tun/tap? I've been looking at ipip or maybe using sockets, but I'm pretty sure those won't be very fast. Speed is what I'm most concerned with.
Remember that tunnel.py is a really, really rough example, and hasn't been updated in a while. It's not really meant to be a basis for other code, so be careful of how much you rely on the code.
Also, remember that TCP over unreliable radio links has significant issues:
http://en.wikipedia.org/wiki/Transmission_Control_Protocol#TCP_over_wireless_networks
Related
I have been trying to establish CAN communication between my laptop (Ubuntu 20 Virtualbox) and Raspberry Pi (Ubuntu 20) without any CAN hardware, because that will not get the CAN message in the simulation environment. I want to send the CAN data as payload through wifi or USB. Then my python simulation environment should be able to interpret these as CAN messages and forward appropriately.
I have tried vcan from socketcan but it works only between 2 terminals of the same Linux machine. I have been advised to look at slcan. It seems like there is no other option other than using actual CAN hardware. I can't find any tutorial or any other help anywhere.
I will humbly appreciate if anyone can suggest how to send and receive CAN messages between 2 Linux machines without any CAN hardware through wifi or USB?
Python source code I tried:
import sys
import socket
import argparse
import struct
import errno
class CANSocket(object):
FORMAT = "<IB3x8s"
FD_FORMAT = "<IB3x64s"
CAN_RAW_FD_FRAMES = 5
def __init__(self, interface=None):
self.sock = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW)
if interface is not None:
self.bind(interface)
def bind(self, interface):
self.sock.bind((interface,))
self.sock.setsockopt(socket.SOL_CAN_RAW, self.CAN_RAW_FD_FRAMES, 1)
def send(self, cob_id, data, flags=0):
cob_id = cob_id | flags
can_pkt = struct.pack(self.FORMAT, cob_id, len(data), data)
self.sock.send(can_pkt)
def recv(self, flags=0):
can_pkt = self.sock.recv(72)
if len(can_pkt) == 16:
cob_id, length, data = struct.unpack(self.FORMAT, can_pkt)
else:
cob_id, length, data = struct.unpack(self.FD_FORMAT, can_pkt)
cob_id &= socket.CAN_EFF_MASK
return (cob_id, data[:length])
def format_data(data):
return ''.join([hex(byte)[2:] for byte in data])
def generate_bytes(hex_string):
if len(hex_string) % 2 != 0:
hex_string = "0" + hex_string
int_array = []
for i in range(0, len(hex_string), 2):
int_array.append(int(hex_string[i:i+2], 16))
return bytes(int_array)
def send_cmd(args):
try:
s = CANSocket(args.interface)
except OSError as e:
sys.stderr.write('Could not send on interface {0}\n'.format(args.interface))
sys.exit(e.errno)
try:
cob_id = int(args.cob_id, 16)
except ValueError:
sys.stderr.write('Invalid cob-id {0}\n'.format(args.cob_id))
sys.exit(errno.EINVAL)
s.send(cob_id, generate_bytes(args.body), socket.CAN_EFF_FLAG if args.extended_id else 0)
def listen_cmd(args):
try:
s = CANSocket(args.interface)
except OSError as e:
sys.stderr.write('Could not listen on interface {0}\n'.format(args.interface))
sys.exit(e.errno)
print('Listening on {0}'.format(args.interface))
while True:
cob_id, data = s.recv()
print('%s %03x#%s' % (args.interface, cob_id, format_data(data)))
def parse_args():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers()
send_parser = subparsers.add_parser('send', help='send a CAN packet')
send_parser.add_argument('interface', type=str, help='interface name (e.g. vcan0)')
send_parser.add_argument('cob_id', type=str, help='hexadecimal COB-ID (e.g. 10a)')
send_parser.add_argument('body', type=str, nargs='?', default='',
help='hexadecimal msg body up to 8 bytes long (e.g. 00af0142fe)')
send_parser.add_argument('-e', '--extended-id', action='store_true', default=False,
help='use extended (29 bit) COB-ID')
send_parser.set_defaults(func=send_cmd)
listen_parser = subparsers.add_parser('listen', help='listen for and print CAN packets')
listen_parser.add_argument('interface', type=str, help='interface name (e.g. vcan0)')
listen_parser.set_defaults(func=listen_cmd)
return parser.parse_args()
def main():
args = parse_args()
args.func(args)
if __name__ == '__main__':
main()
CAN is a broadcast protocol and not a connection protocol like TCP or datagram protocol like UDP. However, ff you want abstract the hardware but still simulate the higher layer behaviour of CAN, I'd suggest the best way to do would be with UDP (but TCP as suggested by #M. Spiller also works, just means handling accepting connections which you don't have to bother with for UDP), and send and receive packets with a byte structure the same as a CAN frame. You would also need to account for filters and error frames.
If you're going to use socketcan on linux, your UDP packets should look like:
struct {
uint16_t id; /* CAN ID of the frame */
uint8_t dlc; /* frame payload length in bytes */
uint8_t data[8]; /* CAN frame payload */
} simulated_udp_can_frame;
You should read the socketCAN kernel documentation for more detail.
It seems there is not very much support for what I am trying to do, but it is supposed to be possible since it is demonstrated in temperature sensor and sensor filter tutorial. However, there are no examples for the actual message creation from an edge module in python. That tutorial only shows forwarding messages. There are examples of sending from a device, but devices use a different class than edge modules. From the filter example and from a couple of device examples I have pieced together the following:
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for
# full license information.
import random
import time
import sys
import iothub_client
from iothub_client import IoTHubModuleClient, IoTHubClientError, IoTHubTransportProvider
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError
# messageTimeout - the maximum time in milliseconds until a message times out.
# The timeout period starts at IoTHubModuleClient.send_event_async.
# By default, messages do not expire.
MESSAGE_TIMEOUT = 10000
# global counters
RECEIVE_CALLBACKS = 0
SEND_CALLBACKS = 0
# Choose HTTP, AMQP or MQTT as transport protocol. Currently only MQTT is supported.
PROTOCOL = IoTHubTransportProvider.MQTT
# Callback received when the message that we're forwarding is processed.
def send_confirmation_callback(message, result, user_context):
global SEND_CALLBACKS
print ( "Confirmation[%d] received for message with result = %s" % (user_context, result) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
SEND_CALLBACKS += 1
print ( " Total calls confirmed: %d" % SEND_CALLBACKS )
# receive_message_callback is invoked when an incoming message arrives on the specified
# input queue (in the case of this sample, "input1"). Because this is a filter module,
# we will forward this message onto the "output1" queue.
def receive_message_callback(message, hubManager):
global RECEIVE_CALLBACKS
message_buffer = message.get_bytearray()
size = len(message_buffer)
print ( " Data: <<<%s>>> & Size=%d" % (message_buffer[:size].decode('utf-8'), size) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
RECEIVE_CALLBACKS += 1
print ( " Total calls received: %d" % RECEIVE_CALLBACKS )
hubManager.forward_event_to_output("output1", message, 0)
return IoTHubMessageDispositionResult.ACCEPTED
def construct_message(message_body, topic):
try:
msg_txt_formatted = message_body
message = IoTHubMessage(msg_txt_formatted)
# Add a custom application property to the message.
# An IoT hub can filter on these properties without access to the message body.
prop_map = message.properties()
prop_map.add("topic", topic)
# TODO Use logging
# Send the message.
print( "Sending message: %s" % message.get_string() )
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
return message
class HubManager(object):
def __init__(
self,
protocol=IoTHubTransportProvider.MQTT):
self.client_protocol = protocol
self.client = IoTHubModuleClient()
self.client.create_from_environment(protocol)
# set the time until a message times out
self.client.set_option("messageTimeout", MESSAGE_TIMEOUT)
# sets the callback when a message arrives on "input1" queue. Messages sent to
# other inputs or to the default will be silently discarded.
self.client.set_message_callback("input1", receive_message_callback, self)
# Forwards the message received onto the next stage in the process.
def forward_event_to_output(self, outputQueueName, event, send_context):
self.client.send_event_async(
outputQueueName, event, send_confirmation_callback, send_context)
def send_message(self, message):
# No callback
# TODO what is the third arg?
self.client.send_event_async(
"output1", message, send_confirmation_callback, 0)
self.client.send_message()
def mypublish(self, topic, msg):
message = construct_message(msg, topic)
self.send_message(message)
print('publishing %s', msg)
def main(protocol):
try:
print ( "\nPython %s\n" % sys.version )
print ( "IoT Hub Client for Python" )
hub_manager = HubManager(protocol)
print ( "Starting the IoT Hub Python sample using protocol %s..." % hub_manager.client_protocol )
print ( "The sample is now waiting for messages and will indefinitely. Press Ctrl-C to exit. ")
while True:
hub_manager.mypublish('testtopic', 'hello world this is a module')
time.sleep(1)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubModuleClient sample stopped" )
if __name__ == '__main__':
main(PROTOCOL)
When I build and deploy this it executes on the edge device without errors and in the log, the callback reports that the messages are sent ok. However, no messages come through when I attempt to monitor D2C messages.
I used this to create and send a message from a JSON dict.
new_message = json.dumps(json_obj)
new_message = IoTHubMessage(new_message)
hubManager.forward_event_to_output("output1", new_message, 0)
You can send anything you need, even strings or whatever.
To narrow down the issue, you can install the azureiotedge-simulated-temperature-sensor module published by Microsoft to see whether the issue relative to the Edge environment issue or coding.
I also wrote a sample Python code module based on the Python Module templates which works well for me, you can refer the code below:
# Copyright (c) Microsoft. All rights reserved.
# Licensed under the MIT license. See LICENSE file in the project root for
# full license information.
import random
import time
import sys
import iothub_client
from iothub_client import IoTHubModuleClient, IoTHubClientError, IoTHubTransportProvider
from iothub_client import IoTHubMessage, IoTHubMessageDispositionResult, IoTHubError
# messageTimeout - the maximum time in milliseconds until a message times out.
# The timeout period starts at IoTHubModuleClient.send_event_async.
# By default, messages do not expire.
MESSAGE_TIMEOUT = 10000
# global counters
RECEIVE_CALLBACKS = 0
SEND_CALLBACKS = 0
# Choose HTTP, AMQP or MQTT as transport protocol. Currently only MQTT is supported.
PROTOCOL = IoTHubTransportProvider.MQTT
# Callback received when the message that we're forwarding is processed.
def send_confirmation_callback(message, result, user_context):
global SEND_CALLBACKS
print ( "Confirmation[%d] received for message with result = %s" % (user_context, result) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
SEND_CALLBACKS += 1
print ( " Total calls confirmed: %d" % SEND_CALLBACKS )
# receive_message_callback is invoked when an incoming message arrives on the specified
# input queue (in the case of this sample, "input1"). Because this is a filter module,
# we will forward this message onto the "output1" queue.
def receive_message_callback(message, hubManager):
global RECEIVE_CALLBACKS
message_buffer = message.get_bytearray()
size = len(message_buffer)
print ( " Data: <<<%s>>> & Size=%d" % (message_buffer[:size].decode('utf-8'), size) )
map_properties = message.properties()
key_value_pair = map_properties.get_internals()
print ( " Properties: %s" % key_value_pair )
RECEIVE_CALLBACKS += 1
print ( " Total calls received: %d" % RECEIVE_CALLBACKS )
hubManager.forward_event_to_output("output1", message, 0)
return IoTHubMessageDispositionResult.ACCEPTED
class HubManager(object):
def __init__(
self,
protocol=IoTHubTransportProvider.MQTT):
self.client_protocol = protocol
self.client = IoTHubModuleClient()
self.client.create_from_environment(protocol)
# set the time until a message times out
self.client.set_option("messageTimeout", MESSAGE_TIMEOUT)
# sets the callback when a message arrives on "input1" queue. Messages sent to
# other inputs or to the default will be silently discarded.
self.client.set_message_callback("input1", receive_message_callback, self)
# Forwards the message received onto the next stage in the process.
def forward_event_to_output(self, outputQueueName, event, send_context):
self.client.send_event_async(
outputQueueName, event, send_confirmation_callback, send_context)
def SendSimulationData(self, msg):
print"sending message..."
message=IoTHubMessage(msg)
self.client.send_event_async(
"output1", message, send_confirmation_callback, 0)
print"finished sending message..."
def main(protocol):
try:
print ( "\nPython %s\n" % sys.version )
print ( "IoT Hub Client for Python" )
hub_manager = HubManager(protocol)
print ( "Starting the IoT Hub Python sample using protocol %s..." % hub_manager.client_protocol )
print ( "The sample is now waiting for messages and will indefinitely. Press Ctrl-C to exit. ")
while True:
hub_manager.SendSimulationData("test msg")
time.sleep(1)
except IoTHubError as iothub_error:
print ( "Unexpected error %s from IoTHub" % iothub_error )
return
except KeyboardInterrupt:
print ( "IoTHubModuleClient sample stopped" )
if __name__ == '__main__':
main(PROTOCOL)
If it can help someone, I think you miss await send_message.
Seems the same problem I answered here
I know only the very basics of python. I have this project for my INFORMATION STORAGE AND MANAGEMENT subject. I have to give an explanation the following code.
I searched every command used in this script but could not find most of them. The code can be found here:
import glob
import json
import os
import re
import string
import sys
from oslo.config import cfg
from nova import context
from nova.db.sqlalchemy import api as db_api
from nova.db.sqlalchemy import models
from nova import utils
CONF = cfg.CONF
def usage():
print("""
Usage:
python %s --config-file /etc/nova/nova.conf
Note: This script intends to clean up the iSCSI multipath faulty devices
hosted by VNX Block Storage.""" % sys.argv[0])
class FaultyDevicesCleaner(object):
def __init__(self):
# Get host name of Nova computer node.
self.host_name = self._get_host_name()
def _get_host_name(self):
(out, err) = utils.execute('hostname')
return out
def _get_ncpu_emc_target_info_list(self):
target_info_list = []
# Find the targets used by VM on the compute node
bdms = db_api.model_query(context.get_admin_context(),
models.BlockDeviceMapping,
session = db_api.get_session())
bdms = bdms.filter(models.BlockDeviceMapping.connection_info != None)
bdms = bdms.join(models.BlockDeviceMapping.instance).filter_by(
host=string.strip(self.host_name))
for bdm in bdms:
conn_info = json.loads(bdm.connection_info)
if 'data' in conn_info:
if 'target_iqns' in conn_info['data']:
target_iqns = conn_info['data']['target_iqns']
target_luns = conn_info['data']['target_luns']
elif 'target_iqn' in conn_info['data']:
target_iqns = [conn_info['data']['target_iqn']]
target_luns = [conn_info['data']['target_lun']]
else:
target_iqns = []
target_luns = []
for target_iqn, target_lun in zip(target_iqns, target_luns):
if 'com.emc' in target_iqn:
target_info = {
'target_iqn': target_iqn,
'target_lun': target_lun,
}
target_info_list.append(target_info)
return target_info_list
def _get_ncpu_emc_target_info_set(self):
target_info_set = set()
for target_info in self._get_ncpu_emc_target_info_list():
target_iqn = target_info['target_iqn']
target_lun = target_info['target_lun']
target_info_key = "%s-%s" % (target_iqn.rsplit('.', 1)[0],
target_lun)
# target_iqn=iqn.1992-04.com.emc:cx.fnm00130200235.a7
# target_lun=203
# target_info_key=iqn.1992-04.com.emc:cx.fnm00130200235-203
target_info_set.add(target_info_key)
return target_info_set
def _get_target_info_key(self, path):
temp_tuple = path.split('-lun-', 1)
target_lun = temp_tuple[1]
target_iqn = temp_tuple[0].split('-iscsi-')[1]
target_info_key = "%s-%s" % (target_iqn.rsplit('.', 1)[0], target_lun)
# path=/dev/disk/by-path/ip-192.168.3.52:3260-iscsi-iqn.1992-
# 04.com.emc:cx.fnm00130200235.a7-lun-203
# target_info_key=iqn.1992-04.com.emc:cx.fnm00130200235-203
return target_info_key
def _get_non_ncpu_target_info_map(self):
# Group the paths by target_info_key
ncpu_target_info_set = self._get_ncpu_emc_target_info_set()
device_paths = self._get_emc_device_paths()
target_info_map = {}
for path in device_paths:
target_info_key = self._get_target_info_key(path)
if target_info_key in ncpu_target_info_set:
continue
if target_info_key not in target_info_map:
target_info_map[target_info_key] = []
target_info_map[target_info_key].append(path)
return target_info_map
def _all_related_paths_faulty(self, paths):
for path in paths:
real_path = os.path.realpath(path)
out, err = self._run_multipath(['-ll', real_path],
run_as_root=True,
check_exit_code=False)
if 'active ready' in out:
# At least one path is still working
return False
return True
def _delete_all_related_paths(self, paths):
for path in paths:
real_path = os.path.realpath(path)
device_name = os.path.basename(real_path)
device_delete = '/sys/block/%s/device/delete' % device_name
if os.path.exists(device_delete):
# Copy '1' from stdin to the device delete control file
utils.execute('cp', '/dev/stdin', device_delete,
process_input='1', run_as_root=True)
else:
print "Unable to delete %s" % real_path
def _cleanup_faulty_paths(self):
non_ncpu_target_info_map = self._get_non_ncpu_target_info_map()
for paths in non_ncpu_target_info_map.itervalues():
if self._all_related_paths_faulty(paths):
self._delete_all_related_paths(paths)
def _cleanup_faulty_dm_devices(self):
out_ll, err_ll = self._run_multipath(['-ll'],
run_as_root=True,
check_exit_code=False)
# Pattern to split the dm device contents as follows
# Each section starts with a WWN and ends with a line with
# " `-" as the prefix
#
# 3600601601bd032007c097518e96ae411 dm-2 ,
# size=1.0G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
# `-+- policy='round-robin 0' prio=0 status=active
# `- #:#:#:# - #:# active faulty running
# 36006016020d03200bb93e048f733e411 dm-0 DGC,VRAID
# size=1.0G features='1 queue_if_no_path' hwhandler='1 alua' wp=rw
# |-+- policy='round-robin 0' prio=130 status=active
# | |- 3:0:0:2 sdd 8:48 active ready running
# | `- 5:0:0:2 sdj 8:144 active ready running
# `-+- policy='round-robin 0' prio=10 status=enabled
# |- 4:0:0:2 sdg 8:96 active ready running
# `- 6:0:0:2 sdm 8:192 active ready running
dm_pat = r'([0-9a-fA-F]{30,})[^\n]+,[^\n]*\n[^,]* `-[^\n]*'
dm_m = re.compile(dm_pat)
path_pat = r'- \d+:\d+:\d+:\d+ '
path_m = re.compile(path_pat)
for m in dm_m.finditer(out_ll):
if not path_m.search(m.group(0)):
# Only #:#:#:# remain in the output, all the paths of the dm
# device should have been deleted. No need to keep the device
out_f, err_f = self._run_multipath(['-f', m.group(1)],
run_as_root=True,
check_exit_code=False)
def cleanup(self):
self._cleanup_faulty_paths()
# Make sure the following configuration is in /etc/multipath.conf
# Otherwise, there may be "map in use" failure when deleting
# dm device
#
# defaults {
# flush_on_last_del yes
# }
#
self._cleanup_faulty_dm_devices()
def _get_emc_device_paths(self):
# Find all the EMC iSCSI devices under /dev/disk/by-path
# except LUNZ and partition reference
pattern = '/dev/disk/by-path/ip-*-iscsi-iqn*com.emc*-lun-*'
device_paths = [path for path in glob.glob(pattern)
if ('lun-0' not in path and '-part' not in path)]
return device_paths
def _run_multipath(self, multipath_command, **kwargs):
check_exit_code = kwargs.pop('check_exit_code', 0)
(out, err) = utils.execute('multipath',
*multipath_command,
run_as_root=True,
check_exit_code=check_exit_code)
print ("multipath %(command)s: stdout=%(out)s stderr=%(err)s"
% {'command': multipath_command, 'out': out, 'err': err})
return out, err
if __name__ == "__main__":
if len(sys.argv) != 3 or sys.argv[1] != '--config-file':
usage()
exit(1)
out, err = utils.execute('which', 'multipath', check_exit_code=False)
if 'multipath' not in out:
print('Info: Multipath tools not installed. No cleanup need be done.')
exit(0)
multipath_flush_on_last_del = False
multipath_conf_path = "/etc/multipath.conf"
if os.path.exists(multipath_conf_path):
flush_on_last_del_yes = re.compile(r'\s*flush_on_last_del.*yes')
for line in open(multipath_conf_path, "r"):
if flush_on_last_del_yes.match(line):
multipath_flush_on_last_del = True
break
if not multipath_flush_on_last_del:
print("Warning: 'flush_on_last_del yes' is not seen in"
" /etc/multipath.conf."
" 'map in use' failure may show up during cleanup.")
CONF(sys.argv[1:])
# connect_volume and disconnect_volume in nova/virt/libvirt/volume.py
# need be adjusted to take the same 'external=True' lock for
# synchronization
#utils.synchronized('connect_volume', external=True)
def do_cleanup():
cleaner = FaultyDevicesCleaner()
cleaner.cleanup()
do_cleanup()
https://wiki.python.org/moin/BeginnersGuide/Programmers
http://www.astro.ufl.edu/~warner/prog/python.html
looks like this python version 3 so. go for the tutorials of version three.
try downloading any IDE. eric5 is good by the way.
try executing this file once.
learn indentations
and dynamic variable declaration
do not jump into the ocean first try swimming pool : )
Also Try to learn method declaration.
Python is a bit different than java.
I will give you a hint looks like system call are also made to execute os commands so try looking at subprocess and how its output is directed to an output stream and error stream.
I'm using this wonderful resource to grab tweets from twitter.
Twitter streams the tweets, and this script grabs them and displays them in the terminal.
I apologize for the beginners question, but how would I limit the number of tweets or the total size of all tweets that are received using this script?
I understand that I can write these to a file and watch the size of the file, but is there a way to do count the bytes received or number of tweets received?
The script is below as well as in the link above.
#!/usr/bin/python
# -*- coding: utf-8 -*-
# Copyright (C) 2012 Gustav Arngården
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import time
import pycurl
import urllib
import json
import oauth2 as oauth
API_ENDPOINT_URL = 'https://stream.twitter.com/1.1/statuses/filter.json'
USER_AGENT = 'TwitterStream 1.0' # This can be anything really
# You need to replace these with your own values
OAUTH_KEYS = {'consumer_key': <Consumer key>,
'consumer_secret': <Consumer secret>,
'access_token_key': <Token key>,
'access_token_secret': <Token secret>}
# These values are posted when setting up the connection
POST_PARAMS = {'include_entities': 0,
'stall_warning': 'true',
'track': 'iphone,ipad,ipod'}
class TwitterStream:
def __init__(self, timeout=False):
self.oauth_token = oauth.Token(key=OAUTH_KEYS['access_token_key'], secret=OAUTH_KEYS['access_token_secret'])
self.oauth_consumer = oauth.Consumer(key=OAUTH_KEYS['consumer_key'], secret=OAUTH_KEYS['consumer_secret'])
self.conn = None
self.buffer = ''
self.timeout = timeout
self.setup_connection()
def setup_connection(self):
""" Create persistant HTTP connection to Streaming API endpoint using cURL.
"""
if self.conn:
self.conn.close()
self.buffer = ''
self.conn = pycurl.Curl()
# Restart connection if less than 1 byte/s is received during "timeout" seconds
if isinstance(self.timeout, int):
self.conn.setopt(pycurl.LOW_SPEED_LIMIT, 1)
self.conn.setopt(pycurl.LOW_SPEED_TIME, self.timeout)
self.conn.setopt(pycurl.URL, API_ENDPOINT_URL)
self.conn.setopt(pycurl.USERAGENT, USER_AGENT)
# Using gzip is optional but saves us bandwidth.
self.conn.setopt(pycurl.ENCODING, 'deflate, gzip')
self.conn.setopt(pycurl.POST, 1)
self.conn.setopt(pycurl.POSTFIELDS, urllib.urlencode(POST_PARAMS))
self.conn.setopt(pycurl.HTTPHEADER, ['Host: stream.twitter.com',
'Authorization: %s' % self.get_oauth_header()])
# self.handle_tweet is the method that are called when new tweets arrive
self.conn.setopt(pycurl.WRITEFUNCTION, self.handle_tweet)
def get_oauth_header(self):
""" Create and return OAuth header.
"""
params = {'oauth_version': '1.0',
'oauth_nonce': oauth.generate_nonce(),
'oauth_timestamp': int(time.time())}
req = oauth.Request(method='POST', parameters=params, url='%s?%s' % (API_ENDPOINT_URL,
urllib.urlencode(POST_PARAMS)))
req.sign_request(oauth.SignatureMethod_HMAC_SHA1(), self.oauth_consumer, self.oauth_token)
return req.to_header()['Authorization'].encode('utf-8')
def start(self):
""" Start listening to Streaming endpoint.
Handle exceptions according to Twitter's recommendations.
"""
backoff_network_error = 0.25
backoff_http_error = 5
backoff_rate_limit = 60
while True:
self.setup_connection()
try:
self.conn.perform()
except:
# Network error, use linear back off up to 16 seconds
print 'Network error: %s' % self.conn.errstr()
print 'Waiting %s seconds before trying again' % backoff_network_error
time.sleep(backoff_network_error)
backoff_network_error = min(backoff_network_error + 1, 16)
continue
# HTTP Error
sc = self.conn.getinfo(pycurl.HTTP_CODE)
if sc == 420:
# Rate limit, use exponential back off starting with 1 minute and double each attempt
print 'Rate limit, waiting %s seconds' % backoff_rate_limit
time.sleep(backoff_rate_limit)
backoff_rate_limit *= 2
else:
# HTTP error, use exponential back off up to 320 seconds
print 'HTTP error %s, %s' % (sc, self.conn.errstr())
print 'Waiting %s seconds' % backoff_http_error
time.sleep(backoff_http_error)
backoff_http_error = min(backoff_http_error * 2, 320)
def handle_tweet(self, data):
""" This method is called when data is received through Streaming endpoint.
"""
self.buffer += data
if data.endswith('\r\n') and self.buffer.strip():
# complete message received
message = json.loads(self.buffer)
self.buffer = ''
msg = ''
if message.get('limit'):
print 'Rate limiting caused us to miss %s tweets' % (message['limit'].get('track'))
elif message.get('disconnect'):
raise Exception('Got disconnect: %s' % message['disconnect'].get('reason'))
elif message.get('warning'):
print 'Got warning: %s' % message['warning'].get('message')
else:
print 'Got tweet with text: %s' % message.get('text')
if __name__ == '__main__':
ts = TwitterStream()
ts.setup_connection()
ts.start()
Why not a counter in your __init__:
def __init__(self, timeout=False):
...
self.tweetsSoFar = 0
def handleTweet(self, data):
...
else:
self.tweetsSoFar += 1 # we've seen another tweet!
print 'Got tweet with text: %s' % message.get('text')
def start(self):
...
while self.tweetsSoFar < 50: # stop at 50 tweets
...
Hope this helps
I'm not too familiar with the API, but assuming each complete message represents a tweet, then what you can do is to first add a counter in the class:
self.tweets = 0
Then increment it in handle_tweet after a full message is received:
if data.endswith('\r\n') and self.buffer.strip():
# complete message received
message = json.loads(self.buffer)
...
...
else:
print 'Got tweet with text: %s' % message.get('text')
self.tweets += 1
if self.tweets == SOME_LIMIT:
self.conn.close() # or exit()
I'd like to make something like this:
10.1.1.0/24 10.1.2.0/24
+------------+ +------------+ +------------+
| | | | | |
| | | | | |
| A d +-------+ e B f +-------+ g C |
| | | | | |
| | | | | |
+------------+ +------------+ +------------+
d e f g
10.1.1.1 10.1.1.2 10.1.2.1 10.1.2.2
So that Acan send packets to C through B.
I attempted to build this thing by running a scapy program on B that would sniff ports e and f, and in each case modify the destination IP and MAC address in the packet and then send it along through the other interface. Something like:
my_macs = [get_if_hwaddr(i) for i in get_if_list()]
pktcnt = 0
dest_mac_address = discover_mac_for_ip(dest_ip) #
output_mac = get_if_hwaddr(output_interface)
def process_packet(pkt):
# ignore packets that were sent from one of our own interfaces
if pkt[Ether].src in my_macs:
return
pktcnt += 1
p = pkt.copy()
# if this packet has an IP layer, change the dst field
# to our final destination
if IP in p:
p[IP].dst = dest_ip
# if this packet has an ethernet layer, change the dst field
# to our final destination. We have to worry about this since
# we're using sendp (rather than send) to send the packet. We
# also don't fiddle with it if it's a broadcast address.
if Ether in p \
and p[Ether].dst != 'ff:ff:ff:ff:ff:ff':
p[Ether].dst = dest_mac_address
p[Ether].src = output_mac
# use sendp to avoid ARP'ing and stuff
sendp(p, iface=output_interface)
sniff(iface=input_interface, prn=process_packet)
However, when I run this thing (full source here) all sorts of crazy things start to happen... Some of the packets get through, and I even get some responses (testing with ping) but there's some type of feedback loop that's causing a bunch of duplicate packets to get sent...
Any ideas what's going on here? Is it crazy to try to do this?
I'm kind of suspicious that the feedback loops are being caused by the fact that B is doing some processing of its own on the packets... Is there any way to prevent the OS from processing a packet after I've sniffed it?
IP packets bridging using scapy:
first make sure you have ip forwarding disabled otherwise duplicate packets will be noticed:
echo "0" > /proc/sys/net/ipv4/ip_forward <br>
second run the following python/scapy script:
!/usr/bin/python2
from optparse import OptionParser
from scapy.all import *
from threading import Thread
from struct import pack, unpack
from time import sleep
def sp_byte(val):
return pack("<B", val)
def su_nint(str):
return unpack(">I", str)[0]
def ipn2num(ipn):
"""ipn(etwork) is BE dotted string ip address
"""
if ipn.count(".") != 3:
print("ipn2num warning: string < %s > is not proper dotted IP address" % ipn)
return su_nint( "".join([sp_byte(int(p)) for p in ipn.strip().split(".")]))
def get_route_if(iface):
try:
return [route for route in conf.route.routes if route[3] == iface and route[2] == "0.0.0.0"][0]
except IndexError:
print("Interface '%s' has no ip address configured or link is down?" % (iface));
return None;
class PacketCapture(Thread):
def __init__(self, net, nm, recv_iface, send_iface):
Thread.__init__(self)
self.net = net
self.netmask = nm
self.recv_iface = recv_iface
self.send_iface = send_iface
self.recv_mac = get_if_hwaddr(recv_iface)
self.send_mac = get_if_hwaddr(send_iface)
self.filter = "ether dst %s and ip" % self.recv_mac
self.arp_cache = []
self.name = "PacketCapture(%s on %s)" % (self.name, self.recv_iface)
self.fw_count = 0
def run(self):
print("%s: waiting packets (%s) on interface %s" % (self.name, self.filter, self.recv_iface))
sniff(count = 0, prn = self.process, store = 0, filter = self.filter, iface = self.recv_iface)
def process(self, pkt):
# only bridge IP packets
if pkt.haslayer(Ether) and pkt.haslayer(IP):
dst_n = ipn2num(pkt[IP].dst)
if dst_n & self.netmask != self.net:
# don't forward if the destination ip address
# doesn't match the destination network address
return
# update layer 2 addresses
rmac = self.get_remote_mac(pkt[IP].dst)
if rmac == None:
print("%s: packet not forwarded %s %s -) %s %s" % (self.name, pkt[Ether].src, pkt[IP].src, pkt[Ether].dst, pkt[IP].dst))
return
pkt[Ether].src = self.send_mac
pkt[Ether].dst = rmac
#print("%s: forwarding %s %s -> %s %s" % (self.name, pkt[Ether].src, pkt[IP].src, pkt[Ether].dst, pkt[IP].dst))
sendp(pkt, iface = self.send_iface)
self.fw_count += 1
def get_remote_mac(self, ip):
mac = ""
for m in self.arp_cache:
if m["ip"] == ip and m["mac"]:
return m["mac"]
mac = getmacbyip(ip)
if mac == None:
print("%s: Could not resolve mac address for destination ip address %s" % (self.name, ip))
else:
self.arp_cache.append({"ip": ip, "mac": mac})
return mac
def stop(self):
Thread._Thread__stop(self)
print("%s stopped" % self.name)
if __name__ == "__main__":
parser = OptionParser(description = "Bridge packets", prog = "brscapy", usage = "Usage: brscapy -l <intf> (--left= <intf>) -r <inft> (--right=<intf>)")
parser.add_option("-l", "--left", action = "store", dest = "left", default = None, choices = get_if_list(), help = "Left side network interface of the bridge")
parser.add_option("-r", "--right", action = "store", dest = "right", default = None, choices = get_if_list(), help = "Right side network interface of the bridge")
args, opts = parser.parse_args()
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
lif = args.left
rif = args.right
lroute = get_route_if(lif)
rroute = get_route_if(rif)
if (lroute == None or rroute == None):
print("Invalid ip addressing on given interfaces");
exit(1)
if (len(lroute) != 5 or len(rroute) != 5):
print("Invalid scapy routes")
exit(1)
conf.verb = 0
lthread = PacketCapture(rroute[0], rroute[1], lif, rif)
rthread = PacketCapture(lroute[0], lroute[1], rif, lif)
lthread.start()
rthread.start()
try:
while True:
sys.stdout.write("FORWARD count: [%s -> %s %d] [%s <- %s %d]\r" % (lif, rif, lthread.fw_count, lif, rif, rthread.fw_count))
sys.stdout.flush()
sleep(0.1)
except KeyboardInterrupt:
pass
lthread.stop()
rthread.stop()
lthread.join()
rthread.join()
On my pc:
# ./brscapy.py --help
Usage: brscapy -l <intf> (--left= <intf>) -r <inft> (--right=<intf>)
Bridge packets
Options:
-h, --help show this help message and exit
-l LEFT, --left=LEFT Left side network interface of the bridge
-r RIGHT, --right=RIGHT
Right side network interface of the bridge
# ./brscapy.py -l e0 -r e2
PacketCapture(Thread-1 on e0): waiting packets (ether dst 00:16:41:ea:ff:dc and ip) on interface e0
PacketCapture(Thread-2 on e2): waiting packets (ether dst 00:0d:88:cc:ed:15 and ip) on interface e2
FORWARD count: [e0 -> e2 5] [e0 <- e2 5]
It is kinda crazy to do this, but it's not a bad way to spend your time. You'll learn a bunch of interesting stuff. However you might want to think about hooking the packets a little lower - I don't think scapy is capable of actually intercepting packets - all libpcap does is set you promisc and let you see everything, so you and the kernel are both getting the same stuff. If you're turning around and resending it, thats likely the cause of your packet storm.
However, you could set up some creative firewall rules that partition each interface off from each-other and hand the packets around that way, or use something like divert sockets to actually thieve the packets away from the kernel so you can have your way with them.