We Keep Coding

Linux NoHup fails for Streaming API IG Markets where file is python

This is quite a specific question regarding nohup in linux, which runs a python file.
Back-story, I am trying to save down streaming data (from IG markets broadcast signal). And, as I am trying to run it via a remote-server (so I don't have to keep my own local desktop up 24/7),
somehow, the nohup will not engage when it 'listen's to a broadcast signal.
Below, is the example python code
#!/usr/bin/env python
#-*- coding:utf-8 -*-
"""
IG Markets Stream API sample with Python
"""
user_ = 'xxx'
password_ = 'xxx'
api_key_ = 'xxx' # this is the 1st api key
account_ = 'xxx'
acc_type_ = 'xxx'
fileLoc = 'marketdata_IG_spx_5min.csv'
list_ = ["CHART:IX.D.SPTRD.DAILY.IP:5MINUTE"]
fields_ = ["UTM", "LTV", "TTV", "BID_OPEN", "BID_HIGH", \
"BID_LOW", "BID_CLOSE",]
import time
import sys
import traceback
import logging
import warnings
warnings.filterwarnings('ignore')
from trading_ig import (IGService, IGStreamService)
from trading_ig.lightstreamer import Subscription
cols_ = ['timestamp', 'data']
# A simple function acting as a Subscription listener
def on_prices_update(item_update):
# print("price: %s " % item_update)
print("xxxxxxxx
))
# A simple function acting as a Subscription listener
def on_charts_update(item_update):
# print("price: %s " % item_update)
print(xxxxxx"\
.format(
stock_name=item_update["name"], **item_update["values"]
))
res_ = [xxxxx"\
.format(
stock_name=item_update["name"], **item_update["values"]
).split(' '))]
# display(pd.DataFrame(res_))
try:
data_ = pd.read_csv(fileLoc)[cols_]
data_ = data_.append(pd.DataFrame(res_, columns = cols_))
data_.to_csv(fileLoc)
print('there is data and we are reading it')
# display(data_)
except:
pd.DataFrame(res_, columns = cols_).to_csv(fileLoc)
print('there is no data and we are saving first time')
time.sleep(60) # sleep for 1 min
def main():
logging.basicConfig(level=logging.INFO)
# logging.basicConfig(level=logging.DEBUG)
ig_service = IGService(
user_, password_, api_key_, acc_type_
)
ig_stream_service = IGStreamService(ig_service)
ig_session = ig_stream_service.create_session()
accountId = account_
################ my code to set sleep function to sleep/read at only certain time intervals
s_time = time.time()
############################
# Making a new Subscription in MERGE mode
subscription_prices = Subscription(
mode="MERGE",
# make sure to put L1 in front of the instrument name
items= list_,
fields= fields_
)
# adapter="QUOTE_ADAPTER")
# Adding the "on_price_update" function to Subscription
subscription_prices.addlistener(on_charts_update)
# Registering the Subscription
sub_key_prices = ig_stream_service.ls_client.subscribe(subscription_prices)
print('this is the line here')
input("{0:-^80}\n".format("HIT CR TO UNSUBSCRIBE AND DISCONNECT FROM \
LIGHTSTREAMER"))
# Disconnecting
ig_stream_service.disconnect()
if __name__ == '__main__':
main()
#######
Then, I try to run it on linux using this command : nohup python marketdata.py
where marketdata.py is basically the python code above.
Somehow, the nohup will not engage....... Any experts/guru who might see what I am missing in my code?

Deterministic runs in Carla

I'd like to be able to run the exact same run, given I didn't change any parameters of the simulation, in the autonomous driving simulator Carla. Before I paste my code, my logic is that I have to set a specific seed for any Random operation to be repeatable, set a specific seed for the traffic manager, and in general to work in synchronous_mode=True so the lag in my computer won't interrupt(?). As you'll see, I log the x,y,z location of the ego vehicle, and run the simulation twice. It is similar, but not the same. What can I do to make it repeatable (not in recording mode, actual live runs)?
Additional info: Carla 0.9.14 on Ubuntu 20.04, Python 3.8.
import random
import numpy as np
import sys
import os
try:
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/carla')
except IndexError:
pass
import carla
from agents.navigation.behavior_agent import BehaviorAgent # pylint: disable=import-error
seed = 123
N_vehicles = 50
camera = None
telemetry = []
random.seed(seed)
try:
# Connect the client and set up bp library and spawn points
client = carla.Client('localhost', 2000)
client.set_timeout(60.0)
world = client.get_world()
bp_lib = world.get_blueprint_library()
spawn_points = world.get_map().get_spawn_points()
settings = world.get_settings()
settings.synchronous_mode = True
settings.fixed_delta_seconds = 0.10
world.apply_settings(settings)
traffic_manager = client.get_trafficmanager()
traffic_manager.set_random_device_seed(seed)
traffic_manager.set_synchronous_mode(True)
# Spawn ego vehicle
vehicle_bp = bp_lib.find('vehicle.audi.a2')
# breakpoint()
vehicle = world.try_spawn_actor(vehicle_bp, random.choice(spawn_points))
# Move spectator behind vehicle to motion
spectator = world.get_spectator()
transform = carla.Transform(vehicle.get_transform().transform(carla.Location(x=-6,z=2.5)),vehicle.get_transform().rotation)
spectator.set_transform(transform)
world.tick()
# set the car's controls
agent = BehaviorAgent(vehicle, behavior="normal")
destination = random.choice(spawn_points).location
agent.set_destination(destination)
print('destination:')
print(destination)
print('current location:')
print(vehicle.get_location())
#Iterate this cell to find desired camera location
camera_bp = bp_lib.find('sensor.camera.rgb')
# Spawn camera
camera_init_trans = carla.Transform(carla.Location(z=2))
camera = world.spawn_actor(camera_bp, camera_init_trans, attach_to=vehicle)
# Callback stores sensor data in a dictionary for use outside callback
def camera_callback(image, data_dict):
data_dict['image'] = np.reshape(np.copy(image.raw_data), (image.height, image.width, 4))
# Get gamera dimensions and initialise dictionary
image_w = camera_bp.get_attribute("image_size_x").as_int()
image_h = camera_bp.get_attribute("image_size_y").as_int()
camera_data = {'image': np.zeros((image_h, image_w, 4))}
# Start camera recording
camera.listen(lambda image: camera_callback(image, camera_data))
# Add traffic to the simulation
SpawnActor = carla.command.SpawnActor
SetAutopilot = carla.command.SetAutopilot
FutureActor = carla.command.FutureActor
vehicles_list, batch = [], []
for i in range(N_vehicles):
ovehicle_bp = random.choice(bp_lib.filter('vehicle'))
npc = world.try_spawn_actor(ovehicle_bp, random.choice(spawn_points))
# add it if it was successful
if(npc):
vehicles_list.append(npc)
print(f'only {len(vehicles_list)} cars were spawned')
world.tick()
# Set the all vehicles in motion using the Traffic Manager
for idx, v in enumerate(vehicles_list):
try:
v.set_autopilot(True)
except:
pass
# Game loop
while True:
world.tick()
pose = vehicle.get_location()
telemetry.append([pose.x, pose.y, pose.z])
# keep following the car
transform = carla.Transform(vehicle.get_transform().transform(carla.Location(x=-6,z=2.5)),vehicle.get_transform().rotation)
spectator.set_transform(transform)
if agent.done():
print("The target has been reached, stopping the simulation")
break
control = agent.run_step()
control.manual_gear_shift = False
vehicle.apply_control(control)
finally:
# Stop the camera when we've recorded enough data
if(camera):
camera.stop()
camera.destroy()
settings = world.get_settings()
settings.synchronous_mode = False
settings.fixed_delta_seconds = None
world.apply_settings(settings)
traffic_manager.set_synchronous_mode(True)
if(vehicles_list):
client.apply_batch([carla.command.DestroyActor(v) for v in vehicles_list])
vehicle.destroy()
np.savetxt('telemetry.txt', np.array(telemetry), delimiter=',')
y-axis is the error between two runs, x-axis is the time index of the run

What is the pymodbus syntax to assign values to TCP server registers?

I am trying to implement a simple synchronous TCP server using the Synchronous Server Example. However, I do not understand the syntax explanations in the documentation. The example includes the following code block:
store = ModbusSlaveContext(
di=ModbusSequentialDataBlock(0, [17]*100),
co=ModbusSequentialDataBlock(0, [17]*100),
hr=ModbusSequentialDataBlock(0, [17]*100),
ir=ModbusSequentialDataBlock(0, [17]*100))
context = ModbusServerContext(slaves=store, single=True)
Suppose I want to store a value of 152 to Input Register (ir) address 30001 and a value of 276 to address 30002? How should I adapt the above code?

Suppose I want to store a value of 152 to 'Input Register (ir)' address 30001 and a value of 276 to address 30002? How should I adapt the above code?
Following code is what you want:
from pymodbus.server.sync import StartTcpServer
from pymodbus.datastore import ModbusSequentialDataBlock
from pymodbus.datastore import ModbusSlaveContext, ModbusServerContext
import logging
FORMAT = ('%(asctime)-15s %(threadName)-15s'
' %(levelname)-8s %(module)-15s:%(lineno)-8s %(message)s')
logging.basicConfig(format=FORMAT)
log = logging.getLogger()
log.setLevel(logging.DEBUG)
def run_server():
store = ModbusSlaveContext(
ir=ModbusSequentialDataBlock(30001, [152, 276]),
zero_mode=True
)
context = ModbusServerContext(slaves=store, single=True)
StartTcpServer(context, address=("localhost", 5020))
if __name__ == "__main__":
run_server()
Test Case:
from pymodbus.client.sync import ModbusTcpClient as ModbusClient
cli = ModbusClient('127.0.0.1', port=5020)
assert cli.connect()
res = cli.read_input_registers(30001, count=2, unit=1)
assert not res.isError()
print(res.registers)
Out:
[152, 276]

From the Pymodbus docs.
The datastores only respond to the addresses that they are initialized to
Therefore, if you initialize a DataBlock to addresses of 0x00 to 0xFF, a
request to 0x100 will respond with an invalid address exception. This is
because many devices exhibit this kind of behavior (but not all)::
block = ModbusSequentialDataBlock(0x00, [0]*0xff)
The ModbusSequentialDataBlock takes two arugments during initialisation address and value.
address – The starting address
values – List of values to initialise each address with.
The total size of the datablock depends on the len(values) .
So looking at the above example , we are trying to create a sequential datablock of size 0xFF and each address initialised with value 0. Similarly
In your case , if you want to store a value of 152 to 'Input Register (ir)' address 30001 and a value of 276 to address 30002, here is what you will have to do.
store = ModbusSlaveContext(
di=ModbusSequentialDataBlock(0, [17]*100),
co=ModbusSequentialDataBlock(0, [17]*100),
hr=ModbusSequentialDataBlock(0, [17]*100),
ir=ModbusSequentialDataBlock(0, [152, 276]), zero_mode=True)
context = ModbusServerContext(slaves=store, single=True)
Please note, the use of kwarg zero_mode=True with out this, read_input_registers request against offset 0 would return 276 instead of 152, this is because with out zero_mode=True pymodbus tries to store values based on section 4.4 of the specification, so address(0-7) will map to (1-8). If you don't want to use zero_mode then initialise the datablock as ir=ModbusSequentialDataBlock(1, [152, 276]).

socket programming Reliable Data Protocol

For part 1 of the project, you will implement a simple go-back-N protocol similar to TCP. This protocol is called the 352RDPv1.
(unfortunately my python knowledge is not that strong, I am being forced to code it in python)
I must implement: init(udp_port1, udpport2)
socket()
connect(address)
I am given the following pseudo code:
import binascii
import socket as syssock
import struct
import sys
# this init function is global to the class and
# defines the UDP ports all messages are sent
# and received from.
def init(UDPportTx,UDPportRx): # initialize your UDP socket here
# create a UDP/datagram socket
# bind the port to the Rx (receive) port number
pass
class socket:
def __init__(self): # fill in your code here
# create any lists/arrays/hashes you need
return
def connect(self,address): # fill in your code here
global UDPportTx # example using a variable global to the Python module
# create a new sequence number
# create a new packet header with the SYN bit set in the flags (use the Struct.pack method)
# also set the other fields (e.g sequence #)
# add the packet to the outbound queue
# set the timeout
# wait for the return SYN
# if there was a timeout, retransmit the SYN packet
# set the outbound and inbound sequence numbers
return
I have given a shot so far at a few methods but I know I have errors and my program does not work.
import binascii
import socket as syssock
import struct
from collections import namedtuple
import sys
import select
version = 0x1
header_len = 7
payload_len = 0
flags = 0
SOCK352_SYN = 0x01
SOCK352_FIN = 0x02
SOCK352_ACK = 0x04
SOCK352_RESET = 0x08
SOCK352_HAS_OPT = 0xA0
sequence_no = 0
ack_no = 0
timeout = 0.2
#given these values to set them to
def init(UDPportTx,UDPportRx):
global sock
global useRx
global useTx
useTx = int(UDPportTx)
useRx = int(UDPportRx)
sock = syssock.syssock(syssock.AF_INET, syssock.SOCK_DGRAM)
sock.bind(('', useRx))
print "Listening on port :", useRx
pass
class socket:
def __init__(self):
global pkt
pkt = namedtuple("pkt",["version", "flags", "sequence_no", "ack_no", "payload_len"])
return
def connect(self, address):
global header_raw
udpPkt = struct.Struct('!BLBBB')
header_raw = udpPkt.pack(version, SOCK352_SYN, sequence_no, ack_no, payload_len)
sock.sendto(header_raw, ('', useTx))
return
I believe I am having errors in these first few methods and before I move onto the others I need to figure in I want to see if anyone is able to help me understand how to handle these few to begin.

How to detect SSTP traffic?

I'm writing a program which should detect a VPN traffic.
As far as I read about the subject, it seems the detection of the tunneling protocol is easy as firewall rules, using their dedicated ports:
PPTP: port 1723/TCP
OpenVPN: port 1194
L2TP: port 1701/UDP
My problem is with the SSTP, because it is using port 443, which is widely used.
So I have 2 questions:
Am I too naive to think I can detect those VPNs tunneling protocols only by their ports?
Does anyone has any idea how to detect SSTP and differ its traffic from any other type / app which uses TLS/SSL (even using DPI)?
I'm attaching piece of Python code which detects the communication in the above ports
import dpkt
import socket
# -------------------------- Globals
# VPN PORTS
import common
import dal
protocols_strs = {"pp2e_gre": "1723/TCP PP2P_GRE_PORT",
"openvpn": "1194 OPENVPN_PORT",
"ike": "500/UDP IKE_PORT",
"l2tp_ipsec": "1701/UDP L2TP_IPSEC_PORT"
}
port_protocols = {1723: 'pp2e_gre',
1194: 'openvpn',
500: 'ike',
1701: 'l2tp_ipsec'
}
# Dict of sets holding the protocols sessions
protocol_sessions = {"pp2e_gre": [],
"openvpn": [],
"ike": [],
"l2tp_ipsec": []}
# -------------------------- Functions
def is_bidirectional(five_tuple, protocol):
"""
Given a tuple and protocol check if the connection is bidirectional in the protocol
:param five_tuple:
:return: True of the connection is bidirectional False otherwise
"""
src_ip = five_tuple['src_ip']
dest_ip = five_tuple['dest_ip']
# Filter the sessions the five tuple's ips spoke in
ike_sessions = filter(lambda session: (session['src_ip'] == src_ip and session['dest_ip'] == dest_ip)
or
(session['dest_ip'] == src_ip and session['src_ip'] == dest_ip),
protocol_sessions[protocol])
# Return true if 2 session (1 for each direction) were found
return len(ike_sessions) == 2
def print_alert(timestamp, protocol, five_tuple):
"""
Print alert description to std
:param timestamp:
:param protocol:
:param five_tuple:
:return:
"""
print timestamp, ":\t detected port %s communication (%s:%s ---> %s:%s)" % \
(protocol, five_tuple['src_ip'], five_tuple['src_port'], five_tuple['dest_ip'],
five_tuple['dest_port'])
def pp2e_gre_openvpn_ike_handler(five_tuple):
# Get protocol
protocol = five_tuple['protocol']
# Clear old sessions in db
dal.remove_old_sessions(five_tuple['timestamp'], 'vpn_sessions')
# Clear old sessions in cache
protocol_sessions[protocol] = common.clear_old_sessions(five_tuple, protocol_sessions[protocol])
# If session already exists - return
if common.check_if_session_exists(five_tuple, protocol_sessions[protocol]):
session_to_update = common.get_session(five_tuple, protocol_sessions[protocol])
session_to_update['timestamp'] = five_tuple['timestamp']
return
# Update DB
dal.upsert_vpn_session(five_tuple)
# Add to cache
protocol_sessions[protocol].append(five_tuple)
# Print alert
print_alert(five_tuple['timestamp'], protocols_strs[protocol], five_tuple)
def l2tp_ipsec_handler(five_tuple):
if five_tuple in protocol_sessions['l2tp_ipsec']:
return
# If bi-directional IKE protocol performed earlier - alert
if not is_bidirectional(five_tuple, 'ike'):
return
protocol_sessions['l2tp_ipsec'].append(five_tuple)
print_alert(five_tuple['timestamp'], protocols_strs['l2tp_ipsec'], five_tuple)
# -------------------------- VPN ports jump tables
tcp_vpn_ports = {1723: pp2e_gre_openvpn_ike_handler,
1194: pp2e_gre_openvpn_ike_handler}
udp_vpn_ports = {500: pp2e_gre_openvpn_ike_handler,
1701: l2tp_ipsec_handler,
1194: pp2e_gre_openvpn_ike_handler}
# -------------------------- Functions
def process_packet(timestamp, packet):
"""
Given a packet process it for detecting a VPN communication
:param packet:
:param timestamp:
:return:
"""
# Parse the input
eth_frame = dpkt.ethernet.Ethernet(packet)
# Check if IP
if eth_frame.type != dpkt.ethernet.ETH_TYPE_IP:
return
# If not IP return
ip_frame = eth_frame.data
# if TCP or UDP
if ip_frame.p not in (dpkt.ip.IP_PROTO_TCP, dpkt.ip.IP_PROTO_UDP):
return
# Extract L3 frame
frame = ip_frame.data
# Extract ports
frame_ports = (frame.sport, frame.dport)
# get VPN ports in session
detected_ports = set(tcp_vpn_ports).intersection(frame_ports)
# If TCP VPN port was not detected return
if not detected_ports:
return
# Get detected port
port = detected_ports.pop()
# Translate port to str
protocol_str = port_protocols[port]
# Choose handler by port
handler = tcp_vpn_ports[port]
# Extract 5-tuple parameters from frames
five_tuple = {'src_ip': socket.inet_ntoa(ip_frame.src),
'dest_ip': socket.inet_ntoa(ip_frame.dst),
'src_port': frame.sport,
'dest_port': frame.dport,
'protocol': protocol_str,
'timestamp': timestamp}
# Invoke the chosen handler
handler(five_tuple)

"Am I too naive to think I can detect those VPNs tunneling protocols only by their ports?:
"The official OpenVPN port number is 1194, but any port number between 1 and 65535 will work. If you don't provide the 'port' option, 1194 will be used.
So if your code is looking for 1194 traffic, as per the dictionary entries, you will capture default Open VPN flows only.
The SSTP message is encrypted with the SSL channel of the HTTPS protocol. So I don't see how you would identify this traffic as it is encrypted. (Source)