We Keep Coding

socketcan J1939 filter use in python

In Python, I am trying to use the J1939 filtering as mentionned in the linux kernel docs: https://www.kernel.org/doc/html/latest/networking/j1939.html
The following code fails at the setsockopt() line (setting up filters):
import socket
import struct
def pack_J1939_filters(can_filters):
can_filter_fmt = "=" + "2Q2B2I" * len(can_filters)
filter_data = []
for can_filter in can_filters:
name = can_filter['name']
name_mask = can_filter['name_mask']
addr = can_filter['addr']
addr_mask = can_filter['addr_mask']
pgn = can_filter['pgn']
pgn_mask = can_filter['pgn_mask']
filter_data.append(name)
filter_data.append(name_mask)
filter_data.append(addr)
filter_data.append(addr_mask)
filter_data.append(pgn)
filter_data.append(pgn_mask)
return struct.pack(can_filter_fmt, *filter_data)
s = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939)
interface = "vcan0"
src_name = socket.J1939_NO_NAME
src_pgn = socket.J1939_NO_PGN
src_addr = 0x81
src_sck_addr = (interface, src_name, src_pgn, src_addr)
s.bind(src_sck_addr)
filters = [{"name": 0, "name_mask":0, "addr":0, "addr_mask":0, "pgn": 0, "pgn_mask": 0}]
packed_filters = pack_J1939_filters(filters)
# socket.SOL_CAN_J1939 does not seem to exist
SOL_CAN_BASE = 100
CAN_J1939 = 7
SOL_CAN_J1939 = SOL_CAN_BASE + CAN_J1939
s.setsockopt(SOL_CAN_J1939, socket.SO_J1939_FILTER , packed_filters)
s.recvfrom(128)
s.close()
First, the kernel documentation mentions to use SOL_CAN_J1939 as the first argument. However socket.SOL_CAN_J1939 does not exist in the socket package. So looking at the code at this location I was able to understand that this int value should be 107: http://socket-can.996257.n3.nabble.com/RFC-v3-0-6-CAN-add-SAE-J1939-protocol-td7571.html
As for the setsockopt() third argument, I packed the filters to match the j1939_filter structure (26 bytes as described in the code from the previous link). This is similar to what is done in can.interfaces.socketcan.utils for raw CAN.
What am I doing wrong to cause setsockopt() to fail?

The first issue was with the struct.pack format (can_filter_fmt) being wrong. I first assumed that the kernel j1939_filter structure size was the sum of the members. This is wrong since the compiler adds padding. This can be added to the struct.pack format as x such as 2Q2I2B6x. Please see Why isn't sizeof for a struct equal to the sum of sizeof of each member?
The second issue was that can_filter_fmt is not packed as 2Q2B2I but as 2Q2I2B6x (the addr member is in the middle).
As for SOL_CAN_J1939 I was correct and needs to be created in file because it is not yet in the package.
The final code is the following:
#!/usr/bin/env python3
import socket
import struct
def pack_J1939_filters(can_filters=None):
if can_filters is None:
# Pass all messages
can_filters = [{}]
can_filter_fmt = "=" + "2Q2I2B6x" * len(can_filters)
filter_data = []
for can_filter in can_filters:
if 'name' in can_filter:
name = can_filter['name']
else:
name = 0
if 'name_mask' in can_filter:
name_mask = can_filter['name_mask']
else:
name_mask = 0
if 'pgn' in can_filter:
pgn = can_filter['pgn']
else:
pgn = 0
if 'pgn_mask' in can_filter:
pgn_mask = can_filter['pgn_mask']
else:
pgn_mask = 0
if 'addr' in can_filter:
addr = can_filter['addr']
else:
addr = 0
if 'addr_mask' in can_filter:
addr_mask = can_filter['addr_mask']
else:
addr_mask = 0
filter_data.append(name)
filter_data.append(name_mask)
filter_data.append(pgn)
filter_data.append(pgn_mask)
filter_data.append(addr)
filter_data.append(addr_mask)
return struct.pack(can_filter_fmt, *filter_data)
def print_msg(data, sck_addr):
print(f"SA:{hex(sck_addr[3])} PGN:{hex(sck_addr[2])}")
for j in range(len(data)):
if j % 8 == 0 and j != 0:
print()
if j % 8 == 0:
print(f"bytes {j} to {j+7}: ", end="")
print(f"{hex(data[j])} ", end="")
print()
print()
def main():
s = socket.socket(socket.PF_CAN, socket.SOCK_DGRAM, socket.CAN_J1939)
# allows to receive broadcast messages
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
interface = "vcan0"
src_name = socket.J1939_NO_NAME
src_pgn = socket.J1939_NO_PGN # always no PGN for source, unless filtering is needed
src_addr = 0x81 # recvfrom() will not return destination specific messages for other addresses
src_sck_addr = (interface, src_name, src_pgn, src_addr)
s.bind(src_sck_addr)
packed_filters = pack_J1939_filters()
SOL_CAN_BASE = 100
CAN_J1939 = 7
SOL_CAN_J1939 = SOL_CAN_BASE + CAN_J1939
s.setsockopt(SOL_CAN_J1939, socket.SO_J1939_FILTER , packed_filters)
(recv_data, recv_sck_addr) = s.recvfrom(128)
print_msg(recv_data, recv_sck_addr)
s.close()
if __name__ == "__main__":
main()
Thank you.

For J1939 to work with SocketCAN you need two things:
kernel 5.4+
can-j1939 kernel module enabled
Testing for can-1939:
If you install can-utils and after sudo modprobe can-j1939 all you get is fatal error, or if you start testj1939 from can-utils and you get error that protocol is not supported, then it means that can-j1939 was not enabled in your kernel and you need to compile it manually.
Here are my instructions for enabling can-j1939 in Debian 10 kernel:
https://github.com/linux-can/can-utils/blob/master/can-j1939-install-kernel-module.md

Filtering packets by src mac in scapy

When I filter the packets using this filter in wireshark:
wlan.sa == 04.b1.67.14.bd.64
All goes perfect.
However, I'm trying to do it with the following python script using scapy, but it never filter by the source mac:
from scapy.all import *
from datetime import datetime
import traceback
# import MySQLdb
def getAverageSSI():
global ssiFinal
return ssiFinal
def setParams():
global window
global timestamp
global SSID
global datetime
global iterator1
window = 1
timestamp = datetime.now()
SSID='DefaultName'
iterator1 = 0
global ssiArray
ssiArray = []
def myPacketHandler(pkt) :
global SSID
global timestamp
global iterator1
global ssiArray
try :
if pkt.haslayer(Dot11) :
ssiNew = -(256-ord(pkt.notdecoded[-4:-3]))
ssiArray.append(ssiNew)
diffT=(datetime.now()-timestamp).seconds
if diffT>window:
print 'With MAC dst = %s with SSI Power= %s' %(pkt.addr1, sum(ssiArray)/len(ssiArray))
print ssiArray
ssiArray = []
timestamp=datetime.now()
except Exception as e:
print 'Exception'
print e
traceback.print_exc()
sys.exit(0)
setParams()
try:
sniff(iface="wlan1", filter="ether src 04:b1:67:14:bd:64", prn = myPacketHandler, store=0)
except Exception as e:
print e
print "Sniff AP1 Off"
I have also tried to remove the filter in sniff, and put an if like the following:
if pkt.addr1 == '04:b1:67:14:bd:64' : # mac xiaomi mi a1
# SSID = pkt.info;
ssiNew = -(256-ord(pkt.notdecoded[-4:-3]))
ssiArray.append(ssiNew)
diffT=(datetime.now()-timestamp).seconds
if diffT>window:
# query = "START TRANSACTION;"
# queryBack=cur.execute(query)
# query = "INSERT INTO RSSI VALUES(%d,\"AP1\",%d);"%(iterator1,ssiNew)
# queryBack = cur.execute(query)
print 'MAC = %s with SSI Power= %s' %(pkt.addr1, sum(ssiArray)/len(ssiArray))
ssiArray = []
# Conexion.commit()
# iterator1+=1
timestamp=datetime.now()
But it is only filtering by destination mac.
Do you know how to properly filter by mac like in the following wireshark image? (it needs to be exactly the same behaviour than in the wireshark filter):

Your second method should be working well, if you used addr2 instead of addr1
Here is how it works in 802.11 (yes it’s really messy)
Also, you should update to the github scapy version, which has support for RSSI directly (so you don’t have to parse notdecoded)
See https://github.com/secdev/scapy/archive/master.zip

Operands missing in MySQL statement

I'm getting an arguments error from the MySQL module looking for 4 operands. I just don't see which operands it's looking for. It works for some case types and not others. Line numbers of error (trimmed at bottom which refers to MySQL library):
Traceback (most recent call last):
File "/Users/christoph/PycharmProjects/physicianWorkQueueProject/physicianWorkQueueProject.py", line 158, in <module>
parse()
File "/Users/christoph/PycharmProjects/physicianWorkQueueProject/physicianWorkQueueProject.py", line 138, in parse
c.execute(getPhysiciansql_cmd)
...
mysql.connector.errors.DataError: 1241 (21000): Operand should contain 4 column(s)
#!/usr/bin/python
##################################################
# This is a prototype pathologist work queue management system
# The program calls out to a database of pathologists, their specialities,
# and an index of case types and their intended specialists
#
# Usage: Run at your command line. You will then enter case numbers (which aren't validated (currently))
# and case types (which are validated). The program will distribute the entered case as follows:
#
# If the case is intended for generalists, a system that amounts to names being pulled from a hat
# is employed. When a name is selected (at random), the case is entered into that pathologists' work queue.
# That pathologists' name is not returned to the pool.
# The cycle will then repeat with a random name picked every time, in this same way, for generalist-requiring cases,
# until the entirety of names have been pulled. At this point, all names are returned to the pool and the whole cycle
# begins again
#
# For the specialist requiring cases, the name-draw system is bypassed and the case is directly entered
# into the pathologists' work queue.
# 16 Aug 2018
# My Real Name
##################################################
import mysql.connector as mariaDB
import time
import pandas as pd
from random import choice
def distributefairly(inputCaseNumber, inputcasetype):
# function distrbutefairly does a draw out of a hat, with each name being pulled and the pot shrinking until none are
# left at which point all names are added back
c.execute("SELECT physicianName FROM physicianNamesSpecialties;")
originalPhysicianList = c.fetchall()
physicianList = originalPhysicianList
# print("counter at:",count)
chosenPhysician = choice(physicianList)
pos = physicianList.index(chosenPhysician)
physicianList.pop(pos)
cleanedUpChosenPhysician = chosenPhysician[0]
insert(cleanedUpChosenPhysician)
print("This case is going to", cleanedUpChosenPhysician + ".")
select()
increment()
if len(physicianList) == 0:
reset()
def reset():
# this resets the counter the distributefairly module calls
global count
global physicianList
count = 0
c.execute("SELECT physicianName FROM physicianNamesSpecialties;")
physicianList = c.fetchall()
def increment():
# increments the counter of the distributefairly module
global count
count +=
def print_count():
print(count)
def insert(cleanedUpPhysResult):
# adds inputted cases into the workQueue table
global inputCaseNumber
global inputCaseType
ts = time.gmtime()
readableTs = time.strftime("%Y-%m-%d %H:%M:%S", ts)
c2.execute("INSERT INTO workQueue (name, caseNumber, timestamp, tableCaseType) values (%s,%s,%s,%s)", (cleanedUpPhysResult, inputCaseNumber, readableTs, str(inputCaseType)))
conn2.commit()
def select():
# this outputs the workQueue after every addition
sql = "SELECT * FROM workQueue"
c2.execute(sql)
rWQ = c2.fetchall()
print(pd.DataFrame(rWQ, columns= ['Name','Case Number','Time Stamp','Specialty','Row ID'])) # .set_index('Row ID')
def startup():
# create()
global inputCaseType
global inputCaseNumber
inputCaseNumber = input("Enter Case Number: ")
inputCaseType = input("Enter case type (use proper abbreviations): ")
def parse():
global inputCaseNumber
global inputCaseType
caseInputsql_cmd = "SELECT specialtyRequiredToProcess,Description FROM caseTypes WHERE caseTypeName='{}'".format(inputCaseType)
c.execute(caseInputsql_cmd)
rows_returned = c.fetchall()
if not rows_returned:
print("No match to table of specimen types returned. Check the case type abbreviation and try again.")
return
else:
for row in rows_returned:
r = row[0]
d = row[1]
print("This is a", r, "service case. It is a", d,"type case.")
if r != "GENERALIST":
getPhysiciansql_cmd = "SELECT physicianName FROM physicianNamesSpecialties WHERE (specialty, specialty2, specialty3, specialty4) ='{}'".format(r)
c.execute(getPhysiciansql_cmd)
physResult = choice(c.fetchall())
cleanedUpPhysResult = physResult[0]
print("This case is going to", cleanedUpPhysResult+".")
insert(cleanedUpPhysResult)
select()
else:
distributefairly(inputCaseNumber, inputCaseType)
conn = mariaDB.connect(host='xxxxx', user='xxxxx',password='xxxxxx',db='lookupDB')
conn2 = mariaDB.connect(host='xxxxxxl', user='xxxxx',password='xxxxxxx',db='workQueue')
c = conn.cursor()
c2 = conn2.cursor()
count = 0
while True:
startup()
parse()
This line of code .execute(...) will fail:
getPhysiciansql_cmd =
"SELECT physicianName FROM physicianNamesSpecialties WHERE (specialty, specialty2, specialty3, specialty4) ='{}'".format(r)
c.execute(getPhysiciansql_cmd)
My Var r holds this data:
print("This is a", r, "service case. It is a", d,"type case.")
This is a THORACIC service case. It is a TRANSBRONCHIAL WANG NEEDLE ASPIRATION type case.
My Var getPhysiciansql_cmd holds this data:
getPhysiciansql_cmd = SELECT physicianName FROM
physicianNamesSpecialties WHERE (specialty, specialty2, specialty3, specialty4) ='THORACIC'

I think this is just a matter of how I'm using the WHERE clause. I saw somewhere that the WHERE with multiple fields is written, you use parens around the fields. I've experimented in reducing this to the bare minimum and using the OR operator in the WHERE clause and I have successfully received a proper response.
import mysql.connector as mariaDB
conn = mariaDB.connect(host='xxxxx', user='xxxx',password='xxxxx',db='lookupDB')
conn2 = mariaDB.connect(host='xxxxx', user='xxxx',password='xxxxxx',db='workQueue')
c = conn.cursor()
c2 = conn2.cursor()
inputSpecialty = input("specialty? ")
r = inputSpecialty
c.execute("SELECT physicianName FROM physicianNamesSpecialties WHERE specialty = %s OR specialty2 = %s OR specialty3 = %s OR specialty4 = %s", (r,r,r,r))
physResult = c.fetchall()
cleanedUpPhysResult = physResult
print(cleanedUpPhysResult)
Output:
specialty? THORACIC
[('Song',), ('Han',), ('He',), ('Goldfischer',)]

"Not all parameters were used in the SQL statement" on BHV1750vi database input

I got an error on the following python code. I execute this code on raspberry Pi, and I want to get a input for database (MySQL) from BHV1750vi (light emittance sensor).
import smbus
import time
from mysql.connector import MySQLConnection, Error
from test_dbconfig import read_db_config
# Define some constants from the datasheet
DEVICE = 0x23 # Default device I2C address
POWER_DOWN = 0x00 # No active state
POWER_ON = 0x01 # Power on
RESET = 0x07 # Reset data register value
# Start measurement at 4lx resolution. Time typically 16ms.
CONTINUOUS_LOW_RES_MODE = 0x13
# Start measurement at 1lx resolution. Time typically 120ms
CONTINUOUS_HIGH_RES_MODE_1 = 0x10
# Start measurement at 0.5lx resolution. Time typically 120ms
CONTINUOUS_HIGH_RES_MODE_2 = 0x11
# Start measurement at 1lx resolution. Time typically 120ms
# Device is automatically set to Power Down after measurement.
ONE_TIME_HIGH_RES_MODE_1 = 0x20
# Start measurement at 0.5lx resolution. Time typically 120ms
# Device is automatically set to Power Down after measurement.
ONE_TIME_HIGH_RES_MODE_2 = 0x21
# Start measurement at 1lx resolution. Time typically 120ms
# Device is automatically set to Power Down after measurement.
ONE_TIME_LOW_RES_MODE = 0x23
#bus = smbus.SMBus(0) # Rev 1 Pi uses 0
bus = smbus.SMBus(1) # Rev 2 Pi uses 1
def convertToNumber(data):
# Simple function to convert 2 bytes of data
# into a decimal number
return ((data[1] + (256 * data[0])) / 1.2)
def readLight(addr=DEVICE):
data = bus.read_i2c_block_data(addr,ONE_TIME_HIGH_RES_MODE_1)
return convertToNumber(data)
def insert_lightlux(l):
query = "INSERT INTO light(lux) VALUES(%s)"
args = (l)
try:
db_config = read_db_config()
conn = MySQLConnection(**db_config)
cursor = conn.cursor()
cursor.executemany(query, args)
if cursor.lastrowid:
print('last insert id', cursor.lastrowid)
else:
print('last insert id not found')
conn.commit()
except Error as error:
print(error)
finally:
cursor.close()
conn.close()
def main():
while True:
if readLight() is not None:
print "luminous emmittance " + str(readLight()) + " lx"
time.sleep(10)
l = [(str(readLight()))]
insert_lightlux(l)
else:
print('Failed to get reading. Try again!')
sys.exit(1)
if __name__=="__main__":
main()
The error message is
luminous emmittance 9.16666666667 lx
Not all parameters were used in the SQL statement
Is there anyone know what is wrong in my code?

The problem is how you build args for executemany, which expects a list of tuples while you have a tuple of list.
In other words your arg is ([(value)]) but it should be [(value)]
Try changing args = (l) to
args = l

why does python loop continue work differently from other programming languages

in other programming languages when a continue is met in a loop it doesn't run the code below it and just does the next loop based on the condition set.
Yet in python it would actually not trigger the continue as much as 3 times on the same exact values until the continue actually triggers finally can someone tell me why it's like this?
function
def get_section(self, address):
for section in self.sections:
section_base = section.image_base + section.VirtualAddress
section_end = section_base + section.Misc_VirtualSize
print 'section_base= 0x%x' % section_base, ' section_end = 0x%x' % section_end
print 'VirtualAdderss = 0x%x' % section.VirtualAddress, 'Misc_virtualSize = 0x%x' % section.Misc_VirtualSize
if address < section_base or address >= section_end:
print 'continuued'
continue
print 'not continuued'
print 'Section name = ', section.section_name
return section
raise NotImplementedError()
Here is the log
address = 0x4013f8
section_base= 0x401000 section_end = 0x5574e5
VirtualAdderss = 0x1000 Misc_virtualSize = 0x1564e5
not continuued
Section name = text
address = 0x4013f8
section_base= 0x401000 section_end = 0x5574e5
VirtualAdderss = 0x1000 Misc_virtualSize = 0x1564e5
not continuued
Section name = text
address = 0x55869c
section_base= 0x401000 section_end = 0x5574e5
VirtualAdderss = 0x1000 Misc_virtualSize = 0x1564e5
continuued
section_base= 0x558000 section_end = 0x5818ac
VirtualAdderss = 0x158000 Misc_virtualSize = 0x298ac
not continuued
Section name = rdata
as you can see it didn't continue 2 times only on the 3rd time it continued and I can't figure out why shouldn't it work from the first time?

The first two times, the if condition was not met; therefore the continue statement was not executed.