We Keep Coding

pyparsing: how to parse nested function which start with particular function name?

I want to use pyparsing to parse a nested function which start with particular function name.
Just like this:
tag("tag_name_1", value_equal("proxy.province", "value", "return_value", test(1,2)))
The string waited to be parsed starts with the function named 'tag'.
The problem is that why exprStack doesn't contain "tag" function?
import pyparsing as pp
from typing import Any, List, Dict
def debug(*args, **kwargs):
print("debug"+"---"*10)
print(*args, **kwargs)
print("debug"+"---"*10)
# return "debug"
return "debug"
def insert_fn_argcount_tuple(t):
fn = t.pop(0)
num_args = len(t)
print((fn, num_args))
t.insert(0, (fn, num_args))
def push_first(toks):
exprStack.append(toks[0])
def to_string(toks):
pass
LPAREN, RPAREN, COMMA = map(pp.Suppress, '(),')
ident = pp.Word(pp.alphas, pp.alphanums+"_")
integer = pp.Word(pp.nums)
string = (pp.QuotedString("'") | pp.QuotedString('"')).setParseAction()
expr = pp.Forward()
expr_list = pp.delimitedList(pp.Group(expr))
tag_fn = ("tag" + LPAREN + expr_list + RPAREN).setParseAction(insert_fn_argcount_tuple)
fn_call = (ident + LPAREN + expr_list + RPAREN).setParseAction(insert_fn_argcount_tuple)
atom = ( (fn_call | string | integer) | pp.Group(LPAREN+expr+RPAREN)).addParseAction(push_first)
# atom = ( fn_call | pp.Group(LPAREN+expr+RPAREN)).addParseAction(push_first)
expr <<= atom
bnf = pp.Forward()
bnf <<= tag_fn
funcs = """tag
value_equal
value_contain
value_match
value
"""
# functions
def tag(tag_name: str, value:Any)->Dict:
if not tag_name or not value:
return {}
return {"tag_name": tag_name, "tag_value": value}
def test(*args, **kwargs):
return ""
def value_equal(key: str, value, default=None, test=None):
print(f"---{value_equal}---")
print(f"key: {key}, value: {value}, defaul: {default}, test:{test}")
return "value-1"
fns = {
"tag": tag,
"value_equal": value_equal,
"test": test
}
exprStack = []
def evaluate_stack(s:List): # List param will be changed after invoke evaluate_stack function
fn, arg_nums = s.pop(), 0
if isinstance(fn, tuple):
fn, arg_nums = fn
if fn in fns:
args = reversed([evaluate_stack(s) for _ in range(arg_nums)])
return fns[fn](*args)
else:
return fn
test_str = """tag("tag_name_1", value_equal("proxy.province", "value", "return_value", test(1,2)))"""
# test_str = "123"
p = bnf.parse_string(test_str)
print(f"\nexprStack:{exprStack}\n")
t = evaluate_stack(exprStack)
print(f"tag:{t}")
The output of above code is:
('test', 2)
('value_equal', 4)
('tag', 2)
exprStack:['tag_name_1', 'proxy.province', 'value', 'return_value', '1', '2', ('test', 2), ('value_equal', 4)\]
I expect that exprStack contains tag function. maybe like this:
exprStack:['tag_name_1', 'proxy.province', 'value', 'return_value', '1', '2', ('test', 2), ('value_equal', 4), ('tag', 2)\]

You are really pretty close. The thing is, the push_first parse action is attached to atoms, but tag_fn is not an atom. So it won't get its data pushed to expr_stack.
To fix this:
Change atom to include tag_fn, something like this:
atom = ((tag_fn | fn_call | string | integer) | pp.Group(LPAREN+expr+RPAREN)).addParseAction(push_first)
Change bnf to expr instead of tag_fn:
bnf <<= expr
With these two changes, I get this for expr_stack:
exprStack:['tag_name_1', 'proxy.province', 'value', 'return_value', '1', '2', ('test', 2), ('value_equal', 4), ('tag', 2)]

python BCD format into Struct

I have the below code in CPP that I am trying to port to Python to send data to a UDPS
#define VERSION_MAIN "V6.60"
#define VERSION_BIN_MAJOR 0x06
#define VERSION_BIN_MINOR 0x60
unsigned char temp[3];
temp[0] = VERSION_MAIN[0];
temp[1] = VERSION_BIN_MAJOR;
temp[2] = VERSION_BIN_MINOR;
I have tried code like the following:
byteone = bytes(VERSION_MAIN, 'utf-8')
hex_string = '0x06'
decimal_int = int(hex_string, 16)
decimal_string = str(decimal_int)
digits = [int(c) for c in decimal_string]
zero_padded_BCD_digits = [format(d, '04b') for d in digits]
s = ''.join(zero_padded_BCD_digits)
bytetwo = bytes(int(s[i : i + 8], 2) for i in range(0, len(s), 8))
hex_string = '0x60'
decimal_int = int(hex_string, 16)
decimal_string = str(decimal_int)
digits = [int(c) for c in decimal_string]
zero_padded_BCD_digits = [format(d, '04b') for d in digits]
s = ''.join(zero_padded_BCD_digits)
bytethree = bytes(int(s[i : i + 8], 2) for i in range(0, len(s), 8))
values = (byteonw,bytetwo,bytethree )
s= struct.Struct(f'!3B')
packed_data = s.pack(*values)
but I keep getting pesky errors
struct.error: required argument is not an integer
Can anyone give me a hand please.
Thanks

No need to convert V into an int, pack can manage the char type as well.
from struct import pack
VERSION_MAIN = "V6.60"
VERSION_BIN_MAJOR = 0x06
VERSION_BIN_MINOR = 0x60
version = pack("!c2B", VERSION_MAIN[0].encode('utf-8'), VERSION_BIN_MAJOR, VERSION_BIN_MINOR)
# version = b'V\x06`'

Difference in Precision Between Python Function and C Function

I'm working with some code that does a mathematical conversion from Earth Centered Earth Fixed Coordinates to Latitude/Longitude/Altitude Coordinates. One iteration of this function is written in Python- another in C. The C method came after the Python method was written and is supposed to be a direct translation. However, I'm finding that the Python method, although it's using floating point values for it's calculations, gets a correct answer, while the C method, even when using doubles, seems to only be rounding after only 5 decimal places and therefore gets an incorrect answer when attempting to convert ECEF to LLA coordinates. Here is the Python method for the conversion, along with an example coordinate:
import math
import numpy as np
def ecef2lla(eceflist):
x = float(eceflist[0])
y = float(eceflist[1])
z = float(eceflist[2])
a = 6378137 # radius
e = 8.1819190842622e-2 # eccentricity
e_string = "E value:" + " " + str(e)
print(e_string)
asq = math.pow(a,2)
asq_string = "asq value:" + " " + str(asq)
print(asq_string)
esq=math.pow(e,2)
esq_string = "esq value:" + " " + str(esq)
print(esq_string)
b = math.sqrt( asq * (1-esq) )
b_string = "b value:" + " " + str(b)
print(b_string)
bsq = math.pow(b,2)
ep = math.sqrt( (asq - bsq)/bsq)
ep_string = "ep value:" + " " + str(ep)
print(ep_string)
p = math.sqrt( math.pow(x,2) + math.pow(y,2) )
p_string = "p value:" + " " + str(p)
print(p_string)
th = math.atan2(a*z, b*p)
th_string = "th value:" + " " + str(th)
print(th_string)
lon = math.atan2(y,x)
lon_string = "lon value:" + " " + str(lon)
print(lon_string)
lat = math.atan2( (z + math.pow(ep,2)*b*math.pow(math.sin(th),3) ), (p - esq*a*math.pow(math.cos(th),3)))
lat_string = "lat value:" + " " + str(lat)
print(lat_string)
N = a/( math.sqrt(1-esq*math.pow(math.sin(lat),2)) )
N_divisor = math.sqrt(1-esq*math.pow(math.sin(lat),2))
ND_string = "N divisor value is" + " " + str(N_divisor)
print(ND_string)
N_string = "N value:" + " " + str(N)
print(N_string)
alt = p / math.cos(lat) - N
alt_string = "alt value:" + " " + str(alt)
print(alt_string)
lon = lon % (2*math.pi)
ret = [lat*180/math.pi, lon*180/math.pi, alt]
return ret
if __name__ == '__main__':
lla_coords = ecef2lla([2155172.63, 2966340.65, 5201390])
print(lla_coords)
The following method is the translated C method, using double data types, with thinking being this would ensure maximum precision:
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdio.h>
#include <unistd.h>
void ecef_to_lla(double *coordinate){
double ecef_latitude = coordinate[0];
double ecef_longitude = coordinate[1];
double ecef_altitude = coordinate[2];
double a = 6378137;
//double e = 8.1819190842622e-2;
double e = 0.081819190842622;
printf("E value: %.16f\n",e);
double asq = pow(a,2);
printf("asq string: %.16f\n",asq);
double esq = pow(e,2);
printf("esq string: %.16f\n",esq);
double b = sqrt(asq * (1-esq));
printf("b string: %.16f\n",b);
double bsq = pow(b,2);
double ep = sqrt((asq-bsq)/bsq);
printf("ep string: %.16f\n",ep);
double p = sqrt(pow(ecef_latitude,2) + pow(ecef_longitude,2));
printf("p string: %.16f\n",p);
double th = atan2(a*ecef_altitude,b*p);
printf("th string: %.16f\n",th);
double lla_longitude = atan2(ecef_longitude,ecef_latitude);
double lla_latitude = atan2((ecef_altitude + pow(ep,2)*b*pow(sin(th),2)),p-esq*a*pow(cos(th),3));
printf("Lat is %.16f\n",lla_latitude);
double n = a/(sqrt(1-esq*pow(sin(lla_latitude),2)));
double n_divisor = sqrt(1-esq*pow(sin(lla_latitude),2));
printf("n_divisor is %.16f\n",n_divisor);
printf("n is %.16f\n",n);
double lla_altitude = p / cos(lla_latitude) - n;
printf("alt is %.16f\n",lla_altitude);
lla_longitude = fmod(lla_longitude ,(2 * M_PI));
lla_latitude = fmod(lla_latitude ,(2 * M_PI));
lla_latitude = lla_latitude * 180/M_PI;
lla_longitude = lla_longitude * 180/M_PI;
coordinate[0] = lla_latitude;
coordinate[1] = lla_longitude;
coordinate[2] = lla_altitude;
}
int main(void){
double coordinates[3] = {2155172.63, 2966340.65, 5201390};
ecef_to_lla(coordinates);
}
The LLA coordinates returned by the Python method are:
54.99999538240099, 54.00000006037918, 8.26665045041591
while those returned by the C method are:
55.0268382213345930,54.0000000603791790,4279.4874338442459702
So the altitude calculationseems to be the issue

There's a tiny transcription error. Change the line
double lla_latitude = atan2((ecef_altitude + pow(ep,2)*b*pow(sin(th),2)),p-esq*a*pow(cos(th),3));
to
double lla_latitude = atan2((ecef_altitude + pow(ep,2)*b*pow(sin(th),3)),p-esq*a*pow(cos(th),3));
When I make that change, I get 8.2666504495 for the altitude.

How do I Use re() in Python and Return Capture Groups within an "If" Statement?

Although I've been using Perl for many years, I've always had trouble with anything more than fairly basic use of Regular Expresions in the language. This is
only a worse situation now, as I'm trying to learn Python... and the use of re() is even more unclear to me.
I'm trying to check for a match if a substring is in a string, using re()
and also am using capture groups to extract some info from the matching process. However, I can't get things to work in a couple of
contexts; when using a re() call and assigning the returned values all
within an "if" statement.. and how to handle the situation when .groups items are not defined
in the match objects (when a match is not made).
So, what follows are examples of what I'm trying to do coded in Perl and Python, with their respective outputs.
I'd appreciate any pointers on how I might better approach the problem using Python.
Perl Code:
use strict;
use warnings;
my ($idx, $dvalue);
while (my $rec = <DATA>) {
chomp($rec);
if ( ($idx, $dvalue) = ($rec =~ /^XA([0-9]+)=(.*?)!/) ) {
printf(" Matched:\n");
printf(" rec: >%s<\n", $rec);
printf(" index = >%s< value = >%s<\n", $idx, $dvalue);
} elsif ( ($idx, $dvalue) = ($rec =~ /^PZ([0-9]+)=(.*?[^#])!/) ) {
printf(" Matched:\n");
printf(" rec: >%s<\n", $rec);
printf(" index = >%s< value = >%s<\n", $idx, $dvalue);
} else {
printf("\n Unknown Record format, \\%s\\\n\n", $rec);
}
}
close(DATA);
exit(0)
__DATA__
DUD=ABC!QUEUE=D23!
XA32=7!P^=32!
PZ112=123^!PQ=ABC!
Perl Output:
Unknown Record format, \DUD=ABC!QUEUE=D23!\
Matched:
rec: >XA32=7!P^=32!<
index = >32< value = >7<
Matched:
rec: >PZ112=123^!PQ=ABC!<
index = >112< value = >123^<
Python Code:
import re
string = 'XA32=7!P^=32!'
with open('data.dat', 'r') as fh:
for rec in fh:
orec = ' rec: >' + rec.rstrip('\n') + '<'
print(orec)
# always using 'string' at least lets this program run
(index, dvalue) = re.search(r'^XA([0-9]+)=(.*?[^#])!', string).groups()
# The following works when there is a match... but fails with an error when
# a match is NOT found, viz:-
# ...
# (index, dvalue) = re.search(r'^XA([0-9]+)=(.*?[^#])!', rec).groups()
#
# Traceback (most recent call last):
# File "T:\tmp\a.py", line 13, in <module>
# (index, dvalue) = re.search(r'^XA([0-9]+)=(.*?[^#])!', rec).groups()
# AttributeError: 'NoneType' object has no attribute 'groups'
#
buf = ' index = >' + index + '<' + ' value = >' + dvalue + '<'
print(buf)
exit(0)
data.dat contents:
DUD=ABC!QUEUE=D23!
XA32=7!P^=32!
PZ112=123^!PQ=ABC!
Python Output:
rec: >DUD=ABC!QUEUE=D23!<
index = >32< value = >7<
rec: >XA32=7!P^=32!<
index = >32< value = >7<
rec: >PZ112=123^!PQ=ABC!<
index = >32< value = >7<
Another development: Some more code to help me understand this better... but I'm unsure about when/how to use the match.group() or match.groups() ...
Python Code:
import re
rec = 'XA22=11^!S^=64!ABC=0,0!PX=0!SP=12B!'
print("rec = >{}<".format(rec))
# ----
index = 0 ; dvalue = 0 ; x = 0
match = re.match(r'XA([0-9]+)=(.*?[^#])!(.*?)!', rec)
if match:
(index, dvalue, x) = match.groups()
print("3 (): index = >{}< value = >{}< x = >{}<".format(index, dvalue, x))
# ----
index = 0 ; dvalue = 0 ; x = 0
match = re.match(r'XA([0-9]+)=(.*?[^#])!', rec)
if match:
(index, dvalue) = match.groups()
print("2 (): index = >{}< value = >{}< x = >{}<".format(index, dvalue, x))
# ----
index = 0 ; dvalue = 0 ; x = 0
match = re.match(r'XA([0-9]+)=', rec)
if match:
#(index) = match.groups() # Why doesn't this work like above examples!?
(index, ) = match.groups() # ...and yet this works!?
# Does match.groups ALWAYS returns a tuple!?
#(index) = match.group(1) # This also works; 0 = entire matched string?
print("1 (): index = >{}< value = >{}< x = >{}<".format(index, dvalue, x))
# ----
index = 0 ; dvalue = 0 ; x = 0
match = re.search(r'S\^=([0-9]+)!', rec)
if match:
(index, ) = match.groups() # Returns tuple(?!)
print("1 (): index = >{}< value = >{}< x = >{}<".format(index, dvalue, x))
Again, I'd appreciate any thoughts on which is the 'preferred' way.. or if there's another way to deal with the groups.

You need to check for a match first, then use the groups. I.e.
compile the regexes (optional for most cases nowadays, according to the documentation)
apply each regex to the string to generate a match object
match() only matches at the beginning of a string, i.e. with an implicit ^ anchor
search() matches anywhere in the string
check if the match object is valid
extract the groups
skip to next loop iteration
# works with Python 2 and Python 3
import re
with open('dummy.txt', 'r') as fh:
for rec in fh:
orec = ' rec: >' + rec.rstrip('\n') + '<'
print(orec)
match = re.match(r'XA([0-9]+)=(.*?[^#])!', rec)
if match:
(index, dvalue) = match.groups()
print(" index = >{}< value = >{}<".format(index, dvalue))
continue
match = re.match(r'PZ([0-9]+)=(.*?[^#])!', rec)
if match:
(index, dvalue) = match.groups()
print(" index = >{}< value = >{}<".format(index, dvalue))
continue
print(" Unknown Record format")
Output:
$ python dummy.py
rec: >DUD=ABC!QUEUE=D23!<
Unknown Record format
rec: >XA32=7!P^=32!<
index = >32< value = >7<
rec: >PZ112=123^!PQ=ABC!<
index = >112< value = >123^<
But I'm wondering why you don't simplify your Perl & Python code to just use a single regex instead? E.g.:
match = re.match(r'(?:XA|PZ)([0-9]+)=(.*?[^#])!', rec)
if match:
(index, dvalue) = match.groups()
print(" index = >{}< value = >{}<".format(index, dvalue))
else:
print(" Unknown Record format")

Get correct brace grouping from string

I have files with incorrect JSON that I want to start fixing by getting it into properly grouped chunks.
The brace grouping {{ {} {} } } {{}} {{{}}} should already be correct
How can I grab all the top-level braces, correctly grouped, as separate strings?

If you don't want to install any extra modules simple function will do:
def top_level(s):
depth = 0
start = -1
for i, c in enumerate(s):
if c == '{':
if depth == 0:
start = i
depth += 1
elif c == '}' and depth:
depth -= 1
if depth == 0:
yield s[start:i+1]
print(list(top_level('{{ {} {} } } {{}} {{{}}}')))
Output:
['{{ {} {} } }', '{{}}', '{{{}}}']
It will skip invalid braces but could be easily modified to report an error when they are spotted.

Using the regex module:
In [1]: import regex
In [2]: braces = regex.compile(r"\{(?:[^{}]++|(?R))*\}")
In [3]: braces.findall("{{ {} {} } } {{}} {{{}}}")
Out[3]: ['{{ {} {} } }', '{{}}', '{{{}}}']

pyparsing can be really helpful here. It will handle pathological cases where you have braces inside strings, etc. It might be a little tricky to do all of this work yourself, but fortunately, somebody (the author of the library) has already done the hard stuff for us.... I'll reproduce the code here to prevent link-rot:
# jsonParser.py
#
# Implementation of a simple JSON parser, returning a hierarchical
# ParseResults object support both list- and dict-style data access.
#
# Copyright 2006, by Paul McGuire
#
# Updated 8 Jan 2007 - fixed dict grouping bug, and made elements and
# members optional in array and object collections
#
json_bnf = """
object
{ members }
{}
members
string : value
members , string : value
array
[ elements ]
[]
elements
value
elements , value
value
string
number
object
array
true
false
null
"""
from pyparsing import *
TRUE = Keyword("true").setParseAction( replaceWith(True) )
FALSE = Keyword("false").setParseAction( replaceWith(False) )
NULL = Keyword("null").setParseAction( replaceWith(None) )
jsonString = dblQuotedString.setParseAction( removeQuotes )
jsonNumber = Combine( Optional('-') + ( '0' | Word('123456789',nums) ) +
Optional( '.' + Word(nums) ) +
Optional( Word('eE',exact=1) + Word(nums+'+-',nums) ) )
jsonObject = Forward()
jsonValue = Forward()
jsonElements = delimitedList( jsonValue )
jsonArray = Group(Suppress('[') + Optional(jsonElements) + Suppress(']') )
jsonValue << ( jsonString | jsonNumber | Group(jsonObject) | jsonArray | TRUE | FALSE | NULL )
memberDef = Group( jsonString + Suppress(':') + jsonValue )
jsonMembers = delimitedList( memberDef )
jsonObject << Dict( Suppress('{') + Optional(jsonMembers) + Suppress('}') )
jsonComment = cppStyleComment
jsonObject.ignore( jsonComment )
def convertNumbers(s,l,toks):
n = toks[0]
try:
return int(n)
except ValueError, ve:
return float(n)
jsonNumber.setParseAction( convertNumbers )
Phew! That's a lot ... Now how do we use it? The general strategy here will be to scan the string for matches and then slice those matches out of the original string. Each scan result is a tuple of the form (lex-tokens, start_index, stop_index). For our use, we don't care about the lex-tokens, just the start and stop. We could do: string[result[1], result[2]] and it would work. We can also do string[slice(*result[1:])] -- Take your pick.
results = jsonObject.scanString(testdata)
for result in results:
print '*' * 80
print testdata[slice(*result[1:])]