stringExp = "2^4"
intVal = int(stringExp) # Expected value: 16
This returns the following error:
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
ValueError: invalid literal for int()
with base 10: '2^4'
I know that eval can work around this, but isn't there a better and - more importantly - safer method to evaluate a mathematical expression that is being stored in a string?
eval is evil
eval("__import__('os').remove('important file')") # arbitrary commands
eval("9**9**9**9**9**9**9**9", {'__builtins__': None}) # CPU, memory
Note: even if you use set __builtins__ to None it still might be possible to break out using introspection:
eval('(1).__class__.__bases__[0].__subclasses__()', {'__builtins__': None})
Evaluate arithmetic expression using ast
import ast
import operator as op
# supported operators
operators = {ast.Add: op.add, ast.Sub: op.sub, ast.Mult: op.mul,
ast.Div: op.truediv, ast.Pow: op.pow, ast.BitXor: op.xor,
ast.USub: op.neg}
def eval_expr(expr):
"""
>>> eval_expr('2^6')
4
>>> eval_expr('2**6')
64
>>> eval_expr('1 + 2*3**(4^5) / (6 + -7)')
-5.0
"""
return eval_(ast.parse(expr, mode='eval').body)
def eval_(node):
if isinstance(node, ast.Num): # <number>
return node.n
elif isinstance(node, ast.BinOp): # <left> <operator> <right>
return operators[type(node.op)](eval_(node.left), eval_(node.right))
elif isinstance(node, ast.UnaryOp): # <operator> <operand> e.g., -1
return operators[type(node.op)](eval_(node.operand))
else:
raise TypeError(node)
You can easily limit allowed range for each operation or any intermediate result, e.g., to limit input arguments for a**b:
def power(a, b):
if any(abs(n) > 100 for n in [a, b]):
raise ValueError((a,b))
return op.pow(a, b)
operators[ast.Pow] = power
Or to limit magnitude of intermediate results:
import functools
def limit(max_=None):
"""Return decorator that limits allowed returned values."""
def decorator(func):
#functools.wraps(func)
def wrapper(*args, **kwargs):
ret = func(*args, **kwargs)
try:
mag = abs(ret)
except TypeError:
pass # not applicable
else:
if mag > max_:
raise ValueError(ret)
return ret
return wrapper
return decorator
eval_ = limit(max_=10**100)(eval_)
Example
>>> evil = "__import__('os').remove('important file')"
>>> eval_expr(evil) #doctest:+IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
TypeError:
>>> eval_expr("9**9")
387420489
>>> eval_expr("9**9**9**9**9**9**9**9") #doctest:+IGNORE_EXCEPTION_DETAIL
Traceback (most recent call last):
...
ValueError:
Pyparsing can be used to parse mathematical expressions. In particular, fourFn.py
shows how to parse basic arithmetic expressions. Below, I've rewrapped fourFn into a numeric parser class for easier reuse.
from __future__ import division
from pyparsing import (Literal, CaselessLiteral, Word, Combine, Group, Optional,
ZeroOrMore, Forward, nums, alphas, oneOf)
import math
import operator
__author__ = 'Paul McGuire'
__version__ = '$Revision: 0.0 $'
__date__ = '$Date: 2009-03-20 $'
__source__ = '''http://pyparsing.wikispaces.com/file/view/fourFn.py
http://pyparsing.wikispaces.com/message/view/home/15549426
'''
__note__ = '''
All I've done is rewrap Paul McGuire's fourFn.py as a class, so I can use it
more easily in other places.
'''
class NumericStringParser(object):
'''
Most of this code comes from the fourFn.py pyparsing example
'''
def pushFirst(self, strg, loc, toks):
self.exprStack.append(toks[0])
def pushUMinus(self, strg, loc, toks):
if toks and toks[0] == '-':
self.exprStack.append('unary -')
def __init__(self):
"""
expop :: '^'
multop :: '*' | '/'
addop :: '+' | '-'
integer :: ['+' | '-'] '0'..'9'+
atom :: PI | E | real | fn '(' expr ')' | '(' expr ')'
factor :: atom [ expop factor ]*
term :: factor [ multop factor ]*
expr :: term [ addop term ]*
"""
point = Literal(".")
e = CaselessLiteral("E")
fnumber = Combine(Word("+-" + nums, nums) +
Optional(point + Optional(Word(nums))) +
Optional(e + Word("+-" + nums, nums)))
ident = Word(alphas, alphas + nums + "_$")
plus = Literal("+")
minus = Literal("-")
mult = Literal("*")
div = Literal("/")
lpar = Literal("(").suppress()
rpar = Literal(")").suppress()
addop = plus | minus
multop = mult | div
expop = Literal("^")
pi = CaselessLiteral("PI")
expr = Forward()
atom = ((Optional(oneOf("- +")) +
(ident + lpar + expr + rpar | pi | e | fnumber).setParseAction(self.pushFirst))
| Optional(oneOf("- +")) + Group(lpar + expr + rpar)
).setParseAction(self.pushUMinus)
# by defining exponentiation as "atom [ ^ factor ]..." instead of
# "atom [ ^ atom ]...", we get right-to-left exponents, instead of left-to-right
# that is, 2^3^2 = 2^(3^2), not (2^3)^2.
factor = Forward()
factor << atom + \
ZeroOrMore((expop + factor).setParseAction(self.pushFirst))
term = factor + \
ZeroOrMore((multop + factor).setParseAction(self.pushFirst))
expr << term + \
ZeroOrMore((addop + term).setParseAction(self.pushFirst))
# addop_term = ( addop + term ).setParseAction( self.pushFirst )
# general_term = term + ZeroOrMore( addop_term ) | OneOrMore( addop_term)
# expr << general_term
self.bnf = expr
# map operator symbols to corresponding arithmetic operations
epsilon = 1e-12
self.opn = {"+": operator.add,
"-": operator.sub,
"*": operator.mul,
"/": operator.truediv,
"^": operator.pow}
self.fn = {"sin": math.sin,
"cos": math.cos,
"tan": math.tan,
"exp": math.exp,
"abs": abs,
"trunc": lambda a: int(a),
"round": round,
"sgn": lambda a: abs(a) > epsilon and cmp(a, 0) or 0}
def evaluateStack(self, s):
op = s.pop()
if op == 'unary -':
return -self.evaluateStack(s)
if op in "+-*/^":
op2 = self.evaluateStack(s)
op1 = self.evaluateStack(s)
return self.opn[op](op1, op2)
elif op == "PI":
return math.pi # 3.1415926535
elif op == "E":
return math.e # 2.718281828
elif op in self.fn:
return self.fn[op](self.evaluateStack(s))
elif op[0].isalpha():
return 0
else:
return float(op)
def eval(self, num_string, parseAll=True):
self.exprStack = []
results = self.bnf.parseString(num_string, parseAll)
val = self.evaluateStack(self.exprStack[:])
return val
You can use it like this
nsp = NumericStringParser()
result = nsp.eval('2^4')
print(result)
# 16.0
result = nsp.eval('exp(2^4)')
print(result)
# 8886110.520507872
Some safer alternatives to eval() and sympy.sympify().evalf()*:
asteval
numexpr
*SymPy sympify is also unsafe according to the following warning from the documentation.
Warning: Note that this function uses eval, and thus shouldn’t be used on unsanitized input.
The reason eval and exec are so dangerous is that the default compile function will generate bytecode for any valid python expression, and the default eval or exec will execute any valid python bytecode. All the answers to date have focused on restricting the bytecode that can be generated (by sanitizing input) or building your own domain-specific-language using the AST.
Instead, you can easily create a simple eval function that is incapable of doing anything nefarious and can easily have runtime checks on memory or time used. Of course, if it is simple math, than there is a shortcut.
c = compile(stringExp, 'userinput', 'eval')
if c.co_code[0]==b'd' and c.co_code[3]==b'S':
return c.co_consts[ord(c.co_code[1])+ord(c.co_code[2])*256]
The way this works is simple, any constant mathematic expression is safely evaluated during compilation and stored as a constant. The code object returned by compile consists of d, which is the bytecode for LOAD_CONST, followed by the number of the constant to load (usually the last one in the list), followed by S, which is the bytecode for RETURN_VALUE. If this shortcut doesn't work, it means that the user input isn't a constant expression (contains a variable or function call or similar).
This also opens the door to some more sophisticated input formats. For example:
stringExp = "1 + cos(2)"
This requires actually evaluating the bytecode, which is still quite simple. Python bytecode is a stack oriented language, so everything is a simple matter of TOS=stack.pop(); op(TOS); stack.put(TOS) or similar. The key is to only implement the opcodes that are safe (loading/storing values, math operations, returning values) and not unsafe ones (attribute lookup). If you want the user to be able to call functions (the whole reason not to use the shortcut above), simple make your implementation of CALL_FUNCTION only allow functions in a 'safe' list.
from dis import opmap
from Queue import LifoQueue
from math import sin,cos
import operator
globs = {'sin':sin, 'cos':cos}
safe = globs.values()
stack = LifoQueue()
class BINARY(object):
def __init__(self, operator):
self.op=operator
def __call__(self, context):
stack.put(self.op(stack.get(),stack.get()))
class UNARY(object):
def __init__(self, operator):
self.op=operator
def __call__(self, context):
stack.put(self.op(stack.get()))
def CALL_FUNCTION(context, arg):
argc = arg[0]+arg[1]*256
args = [stack.get() for i in range(argc)]
func = stack.get()
if func not in safe:
raise TypeError("Function %r now allowed"%func)
stack.put(func(*args))
def LOAD_CONST(context, arg):
cons = arg[0]+arg[1]*256
stack.put(context['code'].co_consts[cons])
def LOAD_NAME(context, arg):
name_num = arg[0]+arg[1]*256
name = context['code'].co_names[name_num]
if name in context['locals']:
stack.put(context['locals'][name])
else:
stack.put(context['globals'][name])
def RETURN_VALUE(context):
return stack.get()
opfuncs = {
opmap['BINARY_ADD']: BINARY(operator.add),
opmap['UNARY_INVERT']: UNARY(operator.invert),
opmap['CALL_FUNCTION']: CALL_FUNCTION,
opmap['LOAD_CONST']: LOAD_CONST,
opmap['LOAD_NAME']: LOAD_NAME
opmap['RETURN_VALUE']: RETURN_VALUE,
}
def VMeval(c):
context = dict(locals={}, globals=globs, code=c)
bci = iter(c.co_code)
for bytecode in bci:
func = opfuncs[ord(bytecode)]
if func.func_code.co_argcount==1:
ret = func(context)
else:
args = ord(bci.next()), ord(bci.next())
ret = func(context, args)
if ret:
return ret
def evaluate(expr):
return VMeval(compile(expr, 'userinput', 'eval'))
Obviously, the real version of this would be a bit longer (there are 119 opcodes, 24 of which are math related). Adding STORE_FAST and a couple others would allow for input like 'x=5;return x+x or similar, trivially easily. It can even be used to execute user-created functions, so long as the user created functions are themselves executed via VMeval (don't make them callable!!! or they could get used as a callback somewhere). Handling loops requires support for the goto bytecodes, which means changing from a for iterator to while and maintaining a pointer to the current instruction, but isn't too hard. For resistance to DOS, the main loop should check how much time has passed since the start of the calculation, and certain operators should deny input over some reasonable limit (BINARY_POWER being the most obvious).
While this approach is somewhat longer than a simple grammar parser for simple expressions (see above about just grabbing the compiled constant), it extends easily to more complicated input, and doesn't require dealing with grammar (compile take anything arbitrarily complicated and reduces it to a sequence of simple instructions).
Okay, so the problem with eval is that it can escape its sandbox too easily, even if you get rid of __builtins__. All the methods for escaping the sandbox come down to using getattr or object.__getattribute__ (via the . operator) to obtain a reference to some dangerous object via some allowed object (''.__class__.__bases__[0].__subclasses__ or similar). getattr is eliminated by setting __builtins__ to None. object.__getattribute__ is the difficult one, since it cannot simply be removed, both because object is immutable and because removing it would break everything. However, __getattribute__ is only accessible via the . operator, so purging that from your input is sufficient to ensure eval cannot escape its sandbox.
In processing formulas, the only valid use of a decimal is when it is preceded or followed by [0-9], so we just remove all other instances of ..
import re
inp = re.sub(r"\.(?![0-9])","", inp)
val = eval(inp, {'__builtins__':None})
Note that while python normally treats 1 + 1. as 1 + 1.0, this will remove the trailing . and leave you with 1 + 1. You could add ),, and EOF to the list of things allowed to follow ., but why bother?
You can use the ast module and write a NodeVisitor that verifies that the type of each node is part of a whitelist.
import ast, math
locals = {key: value for (key,value) in vars(math).items() if key[0] != '_'}
locals.update({"abs": abs, "complex": complex, "min": min, "max": max, "pow": pow, "round": round})
class Visitor(ast.NodeVisitor):
def visit(self, node):
if not isinstance(node, self.whitelist):
raise ValueError(node)
return super().visit(node)
whitelist = (ast.Module, ast.Expr, ast.Load, ast.Expression, ast.Add, ast.Sub, ast.UnaryOp, ast.Num, ast.BinOp,
ast.Mult, ast.Div, ast.Pow, ast.BitOr, ast.BitAnd, ast.BitXor, ast.USub, ast.UAdd, ast.FloorDiv, ast.Mod,
ast.LShift, ast.RShift, ast.Invert, ast.Call, ast.Name)
def evaluate(expr, locals = {}):
if any(elem in expr for elem in '\n#') : raise ValueError(expr)
try:
node = ast.parse(expr.strip(), mode='eval')
Visitor().visit(node)
return eval(compile(node, "<string>", "eval"), {'__builtins__': None}, locals)
except Exception: raise ValueError(expr)
Because it works via a whitelist rather than a blacklist, it is safe. The only functions and variables it can access are those you explicitly give it access to. I populated a dict with math-related functions so you can easily provide access to those if you want, but you have to explicitly use it.
If the string attempts to call functions that haven't been provided, or invoke any methods, an exception will be raised, and it will not be executed.
Because this uses Python's built in parser and evaluator, it also inherits Python's precedence and promotion rules as well.
>>> evaluate("7 + 9 * (2 << 2)")
79
>>> evaluate("6 // 2 + 0.0")
3.0
The above code has only been tested on Python 3.
If desired, you can add a timeout decorator on this function.
I think I would use eval(), but would first check to make sure the string is a valid mathematical expression, as opposed to something malicious. You could use a regex for the validation.
eval() also takes additional arguments which you can use to restrict the namespace it operates in for greater security.
[I know this is an old question, but it is worth pointing out new useful solutions as they pop up]
Since python3.6, this capability is now built into the language, coined "f-strings".
See: PEP 498 -- Literal String Interpolation
For example (note the f prefix):
f'{2**4}'
=> '16'
Based on Perkins' amazing approach, I've updated and improved his "shortcut" for simple algebraic expressions (no functions or variables). Now it works on Python 3.6+ and avoids some pitfalls:
import re
# Kept outside simple_eval() just for performance
_re_simple_eval = re.compile(rb'd([\x00-\xFF]+)S\x00')
def simple_eval(expr):
try:
c = compile(expr, 'userinput', 'eval')
except SyntaxError:
raise ValueError(f"Malformed expression: {expr}")
m = _re_simple_eval.fullmatch(c.co_code)
if not m:
raise ValueError(f"Not a simple algebraic expression: {expr}")
try:
return c.co_consts[int.from_bytes(m.group(1), sys.byteorder)]
except IndexError:
raise ValueError(f"Expression not evaluated as constant: {expr}")
Testing, using some of the examples in other answers:
for expr, res in (
('2^4', 6 ),
('2**4', 16 ),
('1 + 2*3**(4^5) / (6 + -7)', -5.0 ),
('7 + 9 * (2 << 2)', 79 ),
('6 // 2 + 0.0', 3.0 ),
('2+3', 5 ),
('6+4/2*2', 10.0 ),
('3+2.45/8', 3.30625),
('3**3*3/3+3', 30.0 ),
):
result = simple_eval(expr)
ok = (result == res and type(result) == type(res))
print("{} {} = {}".format("OK!" if ok else "FAIL!", expr, result))
OK! 2^4 = 6
OK! 2**4 = 16
OK! 1 + 2*3**(4^5) / (6 + -7) = -5.0
OK! 7 + 9 * (2 << 2) = 79
OK! 6 // 2 + 0.0 = 3.0
OK! 2+3 = 5
OK! 6+4/2*2 = 10.0
OK! 3+2.45/8 = 3.30625
OK! 3**3*3/3+3 = 30.0
Testing bad input:
for expr in (
'foo bar',
'print("hi")',
'2*x',
'lambda: 10',
'2**1234',
):
try:
result = simple_eval(expr)
except ValueError as e:
print(e)
continue
print("OK!") # will never happen
Malformed expression: foo bar
Not a simple algebraic expression: print("hi")
Expression not evaluated as constant: 2*x
Expression not evaluated as constant: lambda: 10
Expression not evaluated as constant: 2**1234
This is a massively late reply, but I think useful for future reference. Rather than write your own math parser (although the pyparsing example above is great) you could use SymPy. I don't have a lot of experience with it, but it contains a much more powerful math engine than anyone is likely to write for a specific application and the basic expression evaluation is very easy:
>>> import sympy
>>> x, y, z = sympy.symbols('x y z')
>>> sympy.sympify("x**3 + sin(y)").evalf(subs={x:1, y:-3})
0.858879991940133
Very cool indeed! A from sympy import * brings in a lot more function support, such as trig functions, special functions, etc., but I've avoided that here to show what's coming from where.
Use eval in a clean namespace:
>>> ns = {'__builtins__': None}
>>> eval('2 ** 4', ns)
16
The clean namespace should prevent injection. For instance:
>>> eval('__builtins__.__import__("os").system("echo got through")', ns)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "<string>", line 1, in <module>
AttributeError: 'NoneType' object has no attribute '__import__'
Otherwise you would get:
>>> eval('__builtins__.__import__("os").system("echo got through")')
got through
0
You might want to give access to the math module:
>>> import math
>>> ns = vars(math).copy()
>>> ns['__builtins__'] = None
>>> eval('cos(pi/3)', ns)
0.50000000000000011
Here's my solution to the problem without using eval. Works with Python2 and Python3. It doesn't work with negative numbers.
$ python -m pytest test.py
test.py
from solution import Solutions
class SolutionsTestCase(unittest.TestCase):
def setUp(self):
self.solutions = Solutions()
def test_evaluate(self):
expressions = [
'2+3=5',
'6+4/2*2=10',
'3+2.45/8=3.30625',
'3**3*3/3+3=30',
'2^4=6'
]
results = [x.split('=')[1] for x in expressions]
for e in range(len(expressions)):
if '.' in results[e]:
results[e] = float(results[e])
else:
results[e] = int(results[e])
self.assertEqual(
results[e],
self.solutions.evaluate(expressions[e])
)
solution.py
class Solutions(object):
def evaluate(self, exp):
def format(res):
if '.' in res:
try:
res = float(res)
except ValueError:
pass
else:
try:
res = int(res)
except ValueError:
pass
return res
def splitter(item, op):
mul = item.split(op)
if len(mul) == 2:
for x in ['^', '*', '/', '+', '-']:
if x in mul[0]:
mul = [mul[0].split(x)[1], mul[1]]
if x in mul[1]:
mul = [mul[0], mul[1].split(x)[0]]
elif len(mul) > 2:
pass
else:
pass
for x in range(len(mul)):
mul[x] = format(mul[x])
return mul
exp = exp.replace(' ', '')
if '=' in exp:
res = exp.split('=')[1]
res = format(res)
exp = exp.replace('=%s' % res, '')
while '^' in exp:
if '^' in exp:
itm = splitter(exp, '^')
res = itm[0] ^ itm[1]
exp = exp.replace('%s^%s' % (str(itm[0]), str(itm[1])), str(res))
while '**' in exp:
if '**' in exp:
itm = splitter(exp, '**')
res = itm[0] ** itm[1]
exp = exp.replace('%s**%s' % (str(itm[0]), str(itm[1])), str(res))
while '/' in exp:
if '/' in exp:
itm = splitter(exp, '/')
res = itm[0] / itm[1]
exp = exp.replace('%s/%s' % (str(itm[0]), str(itm[1])), str(res))
while '*' in exp:
if '*' in exp:
itm = splitter(exp, '*')
res = itm[0] * itm[1]
exp = exp.replace('%s*%s' % (str(itm[0]), str(itm[1])), str(res))
while '+' in exp:
if '+' in exp:
itm = splitter(exp, '+')
res = itm[0] + itm[1]
exp = exp.replace('%s+%s' % (str(itm[0]), str(itm[1])), str(res))
while '-' in exp:
if '-' in exp:
itm = splitter(exp, '-')
res = itm[0] - itm[1]
exp = exp.replace('%s-%s' % (str(itm[0]), str(itm[1])), str(res))
return format(exp)
Using lark parser library https://stackoverflow.com/posts/67491514/edit
from operator import add, sub, mul, truediv, neg, pow
from lark import Lark, Transformer, v_args
calc_grammar = f"""
?start: sum
?sum: product
| sum "+" product -> {add.__name__}
| sum "-" product -> {sub.__name__}
?product: power
| product "*" power -> {mul.__name__}
| product "/" power -> {truediv.__name__}
?power: atom
| power "^" atom -> {pow.__name__}
?atom: NUMBER -> number
| "-" atom -> {neg.__name__}
| "(" sum ")"
%import common.NUMBER
%import common.WS_INLINE
%ignore WS_INLINE
"""
#v_args(inline=True)
class CalculateTree(Transformer):
add = add
sub = sub
neg = neg
mul = mul
truediv = truediv
pow = pow
number = float
calc_parser = Lark(calc_grammar, parser="lalr", transformer=CalculateTree())
calc = calc_parser.parse
def eval_expr(expression: str) -> float:
return calc(expression)
print(eval_expr("2^4"))
print(eval_expr("-1*2^4"))
print(eval_expr("-2^3 + 1"))
print(eval_expr("2**4")) # Error
I came here looking for a mathematic expression parser as well. Reading through some of the answers and looking up libraries, I came across py-expression which I am now using. It basically handles a lot of operators and formula constructs, but if you're missing something you can easily add new operators/functions to it.
The basic syntax is:
from py_expression.core import Exp
exp = Exp()
parsed_formula = exp.parse('a+4')
result = exp.eval(parsed_formula, {"a":2})
The only issue that I've had with it so far is that it doesn't come with built-in mathematical constants nor a mechanism to add them in. I just proposed a solution to that however: https://github.com/FlavioLionelRita/py-expression/issues/7
I want to parse some C files for functions and a specific command.
My aim is to get all functions that use the specific command as well as all times the command gets called.
Thus I decided to generate extra token for that by using multiple conditions.
Here is my lexer and parser code:
import os
import ply.lex as lex
import ply.yacc as yacc
results = []
calls = []
# Declare the state
states = (
('func', 'exclusive'),
('parameter', 'exclusive'),
('httpgettext', 'exclusive')
)
reserved = {
'void': 'VOID',
'int': 'INT',
'uint8': 'UINT8',
'uint16': 'UINT16',
'uint32': 'UINT32',
'TRet': 'TRET',
'TBool': 'TBOOL',
'bool': 'BOOL',
}
tokens = [
'ID',
'FUNC',
'PARAMETERLIST',
'CALL',
'SEMICOLON'
] + list(reserved.values())
# Start of token description for INITIAL mode (inclusive)
def t_ID(t):
r'[a-zA-Z_][a-zA-Z_0-9]*'
t.type = reserved.get(t.value, 'ID')
return t
# Start of token description for HttpGetText condition
def t_httpgettext_SEMICOLON(t):
r'\;'
t.value = t.lexer.lexdata[t.lexer.call_start:t.lexer.lexpos-1]
t.type = 'CALL'
t.lexer.pop_state()
global calls
arguments = str(t.value).split(',')
calls.append([arguments[1], arguments[2]])
# Start of token description for parameter list condition
def t_parameter(t):
r'\('
t.lexer.parameter_start = t.lexer.lexpos
t.lexer.paren_level = 1
t.lexer.push_state('parameter')
def t_parameter_lparen(t):
r'\('
t.lexer.paren_level += 1
def t_parameter_rparen(t):
r'\)'
t.lexer.paren_level -= 1
if t.lexer.paren_level == 0:
t.value = t.lexer.lexdata[t.lexer.parameter_start:t.lexer.lexpos - 1]
t.type = 'PARAMETERLIST'
t.lexer.pop_state()
return t
# Start of token description for function block condition
def t_func(t):
r'\{'
t.lexer.code_start = t.lexer.lexpos # Record the starting position
t.lexer.brace_level = 1 # Initial brace level
t.lexer.push_state('func') # Enter 'ccode' state
# Rules for the ccode state
def t_func_lbrace(t):
r'\{'
t.lexer.brace_level += 1
def t_func_rbrace(t):
r'\}'
t.lexer.brace_level -= 1
if t.lexer.brace_level == 0:
t.value = t.lexer.lexdata[t.lexer.code_start:t.lexer.lexpos - 1]
t.type = "FUNC"
t.lexer.lineno += t.value.count('\n')
t.lexer.pop_state()
return t
# Start of token description valid for all conditions
t_ANY_ignore = " \t§$%&+#-_:.<<|',\0"
def t_ANY_HttpGetText(t):
r'HttpGetText'
t.lexer.call_start = t.lexer.lexpos
t.lexer.push_state('httpgettext')
# For bad characters, we just skip over it
def t_ANY_error(t):
t.lexer.skip(1)
def t_ANY_comment(t):
r'(/\*(.|\n)*?\*/)|(//.*)'
pass
def t_ANY_ignore_comments(t):
r'//.*'
pass
def t_ANY_newline(t):
r'\n+'
t.lexer.lineno += len(t.value)
lexer = lex.lex()
def p_statement_function(p):
'statement : type identifier parameter function'
p[0] = p[2]
global results
identifier = str(p[2])
parameter_list = str(p[3]).replace('\n', '')
function_block = str(p[4])
if function_block.find('HttpGetText') != -1:
results.append([identifier, parameter_list, function_block])
print(identifier)
# while True:
# tok = parser.token()
# print(tok)
# if not tok:
# break
def p_parameter_PARAMETERLIST(p):
'parameter : PARAMETERLIST'
p[0] = p[1]
def p_function_FUNC(p):
'function : FUNC'
p[0] = p[1]
def p_identifier_ID(p):
'identifier : ID '
p[0] = p[1]
def p_type_TBOOL(p):
'type : TBOOL'
p[0] = p[1]
def p_type_VOID(p):
'type : VOID'
p[0] = p[1]
def p_type_TRET(p):
'type : TRET'
p[0] = p[1]
def p_type_BOOL(p):
'type : BOOL'
def p_type_INT(p):
'type : INTEGER'
p[0] = p[1]
def p_INTEGER_INT(p):
'INTEGER : INT'
p[0] = p[1]
def p_INTEGER_UINT8(p):
'INTEGER : UINT8'
p[0] = p[1]
def p_INTEGER_UINT16(p):
'INTEGER : UINT16'
p[0] = p[1]
def p_INTEGER_UINT32(p):
'INTEGER : UINT32'
p[0] = p[1]
def p_error(p):
print('Syntax error in input: ', p)
parser.restart()
parser = yacc.yacc()
with open('C:/Users/z0046abb/Desktop/Bachelorarbeit/TextLibraryAnalysis/test_file.txt', 'r') as f:
read_data = f.read()
parser.parse(read_data)
print(results)
print(calls)
This is the content of my test_file.txt:
int main(argv)
{
HttpGetText(Arg1, Arg2, Arg3, Arg4);
return 0
}
void func2(bla, bla, bla)
{
something = random();
HttpGetText(1,2,3,4);
}
void func3(bla, bla, bla)
{
something = random();
HttpGetText(1,21,31,4);
}
void func4(bla, bla, bla)
{
HttpGetText(1, 22, 32, 4);
}
void func5(bla, bla, bla)
{
something();
}
void func6(bla)
{
HttpGetText(1, 23, 33, 4);
}
HttpGetText(1, 24, 34, 4);
HtppGetText(1, 25, 35, 4);
But somehow not all matches are found/processed.
This is the output of a test run:
main
Syntax error in input: LexToken(VOID,'void',12,75)
func3
Syntax error in input: LexToken(VOID,'void',30,243)
Syntax error in input: LexToken(VOID,'void',44,353)
[['main', 'argv', '\n HttpGetText(Arg1, Arg2, Arg3, Arg4);\n\n return 0\n'], ['func3', 'bla, bla, bla', '\n something = random();\n HttpGetText(1,21,31,4);\n']]
[[' Arg2', ' Arg3'], ['2', '3'], ['21', '31'], [' 22', ' 32'], [' 23', ' 33']]
As you can see there is a error at void despite it being a reserved token.
I am not sure if the problem is in my lexer or parser implementation.
If I use the 'lookahead' functionality (part of the function that is a comment) from p_statement_function(p): it seems like all token are correctly labeled.
However the above output only seems to identify main() and func3().
Additionally the last two lines of test_file.txt should be appended as well.
My first idea was to switch from t.lexer.begin(state) to t.lexer.push_state(state) so I could return to the last state the lexer had would help here but it doesn't seem so.
Now I ran out of ideas. It doesn't seem to fail because of the global lists I use to store results (I am aware that global vars are a bit risky).
In addition I am suprised by the fact that main() and func3() are found to be fitting matches other than the rest of implemented functions.
I would be happy if anyone of you has an idea for me.
Edit:
I tried to modify test_file.txt. If there is some nonsense word between every function I can record all functions in my global result list. Though this isn't the solution I wish for.
The immediate problem you have is that the starting point you have given your parser is statement. As it name suggests, that non-terminal matches a single statement (actually, a single function definition). It does not match anything more, so once the statement is complete, the parser expects to see the end of input. Any token will therefore be a syntax error. (Some of these error messages are suppressed by Ply's error recovery mechanism.)
To fix that, you need to add a start non-terminal which recognises a sequence of statements:
program : | program statement
The other relevant error is in your lexer. Since t_ID precedes t_ANY_HttpGetText, it takes priority. That means that in INITIAL state, the HttpGetText token is recognised as an ID something which should have been visible when you tested the scanner. I don't think this is necessarily serious since top-level function calls are illegal in C (even in initialisers of global variables). But you can easily fix it by reordering those two lexer rules.
In an answer to a previous question of yours which I think concerns the same project, I said:
Note that trying to do all this work in the lexer is not generally recommended. Lexers should normally return simple atomic tokens, leaving it to the parser's grammar to do the work of putting the tokens together into a useful structure.
I regret not having made this warning stronger. Although it may seem like a lot of work to correctly tokenise C programs, it's actually not that difficult, and there are lots of examples floating around. A full parse is complicated, but most of the complications have to do with declarations and simplifications are possible if you don't need all that information.
Alternatively, complete open source C parsing solutions exist. There's a certain learning curve associated with using them, but the payoff is the flexibility to analyse program structure systematically without having to immerse yourself in the quirks of C syntax. (These aspects are much more acute if the code you are analysing is C++.)
I'm trying to parse a simple template language with pyparsing which looks kind of like this:
if <expr>: <statements>
elif <expr>:
if <expr>:
<statements>
else:
<statements>
<statements>
In which statements are either if-blocks with being valid Python code or any text otherwise. If a string starts with an "if" keyword, that means it's an if-block right away. In fact, lines of text can also contain interpolations of the type {variable}, but that's about it.
A full valid example of the this syntax:
if a == b: do some {interpolation} stuff
elif a.x > b.y:
if True:
some {more} interpolation
and some more text
else: foo {bar}
arbitrary {text} here
lorem {ipsum}
To detect Python expressions, I've subclassed pyparsing.Token and it sort of seems to work:
import ast
from pyparsing import *
class PythonExpression(Token):
name = 'PythonExpression'
def parseImpl(self, s, loc, doActions=True):
max_loc = s.find('\n') if '\n' in s else len(s)
best_loc = None
for n in range(loc + 1, max_loc + 1):
try:
tree = ast.parse(s[loc:n])
except:
continue
if isinstance(tree, ast.Module):
if len(tree.body) is 1:
if isinstance(tree.body[0], ast.Expr):
best_loc = n
if best_loc is not None:
return best_loc, s[loc:best_loc]
raise ParseException(s, loc, 'invalid Python expression')
expr = 'if foo[1 : "bar:baz"] == 1 : passqwe'
print (Keyword('if') + PythonExpression()).parseString(expr).asList()
results in
['if', 'foo[1 : "bar:baz"] == 1 ']
However, I'm a bit lost with the use of pyparsing.indentedBlock, cannot seem to make it parse the entire grammar no matter what. My last attempt is this (note that it only contains the if statement implementation; there are also optional elif and else blocks):
ParserElement.setDefaultWhitespaceChars(' \t')
colon = Literal(':').suppress()
if_clause = PythonExpression() + colon
if_statement = Group(Keyword('if') + if_clause)
non_white = Regex(r'\S+')
anything = Combine(non_white + restOfLine)
statement = Forward()
indent_stack = [1]
if_block = Group(if_statement + ((anything) | indentedBlock(statement, indent_stack)))
other = ~Keyword('if') + anything
statement << (if_block | other)
parser = OneOrMore(indentedBlock(statement, indent_stack, False))
data = """\
if foo[1:2]:
bar
baz
if foo[3]:
bar : baz
if foo[4]: bar
baz
foo
"""
pprint.pprint(parser.parseString(data).asList())
Which starts parsing correctly but then stops:
[[[[['if', 'foo[1:2]'], [['bar'], ['baz']]]]]]
I've also tried explicitly adding + lineEnd.suppress() to anything but that didn't seem to help. I'm sure I'm doing something stupid here, probably related to newlines, but can't really figure it out.
On a side note, how do I detect interpolation patterns in anything in the example above (if the example above was to work), so that foo {bar} baz gets parsed to a ['foo', Var('bar'), 'baz']? It's easy to detect {var}, but what's the correct expression for plain text then that's not greedy enough to consume everything thrown at it and that doesn't mess up the if/elif/else logic (I tried using SkipTo but that became quite cumbersome)?
EDIT: adding the separate grammar for parsing the interpolation with an example:
class Substitution(object):
def __init__(self, s, l, t):
self.name = t[0]
def __repr__(self):
return 'Substitution(%r)' % self.name
ParserElement.setDefaultWhitespaceChars(' \t')
lbrace = Literal('{').suppress()
rbrace = Literal('}').suppress()
name = Word(alphas, alphanums + '_')
substitution = Combine(lbrace + name + rbrace).setParseAction(Substitution)
text = SkipTo(substitution | lineEnd.suppress(), include=True).leaveWhitespace()
parser = OneOrMore(text | substitution)
parser.parseString('hello \n {world} {invalid.sub} \n foo {bar} baz ').asList()
outputs
['hello ',
[' ', Substitution('world')],
' {invalid.sub} ',
[' foo ', Substitution('bar')],
' baz ']
As you can see, in this case the parser is not being quite correct at grouping lines together (it doesn't allow text to follow substitution somewhy), but it shows the point. These Substitution objects are then to be later processed at runtime.
This is a class which will take in as input and then output a polynomial in string form (both ways same format). Some arithmetic is performed in the various methods. I've been trying to inherit this class into another class that will then use the __mod__() special method of the first class (or make it's own special method if necessary but I don't see how you can't just use the original method) to perform the mod on intake. Seems like this goes into __init__() but I've tried 5 different versions of this, even going so far as to change the parent class, and I'm getting nowhere. I'm teaching myself Python so I'm sure that even a junior Python dev can see where I'm going totally wrong.
import re
class GF2Polynomial(object): #classes should generally inherit from object
def __init__(self, string):
'''__init__ is a standard special method used to initialize objects.
Here __init__ will initialize a gf2infix object based on a string.'''
self.string = string #basically the initial string (polynomial)
self.key,self.lst = self.parsePolyVariable(string) # key determines polynomial compatibility
self.bin = self.prepBinary(string) #main value used in operations
def id(self,lst):
"""returns modulus 2 (1,0,0,1,1,....) for input lists"""
return [int(lst[i])%2 for i in range(len(lst))]
def listToInt(self,lst):
"""converts list to integer for later use"""
result = self.id(lst)
return int(''.join(map(str,result)))
def parsePolyToListInput(self,poly):
"""
replaced by parsePolyVariable. still functional but not needed.
performs regex on raw string and converts to list
"""
c = [int(i.group(0)) for i in re.finditer(r'\d+', poly)]
return [1 if x in c else 0 for x in xrange(max(c), -1, -1)]
def parsePolyVariable(self,poly):
"""
performs regex on raw string, converts to list.
also determines key (main variable used) in each polynomial on intake
"""
c = [int(m.group(0)) for m in re.finditer(r'\d+', poly)] #re.finditer returns an iterator
letter = [str(m.group(0)) for m in re.finditer(r'[a-z]', poly)]
m = max(c); varmatch = True; key = letter[0]
for i in range(len(letter)):
if letter[i] != key: varmatch = False
else: varmatch = True
if varmatch == False: return "error: not all variables in %s are the same"%a
d = [1 if x in c else (1 if x==0 else (1 if x=='x' else 0)) for x in xrange(m, -1, -1)]
return key,d
def polyVariableCheck(self,other):
return self.key == other.key
def prepBinary(self,poly):
"""converts to base 2; bina,binb are binary values like 110100101100....."""
x = self.lst; a = self.listToInt(x)
return int(str(a),2)
def __mod__(self,other):
"""
__mod__ is the special method for overriding the % operator
returns remainder formatted as polynomial
"""
if self.polyVariableCheck(other) == False:
return "error: variables of %s and %s do not match"%(self.string,other.string)
if self.bin == other.bin: return 0
return GF2Polynomial(self.outFormat(self.bin%other.bin))
def __str__(self):
return self.string
def outFormat(self,raw):
"""process resulting values into polynomial format"""
raw = "{0:b}".format(raw); raw = str(raw[::-1]); g = [] #reverse binary string for enumeration
g = [i for i,c in enumerate(raw) if c == '1']
processed = "x**"+" + x**".join(map(str, g[::-1]))
proc1 = processed.replace("x**1","x"); proc2 = proc1.replace("x**0","1")
if len(g) == 0: return 0 #return 0 if list empty
return proc2 #returns result in gf(2) polynomial form
The desired result is to be able to call it on a new (child) class with the parent type and while changing the parent class as little as possible (if even at all). Note that class "BinaryField" is the intended child class:
p=GF2Polynomial("x**2+x**1+x**0")
a=BinaryField("x**1+x**0", p)
b=BinaryField("x**1", p)
On intake, the given polynomial should be modulus divided by the 2nd element (here it's 'p'). This is necessary for finite field math.
EDIT:
when running it with --
## "x**1 + x**0" polynomial string style input
poly1 = "x**14 + x**1 + x**0"; poly2 = "x**6 + x**2 + x**1"; poly3 = "y**6 + y**2 + y**1"
a = GF2Polynomial(poly1); b = GF2Polynomial(poly2); c = GF2Polynomial(poly3)
## "x+1" polynomial string style input
poly4 = "x**14 + x + 1"; poly5 = "x**6 + x**2 + x"; poly6 = "y**6 + y**2 + 1"
d = GF2Polynomial(poly4); e = GF2Polynomial(poly5); f = GF2Polynomial(poly6)
bf1 = BinaryField(poly1,b); print bf1
bf2 = BinaryField(poly4,e); print bf2
Both of these styles are possible because of the way I coded it, but they should both return the same answer. However the result on that code is:
>>>
x**5 + x**4 + x**3 + 1
x**5 + x
Also, when using BinaryField(poly4,d), which is just the same string with it's GF2Polynomial() initialization, this errors as:
AttributeError: 'int' object has no attribute 'string'
Does this solves your problem?
class BinaryField(GF2Polynomial):
def __init__(self, string, mod):
modded = GF2Polynomial(string) % mod
super(BinaryField, self).__init__(modded.string)
>>> p = GF2Polynomial("x**2+x**1+x**0")
>>> a = BinaryField("x**1+x**0", p)
>>> print a
x + 1
You can also make the BinaryField class to be just a factory method:
def BinaryField(string, mod):
return GF2Polynomial(string) % mod