We Keep Coding

How do you obtain the body of a function?

I would need to separate the signature from the body of a function in Python, i.e. :
def mult(a, b):
"Multiplication"
if a or b is None:
# border case
return None
else:
return a * b
(the function is there for demonstration only).
I know that inspect.getsourcelines(mult) will get me the full code, but I would like to have only the body, i.e. less the signature and the docstring:
if a or b is None:
# border case
return None
else:
return a * b
Is there an elegant way to obtain that, ideally using Python3 built-in parsing tools, rather than string manipulation?

As I don't know any function that does this, here is a homemade function that should work.
I don't have time for more in-depth use-cases. So i'll leave my incomplete answer here.
def get_body(func):
""" Using the magic method __doc__, we KNOW the size of the docstring.
We then, just substract this from the total length of the function
"""
try:
lines_to_skip = len(func.__doc__.split('\n'))
except AttributeError:
lines_to_skip = 0
lines = getsourcelines(func)[0]
return ''.join( lines[lines_to_skip+1:] )
def ex_0(a, b):
""" Please !
Don't
Show
This
"""
if a or b is None:
# border case
return None
else:
return a * b
def ex_1(a, b):
''' Please !Don'tShowThis'''
if a or b is None:
# border case
return None
else:
return a * b
def ex_2(a, b):
if a or b is None:
# border case
return None
else:
return a * b
def ex_3(bullshit, hello):
pass
get_body(ex_0)
# if a or b is None:
# # border case
# return None
# else:
# return a * b
get_body(ex_1)
# if a or b is None:
# # border case
# return None
# else:
# return a * b
get_body(ex_2)
# if a or b is None:
# # border case
# return None
# else:
# return a * b
get_body(ex_3)
# pass

you can use this also
i = inspect.getsource(fun_name).index('\n')
j = inspect.getsource(fun_name).rindex(':',0,i)
print(inspect.getsource(fun_name)[j+1:])

If you know your code will always have a non-empty docstring like a above, You can do this with the built in inspect library.
The below snippet should get you acquainted with how to see the source code body of a function.
def hello_world():
"""Python port of original Fortran code"""
print("hello_world")
import inspect
source = inspect.getsource(hello_world)
doc = hello_world.__doc__
code = source.split(doc)
body = code[1]
print(body)
EDIT:
I thought about this some more, you can first remove the defintion line and then strip out the docstring if it exists:
def hello_world():
"""Python port of original Fortran code"""
print("hello_world")
import inspect
source = inspect.getsource(hello_world)
doc = hello_world.__doc__
code = source.split(':',1)
body= code[1].replace(doc, "")
body = body.replace('""""""',"")
print(body)

Thanks a lot for the inspiring answers! I tried to take this further and I found a solution that uses Python's ast facilities.
import inspect, ast
from astunparse import unparse
def mult(a,
b): # comment
"""
This is a 'doctstring', "hello"
"""
if not a is None or b is None:
# border case
return None
else:
return a * b
def get_body(f, doc_string=False):
"""
Get the body text of a function, i.e. without the signature.
NOTE: Comments are stripped.
"""
COMMENT_START = ("'", '"')
code = inspect.getsource(mult)
# print("Function's code:\n", code)
module_tree = ast.parse(code) # this creates a module
# print("Dump:\n", ast.dump(module_tree))
# the first element of the module is evidently the function:
function_body = module_tree.body[0].body
# strip the code lines to remove the additional lines:
lines = [unparse(code_line).strip() for code_line in function_body]
# for i, line in enumerate(lines): print("%s: %s" % (i, line.strip()))
# in principle the notion of body contains the docstring:
if not doc_string:
lines = (line for line in lines if not line.startswith(COMMENT_START))
return '\n'.join(lines).strip()
s =get_body(mult)
print("---------")
print(s)
Here is the result:
$ python3 function_body.py
---------
if ((not (a is None)) or (b is None)):
return None
else:
return (a * b)
You can choose if you want the docstring or not. The downside of this approach (which should not be a downside in my use case) is that comments are stripped.
I also left some commented print statements, in case someone would like to explore the various steps.

getter setter as function in python class giving "no attribute found" error

import operator
import re
from ply import lex, yacc
class Lexer(object):
tokens = [
'COMMA',
'TILDE',
'PARAM',
'LP',
'RP',
'FUNC'
]
# Regular expression rules for simple tokens
t_COMMA = r'\,'
t_TILDE = r'\~'
t_PARAM = r'[^\s\(\),&:\"\'~]+'
def __init__(self, dict_obj):
self.dict_obj = dict_obj
def t_LP(self, t):
r'\('
return t
def t_RP(self, t):
r'\)'
return t
def t_FUNC(self, t):
# I want to generate token for this FUNC from the keys of model map
# For eg: r'key1|key2'
r'(?i)FUNC'
return t
# Define a rule so we can track line numbers
def t_newline(self, t):
r'\n+'
t.lexer.lineno += len(t.value)
# A string containing ignored characters (spaces and tabs)
t_ignore = ' \t'
# Error handling rule
def t_error(self, t):
print("Illegal character '%s' on line %d, column %d" % (t.value[0], t.lexer.lineno, t.lexer.lexpos))
t.lexer.skip(1)
# Build the lexer
def build_lexer(self, **kwargs):
self.lexer = lex.lex(module=self, **kwargs)
return self.lexer
class Parser(object):
tokens = Lexer.tokens
def __init__(self, **kwargs):
self.parser = yacc.yacc(module=self, **kwargs)
self.lexer = None
self._dict_obj = None
self.error = ""
self.result = ""
#property
def dict_obj(self):
return self._dict_obj
#dict_obj.setter
def dict_obj(self, dict_obj):
self._dict_obj = dict_obj
self.lexer = Lexer(self._dict_obj).build_lexer()
# Handles LP expression RP
def p_expression(self, p):
"""
expression : LP expression RP
"""
# Handles TILDE PARAM - call search
def p_tilde_param(self, p):
"""
expression : TILDE PARAM
"""
p[0] = p[2]
return p[0]
# Handles ANY LP PARAM RP - call search
def p_expression_any(self, p):
"""
expression : FUNC LP PARAM RP
"""
p[0] = p[3]
return p[0]
# Error handling rule
def p_error(self, p):
if p:
stack_state_str = " ".join([symbol.type for symbol in self.parser.symstack[1:]])
self.error = "Syntax error at %s, type %s, on line %d, Parser state: %s %s . %s" % (
p.value, p.type, p.lineno, self.parser.state, stack_state_str, p
)
else:
self.error = "SYNTAX ERROR IN INPUT"
def get_result(self, input_):
input_ = input_.strip()
if input_:
self.result = self.parser.parse(input_, lexer=self.lexer)
return self.result
else:
raise ValueError("EMPTY EXPRESSION ERROR")
def parser(input_):
par_obj = Parser()
par_obj.dict_obj = {
'key1' : 'value1',
'key2' : 'value2'
}
return par_obj.get_result(input_)
result = parser("~hello")
Above is the code of lexer and parser using ply library. I have just encapsulated all of my code in the class form. Problems which i am facing:
1.) I'm trying to pass a dict_obj to the parser class. I don't know what i am doing wrong and getting an error like :
AttributeError: 'Parser' object has no attribute 'dict_obj'
2.) What I'm trying to do?
I want to pass this dict_obj to the parser class and then pass it to the lexer class as well and then make use of it in the lexer one of the tokens methods (t_FUNC) method. In this method my regex will return keys of the this dict obj.
I think i'm doing something wrong and hence failing to implement it. Please help.

In your constructor (__init__) for the Parser object, you ask Ply to generate a parser before the Parser object is fully constructed:
def __init__(self, **kwargs):
self.parser = yacc.yacc(module=self, **kwargs)
# This is the critical line:
self._dict_obj = None
In order to construct a parser from the object (yacc.yacc(module=self)), Ply needs to iterate over all the object's attributes. For example, it needs to find all the parser functions in order to extract their docstrings in order to determine the grammar.
Ply uses the dir built-in function to make a dictionary of all the object's attributes. Because your Parser object has a custom attribute dict_obj, that key is returned from dir and so Ply tries to cache that attribute with its value. But when it calls gettattr(module, 'dict_obj'), the getter is called, and the getter tries to return self._dict_obj. However, self._dict_obj has not yet been defined, so that ends up throwing an error:
AttributeError: 'Parser' object has no attribute '_dict_obj'
Note that this is not the error message you reported in your question; that error says that there is no attribute dict_obj. Perhaps that was a copy-and-paste error.
If you move the call to yacc.yacc to the end of the initialiser, that particular problem goes away:
def __init__(self, **kwargs):
self.lexer = None
self._dict_obj = None
self.error = ""
self.result = ""
self.parser = yacc.yacc(module=self, **kwargs)
However, there are a number of other problems in the code excerpt which make it difficult to verify this solution. These include:
There is no LexerNmsysSearch. I assumed you meant Lexer.
There is no node_expression. I have no idea what that is supposed to be so I just removed the test.
Your grammar does not match the input you are testing, so the parser immediately throws a syntax error. I changed the input to "(~hello)" in an attempt to produce something parseable.
The parser actions do not set semantic values, so self.parse.parse() doesn't return any value. This causes get_result to throw an error.
At that point, I gave up on trying to produce anything sensible out of the code. For future reference, please ensure that error messages are quoted exactly and that sample code included in the question can be run.

python - importing module from within a module

I have this file pluralizer.py containing functions and a class which use the re module:
from re import *
def pluralize(noun, funcs):
for matches_rule, apply_rule in funcs:
if matches_rule(noun):
return apply_rule(noun)
raise ValueError("no matching rule for {0}".format(noun))
def build_match_and_apply_functions(pattern, search, replace):
def matches_rule(word):
return re.search(pattern, word)
def apply_rule(word):
return re.sub(search, replace, word)
return (matches_rule, apply_rule)
class LazyRules:
rules_filename = 'rules.txt' #a class variable - shared across all instances of the LazyRules class
def __init__(self):
self.pattern_file = open(self.rules_filename, encoding="utf-8")
self.cache=[]
def __iter__(self):
self.cache_index=0
return self #returning self signals that this class defines a __next__ method
def __next__(self):
self.cache_index += 1
if len(self.cache) >= self.cache_index:
return self.cache[self.cache_index-1]
if self.pattern_file.closed:
raise StopIteration
line = self.pattern_file.readline()
if not line: #if there's a line to read, it will not be an empty string (even if new row, it will be "\n")
self.pattern_file.close()
raise StopIteration
pattern,search,replace= line.split(None,3)
funcs = build_match_and_apply_functions(pattern,search,replace)
self.cache.append(funcs) # before returning the match&apply functions, we save them in the list self.cache
return funcs
There's also the data file rules.txt:
[sxz]$ $ es
[^aeioudgkprt]h$ $ es
[^aeiou]y$ y$ ies
$ $ s
The way it's supposed to work is:
import pluralizer
funcs = pluralizer.LazyRules()
p = pluralizer.pluralize("baby", funcs)
from which the expected output is "babies", but I get:
NameError: name 're' is not defined
Placing import re inside pluralize function didn't work either. How come the re module 'refuses' to import? I searched old questions but didn't find an answer, sorry if I overlooked it. Thanks!
P.S. Code is from 'Dive Into Python 3' by Mark Pilgrim

works for me as follows, before running it, I make sure to change the working directory within the python shell
import os
os.chdir('whatever your working directory and files are')
The code in my 'lazyrules.py' file looks like
import re
def build_match_and_apply_functions(pattern, search, replace):
def matches_rule(word):
return re.search(pattern, word)
def apply_rule(word):
return re.sub(search, replace, word)
return (matches_rule, apply_rule)
def plural(noun, funcs):
for matches_rule, apply_rule in funcs:
if matches_rule(noun):
return apply_rule(noun)
raise ValueError('no matching rule for {0}'.format(noun))
class LazyRules:
rules_filename = 'plural6-rules.txt'
def __init__(self):
self.pattern_file = open(self.rules_filename, encoding='utf-8')
self.cache = []
def __iter__(self):
self.cache_index = 0
return self
def __next__(self):
self.cache_index += 1
if len(self.cache) >= self.cache_index:
return self.cache[self.cache_index - 1]
if self.pattern_file.closed:
raise StopIteration
line = self.pattern_file.readline()
if not line:
self.pattern_file.close()
raise StopIteration
pattern, search, replace = line.split(None, 3)
funcs = build_match_and_apply_functions(pattern, search, replace)
self.cache.append(funcs)
return funcs
rules = LazyRules()

Enforce structure of file with PLY

I am trying to process input file with description of algorithm behavior. I am using python's PLY module for defining lexer and parser. I stumbled upon problem of defining grammar which will enforce user to correctly write this file.
File
# Beginning of the first section
STATES = INITIATOR, IDLE, DONE;
INIT = INITIATOR, IDLE;
TERM = DONE;
# End of first section
# Beginning of the second section
INITIATOR
RANDOM
begin
SEND(x, NEIGHBORS);
BECOME(DONE);
end
IDLE
RECEIVE(x)
begin
SEND(x, NEIGHBORS);
BECOME(DONE);
end
# End of second section
Lexer
import ply.lex as lex
from soda.helpers import prepare_file
class Lexer(object):
keywords = (
'INIT', 'TERM', 'STATES', 'REGISTERS',
'begin', 'end',
'SEND', 'BECOME'
)
tokens = keywords + (
'NAME', 'EQUALS', 'COMMA', 'SEMICOLON',
'LPAREN', 'RPAREN'
)
# Tokens
t_EQUALS = r'='
t_COMMA = r','
t_SEMICOLON = r';'
t_STATES = r'STATES'
t_REGISTERS = r'REGISTERS'
t_INIT = r'INIT'
t_TERM = r'TERM'
t_begin = r'begin'
t_end = r'end'
t_SEND = r'SEND'
t_BECOME = r'BECOME'
t_LPAREN = r'\('
t_RPAREN = r'\)'
# Ignored characters
t_ignore = ' \t\n'
def t_NAME(self, t):
r'[a-zA-Z][a-zA-Z]*'
if t.value in self.keywords: # is this a keyword?
t.type = t.value
return t
def t_error(self, t):
print ("Illegal character {0} at line {1}".format(t.value[0], t.lineno))
t.lexer.skip(1)
def build(self, **kwargs):
self._lexer = lex.lex(module=self, **kwargs)
#prepare_file
def lexical_analysis(self, file):
print ("Started lexical analysis...")
for line in file:
try:
lex_input = line
except EOFError:
break
self._lexer.input(lex_input)
while True:
token = self._lexer.token()
if not token:
break
print (" ", token)
Parser
import ply.yacc as yacc
from soda.helpers import prepare_file
class Parser(object):
def p_algorithm(self, p):
''' algorithm : first_section second_section'''
def p_first_section(self, p):
''' first_section : STATES EQUALS states_list SEMICOLON
| REGISTERS EQUALS register_list SEMICOLON
| INIT EQUALS init_list SEMICOLON
| TERM EQUALS term_list SEMICOLON'''
def p_states_list(self, p):
''' states_list : state_term
| states_list COMMA state_term'''
def p_state_term(self, p):
''' state_term : NAME'''
self.behavior.states.append(p[1])
def p_register_list(self, p):
''' register_list : register_term
| register_list COMMA register_term'''
def p_register_term(self, p):
''' register_term : NAME'''
self.behavior.registers.append(p[1])
def p_init_list(self, p):
''' init_list : init_term
| init_list COMMA init_term'''
def p_init_term(self, p):
''' init_term : NAME'''
self.behavior.init_states.append(p[1])
def p_term_list(self, p):
''' term_list : term_term
| term_list COMMA term_term'''
def p_term_term(self, p):
''' term_term : NAME'''
self.behavior.term_states.append(p[1])
def p_second_section(self, p):
''' second_section : NAME begin commands end'''
def p_error(self, p):
print("Syntax error in input! -> {}".format(p))
def build(self, lexer, behavior):
self.lexer = lexer
self.behavior = behavior
self.tokens = lexer.tokens
self._parser = yacc.yacc(module=self)
#prepare_file
def parsing(self, file):
for line in file:
try:
parser_input = line
print (line)
except EOFError:
break
self._parser.parse(parser_input, lexer=self.lexer._lexer)
Parsing results in syntax error and I am not sure how to define rules to enforce the consistency of file with algorithm behavior. first_section is parsed ok and problem is second_section. My solution defines that algorithm : first_section second_section and it is not working. I tried to define it like algorithm: first_section | second_section and it works good but this rule states that first and second section can be switched in file.
So my question is how to enforce it with rules so user will keep the input file consistent.
Error output
enter STATES = INITIATOR, IDLE, DONE;
Syntax error in input! -> None
INIT = INITIATOR, IDLE;
Syntax error in input! -> None
TERM = DONE;
Syntax error in input! -> None
INITIATOR
Syntax error in input! -> LexToken(NAME,'INITIATOR',1,0)
begin
Syntax error in input! -> LexToken(begin,'begin',1,0)
Program just states there is error in syntax. Problem is not with lexical analysis but with defined grammar. I can define it in such way that input is accepted but for example user would be able to switch first_section with second_section.
Edit
I think it is not clear from this question what I want to achieve or my problem so I voted to close it. I came up with idea how to better state what I am looking for so I want to raise new question.

Oups! Your grammar parses the file line by line, which is at least uncommon and does not allow to control the ordering of lines. IMHO, you should parse the file as a whole. The trick is to pass the parser a tokenfunc function that will feed the lexer with one line at a time, and declare each section to be composed of lines:
class Parser(object):
def p_algorithm(self, p):
''' algorithm : first_section second_section'''
def p_first_section(self, p):
''' first_section : first_section_line
| first_section_line first_section'''
def p_first_section_line(self, p):
''' first_section_line : STATES EQUALS states_list SEMICOLON
| REGISTERS EQUALS register_list SEMICOLON
| INIT EQUALS init_list SEMICOLON
| TERM EQUALS term_list SEMICOLON'''
...
# same for second section...
#prepare_file
def parsing(self, file):
def get_token():
'a tokenizer that automatically feeds the lexer with the next line'
while True:
tok = self.lexer._lexer.token()
if tok is not None: return tok
try:
line = next(file)
self.lexer._lexer.input(line)
except StopIteration:
return None
self._parser.parse("", lexer=self.lexer._lexer, tokenfunc = get_token)

Python PLY Yacc "syntax error"

Okay, so I'm trying to build a parser of my mini-language (obviously), and setting variables seems to be properly working. But as soon as Yacc comes across a function definition, it just gives me a syntax error, and a couple of EOF errors (which I know are from when Yacc has no remaining rules to set) and nothing else happens... Where did I go wrong?
Here's an example of the syntax I'm parsing:
$name = "John Doe"
$age = 72
$waterInOceans = 95.4
!testFunction {
}
Where the !testFunction { } section is defining a function (based off of the exclamation point). I don't know if that's going to be useful in debugging.
# The Lexer
import ply.lex as lex
tokens = ["MINUS", "SEPARATOR", "MODIFIER", "FUNCTION_NAME", "UNDEF_BLOCK", "VARIABLE_NAME", "EQUALS", "STRING", "FLOAT", "INNER_CONTENT", "ARGUMENTS", "INTEGER", "PLUS"]
def t_ARGUMENTS(t): # Finds arguments in calls and function definitions
r'\(.*\)'
t.value = t.value[1:-1] # strip parenthesis
t.value = t.value.split(" && ")
return t
def t_STRING(t): # finds strings
r'"\w.+"'
t.value = t.value[1:-1] # strips the quotation marks of the string
return t
def t_FLOAT(t): # finds floats
r'\d+.\d+'
t.value = float(t.value)
return t
def t_INTEGER(t):
r'\d+'
t.value = int(t.value)
return t
def t_VARIABLE_NAME(t):
r'\$\w*\b'
t.value = t.value[1:]
return t
def t_INNER_CONTENT(t):
r'\{\n.*\n\}|\{.*\}'
t.value = t.value[1:-1]
return t
def t_FUNCTION_NAME(t):
r'!\w+'
t.value = t.value[1:]
return t
t_ignore = r"\n|\t|\r"
t_EQUALS = r"\="
t_PLUS = r"\+"
t_MINUS = r"-"
t_MODIFIER = r"\."
t_SEPARATOR = r"\,"
t_UNDEF_BLOCK = r"\w+" # Any block of text that is left over and isn't assigned by the end (used by functions)
def t_error(t):
t.lexer.skip(1)
lex.lex()
#opened = open("example.zeq", "r")
#content = opened.read()
#opened.close()
#lex.input(content)
And then the Yacc half:
# The Yacc parser
import ply.yacc as yacc
import compiler # Get the compiler (tokenizer; compiler.py) which generates tokens
import sys
from os import system
##############
### IGNORE ###
tokens = compiler.tokens
#system("clear")
print("Executing "+sys.argv[1]+" |\n"+("-"*(len(sys.argv[1])+12)))
### IGNORE ###
##############
VARIABLES = {}
FUNCTIONS = {}
def p_assign(p): # Set new variable
'''assignment : VARIABLE_NAME EQUALS compound
| VARIABLE_NAME EQUALS STRING
| VARIABLE_NAME EQUALS INTEGER
| VARIABLE_NAME EQUALS FLOAT'''
#print("Setting '{}' to '{}'...".format(str(p[1]), str(p[3])))
VARIABLES[p[1]] = p[3]
def p_number(p): # Combines floats and integers into a blanket non-terminal for simplicity sakes
'''number : FLOAT
| INTEGER'''
p[0] = p[1]
def p_compound(p): # Complete the value *before* the variable is assigned!
'''compound : number PLUS number
| number MINUS number'''
type1 = type(p[1])
type2 = type(p[3])
operator = p[2]
if operator == "+":
p[0] = p[1] + p[3]
elif operator == "-":
p[0] = p[1] - p[3]
def p_undefined(p):
'''undefined : UNDEF_BLOCK'''
print("Undefined block")
def p_function(p):
'''function : FUNCTION_NAME INNER_CONTENT'''
print("Creating a function")
name = p[1]
content = p[2]
FUNCTIONS[name] = content
def p_empty(p):
'''empty : '''
#~ def p_error(p):
#~ if p:
#~ print("Syntax error: "+p.type)
#~ else:
#~ pass
parser = yacc.yacc()
opened = open(sys.argv[1], "r")
content = opened.read()
opened.close()
for line in content.splitlines():
parser.parse(line)
print(VARIABLES)
print(FUNCTIONS)
I'm waiting for it to be a simple overlooked detail...

When you ask Ply (or yacc, for that matter) to parse an input, it attempts to recognize a single instance of the top-level non-terminal (or "starting symbol"). This will usually a grammatical description of the entire input, so it will often have a name like program, although there are use cases in which it is useful to parse just a part of the input.
Ply (and yacc) assume that the first grammar production is for the starting symbol. In your case, the first production is assignment, and so that is what it will try to parse (and nothing else). assignment cannot derive a function definition or any other statement type, so those cause syntax errors.
If you want to explicitly tell Ply what the top-level symbol is, you can do so. See the manual section on starting symbols.