We Keep Coding

'file' object has no attribute '__getitem__' and a missing module

I am getting error with this code!!
First the xreadlines module does not exist anymore so i commented the line. Now when i run this code it gives error:
Traceback (most recent call last):
File "/home/integrand/projects/testproject2/testenv/Paragraph.py", line 58, in <module>
show_paragraphs("/home/integrand/projects/testproject2/Files/lease.text")
File "/home/integrand/projects/testproject2/testenv/Paragraph.py", line 50, in show_paragraphs
for p in pp:
File "/home/integrand/projects/testproject2/testenv/Paragraph.py", line 29, in __getitem__
line = self.seq[self.line_num]
TypeError: 'file' object has no attribute '__getitem__'
class Paragraphs:
def __init__(self, fileobj, separator='\n'):
# Ensure that we get a line-reading sequence in the best way possible:
# import xreadlines
try:
# Check if the file-like object has an xreadlines method
self.seq = fileobj.xreadlines()
except AttributeError:
# No, so fall back to the xreadlines module's implementation
self.seq = fileobj.xreadlines()
self.line_num = 0 # current index into self.seq (line number)
self.para_num = 0 # current index into self (paragraph number)
# Ensure that separator string includes a line-end character at the end
if separator[-1:] != '\n': separator += '\n'
self.separator = separator
def __getitem__(self, index):
if index != self.para_num:
raise TypeError, "Only sequential access supported"
self.para_num += 1
# Start where we left off and skip 0+ separator lines
while 1:
# Propagate IndexError, if any, since we're finished if it occurs
line = self.seq[self.line_num]
self.line_num += 1
if line != self.separator: break
# Accumulate 1+ nonempty lines into result
result = [line]
while 1:
# Intercept IndexError, since we have one last paragraph to return
try:
# Let's check if there's at least one more line in self.seq
line = self.seq[self.line_num]
except IndexError:
# self.seq is finished, so we exit the loop
break
# Increment index into self.seq for next time
self.line_num += 1
if line == self.separator: break
result.append(line)
return ''.join(result)
def show_paragraphs(filename,numpars=5):
pp = Paragraphs(open(filename))
for p in pp:
print "Par#%d, line# %d: %s" % (
pp.para_num, pp.line_num, repr(p))
if pp.para_num>numpars: break
if __name__ == '__main__':
#pdfparser("/home/USER/projects/testproject2/Files/Lea.pdf")
show_paragraphs("/home/USER/projects/testproject2/Files/Lea.text")

You can't use subscription on an iterator (which is what file.xreadlines() returns). If you need random access to different lines, just use file.readlines() or list(file) instead.
Note that file.xreadlines() has been deprecated since Python 2.3 (released 13 years ago!); use iteration over the file object instead (which, incidentally, is what list(file) does).

Python Parser Recursion Infinite Reference

Im new to Parsers and i have a problem with my Parser, specifically when i call itself to analize the body of a function.
When it finds another function, just became crazy and messy.
Basically, when analyzing this code
fn test (a, b, c):
fn test2 (c, b, a):
print("Hello world")
end
end
It starts to point the object to itself, not the subfunction:
>>> print(ast[0].value.body[9])
<ast.VariableAST object at 0x7f6285540710>
>>> print(ast[0].value.body[9].value.body[9])
<ast.VariableAST object at 0x7f6285540710>
This is the main parser code:
# Parser Loop
def Parser(tokenList):
global tokens
global size
tokens = tokenList
size = len(tokens)
ast = []
while i < size:
ast.append(MainParser())
return ast
# The Main Parser
def MainParser():
global i
if tokens[i] == 'fn':
i += 1
node = FunctionParser()
else:
node = tokens[i]
i += 1
return node
# Parse a function
def FunctionParser():
global i
checkEnd("function")
if tokens[i][0].isalpha():
node = VariableAST()
node.name = tokens[i]
i += 1
node.value = FunctionBodyParser()
elif tokens[i] == '(':
node = FunctionBodyParser()
else:
syntaxError("Expecting '(' or function name")
return node
# Parse a function body
def FunctionBodyParser():
global i
i += 1
node = FunctionAST()
while True:
checkEnd("function")
if tokens[i][0].isalpha():
node.args.append(tokens[i])
i += 1
elif tokens[i] == ',':
i += 1
elif tokens[i] == ')':
break
else:
syntaxError("Expecting ')' or ',' in function declaration")
i += 1
checkEnd("function")
if tokens[i] != ':' and tokens[i] != '{':
syntaxError("Expecting '{' or ':'")
begin = tokens[i]
while True:
checkEnd("function")
if begin == ':' and tokens[i] == 'end':
break
elif begin == '{' and tokens[i] == '}':
break
else:
node.body.append(MainParser())
i += 1
return node
Edit: I forgot to mention that this is a prototype for a C version. Im avoiding stuff related to object orientation and some good pratices in python to make easier to port the code to C later.

There's a lot of parser implemented in Python https://wiki.python.org/moin/LanguageParsing like PyPEG allowing you to describe the language you're parsing instead of parsing it yourself, which is more readable and less error-prone.
Also using global is typically a source of problems as you can't control the scope of a variable (there's only one), reducing reusability of your functions.
It's probably better to use a class, which is almost the same thing but you can have multiple instances of the same class running at the same time without variable colisions:
class Parser:
def __init__(self, tokenList):
self.tokens = tokenList
self.size = len(tokenList)
self.ast = []
self.position = 0
def parse(tokenList):
while self.position < self.size:
self.ast.append(self.main())
return self.ast
def main(self):
if self.tokens[self.position] == 'fn':
self.position += 1
node = self.function()
else:
node = self.tokens[self.position]
self.position += 1
return node
# And so on...
From this point you can deduplicate self.position += 1 in main:
def main(self):
self.position += 1
if self.tokens[self.position] == 'fn':
node = self.function()
else:
node = self.tokens[self.position]
return node
Then remove the useless "node" variable:
def main(self):
self.position += 1
if self.tokens[self.position] == 'fn':
return self.function()
else:
return self.tokens[self.position]
But the real way to do this is to use a parser, take a look a pyPEG, it's a nice one, but others are nice too.
Oh and last point, avoid useless comments like:
# Parse a function
def FunctionParser():
We know that "FunctionParser" "Parse a function", thanks, that's not an information. The most important is to wisely choose your function names (oh, the PEP8 tells us not to start method name with capitalized letters), and if you want to add meta-information about the function, put it in a string as a first statement in your function like:
def functionParser():
"Delegates the body parsing to functionBodyParser"

I found the solution,
Actually it was a silly error of programming in Python and the above parser is working fine.
In AST i was creating classes like structure to make easier to port the language to C. But i was doing it wrong, for example:
class VariableAST(NodeAST):
name = None
value = None
What i didnt knew about Python is that those parameters inside the class arent attributes for the object, but static variables, wich just made my program work unpredictable when i assing a value to a object, since im assing to a static variable, i also assing to all other variables and theres is my infinite recursion of objects.

Unbound method TypeError

I've just been reading an article that talks about implementing a parser in python:
http://effbot.org/zone/simple-top-down-parsing.htm
The general idea behind the code is described in this paper: http://mauke.hopto.org/stuff/papers/p41-pratt.pdf
Being fairly new to writing parsers in python so I'm trying to write something similar as a learning exercise. However when I attempted to try to code up something similar to what was found in the article I am getting an TypeError: unbound method TypeError. This is the first time I've encountered such an error and I've spent all day trying to figure this out but I haven't solved the issue. Here is a minimal code example (in it's entirety) that has this problem:
import re
class Symbol_base(object):
""" A base class for all symbols"""
id = None # node/token type name
value = None #used by literals
first = second = third = None #used by tree nodes
def nud(self):
""" A default implementation for nud """
raise SyntaxError("Syntax error (%r)." % self.id)
def led(self,left):
""" A default implementation for led """
raise SyntaxError("Unknown operator (%r)." % self.id)
def __repr__(self):
if self.id == "(name)" or self.id == "(literal)":
return "(%s %s)" % (self.id[1:-1], self.value)
out = [self.id, self.first, self.second, self.third]
out = map(str, filter(None,out))
return "(" + " ".join(out) + ")"
symbol_table = {}
def symbol(id, bindingpower=0):
""" If a given symbol is found in the symbol_table return it.
If the symblo cannot be found theni create the appropriate class
and add that to the symbol_table."""
try:
s = symbol_table[id]
except KeyError:
class s(Symbol_base):
pass
s.__name__ = "symbol:" + id #for debugging purposes
s.id = id
s.lbp = bindingpower
symbol_table[id] = s
else:
s.lbp = max(bindingpower,s.lbp)
return s
def infix(id, bp):
""" Helper function for defining the symbols for infix operations """
def infix_led(self, left):
self.first = left
self.second = expression(bp)
return self
symbol(id, bp).led = infix_led
#define all the symbols
infix("+", 10)
symbol("(literal)").nud = lambda self: self #literal values must return the symbol itself
symbol("(end)")
token_pat = re.compile("\s*(?:(\d+)|(.))")
def tokenize(program):
for number, operator in token_pat.findall(program):
if number:
symbol = symbol_table["(literal)"]
s = symbol()
s.value = number
yield s
else:
symbol = symbol_table.get(operator)
if not symbol:
raise SyntaxError("Unknown operator")
yield symbol
symbol = symbol_table["(end)"]
yield symbol()
def expression(rbp = 0):
global token
t = token
token = next()
left = t.nud()
while rbp < token.lbp:
t = token
token = next()
left = t.led(left)
return left
def parse(program):
global token, next
next = tokenize(program).next
token = next()
return expression()
def __main__():
print parse("1 + 2")
if __name__ == "__main__":
__main__()
When I try to run this with pypy:
Traceback (most recent call last):
File "app_main.py", line 72, in run_toplevel
File "parser_code_issue.py", line 93, in <module>
__main__()
File "parser_code_issue.py", line 90, in __main__
print parse("1 + 2")
File "parser_code_issue.py", line 87, in parse
return expression()
File "parser_code_issue.py", line 81, in expression
left = t.led(left)
TypeError: unbound method infix_led() must be called with symbol:+ instance as first argument (got symbol:(literal) instance instead)
I'm guessing this happens because I don't create an instance for the infix operations but I'm not really wanting to create an instance at that point. Is there some way I can change those methods without creating instances?
Any help explaining why this is happening and what I can do to fix the code is greatly appreciated!
Also is this behaviour going to change in python 3?

You forgot to create an instance of the symbol in your tokenize() function; when not a number, yield symbol(), not symbol:
else:
symbol = symbol_table.get(operator)
if not symbol:
raise SyntaxError("Unknown operator")
yield symbol()
With that one change your code prints:
(+ (literal 1) (literal 2))

You haven't bound new function to the instance of your object.
import types
obj = symbol(id, bp)
obj.led = types.MethodType(infix_led, obj)
See accepted answer to another SO question

How to get source corresponding to a Python AST node?

Python AST nodes have lineno and col_offset attributes, which indicate the beginning of respective code range. Is there an easy way to get also the end of the code range? A 3rd party library?

EDIT: Latest code (tested in Python 3.5-3.7) is here: https://bitbucket.org/plas/thonny/src/master/thonny/ast_utils.py
As I didn't find an easy way, here's a hard (and probably not optimal) way. Might crash and/or work incorrectly if there are more lineno/col_offset bugs in Python parser than those mentioned (and worked around) in the code. Tested in Python 3.3:
def mark_code_ranges(node, source):
"""
Node is an AST, source is corresponding source as string.
Function adds recursively attributes end_lineno and end_col_offset to each node
which has attributes lineno and col_offset.
"""
NON_VALUE_KEYWORDS = set(keyword.kwlist) - {'False', 'True', 'None'}
def _get_ordered_child_nodes(node):
if isinstance(node, ast.Dict):
children = []
for i in range(len(node.keys)):
children.append(node.keys[i])
children.append(node.values[i])
return children
elif isinstance(node, ast.Call):
children = [node.func] + node.args
for kw in node.keywords:
children.append(kw.value)
if node.starargs != None:
children.append(node.starargs)
if node.kwargs != None:
children.append(node.kwargs)
children.sort(key=lambda x: (x.lineno, x.col_offset))
return children
else:
return ast.iter_child_nodes(node)
def _fix_triple_quote_positions(root, all_tokens):
"""
http://bugs.python.org/issue18370
"""
string_tokens = list(filter(lambda tok: tok.type == token.STRING, all_tokens))
def _fix_str_nodes(node):
if isinstance(node, ast.Str):
tok = string_tokens.pop(0)
node.lineno, node.col_offset = tok.start
for child in _get_ordered_child_nodes(node):
_fix_str_nodes(child)
_fix_str_nodes(root)
# fix their erroneous Expr parents
for node in ast.walk(root):
if ((isinstance(node, ast.Expr) or isinstance(node, ast.Attribute))
and isinstance(node.value, ast.Str)):
node.lineno, node.col_offset = node.value.lineno, node.value.col_offset
def _fix_binop_positions(node):
"""
http://bugs.python.org/issue18374
"""
for child in ast.iter_child_nodes(node):
_fix_binop_positions(child)
if isinstance(node, ast.BinOp):
node.lineno = node.left.lineno
node.col_offset = node.left.col_offset
def _extract_tokens(tokens, lineno, col_offset, end_lineno, end_col_offset):
return list(filter((lambda tok: tok.start[0] >= lineno
and (tok.start[1] >= col_offset or tok.start[0] > lineno)
and tok.end[0] <= end_lineno
and (tok.end[1] <= end_col_offset or tok.end[0] < end_lineno)
and tok.string != ''),
tokens))
def _mark_code_ranges_rec(node, tokens, prelim_end_lineno, prelim_end_col_offset):
"""
Returns the earliest starting position found in given tree,
this is convenient for internal handling of the siblings
"""
# set end markers to this node
if "lineno" in node._attributes and "col_offset" in node._attributes:
tokens = _extract_tokens(tokens, node.lineno, node.col_offset, prelim_end_lineno, prelim_end_col_offset)
#tokens =
_set_real_end(node, tokens, prelim_end_lineno, prelim_end_col_offset)
# mark its children, starting from last one
# NB! need to sort children because eg. in dict literal all keys come first and then all values
children = list(_get_ordered_child_nodes(node))
for child in reversed(children):
(prelim_end_lineno, prelim_end_col_offset) = \
_mark_code_ranges_rec(child, tokens, prelim_end_lineno, prelim_end_col_offset)
if "lineno" in node._attributes and "col_offset" in node._attributes:
# new "front" is beginning of this node
prelim_end_lineno = node.lineno
prelim_end_col_offset = node.col_offset
return (prelim_end_lineno, prelim_end_col_offset)
def _strip_trailing_junk_from_expressions(tokens):
while (tokens[-1].type not in (token.RBRACE, token.RPAR, token.RSQB,
token.NAME, token.NUMBER, token.STRING,
token.ELLIPSIS)
and tokens[-1].string not in ")}]"
or tokens[-1].string in NON_VALUE_KEYWORDS):
del tokens[-1]
def _strip_trailing_extra_closers(tokens, remove_naked_comma):
level = 0
for i in range(len(tokens)):
if tokens[i].string in "({[":
level += 1
elif tokens[i].string in ")}]":
level -= 1
if level == 0 and tokens[i].string == "," and remove_naked_comma:
tokens[:] = tokens[0:i]
return
if level < 0:
tokens[:] = tokens[0:i]
return
def _set_real_end(node, tokens, prelim_end_lineno, prelim_end_col_offset):
# prelim_end_lineno and prelim_end_col_offset are the start of
# next positioned node or end of source, ie. the suffix of given
# range may contain keywords, commas and other stuff not belonging to current node
# Function returns the list of tokens which cover all its children
if isinstance(node, _ast.stmt):
# remove empty trailing lines
while (tokens[-1].type in (tokenize.NL, tokenize.COMMENT, token.NEWLINE, token.INDENT)
or tokens[-1].string in (":", "else", "elif", "finally", "except")):
del tokens[-1]
else:
_strip_trailing_extra_closers(tokens, not isinstance(node, ast.Tuple))
_strip_trailing_junk_from_expressions(tokens)
# set the end markers of this node
node.end_lineno = tokens[-1].end[0]
node.end_col_offset = tokens[-1].end[1]
# Try to peel off more tokens to give better estimate for children
# Empty parens would confuse the children of no argument Call
if ((isinstance(node, ast.Call))
and not (node.args or node.keywords or node.starargs or node.kwargs)):
assert tokens[-1].string == ')'
del tokens[-1]
_strip_trailing_junk_from_expressions(tokens)
# attribute name would confuse the "value" of Attribute
elif isinstance(node, ast.Attribute):
if tokens[-1].type == token.NAME:
del tokens[-1]
_strip_trailing_junk_from_expressions(tokens)
else:
raise AssertionError("Expected token.NAME, got " + str(tokens[-1]))
#import sys
#print("Expected token.NAME, got " + str(tokens[-1]), file=sys.stderr)
return tokens
all_tokens = list(tokenize.tokenize(io.BytesIO(source.encode('utf-8')).readline))
_fix_triple_quote_positions(node, all_tokens)
_fix_binop_positions(node)
source_lines = source.split("\n")
prelim_end_lineno = len(source_lines)
prelim_end_col_offset = len(source_lines[len(source_lines)-1])
_mark_code_ranges_rec(node, all_tokens, prelim_end_lineno, prelim_end_col_offset)

We had a similar need, and I created the asttokens library for this purpose. It maintains the source in both text and tokenized form, and marks AST nodes with token information, from which text is also readily available.
It works with Python 2 and 3 (tested with 2.7 and 3.5). For example:
import ast, asttokens
st='''
def greet(a):
say("hello") if a else say("bye")
'''
atok = asttokens.ASTTokens(st, parse=True)
for node in ast.walk(atok.tree):
if hasattr(node, 'lineno'):
print atok.get_text_range(node), node.__class__.__name__, atok.get_text(node)
Prints
(1, 50) FunctionDef def greet(a):
say("hello") if a else say("bye")
(17, 50) Expr say("hello") if a else say("bye")
(11, 12) Name a
(17, 50) IfExp say("hello") if a else say("bye")
(33, 34) Name a
(17, 29) Call say("hello")
(40, 50) Call say("bye")
(17, 20) Name say
(21, 28) Str "hello"
(40, 43) Name say
(44, 49) Str "bye"

ast.get_source_segment was added in python 3.8:
import ast
code = """
if 1 == 1 and 2 == 2 and 3 == 3:
test = 1
"""
node = ast.parse(code)
ast.get_source_segment(code, node.body[0])
Produces: if 1 == 1 and 2 == 2 and 3 == 3:\n test = 1
Thanks to Blane for his answer in https://stackoverflow.com/a/62624882/3800552

Hi I know its very late , But I think is this is what you are looking for,
I am doing the parsing only for function definitions in the module.
We can get the first and last line of the ast node by this method. This way the source code lines of a function definition can be obtained by parsing the source file by reading only the lines we need .
This is a very simple example ,
st='def foo():\n print "hello" \n\ndef bla():\n a = 1\n b = 2\n
c= a+b\n print c'
import ast
tree = ast.parse(st)
for function in tree.body:
if isinstance(function,ast.FunctionDef):
# Just in case if there are loops in the definition
lastBody = func.body[-1]
while isinstance (lastBody,(ast.For,ast.While,ast.If)):
lastBody = lastBody.Body[-1]
lastLine = lastBody.lineno
print "Name of the function is ",function.name
print "firstLine of the function is ",function.lineno
print "LastLine of the function is ",lastLine
print "the source lines are "
if isinstance(st,str):
st = st.split("\n")
for i , line in enumerate(st,1):
if i in range(function.lineno,lastLine+1):
print line

Python string assignment issue!

So I'm fairly new to Python but I have absolutely no idea why this strong oldUser is changing to current user after I make the parse call. Any help would be greatly appreciated.
while a < 20:
f = urllib.urlopen("SITE")
a = a+1
for i, line in enumerate(f):
if i == 187:
print line
myparser.parse(line)
if fCheck == 1:
result = oldUser[0] is oldUser[1]
print oldUser[0]
print oldUser[1]
else:
result = user is oldUser
fCheck = 1
print result
user = myparser.get_descriptions(firstCheck)
firstCheck = 1
print user
if result:
print "SAME"
array[index+1] = array[index+1] +0
else:
oldUser = user
elif i > 200:
break
myparser.reset()
I don't understand why result doesn't work either... I print out both values and when they're the same it's telling me they're not equal... Also, why does myparser.parse(line) turn oldUser into a size 2 array? Thanks!
** Here's the definition for myparse...
class MyParser(sgmllib.SGMLParser):
"A simple parser class."
def parse(self, s):
"Parse the given string 's'."
self.feed(s)
self.close()
def __init__(self, verbose=0):
"Initialise an object, passing 'verbose' to the superclass."
sgmllib.SGMLParser.__init__(self, verbose)
self.divs = []
self.descriptions = []
self.inside_div_element = 0
def start_div(self, attributes):
"Process a hyperlink and its 'attributes'."
for name, value in attributes:
if name == "id":
self.divs.append(value)
self.inside_div_element = 1
def end_div(self):
"Record the end of a hyperlink."
self.inside_div_element = 0
def handle_data(self, data):
"Handle the textual 'data'."
if self.inside_div_element:
self.descriptions.append(data)
def get_div(self):
"Return the list of hyperlinks."
return self.divs
def get_descriptions(self, check):
"Return a list of descriptions."
if check == 1:
self.descriptions.pop(0)
return self.descriptions

Don’t compare strings with is. That checks if they’re the same object, not two copies of the same string. See:
>>> string = raw_input()
hello
>>> string is 'hello'
False
>>> string == 'hello'
True
Also, the definition of myparser would be useful.

I'm not quite sure what your code is doing, but I suspect you want to use == instead of is. Using is compares object identity, which is not the same as string equality. Two different string objects may contain the same sequence of characters.
result = oldUser[0] == oldUser[1]
If you're curious, for more information on the behaviour of the is operator see Python “is” operator behaves unexpectedly with integers.