Using regex to parse kindle "My Clippings.txt" file - python

I am currently trying to use python to parse the notes file for my kindle so that I can keep them more organized than the chronologically ordered list that the kindle automatically saves notes in. Unfortunately, I'm having trouble using regex to parse the file. Here's my code so far:
import re
def parse_file(in_file):
read_file = open(in_file, 'r')
file_lines = read_file.readlines()
read_file.close()
raw_note = "".join(file_lines)
# Regex parts
title_regex = "(.+)"
title_author_regex = "(.+) \((.+)\)"
loc_norange_regex = "(.+) (Location|on Page) ([0-9]+)"
loc_range_regex = "(.+) (Location|on Page) ([0-9]+)-([0-9]+)"
date_regex = "([a-zA-Z]+), ([a-zA-Z]+) ([0-9]+), ([0-9]+)" # Date
time_regex = "([0-9]+):([0-9]+) (AM|PM)" # Time
content_regex = "(.*)"
footer_regex = "=+"
nl_re = "\r*\n"
# No author
regex_noauthor_str =\
title_regex + nl_re +\
"- Your " + loc_range_regex + " | Added on " +\
date_regex + ", " + time_regex + nl_re +\
content_regex + nl_re +\
footer_regex
regex_noauthor = re.compile(regex_noauthor_str)
print regex_noauthor.findall(raw_note)
parse_file("testnotes")
Here is the contents of "testnotes":
Title
- Your Highlight Location 3360-3362 | Added on Wednesday, March 21, 2012, 12:16 AM
Note content goes here
==========
What I want:
[('Title', 'Highlight', 'Location', '3360', '3362', 'Wednesday', 'March', '21', '2012', '12', '16', 'AM',
But when I run the program, I get:
[('Title', 'Highlight', 'Location', '3360', '3362', '', '', '', '', '', '', '', '')]
I'm fairly new to regex, but I feel like this should be fairly straightforward.


When you say " | Added on ", you need to escape the |.
Replace that string with " \| Added on "


You need to escape the | in "- Your " + loc_range_regex + " | Added on " +\
to: "- Your " + loc_range_regex + " \| Added on " +\
| is the OR operator in a regex.


Should anyone need an update to this, the following works with Paperwhite & Voyage Kindles in 2017 : https://gist.github.com/laffan/7b945d256028d2ffaacd4d99be40ca34

Related

Convert a list of tab prefixed strings to a dictionary

Text mining attempts here, I would like to turn the below:
a=['Colors.of.the universe:\n',
' Black: 111\n',
' Grey: 222\n',
' White: 11\n'
'Movies of the week:\n',
' Mission Impossible: 121\n',
' Die_Hard: 123\n',
' Jurassic Park: 33\n',
'Lands.categories.said:\n',
' Desert: 33212\n',
' forest: 4532\n',
' grassland : 431\n',
' tundra : 243451\n']
to this:
{'Colors.of.the universe':{Black:111,Grey:222,White:11},
'Movies of the week':{Mission Impossible:121,Die_Hard:123,Jurassic Park:33},
'Lands.categories.said': {Desert:33212,forest:4532,grassland:431,tundra:243451}}
Tried this code below but it was not good:
{words[1]:words[1:] for words in a}
which gives
{'o': 'olors.of.the universe:\n',
' ': ' tundra : 243451\n',
'a': 'ands.categories.said:\n'}
It only takes the first word as the key which is not what's needed.
A dict comprehension is an interesting approach.

a = ['Colors.of.the universe:\n',
' Black: 111\n',
' Grey: 222\n',
' White: 11\n',
'Movies of the week:\n',
' Mission Impossible: 121\n',
' Die_Hard: 123\n',
' Jurassic Park: 33\n',
'Lands.categories.said:\n',
' Desert: 33212\n',
' forest: 4532\n',
' grassland : 431\n',
' tundra : 243451\n']
result = dict()
current_key = None
for w in a:
# If starts with tab - its an item (under category)
if w.startswith(' '):
# Splitting item (i.e. ' Desert: 33212\n' -> [' Desert', ' 33212\n']
splitted = w.split(':')
# Setting the key and the value of the item
# Removing redundant spaces and '\n'
# Converting value to number
k, v = splitted[0].strip(), int(splitted[1].replace('\n', ''))
result[current_key][k] = v
# Else, it's a category
else:
# Removing ':' and '\n' form category name
current_key = w.replace(':', '').replace('\n', '')
# If category not exist - create a dictionary for it
if not current_key in result.keys():
result[current_key] = {}
# {'Colors.of.the universe': {'Black': 111, 'Grey': 222, 'White': 11}, 'Movies of the week': {'Mission Impossible': 121, 'Die_Hard': 123, 'Jurassic Park': 33}, 'Lands.categories.said': {'Desert': 33212, 'forest': 4532, 'grassland': 431, 'tundra': 243451}}
print(result)

That's really close to valid YAML already. You could just quote the property labels and parse. And parsing a known format is MUCH superior to dealing with and/or inventing your own. Even if you're just exploring base python, exploring good practices is just as (probably more) important.
import re
import yaml
raw = ['Colors.of.the universe:\n',
' Black: 111\n',
' Grey: 222\n',
' White: 11\n',
'Movies of the week:\n',
' Mission Impossible: 121\n',
' Die_Hard: 123\n',
' Jurassic Park: 33\n',
'Lands.categories.said:\n',
' Desert: 33212\n',
' forest: 4532\n',
' grassland : 431\n',
' tundra : 243451\n']
# Fix spaces in property names
fixed = []
for line in raw:
match = re.match(r'^( *)(\S.*?): ?(\S*)\s*', line)
if match:
fixed.append('{indent}{safe_label}:{value}'.format(
indent = match.group(1),
safe_label = "'{}'".format(match.group(2)),
value = ' ' + match.group(3) if match.group(3) else ''
))
else:
raise Exception("regex failed")
parsed = yaml.load('\n'.join(fixed), Loader=yaml.FullLoader)
print(parsed)

Set property with whitespace with py2neo

I'm trying to set a property with a value which could have with space.I'm using py2neo and I wrote this, but it doesn't work:
name = mention['name']
screen_name = mention['screen_name']
id = mention['id']
graph.run(
'MATCH (f) '
'WHERE EXISTS(f.account_id) AND f.account_id=\'' + id + '\' '
'SET f.name=\'' + re.escape(name) + '\', f.screen_name=\'#' + screen_name + '\' '
'RETURN null'
)
How can I solve ? Thanks

You should pass the values (including the problematic value for name) as parameters. That avoids messy issues with the Python and Cypher parsers (and escaping the string value should also not be necessary).
This may work for you:
graph.run(
'MATCH (f) '
'WHERE f.account_id=$id '
'SET f.name=$name, f.screen_name=$screen_name',
{"id": mention['id'], "name": mention['name'], "screen_name": '#'+mention['screen_name']}
)

P.S: I assume you are trying to set the property screen_name with
space.
You placed space after \' (single quote), It should be before it:
graph.run(
'MATCH (f) '
'WHERE EXISTS(f.account_id) AND f.account_id=\'' + id + '\''
'SET f.name=\'' + re.escape(name) + '\', f.screen_name=\'#' + screen_name + ' \''
'RETURN null'
)

network device command parsing with pyparsing

I`m developing network device command parser using pyparsing.
I analysed and define the command format as below:
cli ::= string + (next)*
next ::= string|range|group|simple_recursive|selective_recursive|infinite_recursive|keywords
keywords ::= "WORD"
| "LINE"
| "A.B.C.D"
| "A.B.C.D/M"
| "X:X::X:X"
| "X:X::X:X/M"
| "HH:MM:SS"
| "AA:NN"
| "XX:XX:XX:XX:XX:XX"
| "MULTILINE"
inner_recur ::= next + (next)* + ("|")* | ("|" + next + (next)*)*
string ::= alphanums + "_" + "-"
range ::= "<" + nums + "-" nums + ">"
group ::= "(" + inner_recur + ")"
simple_recursive ::= "." + range
selective_recursive ::= "{" + inner_recur + "}"
infinite_recursive ::= "[" + inner_recur + "]"
and implemented written:
# string ::= alphanums + "_" + "-"
string_ = Word(alphanums + "_" + "-").setResultsName("string")
#print(string_.parseString("option82"))
# range ::= "<" + nums + "-" nums + ">"
range_ = Combine(Literal("<") + Word(nums) + Literal("-") + Word(nums) + Literal(">")).setResultsName("range")
#print(range_.parseString("<24-1004>"))
# simple_recursive ::= "." + range
simple_recursive_ = Combine(Literal(".") + range_).setResultsName("simple_recursive")
#print(simple_recursive_.parseString(".<1-60045>"))
# keywords ::= "WORD" | "LINE" | "A.B.C.D" | "A.B.C.D/M" | "X:X::X:X" | "X:X::X:X/M" | "HH:MM:SS" | "AA:NN" | "XX:XX:XX:XX:XX:XX" | "MULTILINE"
keywords_ = Keyword("X:X::X:X/M").setResultsName("X:X::X:/M") | Keyword("A.B.C.D/M").setResultsName("A.B.C.D/M") | Keyword("A.B.C.D").setResultsName("A.B.C.D") | Keyword("X:X::X:X").setResultsName("X:X::X:X") | Keyword("HH:MM:SS").setResultsName("HH:MM:SS") | Keyword("AA:NN").setResultsName("AA:NN") | Keyword("XX:XX:XX:XX:XX:XX").setResultsName("XX:XX:XX:XX:XX:XX") | Keyword("MULTILINE").setResultsName("MULTILINE") | Keyword("WORD").setResultsName("WORD") | Keyword("LINE").setResultsName("LINE")
#print(keywords_.parseString("A.B.C.D").asXML())
#next_ = Forward()
inner_recur = Forward()
# group ::= "(" + inner_recur + ")"
group_ = Combine(Literal("(") + inner_recur + Literal(")"))
# selective_recursive ::= "{" + inner_recur + "}"
selective_recursive_ = Combine(Literal("{") + inner_recur + Literal("}"))
# infinite_recursive ::= "[" + inner_recur + "]"
infinite_recursive_ = Combine(Literal("[") + inner_recur + Literal("]"))
# next ::= string|range|group|simple_recursive|selective_recursive|infinite_recursive|keywords
next_ = keywords_ | string_ | simple_recursive_ | range_ | group_ | selective_recursive_ | infinite_recursive_
# inner_recur ::= next + (next)* + ("|")* | ("|" + next + (next)*)*
inner_recur << next_ + ZeroOrMore(next_) + ZeroOrMore(Literal("|") | ZeroOrMore(Literal("|") + next_ + OneOrMore(next_)))
# cli ::= string + (next)*
cli_ = string_ + ZeroOrMore(next_)
To test my parser, I tried to input datas
>>> test = cli_.parseString("bgp as .<1-200>")
>>> print(test)
>>> ['bgp', 'as', ['.<1-200>']]
test = cli_.parseString("bgp as <1-200> <1-255> <1-255> WORD A.B.C.D A.B.C.D/M (A|(B|C))")
print(test)
>>>
test = cli_.parseString("test (A|<1-200>|(B|{a|b|c} aaa)")
test = cli_.parseString("test (A|<1-200>|(B|{a|b|c|})|)")
when parsed second data, infinite recursion raised. I don't understand this situation and have any solution...
I expect the result:
['bgp', 'as', ['<1-200>'], ['<1-255>'], ['<1-255>'], 'WORD',
'A.B.C.D', 'A.B.C.D/M', ['A', ['B', 'C']]]
what is my problem in format or code? and point be modified?

While you have made a good first step in defining your grammar in conceptual BNF terms before writing code, I'm struggling a bit with making sense of your grammar. The culprit to me seems to be this part:
inner_recur ::= next + (next)* + ("|")* | ("|" + next + (next)*)*
From your posted examples, this looks like you are trying to define some sort of infix notation, using '|' as an operator.
From your tests, it also looks like you need to support multiple inner_recur terms within any grouping ()'s, []'s, or {}'s.
Also, please read the docs (https://pyparsing-docs.readthedocs.io/en/latest/pyparsing.html) to get a clearer picture of the difference between setResultsName and setName. I'm pretty sure in your parser throughout, you are using setResultsName but really want setName. Similarly with using Combine when you really want Group.
Lastly, I rewrote your test code using runTests, and saw that you had mismatched ()'s on the third test.
Here is your parser with these changes:
# string ::= alphanums + "_" + "-"
string_ = Word(alphanums + "_" + "-").setResultsName("string")
#print(string_.parseString("option82"))
# range ::= "<" + nums + "-" nums + ">"
range_ = Group(Literal("<") + Word(nums) + Literal("-") + Word(nums) + Literal(">")).setResultsName("range")
#print(range_.parseString("<24-1004>"))
# simple_recursive ::= "." + range
simple_recursive_ = Group(Literal(".") + range_).setResultsName("simple_recursive")
#print(simple_recursive_.parseString(".<1-60045>"))
# keywords ::= "WORD" | "LINE" | "A.B.C.D" | "A.B.C.D/M" | "X:X::X:X" | "X:X::X:X/M" | "HH:MM:SS" | "AA:NN" | "XX:XX:XX:XX:XX:XX" | "MULTILINE"
keywords_ = Keyword("X:X::X:X/M").setResultsName("X:X::X:/M") | Keyword("A.B.C.D/M").setResultsName("A.B.C.D/M") | Keyword("A.B.C.D").setResultsName("A.B.C.D") | Keyword("X:X::X:X").setResultsName("X:X::X:X") | Keyword("HH:MM:SS").setResultsName("HH:MM:SS") | Keyword("AA:NN").setResultsName("AA:NN") | Keyword("XX:XX:XX:XX:XX:XX").setResultsName("XX:XX:XX:XX:XX:XX") | Keyword("MULTILINE").setResultsName("MULTILINE") | Keyword("WORD").setResultsName("WORD") | Keyword("LINE").setResultsName("LINE")
#print(keywords_.parseString("A.B.C.D").asXML())
#next_ = Forward()
inner_recur = Forward()
# group ::= "(" + inner_recur + ")"
group_ = Group(Literal("(") + OneOrMore(inner_recur) + Literal(")"))
# selective_recursive ::= "{" + inner_recur + "}"
selective_recursive_ = Group(Literal("{") + OneOrMore(inner_recur) + Literal("}"))
# infinite_recursive ::= "[" + inner_recur + "]"
infinite_recursive_ = Group(Literal("[") + OneOrMore(inner_recur) + Literal("]"))
# next ::= string|range|group|simple_recursive|selective_recursive|infinite_recursive|keywords
next_ = keywords_ | string_ | simple_recursive_ | range_ | group_ | selective_recursive_ | infinite_recursive_
#~ next_.setName("next_").setDebug()
# inner_recur ::= next + (next)* + ("|")* | ("|" + next + (next)*)*
#~ inner_recur <<= OneOrMore(next_) + ZeroOrMore(Literal("|")) | ZeroOrMore(Literal("|") + OneOrMore(next_))
inner_recur <<= Group(infixNotation(next_,
[
(None, 2, opAssoc.LEFT),
('|', 2, opAssoc.LEFT),
]) + Optional('|'))
# cli ::= string + (next)*
cli_ = string_ + ZeroOrMore(next_)
tests = """\
bgp as .<1-200>
bgp as <1-200> <1-255> <1-255> WORD A.B.C.D A.B.C.D/M (A|(B|C))
test (A|<1-200>|(B|{a|b|c} aaa))
test (A|<1-200>|(B|{a|b|c|})|)
"""
cli_.runTests(tests)
Which gives:
bgp as .<1-200>
['bgp', 'as', ['.', ['<', '1', '-', '200', '>']]]
- simple_recursive: ['.', ['<', '1', '-', '200', '>']]
- range: ['<', '1', '-', '200', '>']
- string: 'as'
bgp as <1-200> <1-255> <1-255> WORD A.B.C.D A.B.C.D/M (A|(B|C))
['bgp', 'as', ['<', '1', '-', '200', '>'], ['<', '1', '-', '255', '>'], ['<', '1', '-', '255', '>'], 'WORD', 'A.B.C.D', 'A.B.C.D/M', ['(', [['A', '|', ['(', [['B', '|', 'C']], ')']]], ')']]
- A.B.C.D: 'A.B.C.D'
- A.B.C.D/M: 'A.B.C.D/M'
- WORD: 'WORD'
- range: ['<', '1', '-', '255', '>']
- string: 'as'
test (A|<1-200>|(B|{a|b|c} aaa))
['test', ['(', [['A', '|', ['<', '1', '-', '200', '>'], '|', ['(', [['B', '|', [['{', [['a', '|', 'b', '|', 'c']], '}'], 'aaa']]], ')']]], ')']]
- string: 'test'
test (A|<1-200>|(B|{a|b|c|})|)
['test', ['(', [['A', '|', ['<', '1', '-', '200', '>'], '|', ['(', [['B', '|', ['{', [['a', '|', 'b', '|', 'c'], '|'], '}']]], ')']], '|'], ')']]
- string: 'test'
This may be off the mark in some places, but I hope it gives you some ideas to move forward with your project.

prevent pandas from removing spaces in numbers in text columns

I'm trying to load CSV file into pandas dataframe. CSV is semicolon delimited. Values in text columns are in double quotation marks.
File in question: https://www.dropbox.com/s/1xv391gebjzmmco/file_01.csv?dl=0
In one of the text columns ('TYTUL') i have following value:
"00 307 1457 212"
I specify the column as str but when i print or export results to excel I get
003071457212
instead of
00 307 1457 212
How do I prevent pandas from removing spaces?
Here is my code:
import pandas
df = pandas.read_csv(r'file_01.csv'
,sep = ';'
,quotechar = '"'
,names = ['DATA_OPERACJI'
,'DATA_KSIEGOWANIA'
,'OPIS_OPERACJI'
,'TYTUL'
,'NADAWCA_ODBIORCA'
,'NUMER_KONTA'
,'KWOTA'
,'SALDO_PO_OPERACJI'
,'KOLUMNA_9']
,usecols = [0,1,2,3,4,5,6,7]
,skiprows = 38
,skipfooter = 3
,encoding = 'cp1250'
,thousands = ' '
,decimal = ','
,parse_dates = [0,1]
,converters = {'OPIS_OPERACJI': str
,'TYTUL': str
,'NADAWCA_ODBIORCA': str
,'NUMER_KONTA': str}
,engine = 'python'
)
df.TYTUL.replace([' +', '^ +', ' +$'], [' ', '', ''],regex=True,inplace=True) #this only removes excessive spaces
print(df.TYTUL)
I also came up with a workaround (comment #workaround) but I would like to ask if there is a better way.
import pandas
df = pandas.read_csv(r'file_01.csv'
,sep = ';'
,quotechar = '?' #workaround
,names = ['DATA_OPERACJI'
,'DATA_KSIEGOWANIA'
,'OPIS_OPERACJI'
,'TYTUL'
,'NADAWCA_ODBIORCA'
,'NUMER_KONTA'
,'KWOTA'
,'SALDO_PO_OPERACJI'
,'KOLUMNA_9']
,usecols = [0,1,2,3,4,5,6,7]
,skiprows = 38
,skipfooter = 3
,encoding = 'cp1250'
,thousands = ' '
,decimal = ','
,parse_dates = [0,1]
,converters = {'OPIS_OPERACJI': str
,'TYTUL': str
,'NADAWCA_ODBIORCA': str
,'NUMER_KONTA': str}
,engine = 'python'
)
df.TYTUL.replace([' +', '^ +', ' +$'], [' ', '', ''],regex=True,inplace=True) #this only removes excessive spaces
df.TYTUL.replace(['^"', '"$'], ['', ''],regex=True,inplace=True) #workaround
print(df.TYTUL)

remove this line from your read_csv code
,thousands = ' '
I tested it, the output is correct without this option
'00 307 1457 212'

Parse a colon separated string with pyparsing

This is the data:
C:/data/my_file.txt.c:10:0x21:name1:name2:0x10:1:OK
C:/data/my_file2.txt.c:110:0x1:name2:name5:0x12:1:NOT_OK
./data/my_file3.txt.c:110:0x1:name2:name5:0x12:10:OK
And I would like to get this result
[C:/data/my_file.txt.c, 10, 0x21, name1, name2, 0x10, 1, OK]
[C:/data/my_file2.txt.c, 110, 0x1, name2, name5, 0x12, 1, NOT_OK]
[./data/my_file3.txt.c, 110, 0x1, name2, name5, 0x12, 10, OK]
I know how to do that with some code or string split and stuff like that, but I am searching for a nice solution using pyparsing. My problem is the :/ for the file path.
Additional Question I use some code to strip comments and other stuff from the records so the raw data looks like this:
text = """C:/data/my_file.txt.c:10:0x21:name1:name2:0x10:1:OK
C:/data/my_file2.txt.c:110:0x1:name2:name5:0x12:1:NOT_OK
// comment
./data/my_file3.txt.c:110:0x1:name2:name5:0x12:10:OK
----
ok
"""
And i strip the "//", "ok", and "---" before parsing right now
So now I have a next question too the first:
Some addition to the first question. Till now I extracted the lines above from a data file - that works great. So I read the file line by line and parse it. But now I found out it is possible to use parseFile to parse a whole file. So I think I could strip some of my code and use parseFile instead. So the files I would like to parse have an additional footer.
C:/data/my_file.txt.c:10:0x21:name1:name2:0x10:1:OK
C:/data/my_file2.txt.c:110:0x1:name2:name5:0x12:1:NOT_OK
./data/my_file3.txt.c:110:0x1:name2:name5:0x12:10:OK: info message
-----------------------
3 Files 2 OK 1 NOT_OK
NOT_OK
Is it possible to change the parser to get 2 parse results?
Result1:
[['C:/data/my_file.txt.c', '10', '0x21', 'name1', 'name2', '0x10', '1', 'OK'],
['C:/data/my_file2.txt.c', '110', '0x1', 'name2', 'name5', '0x12', '1', 'NOT_OK'],
['./data/my_file3.txt.c', '110', '0x1', 'name2', 'name5', '0x12', '10', 'OK']]
Ignore the blank line
Ignore this line => -----------------------
Result 2:
[['3', 'Files', 2', 'OK’, '1', 'NOT_OK'],
['NOT_OK’],
So I changed the thes Code for that:
# define an expression for your file reference
one_thing = Combine(
oneOf(list(alphas)) + ':/' +
Word(alphanums + '_-./'))
# define a catchall expression for everything else (words of non-whitespace characters,
# excluding ':')
another_thing = Word(printables + " ", excludeChars=':')
# define an expression of the two; be sure to list the file reference first
thing = one_thing | another_thing
# now use plain old pyparsing delimitedList, with ':' delimiter
list_of_things = delimitedList(thing, delim=':')
list_of_other_things = Word(printables).setName('a')
# run it and see...
parse_ret = OneOrMore(Group(list_of_things | list_of_other_things)).parseFile("data.file")
parse_ret.pprint()
And I get this result:
[['C:/data/my_file.txt.c', '10', '0x21', 'name1', 'name2', '0x10', '1', 'OK'],
['C:/data/my_file2.txt.c','110', '0x1', 'name2', 'name5', '0x12', '1', 'NOT_OK'],
['./data/my_file3.txt.c', '110', '0x1', 'name2', 'name5', '0x12', '10', 'OK', 'info message'],
['-----------------------'],
['3 Files 2 OK 1 NOT_OK'],
['NOT_OK']]
So I can go with this but is it possible to split the result into two named results? I searched the docs but I didn´t find anything that works.

See embedded comments for pyparsing description:
from pyparsing import *
text = """C:/data/my_file.txt.c:10:0x21:name1:name2:0x10:1:OK
C:/data/my_file2.txt.c:110:0x1:name2:name5:0x12:1:NOT_OK
// blah-de blah blah blah
./data/my_file3.txt.c:110:0x1:name2:name5:0x12:10:OK"""
# define an expression for your file reference
one_thing = Combine(
oneOf(list(alphas.upper())) + ':/' +
Word(alphanums + '_-./'))
# define a catchall expression for everything else (words of non-whitespace characters,
# excluding ':')
another_thing = Word(printables, excludeChars=':')
# define an expression of the two; be sure to list the file reference first
thing = one_thing | another_thing
# now use plain old pyparsing delimitedList, with ':' delimiter
list_of_things = delimitedList(thing, delim=':')
parser = OneOrMore(Group(list_of_things))
# ignore comments starting with double slash
parser.ignore(dblSlashComment)
# run it and see...
parser.parseString(text).pprint()
prints:
[['C:/data/my_file.txt.c', '10', '0x21', 'name1', 'name2', '0x10', '1', 'OK'],
['C:/data/my_file2.txt.c', '110', '0x1', 'name2', 'name5', '0x12', '1', 'NOT_OK'],
['./data/my_file3.txt.c', '110', '0x1', 'name2', 'name5', '0x12', '10', 'OK']]

So I didn´t found a solution with delimitedList and parseFile but I found a Solution which is okay for me.
from pyparsing import *
data = """
C: / data / my_file.txt.c:10:0x21:name1:name2:0x10:1:OK
C: / data / my_file2.txt.c:110:0x1:name2:name5:0x12:1:NOT_OK
./ data / my_file3.txt.c:110:0x1:name2:name5:0x12:10:OK: info message
-----------------------
3 Files 2 OK 1 NOT_OK
NOT_OK
"""
if __name__ == '__main__':
# define an expression for your file reference
entry_one = Combine(
oneOf(list(alphas)) + ':/' +
Word(alphanums + '_-./'))
entry_two = Word(printables + ' ', excludeChars=':')
entry = entry_one | entry_two
delimiter = Literal(':').suppress()
tc_result_line = Group(entry.setResultsName('file_name') + delimiter + entry.setResultsName(
'line_nr') + delimiter + entry.setResultsName('num_one') + delimiter + entry.setResultsName('name_one') + delimiter + entry.setResultsName(
'name_two') + delimiter + entry.setResultsName('num_two') + delimiter + entry.setResultsName('status') + Optional(
delimiter + entry.setResultsName('msg'))).setResultsName("info_line")
EOL = LineEnd().suppress()
SOL = LineStart().suppress()
blank_line = SOL + EOL
tc_summary_line = Group(Word(nums).setResultsName("num_of_lines") + "Files" + Word(nums).setResultsName(
"num_of_ok") + "OK" + Word(nums).setResultsName("num_of_not_ok") + "NOT_OK").setResultsName(
"info_summary")
tc_end_line = Or(Literal("NOT_OK"), Literal('Ok')).setResultsName("info_result")
# run it and see...
pp1 = tc_result_line | Optional(tc_summary_line | tc_end_line)
pp1.ignore(blank_line | OneOrMore("-"))
result = list()
for l in data.split('\n'):
result.append((pp1.parseString(l)).asDict())
# delete empty results
result = filter(None, result)
for r in result:
print(r)
pass
Result:
{'info_line': {'file_name': 'C', 'num_one': '10', 'msg': '1', 'name_one': '0x21', 'line_nr': '/ data / my_file.txt.c', 'status': '0x10', 'num_two': 'name2', 'name_two': 'name1'}}
{'info_line': {'file_name': 'C', 'num_one': '110', 'msg': '1', 'name_one': '0x1', 'line_nr': '/ data / my_file2.txt.c', 'status': '0x12', 'num_two': 'name5', 'name_two': 'name2'}}
{'info_line': {'file_name': './ data / my_file3.txt.c', 'num_one': '0x1', 'msg': 'OK', 'name_one': 'name2', 'line_nr': '110', 'status': '10', 'num_two': '0x12', 'name_two': 'name5'}}
{'info_summary': {'num_of_lines': '3', 'num_of_ok': '2', 'num_of_not_ok': '1'}}
{'info_result': ['NOT_OK']}

Using re:
myList = ["C:/data/my_file.txt.c:10:0x21:name1:name2:0x10:1:OK", "C:/data/my_file2.txt.c:110:0x1:name2:name5:0x12:1:NOT_OK", "./data/my_file3.txt.c:110:0x1:name2:name5:0x12:10:OK"]
for i in myList:
newTxt = re.sub(r':', ",", i)
newTxt = re.sub(r',/', ":/", newTxt)
print newTxt

Categories