We Keep Coding

How would I extract all names of a similar type in a file using python and regex?

I have a file that is filled with cultures, and each culture has a unique set of surnames for the people of each culture. The problem I have is that there are numerous files, each of them with hundreds, if not thousands of names, so instead of gathering all of these files by hand, I would like to automate this task in a sense using python and regex.
Here is an example of the file's contents:
###Myrman###
360 = { # DUPLICATE §§§§§§
name="of Myr"
culture = myrman
}
300507 = {
name = "of Myr"
culture = myrman
}
300525 = {
name = "Trellos"
culture = myrman
}
300534 = {
name = "Uteuran"
culture = myrman
}
##Lysene##
1386 = {
name="Ormollen"
culture = lysene
coat_of_arms = {
template = 0
layer = {
texture = 14
texture_internal = 9
emblem = 0
color = 0
color = 0
color = 0
}
}
}
300505 = {
name = "of Lys"
culture = lysene
}
300523 = {
name = "Lohar"
culture = lysene
}
300532 = {
name = "Assadyrn"
culture = lysene
}
So as you can see, there are two types of cultures here, each with different surnames for people of the respective cultures. I want to take all of these different names, and sort them into different groups, that are also separated by commas and quotes. Here is an example of what I want to happen:
Myrman: ["of Myr", "of Myr", "Trellos", "Uteuran"]
Lysene: ["Ormollen", "of Lys", "Lohar", "Assadyrn"]
How would I go about doing this with python and it's regex library?

Ooh, a parser problem! Let's use the lark parser generator to figure this out.
First, let's create a syntax for our file - this is cobbled together based on the JSON parser example:
import lark
parser = lark.Lark(r"""
start: (term)*
term: key "=" value "\n"
key: CNAME | SIGNED_NUMBER
value: CNAME | SIGNED_NUMBER | ESCAPED_STRING | map
map: "{" (term)* "}"
%import common.CNAME
%import common.ESCAPED_STRING
%import common.SIGNED_NUMBER
%import common.WS
%ignore WS
%ignore /#.*/
""")
Pretty straightforward; the file is a list of terms, which are key-values, where the key may be a name or a number, and the value can be a name, number, string, or a map, which is a brace-enclosed list of terms.
Then, let's write a transformer to transform the Lark parse tree to a dict:
class TreeTransformer(lark.Transformer):
def start(self, items):
return dict(items)
def term(self, items):
return (items[0], items[1])
def CNAME(self, item):
return item.value
def SIGNED_NUMBER(self, item):
return int(item.value)
def ESCAPED_STRING(self, item):
return item.value[1:-1]
def map(self, items):
return dict(items)
def key(self, item):
return item[0]
def value(self, item):
return item[0]
Could probably be more terse, but this works.
Let's run it against the data:
from pathlib import Path
from pprint import pprint
data = Path("./so75472097-data.txt").read_text()
tree = parser.parse(data)
res = TreeTransformer().transform(tree)
pprint(res)
The output is
{360: {'culture': 'myrman', 'name': 'of Myr'},
1386: {'coat_of_arms': {'layer': {'color': 0,
'emblem': 0,
'texture': 14,
'texture_internal': 9},
'template': 0},
'culture': 'lysene',
'name': 'Ormollen'},
300505: {'culture': 'lysene', 'name': 'of Lys'},
300507: {'culture': 'myrman', 'name': 'of Myr'},
300523: {'culture': 'lysene', 'name': 'Lohar'},
300525: {'culture': 'myrman', 'name': 'Trellos'},
300532: {'culture': 'lysene', 'name': 'Assadyrn'},
300534: {'culture': 'myrman', 'name': 'Uteuran'}}
-- that looks promising!
Then, it's just a matter of dict traversal:
from collections import defaultdict
names_by_culture = defaultdict(list)
for info in res.values():
names_by_culture[info["culture"]].append(info["name"])
pprint(dict(names_by_culture))
... and hey voilà!
{'lysene': ['Ormollen', 'of Lys', 'Lohar', 'Assadyrn'],
'myrman': ['of Myr', 'of Myr', 'Trellos', 'Uteuran']}
Now, all you have to do is wrap that bad boy into a function and call it on all of your files.
(EDIT, now that I read the latest comment and know what to google: you could just use the ClauseWizard library instead of writing the parser yourself, but this was more fun!)
EDIT 2
As discussed in the comments, a grammar and transformer that's also fine with "basically anything" for unquoted values:
parser = lark.Lark(r"""
start: (term)*
term: key "=" value "\n"
key: KEYNAME | SIGNED_NUMBER
value: VALUENAME | SIGNED_NUMBER | ESCAPED_STRING | map
map: "{" (term)* "}"
VALUENAME: /[a-zA-Z][^\s=]*/
KEYNAME: /[a-zA-Z][-a-zA-Z0-9_]*/
%import common.ESCAPED_STRING
%import common.SIGNED_NUMBER
%import common.WS
%ignore WS
%ignore /#.*/
""")
from operator import itemgetter, attrgetter
class TreeTransformer(lark.Transformer):
start = dict
map = dict
key = itemgetter(0)
value = itemgetter(0)
VALUENAME = attrgetter("value")
KEYNAME = attrgetter("value")
term = tuple
def SIGNED_NUMBER(self, item):
return int(item.value)
def ESCAPED_STRING(self, item):
return item.value[1:-1]

Since the file is well structured, just use regex with the appropriate query and treat the tuples that are outputted accordingly.
result = re.findall('name[ ]*=[ ]*"([A-z ]+)"\n[ ]+culture[ ]*=[ ]*([A-z]+)', a)
names_by_culture = {}
for i in result:
name = i[0]
culture = i[1]
try:
names_by_culture[culture].append(name)
except:
names_by_culture[culture] = []
names_by_culture[culture].append(name)
print(names_by_culture)
Output:
{'myrman': ['of Myr', 'of Myr', 'Trellos', 'Uteuran'],
'lysene': ['Ormollen', 'of Lys', 'Lohar', 'Assadyrn']}

replace trademark symbol (™) when alone

I'm trying to remove trademark symbol (™) but only in the case it's not followed by any other symbol for instance I might have ’ which is a bad encoding of quotation mark (') so I don't want to remove trademark symbol (™) and hence broking the pattern that i'm using to replace xx™ with quotation mark.
dict = {};
chars = {
'\xe2\x84\xa2': '', # ™
'\xe2\x80\x99': "'", # ’
}
def stats_change(char, number):
if dict.has_key(char):
dict[char] = dict[char]+number
else:
dict[char] = number # Add new entry
def replace_chars(match):
char = match.group(0)
stats_change(char,1)
return chars[char]
i, nmatches = re.subn("(\\" + '|\\'.join(chars.keys()) + ")", replace_chars, i)
count_matches += nmatches
Input: foo™ oof
Output: foo oof
Input: o’f oof
Output: o'f oof
Any suggestions ?

Python equivalent of Ruby's .find

I'm trying to implement the following Ruby method into a Python Method:
CF = {:metre=>{:kilometre=>0.001, :metre=>1.0, :centimetre=>100.0}, :litre=>{:litre=>1.0, :millilitre=>1000.0, :imperial_pint=>1.75975}}
def common_dimension(from, to)
CF.keys.find do |canonical_unit|
CF[canonical_unit].keys.include?(from) &&
CF[canonical_unit].keys.include?(to)
end
end
Which behaves like:
>> common_dimension(:metre, :centimetre)
=> :metre
>> common_dimension(:litre, :centimetre)
=> nil
>> common_dimension(:millilitre, :imperial_pint)
=> :litre
What is the "Pythonic" way to implement this?

Below code in python for your ruby logic.
CF={"metre":{"kilometre":0.001, "metre":1.0, "centimetre":100.0}, "litre":{"litre":1.0, "millilitre":1000.0, "imperial_pint":1.75975}}
def common(fr,to):
for key,value in CF.items():
if (fr in value) and (to in value):
return key
print(common('metre','centimdetre'))
metre
print(com('metre','centimdetre'))
None
******************
single line function
com = lambda x,y:[key for key,value in CF.items() if (x in value) and (y in value)]
print(com('metre','centimdetre'))
['metre']

Other option both for Ruby and Python.
For Ruby:
cf = {:metre=>{:kilometre=>0.001, :metre=>1.0, :centimetre=>100.0}, :litre=>{:litre=>1.0, :millilitre=>1000.0, :imperial_pint=>1.75975}}
from = :litre
to = :millilitre
cf.select { |k, v| ([from, to] - v.keys).empty? }.keys
#=> [:litre]
For Python:
CF = {'metre': {'kilometre': 0.001, 'metre': 1.0, 'centimetre': 100.0}, 'litre': {'litre': 1.0, 'millilitre': 1000.0, 'imperial_pint': 1.75975}}
_from = 'millilitre'
_to = 'imperial_pint'
res = [ k for k, v in CF.items() if not bool(set([_from, _to]) - set(v.keys())) ]
#=> ['litre']

How to replace some special characters from user input for different Python platforms

I need to replace some special characters from user input for different platform (i.e. Linux and Windows) using Python. Here is my code:
if request.method == 'POST':
rname1 = request.POST.get('react')
Here I am getting the user input by post method. I need to the following characters to remove from the user input (if there is any).
1- Escape or filter special characters for windows, ( ) < > * ‘ = ? ; [ ] ^ ~ ! . ” % # / \ : + , `
2- Escape or filter special characters for Linux, { } ( ) < > * ‘ = ? ; [ ] $ – # ~ ! . ” % / \ : + , `
The special characters are given above. Here I need to remove for both Linux and Windows.

Python strings have a built in method translate for substitution/deletion of characters. You need to build a translation table and then call the function.
import sys
if "win" in sys.platform:
special = """( ) < > * ‘ = ? ; [ ] ^ ~ ! . ” % # / \ : + , `""".split()
else:
special = """{ } ( ) < > * ‘ = ? ; [ ] $ – # ~ ! . ” % / \ : + , `""".split()
trans_dict = {character: None for character in special}
trans_table = str.maketrans(trans_dict)
print("Lo+=r?e~~m ipsum dol;or sit!! amet, consectet..ur ad%".translate(trans_table))
Will print Lorem ipsum dolor sit amet consectetur ad.
If you want to use a replacement character instead of deleting, then replace None above with the character. You can build a translation table with specific substitutions, `{"a": "m", "b": "n", ...}
Edit: The above snippet is indeed in Python3. In Python2 (TiO) it's easier to delete characters:
>>> import sys
>>> import string
>>> if "win" in sys.platform:
... special = """()<>*'=?;[]^~!%#/\:=,`"""
... else:
... special = """{}()<>*'=?;[]$-#~!."%/\:+"""
...
>>> s = "Lo+r?e~~/\#<>m ips()u;m"
>>> string.translate(s, None, special)
'Lorem ipsum'
Note that I've substituted ‘ with ' and similarly replaced ” with " because I think you're only dealing with ascii strings.

Python equivalent of unix cksum function

I've been looking for the equivalent python method for the unix cksum command:
http://pubs.opengroup.org/onlinepubs/7990989775/xcu/cksum.html
$ cksum ./temp.bin
1605138151 712368 ./temp.bin
So far I have found the zlib.crc32() function
>>> import zlib
>>> f = open('./temp.bin','rb')
>>> data = f.read()
>>> zlib.crc32(data)
1128751837
However this code appears to produce different results. As far as I can tell this should be using the same crc polynomial but I imagine there must be some difference in byte order or initial values. Can anyone shed some light on this for me?

Found a snippet here that implements a compatible cksum in python:
"""
This module implements the cksum command found in most UNIXes in pure
python.
The constants and routine are cribbed from the POSIX man page
"""
import sys
crctab = [ 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc,
0x17c56b6b, 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f,
0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a,
0x384fbdbd, 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9,
0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, 0x6a1936c8,
0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3,
0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e,
0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5,
0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84,
0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, 0xd4326d90, 0xd0f37027,
0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022,
0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1,
0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077,
0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c,
0x2e003dc5, 0x2ac12072, 0x128e9dcf, 0x164f8078, 0x1b0ca6a1,
0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca,
0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb,
0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08,
0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d,
0x40d816ba, 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e,
0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, 0x8aad2b2f,
0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044,
0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, 0xe9362689,
0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683,
0xd1799b34, 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59,
0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c,
0x774bb0eb, 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f,
0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, 0x251d3b9e,
0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5,
0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48,
0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623,
0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2,
0xe6ea3d65, 0xeba91bbc, 0xef68060b, 0xd727bbb6, 0xd3e6a601,
0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604,
0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7,
0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6,
0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad,
0x81b02d74, 0x857130c3, 0x5d8a9099, 0x594b8d2e, 0x5408abf7,
0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c,
0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd,
0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e,
0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b,
0x0fdc1bec, 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088,
0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, 0xc5a92679,
0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12,
0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, 0xea23f0af,
0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5,
0x9e7d9662, 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06,
0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03,
0xb1f740b4 ]
UNSIGNED = lambda n: n & 0xffffffff
def memcrc(b):
n = len(b)
i = c = s = 0
for ch in b:
c = ord(ch)
tabidx = (s>>24)^c
s = UNSIGNED((s << 8)) ^ crctab[tabidx]
while n:
c = n & 0377
n = n >> 8
s = UNSIGNED(s << 8) ^ crctab[(s >> 24) ^ c]
return UNSIGNED(~s)
if __name__ == '__main__':
fname = sys.argv[-1]
buffer = open(fname, 'rb').read()
print "%d\t%d\t%s" % (memcrc(buffer), len(buffer), fname)
Tested briefly, seems to work well.

Same for python3
import sys
crctab = [ 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, 0x130476dc,
0x17c56b6b, 0x1a864db2, 0x1e475005, 0x2608edb8, 0x22c9f00f,
0x2f8ad6d6, 0x2b4bcb61, 0x350c9b64, 0x31cd86d3, 0x3c8ea00a,
0x384fbdbd, 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9,
0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, 0x6a1936c8,
0x6ed82b7f, 0x639b0da6, 0x675a1011, 0x791d4014, 0x7ddc5da3,
0x709f7b7a, 0x745e66cd, 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e,
0x95609039, 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5,
0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, 0xad2f2d84,
0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, 0xd4326d90, 0xd0f37027,
0xddb056fe, 0xd9714b49, 0xc7361b4c, 0xc3f706fb, 0xceb42022,
0xca753d95, 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1,
0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, 0x34867077,
0x30476dc0, 0x3d044b19, 0x39c556ae, 0x278206ab, 0x23431b1c,
0x2e003dc5, 0x2ac12072, 0x128e9dcf, 0x164f8078, 0x1b0ca6a1,
0x1fcdbb16, 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca,
0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, 0x6b93dddb,
0x6f52c06c, 0x6211e6b5, 0x66d0fb02, 0x5e9f46bf, 0x5a5e5b08,
0x571d7dd1, 0x53dc6066, 0x4d9b3063, 0x495a2dd4, 0x44190b0d,
0x40d816ba, 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e,
0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, 0x8aad2b2f,
0x8e6c3698, 0x832f1041, 0x87ee0df6, 0x99a95df3, 0x9d684044,
0x902b669d, 0x94ea7b2a, 0xe0b41de7, 0xe4750050, 0xe9362689,
0xedf73b3e, 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2,
0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, 0xd5b88683,
0xd1799b34, 0xdc3abded, 0xd8fba05a, 0x690ce0ee, 0x6dcdfd59,
0x608edb80, 0x644fc637, 0x7a089632, 0x7ec98b85, 0x738aad5c,
0x774bb0eb, 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f,
0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, 0x251d3b9e,
0x21dc2629, 0x2c9f00f0, 0x285e1d47, 0x36194d42, 0x32d850f5,
0x3f9b762c, 0x3b5a6b9b, 0x0315d626, 0x07d4cb91, 0x0a97ed48,
0x0e56f0ff, 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623,
0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, 0xe22b20d2,
0xe6ea3d65, 0xeba91bbc, 0xef68060b, 0xd727bbb6, 0xd3e6a601,
0xdea580d8, 0xda649d6f, 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604,
0xc960ebb3, 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7,
0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, 0x9b3660c6,
0x9ff77d71, 0x92b45ba8, 0x9675461f, 0x8832161a, 0x8cf30bad,
0x81b02d74, 0x857130c3, 0x5d8a9099, 0x594b8d2e, 0x5408abf7,
0x50c9b640, 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c,
0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, 0x68860bfd,
0x6c47164a, 0x61043093, 0x65c52d24, 0x119b4be9, 0x155a565e,
0x18197087, 0x1cd86d30, 0x029f3d35, 0x065e2082, 0x0b1d065b,
0x0fdc1bec, 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088,
0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, 0xc5a92679,
0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, 0xd6ad50a5, 0xd26c4d12,
0xdf2f6bcb, 0xdbee767c, 0xe3a1cbc1, 0xe760d676, 0xea23f0af,
0xeee2ed18, 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4,
0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, 0x9abc8bd5,
0x9e7d9662, 0x933eb0bb, 0x97ffad0c, 0xafb010b1, 0xab710d06,
0xa6322bdf, 0xa2f33668, 0xbcb4666d, 0xb8757bda, 0xb5365d03,
0xb1f740b4 ]
UNSIGNED = lambda n: n & 0xffffffff
def memcrc(b):
n = len(b)
i = c = s = 0
for c in b:
tabidx = (s>>24)^c
s = UNSIGNED((s << 8)) ^ crctab[tabidx]
while n:
c = n & 0o0377
n = n >> 8
s = UNSIGNED(s << 8) ^ crctab[(s >> 24) ^ c]
return UNSIGNED(~s)
if __name__ == '__main__':
fname = sys.argv[-1]
buffer = open(fname, 'rb').read()
print("%d\t%d\t%s" % (memcrc(buffer), len(buffer), fname))