We Keep Coding

How to make replacement in python's dict?

The goal I want to achieve is to exchange all items whose form is #item_name# to the from (item_value) in the dict. I use two dict named test1 and test2 to test my function. Here is the code:
test1={'integer_set': '{#integer_list#?}', 'integer_list': '#integer_range#(?,#integer_range#)*', 'integer_range': '#integer#(..#integer#)?', 'integer': '[+-]?\\d+'}
test2={'b': '#a#', 'f': '#e#', 'c': '#b#', 'e': '#d#', 'd': '#c#', 'g': '#f#', 'a': 'correct'}
def change(pat_dict:{str:str}):
print('Expanding: ',pat_dict)
num=0
while num<len(pat_dict):
inv_pat_dict = {v: k for k, v in pat_dict.items()}
for value in pat_dict.values():
for key in pat_dict.keys():
if key in value:
repl='#'+key+'#'
repl2='('+pat_dict[key]+')'
value0=value.replace(repl,repl2)
pat_dict[inv_pat_dict[value]]=value0
num+=1
print('Result: ',pat_dict)
change(test1)
change(test2)
sometimes I can get correct result like:
Expanding: {'integer': '[+-]?\\d+', 'integer_list': '#integer_range#(?,#integer_range#)*', 'integer_set': '{#integer_list#?}', 'integer_range': '#integer#(..#integer#)?'}
Result: {'integer': '[+-]?\\d+', 'integer_list': '(([+-]?\\d+)(..([+-]?\\d+))?)(?,(([+-]?\\d+)(..([+-]?\\d+))?))*', 'integer_set': '{((([+-]?\\d+)(..([+-]?\\d+))?)(?,(([+-]?\\d+)(..([+-]?\\d+))?))*)?}', 'integer_range': '([+-]?\\d+)(..([+-]?\\d+))?'}
Expanding: {'c': '#b#', 'f': '#e#', 'e': '#d#', 'b': '#a#', 'g': '#f#', 'd': '#c#', 'a': 'correct'}
Result: {'c': '((correct))', 'f': '(((((correct)))))', 'e': '((((correct))))', 'b': '(correct)', 'g': '((((((correct))))))', 'd': '(((correct)))', 'a': 'correct'}
But most of time I get wrong results like that:
Expanding: {'integer_range': '#integer#(..#integer#)?', 'integer': '[+-]?\\d+', 'integer_set': '{#integer_list#?}', 'integer_list': '#integer_range#(?,#integer_range#)*'}
Result: {'integer_range': '([+-]?\\d+)(..([+-]?\\d+))?', 'integer': '[+-]?\\d+', 'integer_set': '{(#integer_range#(?,#integer_range#)*)?}', 'integer_list': '#integer_range#(?,#integer_range#)*'}
Expanding: {'f': '#e#', 'a': 'correct', 'd': '#c#', 'g': '#f#', 'b': '#a#', 'c': '#b#', 'e': '#d#'}
Result: {'f': '(((((correct)))))', 'a': 'correct', 'd': '(((correct)))', 'g': '((((((correct))))))', 'b': '(correct)', 'c': '((correct))', 'e': '((((correct))))'}
How could I update my code to achieve my goal?

Your problem is caused by the fact that python dictionaries are unordered. Try using a OrderedDict instead of dict and you should be fine. The OrderedDict works just like a normal dict but with ordering retained, at a small performance cost.
Note that while you could create an OrderedDict from a dict literal (like I did here at first), that dict would be unordered, so the ordering might not be guaranteed. Using a list of (key, value) pairs preserves the ordering in all cases.
from collections import OrderedDict
test1=OrderedDict([('integer_set', '{#integer_list#?}'), ('integer_list', '#integer_range#(?,#integer_range#)*'), ('integer_range', '#integer#(..#integer#)?'), ('integer', '[+-]?\\d+')])
test2=OrderedDict([('b', '#a#'), ('f', '#e#'), ('c', '#b#'), ('e', '#d#'), ('d', '#c#'), ('g', '#f#'), ('a', 'correct')])
def change(pat_dict:{str:str}):
print('Expanding: ',pat_dict)
num=0
while num<len(pat_dict):
inv_pat_dict = {v: k for k, v in pat_dict.items()}
for value in pat_dict.values():
for key in pat_dict.keys():
if key in value:
repl='#'+key+'#'
repl2='('+pat_dict[key]+')'
value0=value.replace(repl,repl2)
pat_dict[inv_pat_dict[value]]=value0
num+=1
print('Result: ',pat_dict)
change(test1)
change(test2)

Try this one. Your problem is due to mutating starting dict. You need to change its copy.
test1={'integer_set': '{#integer_list#?}', 'integer_list': '#integer_range#(?,#integer_range#)*', 'integer_range': '#integer#(..#integer#)?', 'integer': '[+-]?\\d+'}
test2={'b': '#a#', 'f': '#e#', 'c': '#b#', 'e': '#d#', 'd': '#c#', 'g': '#f#', 'a': 'correct'}
def change(d):
new_d = d.copy()
for k in d.keys():
for nk, v in new_d.items():
if k in v:
new_d[nk] = v.replace('#{}#'.format(k), '({})'.format(new_d[k]))
return new_d
test1 = change(test1)
test2 = change(test2)

Creating a tree/deeply nested dict with lists from an indented text file

I want to iterate through a file and put the contents of each line into a deeply nested dict, the structure of which is defined by leading whitespace. This desire is very much like that documented here. I've solved that but now have the problem of handling the case where repeating keys are overwritten instead of being cast into a list.
Essentially:
a:
b: c
d: e
a:
b: c2
d: e2
d: wrench
is cast into {"a":{"b":"c2","d":"wrench"}} when it should be cast into
{"a":[{"b":"c","d":"e"},{"b":"c2","d":["e2","wrench"]}]}
A self-contained example:
import json
def jsonify_indented_tree(tree):
#convert indentet text into json
parsedJson= {}
parentStack = [parsedJson]
for i, line in enumerate(tree):
data = get_key_value(line)
if data['key'] in parsedJson.keys(): #if parent key is repeated, then cast value as list entry
# stuff that doesn't work
# if isinstance(parsedJson[data['key']],list):
# parsedJson[data['key']].append(parsedJson[data['key']])
# else:
# parsedJson[data['key']]=[parsedJson[data['key']]]
print('Hey - Make a list now!')
if data['value']: #process child by adding it to its current parent
currentParent = parentStack[-1] #.getLastElement()
currentParent[data['key']] = data['value']
if i is not len(tree)-1:
#determine when to switch to next branch
level_dif = data['level']-get_key_value(tree[i+1])['level'] #peek next line level
if (level_dif > 0):
del parentStack[-level_dif:] #reached leaf, process next branch
else:
#group node, push it as the new parent and keep on processing.
currentParent = parentStack[-1] #.getLastElement()
currentParent[data['key']] = {}
newParent = currentParent[data['key']]
parentStack.append(newParent)
return parsedJson
def get_key_value(line):
key = line.split(":")[0].strip()
value = line.split(":")[1].strip()
level = len(line) - len(line.lstrip())
return {'key':key,'value':value,'level':level}
def pp_json(json_thing, sort=True, indents=4):
if type(json_thing) is str:
print(json.dumps(json.loads(json_thing), sort_keys=sort, indent=indents))
else:
print(json.dumps(json_thing, sort_keys=sort, indent=indents))
return None
#nested_string=['a:', '\tb:\t\tc', '\td:\t\te', 'a:', '\tb:\t\tc2', '\td:\t\te2']
#nested_string=['w:','\tgeneral:\t\tcase','a:','\tb:\t\tc','\td:\t\te','a:','\tb:\t\tc2','\td:\t\te2']
nested_string=['a:',
'\tb:\t\tc',
'\td:\t\te',
'a:',
'\tb:\t\tc2',
'\td:\t\te2',
'\td:\t\twrench']
pp_json(jsonify_indented_tree(nested_string))

This approach is (logically) a lot more straightforward (though longer):
Track the level and key-value pair of each line in your multi-line string
Store this data in a level keyed dict of lists:
{level1:[dict1,dict2]}
Append only a string representing the key in a key-only line: {level1:[dict1,dict2,"nestKeyA"]}
Since a key-only line means the next line is one level deeper, process that on the next level: {level1:[dict1,dict2,"nestKeyA"],level2:[...]}. The contents of some deeper level level2 may itself be just another key-only line (and the next loop will add a new level level3 such that it will become {level1:[dict1,dict2,"nestKeyA"],level2:["nestKeyB"],level3:[...]}) or a new dict dict3 such that {level1:[dict1,dict2,"nestKeyA"],level2:[dict3]
Steps 1-4 continue until the current line is indented less than the previous one (signifying a return to some prior scope). This is what the data structure looks like on my example per line iteration.
0, {0: []}
1, {0: [{'k': 'sds'}]}
2, {0: [{'k': 'sds'}, 'a']}
3, {0: [{'k': 'sds'}, 'a'], 1: [{'b': 'c'}]}
4, {0: [{'k': 'sds'}, 'a'], 1: [{'b': 'c'}, {'d': 'e'}]}
5, {0: [{'k': 'sds'}, {'a': {'d': 'e', 'b': 'c'}}, 'a'], 1: []}
6, {0: [{'k': 'sds'}, {'a': {'d': 'e', 'b': 'c'}}, 'a'], 1: [{'b': 'c2'}]}
7, {0: [{'k': 'sds'}, {'a': {'d': 'e', 'b': 'c'}}, 'a'], 1: [{'b': 'c2'}, {'d': 'e2'}]}
Then two things need to happen. 1: the list of dict need to be inspected for containing duplicate keys and any of those duplicated dict's values combined in a list - this will be demonstrated in a moment. 2: as can be seen between iteration 4 and 5, the list of dicts from the deepest level (here 1) are combined into one dict... Finally, to demonstrate duplicate handling observe:
[7b, {0: [{'k': 'sds'}, {'a': {'d': 'e', 'b': 'c'}}, 'a'], 1: [{'b': 'c2'}, {'d': 'e2'}, {'d': 'wrench'}]}]
[7c, {0: [{'k': 'sds'}, {'a': {'d': 'e', 'b': 'c'}}, {'a': {'d': ['wrench', 'e2'], 'b': 'c2'}}], 1: []}]
where wrench and e2 are placed in a list that itself goes into a dict keyed by their original key.
Repeat Steps 1-5, hoisting deeper scoped dicts up and onto their parent keys until the current line's scope (level) is reached.
Handle termination condition to combine the list of dict on the zeroth level into a dict.
Here's the code:
import json
def get_kvl(line):
key = line.split(":")[0].strip()
value = line.split(":")[1].strip()
level = len(line) - len(line.lstrip())
return {'key':key,'value':value,'level':level}
def pp_json(json_thing, sort=True, indents=4):
if type(json_thing) is str:
print(json.dumps(json.loads(json_thing), sort_keys=sort, indent=indents))
else:
print(json.dumps(json_thing, sort_keys=sort, indent=indents))
return None
def jsonify_indented_tree(tree): #convert shitty sgml header into json
level_map= {0:[]}
tree_length=len(tree)-1
for i, line in enumerate(tree):
data = get_kvl(line)
if data['level'] not in level_map.keys():
level_map[data['level']]=[] # initialize
prior_level=get_kvl(tree[i-1])['level']
level_dif = data['level']-prior_level # +: line is deeper, -: shallower, 0:same
if data['value']:
level_map[data['level']].append({data['key']:data['value']})
if not data['value'] or i==tree_length:
if i==tree_length: #end condition
level_dif = -len(list(level_map.keys()))
if level_dif < 0:
for level in reversed(range(prior_level+level_dif+1,prior_level+1)): # (end, start)
#check for duplicate keys in current deepest (child) sibling group,
# merge them into a list, put that list in a dict
key_freq={} #track repeated keys
for n, dictionary in enumerate(level_map[level]):
current_key=list(dictionary.keys())[0]
if current_key in list(key_freq.keys()):
key_freq[current_key][0]+=1
key_freq[current_key][1].append(n)
else:
key_freq[current_key]=[1,[n]]
for k,v in key_freq.items():
if v[0]>1: #key is repeated
duplicates_list=[]
for index in reversed(v[1]): #merge value of key-repeated dicts into list
duplicates_list.append(list(level_map[level].pop(index).values())[0])
level_map[level].append({k:duplicates_list}) #push that list into a dict on the same stack it came from
if i==tree_length and level==0: #end condition
#convert list-of-dict into dict
parsed_nest={k:v for d in level_map[level] for k,v in d.items()}
else:
#push current deepest (child) sibling group onto parent key
key=level_map[level-1].pop() #string
#convert child list-of-dict into dict
level_map[level-1].append({key:{k:v for d in level_map[level] for k,v in d.items()}})
level_map[level]=[] #reset deeper level
level_map[data['level']].append(data['key'])
return parsed_nest
nested_string=['k:\t\tsds', #need a starter key,value pair otherwise this won't work... fortunately I always have one
'a:',
'\tb:\t\tc',
'\td:\t\te',
'a:',
'\tb:\t\tc2',
'\td:\t\te2',
'\td:\t\twrench']
pp_json(jsonify_indented_tree(nested_string))

Rearranging levels of a nested dictionary in python

Is there a library that would help me achieve the task to rearrange the levels of a nested dictionary
Eg: From this:
{1:{"A":"i","B":"ii","C":"i"},2:{"B":"i","C":"ii"},3:{"A":"iii"}}
To this:
{"A":{1:"i",3:"iii"},"B":{1:"ii",2:"i"},"C":{1:"i",2:"ii"}}
ie first two levels on a 3 levelled dictionary swapped. So instead of 1 mapping to A and 3 mapping to A, we have A mapping to 1 and 3.
The solution should be practical for an arbitrary depth and move from one level to any other within.

>>> d = {1:{"A":"i","B":"ii","C":"i"},2:{"B":"i","C":"ii"},3:{"A":"iii"}}
>>> keys = ['A','B','C']
>>> e = {key:{k:d[k][key] for k in d if key in d[k]} for key in keys}
>>> e
{'C': {1: 'i', 2: 'ii'}, 'B': {1: 'ii', 2: 'i'}, 'A': {1: 'i', 3: 'iii'}}
thank god for dict comprehension

One way to think about this would be to consider your data as a (named) array and to take the transpose. An easy way to achieve this would be to use the data analysis package Pandas:
import pandas as pd
df = pd.DataFrame({1: {"A":"i","B":"ii","C":"i"},
2: {"B":"i","C":"ii"},
3: {"A":"iii"}})
df.transpose().to_dict()
{'A': {1: 'i', 2: nan, 3: 'iii'},
'B': {1: 'ii', 2: 'i', 3: nan},
'C': {1: 'i', 2: 'ii', 3: nan}}

I don't really care about performance for my application of this so I haven't bothered checking how efficient this is. Its based on bubblesort so my guess is ~O(N^2).
Maybe this is convoluted, but essentially below works by:
- providing dict_swap_index a nested dictionary and a list. the list should be of the format [i,j,k]. The length should be the depth of the dictionary. Each element corresponds to which position you'd like to move each element to. e.g. [2,0,1] would indicate move element 0 to position 2, element 1 to position 0 and element 2 to position 1.
- this function performs a bubble sort on the order list and dict_, calling deep_swap to swap the levels of the dictionary which are being swapped in the order list
- deep_swap recursively calls itself to find the level provided and returns a dictionary which has been re-ordered
- swap_two_level_dict is called to swap any two levels in a dictionary.
Essentially the idea is to perform a bubble sort on the dictionary, but instead of swapping elements in a list swap levels in a dictionary.
from collections import defaultdict
def dict_swap_index(dict_, order):
for pas_no in range(len(order)-1,0,-1):
for i in range(pas_no):
if order[i] > order[i+1]:
temp = order[i]
order[i] = order[i+1]
order[i+1] = temp
dict_ = deep_swap(dict_, i)
return dict_, order
def deep_swap(dict_, level):
dict_ = deepcopy(dict_)
if level==0:
dict_ = swap_two_level_dict(dict_)
else:
for key in dict_:
dict_[key] = deep_swap(dict_[key], level-1)
return dict_
def swap_two_level_dict(a):
b = defaultdict(dict)
for key1, value1 in a.items():
for key2, value2 in value1.items():
b[key2].update({key1: value2})
return b
e.g.
test_dict = {'a': {'c': {'e':0, 'f':1}, 'd': {'e':2,'f':3}}, 'b': {'c': {'g':4,'h':5}, 'd': {'j':6,'k':7}}}
result = dict_swap_index(test_dict, [2,0,1])
result
(defaultdict(dict,
{'c': defaultdict(dict,
{'e': {'a': 0},
'f': {'a': 1},
'g': {'b': 4},
'h': {'b': 5}}),
'd': defaultdict(dict,
{'e': {'a': 2},
'f': {'a': 3},
'j': {'b': 6},
'k': {'b': 7}})}),
[0, 1, 2])

Flatten Arbitrary Length of Dictionary Items Into List of Paths in Python

So, I have read quite a few posts on flattening dictionaries recursively in Python. None (save one) have come close to what I'm looking for. First, a quick example of what I am trying to accomplish:
Example dictionary with mixed entries: (keys and values will always be of mixed types)
{'a': [{'b': {'c': 'd', 'e': 'f', 'g': 'h',
'i': {'j': {'k': ['l'], 'm': 'n'}},
'o': {'p': {'q': ['r', 's' ], 't': 'u'}}
}
}]
}
Desired output:
{'a/b/c/d',
'a/b/e/f',
'a/b/g/h',
'a/b/i/j/k/l',
'a/b/i/j/m/n',
'a/b/o/p/q/r',
'a/b/o/p/q/s',
'a/b/o/p/t/u'}
The function should (theoretically) work on lists as well.
To explain a bit about what I am doing, I am attempting to search through a Mac plist and other attempts to search by key or value have been shaky at best. To compensate, I want to try a different approach. Convert the dictionary to a list of 'paths' and then just search the paths.
I tried myself (and partially succeeded) and then I found a better solution in the form of this:
def flatten(structure, key="", path="", flattened=None):
if flattened is None:
flattened = {}
if type(structure) not in(dict, list):
flattened[((path + "/") if path else "") + key] = structure
elif isinstance(structure, list):
for i, item in enumerate(structure):
flatten(item, "", "/".join(filter(None,[path,key])), flattened)
else:
for new_key, value in structure.items():
flatten(value, new_key, "/".join(filter(None,[path,key])), flattened)
return flattened
This works well but there are a few undesired effects. First, the output is as follows:
{'a/b/c' : 'd',
'a/b/e' : 'f',
'a/b/g' : 'h',
'a/b/i/j/k/': 'l',
'a/b/i/j/m' : 'n',
'a/b/o/p/q/': 's',
'a/b/o/p/t' : 'u'}
This is returning a dictionary of key/value pairs. I'd rather have a list of string paths. Secondly, and more importantly, you'll notice, the script has stripped values where the value was a list. Only appending the last item of the list.
'a/b/o/p/q/': 's' # there should be another entry with 'r' as the value.
I have spent a fair amount of time fiddling with the output and trying to completely wrap my head around the problem to no avail. It may just be my lac of understanding Python, but the output I want should be possible.
I try not to ask questions unless I have run out of options and here I am. Please do not mark as a duplicate, as other questions aren't quite looking to accomplish what I am looking for.
Thank you for your time and assistance/guidance.

Python 2.7:
def flatten(structure):
if isinstance(structure, basestring):
return [structure]
ret = []
if isinstance(structure, list):
for v in structure:
ret.extend(flatten(v))
elif isinstance(structure, dict):
for k, v in structure.items():
ret.extend(k + '/' + f for f in flatten(v))
return ret
print sorted(flatten(structure))
Output:
['a/b/c/d', 'a/b/e/f', 'a/b/g/h', 'a/b/i/j/k/l', 'a/b/i/j/m/n', 'a/b/o/p/q/r', 'a/b/o/p/q/s', 'a/b/o/p/t/u']
Or, if you don't care about the order, you can simply print flatten(structure).

Here's how I would do it in Python 3.3+:
def flatten(exp):
def sub(exp, res):
if type(exp) == dict:
for k, v in exp.items():
yield from sub(v, res+[k])
elif type(exp) == list:
for v in exp:
yield from sub(v, res)
else:
yield "/".join(res+[exp])
yield from sub(exp, [])
testing:
l={'a': [{'b': {'c': 'd', 'e': 'f', 'g': 'h',
'i': {'j': {'k': ['l'], 'm': 'n'}},
'o': {'p': {'q': ['r', 's' ], 't': 'u'}}
}
}]
}
for i in sorted(flatten(l)):
print(i)
yields
a/b/c/d
a/b/e/f
a/b/g/h
a/b/i/j/k/l
a/b/i/j/m/n
a/b/o/p/q/r
a/b/o/p/q/s
a/b/o/p/t/u
EDIT translation to Python 2 is trivial:
def flatten(exp):
def sub(exp, res):
if type(exp) == dict:
for k, v in exp.items():
for r in sub(v, res+[k]):
yield r
elif type(exp) == list:
for v in exp:
for r in sub(v, res):
yield r
else:
yield "/".join(res+[exp])
for r in sub(exp, []):
yield r
then
>>> for i in sorted(flatten(l)):
... print i
...
a/b/c/d
a/b/e/f
a/b/g/h
a/b/i/j/k/l
a/b/i/j/m/n
a/b/o/p/q/r
a/b/o/p/q/s
a/b/o/p/t/u

Make a python nested list for use in Django's unordered_list

I've got a Django model with a self-referencing foreign key, so my model (as a class in its most basic form) looks like:
class MyObj(object):
def __init__(self, id, ttl, pid):
self.id = id
self.name = ttl
self.parentid = pid
So a sample of my data might look like:
nodes = []
nodes.append(MyObj(1,'a',0))
nodes.append(MyObj(2,'b',0))
nodes.append(MyObj(3,'c',1))
nodes.append(MyObj(4,'d',1))
nodes.append(MyObj(5,'e',3))
nodes.append(MyObj(6,'f',2))
I've got to a point where I can convert this into a nested dictionary:
{'a': {'c': {'e': {}}, 'd': {}}, 'b': {'f': {}}}
using Converting tree list to hierarchy dict as a guide, but I need it in a form that I can use for Django's unordered_list filter.
So my question is, how can I get from (either) a nested dictionary to a nested list/tuple or straight from the source data to a nested list? I can't seem to get a recursive function to nest the lists correctly (as in a list I can't reference "sub trees" by name)
eval(string_rep_of_dictionary.replace(':',',').replace('{','[').replace('}',']')) seems to just about get me there but that seems a horrible solution?

Try
lists = {}
for n in nodes:
b = lists.setdefault(n.id, [])
lists.setdefault(n.parentid, []).extend([n.name, b])
print lists[0]
or, using collections.defaultdict
lists = collections.defaultdict(list)
for n in nodes:
lists[n.parentid] += [n.name, lists[n.id]]
print lists[0]
both of which will print
['a', ['c', ['e', []], 'd', []], 'b', ['f', []]]
Edit:To get rid of the empty lists, iterate through the nodes for a second time:
for n in nodes:
if not lists[n.id]:
lists[n.parentid].remove(lists[n.id])

def nested_dict_to_list(d):
result = []
for key, value in d.iteritems():
try:
value = nested_dict_to_list(value)
except AttributeError:
pass
result += [key, value]
return result
test = {'a': {'c': {'e': {}}, 'd': {}}, 'b': {'f': {}}}
desired_result = ['a', ['c', ['e', []], 'd', []], 'b', ['f', []]]
nested_dict_to_list(test) == desired_result
# True