Purpose this code that works to iterate over a nested dictionary but I'm looking for an output that gives a tuple or list of [keys] and then [values]. Here's the code:
from collections import Mapping, Set, Sequence
string_types = (str, unicode) if str is bytes else (str, bytes)
iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()
def recurse(obj, path=(), memo=None):
if memo is None:
memo = set()
iterator = None
if isinstance(obj, Mapping):
iterator = iteritems
elif isinstance(obj, (Sequence, Set)) and not isinstance(obj, string_types):
iterator = enumerate
if iterator:
if id(obj) not in memo:
memo.add(id(obj))
for path_component, value in iterator(obj):
for result in recurse(value, path + (path_component,), memo):
yield result
memo.remove(id(obj))
else:
yield path, obj
class addNestedDict(dict):
def __missing__(self, key):
value = self[key] = type(self)()
return value
loansDict=addNestedDict()
loansDict[15]['A']['B']=[1,2,3,4]
for k,v in recurse(loansDict):
print(k,v)
The output I'm looking for is one line (15 ,'A','B') [1,2,3,4] so that I can be able to reference k[0],k[1] and v[0] etc...
This seems to work:
results = AddNestedDict()
for k,v in recurse(loansDict):
results.setdefault(k[:-1], []).append(v)
result_key, result_value = results.items()[0]
print('{} {}'.format(result_key, result_value)) # -> (15, 'A', 'B') [1, 2, 3, 4]
I renamed your class AddNestedDict so it follows PEP 8 guidelines.
Related
I am trying to develop the most efficient/comprehensive function with this aim:
Sorting every nested dictionary or list inside a dictionary or list.
Note: I used the collections.OrderedDict because I wanted to make it useful also for python versions before the 3.7, the ones that does not preserve order in dictionaries.
Based on the recursive function from this thread, which sorts only nested dictionaries, I'm trying to build a correspondant recursive function that sorts only nested lists, and then to combine them by using if cycles that identify if the object to be sorted is a dictionary or a list.
This is what I have developed:
from collections import OrderedDict
def recursively_order_dict(d):
ordered_dict = OrderedDict()
for key in sorted(d.keys()):
val = d[key]
if isinstance(val, dict):
val = recursively_order_dict(val)
if isinstance(val, list):
val = recursively_order_list(val)
ordered_dict[key] = val
return ordered_dict
def recursively_order_list(l):
ordered_list = []
for element in sorted(l):
if isinstance(element, list):
element = recursively_order_list(element)
if isinstance(element, dict):
element = recursively_order_dict(element)
ordered_list.append(element)
return ordered_list
def order_all_dicts_and_lists_in_iterable(iterable1):
if isinstance(iterable1, dict):
ordered_iterable = recursively_order_dict(iterable1)
if isinstance(iterable1, list):
ordered_iterable = recursively_order_list(iterable1)
else:
print("%s\n is nor a list nor a dictionary.\nIts type is %s." % (iterable1, type(iterable1)) )
return
return ordered_iterable
It works fine on many examples, but it does not by processing the dictionary dict_2
dict_2 = {
"key9":"value9",
"key5":"value5",
"key3":{
"key3_1":"value3_1",
"key3_5":"value3_5",
"key3_2":[[],"value3_2_1",[] ],
},
"key2":"value2",
"key8":{
"key8_1":"value8_1",
"key8_5":{
"key8_5_4":["value8_5_b", "value8_5_a", "value8_5_c"],
"key8_5_2":[{},{"key8_5_2_4_2":"value8_5_2_4_2", "key8_5_2_4_1":"value8_5_2_4_1", "key8_5_2_4_5":"value8_5_2_4_5"}, "value8_5_2_1",{}],
},
"key8_2":"value8_2",
},
"key1":"value1",
}
sorted_dict_2 = order_all_dicts_and_lists_in_iterable(dict_2)
and throws this error:
--------------------------------------------------------------------------- TypeError Traceback (most recent call last) <ipython-input-12-9cbf4414127d> in <module>
----> 1 order_all_dicts_and_lists_in_iterable(dict_2)
<ipython-input-9-352b10801248> in order_all_dicts_and_lists_in_iterable(iterable1)
26
27 if isinstance(iterable1, dict):
---> 28 ordered_iterable = recursively_order_dict(iterable1)
29 if isinstance(iterable1, list):
30 ordered_iterable = order_all_dicts_and_lists_in_iterable(ordered_iterable)
<ipython-input-9-352b10801248> in recursively_order_dict(d)
6 val = d[key]
7 if isinstance(val, dict):
----> 8 val = recursively_order_dict(val)
9 if isinstance(val, list):
10 val = recursively_order_list(val)
<ipython-input-9-352b10801248> in recursively_order_dict(d)
8 val = recursively_order_dict(val)
9 if isinstance(val, list):
---> 10 val = recursively_order_list(val)
11 ordered_dict[key] = val
12 return ordered_dict
<ipython-input-9-352b10801248> in recursively_order_list(l)
14 def recursively_order_list(l):
15 ordered_list = []
---> 16 for element in sorted(l):
17 if isinstance(element, list):
18 element = recursively_order_list(element)
TypeError: '<' not supported between instances of 'str' and 'list'
So it looks like Python cannot sort iterable made of strings/numbers and lists/dictionaries, because it does not know what to take from lists/dictionaries as a term of comparison.
How could I change my function in order to get lists/dictionaries just being put at the end/start of the sorted iterable, when compared to strings/numbers ?
In few words, how should I change my function to have it turn the above dict_2 into this (hand-edited) sorted_dict_2?
sorted_dict_2 = {
"key1":"value1",
"key2":"value2",
"key3":{
"key3_1":"value3_1",
"key3_2":[ [],[],"value3_2_1" ],
"key3_5":"value3_5",
},
"key5":"value5",
"key8":{
"key8_1":"value8_1",
"key8_2":"value8_2",
"key8_5":{
"key8_5_2":[
{},
{},
"value8_5_2_1",
{
"key8_5_2_4_1":"value8_5_2_4_1",
"key8_5_2_4_2":"value8_5_2_4_2",
"key8_5_2_4_5":"value8_5_2_4_5"
},
],
"key8_5_4":["value8_5_a", "value8_5_b", "value8_5_c"],
},
},
"key9":"value9",
}
So, basically, you need to make a key function that will make all containers compare less than anything else. A handy value is float('inf') for this. However, since we don't know if the thing we are sorting contains numbers or strings, we have to just transform everything into a tuple, and manually map the ordinal values for each string: map(ord, x)
The following is an example if you want containers to move to the front (so negative inf...:
from collections import OrderedDict
def recursively_order_dict(d):
ordered_dict = OrderedDict()
for key in sorted(d.keys()):
val = d[key]
if isinstance(val, dict):
val = recursively_order_dict(val)
if isinstance(val, list):
val = recursively_order_list(val)
ordered_dict[key] = val
return ordered_dict
def _move_containers_to_end(x):
if isinstance(x, (list, dict)):
# to put at the end, use inf, at the start, -inf
return (float('-inf'),)
elif isinstance(x, str):
return tuple(map(ord, x))
else: # assuming we only can get numbers at this point
return (x,)
def recursively_order_list(l):
ordered_list = []
for element in sorted(l, key=_move_containers_to_end):
if isinstance(element, list):
element = recursively_order_list(element)
if isinstance(element, dict):
element = recursively_order_dict(element)
ordered_list.append(element)
return ordered_list
def order_all_dicts_and_lists_in_iterable(iterable1):
if isinstance(iterable1, dict):
ordered_iterable = recursively_order_dict(iterable1)
elif isinstance(iterable1, list):
ordered_iterable = recursively_orded_list(iterable1)
else:
print("%s\n is nor a list nor a dictionary.\nIts type is %s." % (iterable1, type(iterable1)) )
return ordered_iterable
The result of the above is:
OrderedDict([('key1', 'value1'),
('key2', 'value2'),
('key3',
OrderedDict([('key3_1', 'value3_1'),
('key3_2', [[], [], 'value3_2_1']),
('key3_5', 'value3_5')])),
('key5', 'value5'),
('key8',
OrderedDict([('key8_1', 'value8_1'),
('key8_2', 'value8_2'),
('key8_5',
OrderedDict([('key8_5_2',
[OrderedDict(),
OrderedDict([('key8_5_2_4_1',
'value8_5_2_4_1'),
('key8_5_2_4_2',
'value8_5_2_4_2'),
('key8_5_2_4_5',
'value8_5_2_4_5')]),
OrderedDict(),
'value8_5_2_1']),
('key8_5_4',
['value8_5_a',
'value8_5_b',
'value8_5_c'])]))])),
('key9', 'value9')])
Note, you may want to do something like:
import sys:
if sys.version_info.minor < 7:
OrderedMapping = dict
else:
from collections import OrderedDict as OrderedMapping
Then use:
ordered_dict = OrderedMapping()
in recursively_order_dict
I am making a MappingList class which is a list implemented as an OrderedDict.
This is the MappingList class (some methods omitted):
class MappingList(MutableSequence):
"""
A MappingList is a regular list implemented as a dictionary
"""
def __repr__(self):
return str(list(self.seq.values()))
def __getitem__(self, item):
try:
return self.seq[item]
except KeyError:
_traceback_from_none(IndexError, "list index out of range")
def __setitem__(self, key, value, *, usage=None):
if key > max(self.seq.keys()) and usage != "append":
raise IndexError("list index out of range")
self.seq[key] = value
def __delitem__(self, key):
try:
del self.seq[key]
except KeyError:
_traceback_from_none(IndexError, "list index out of range")
def __len__(self):
return len(self.seq)
def __eq__(self, other):
if not isinstance(other, MappingList):
return NotImplemented
return self.seq == other.seq
#classmethod
def _dict_from_seq(cls, seq):
return OrderedDict(enumerate(seq))
def _next_available_slot(self):
return max(self.seq) + 1
def insert(self, index, value): # todo: insert() should not overwrite
"""Insert a value into the MappingList"""
if index > max(self.seq.keys()):
raise IndexError("list index out of range")
for k, v in {k: v for k, v in self.seq.items() if k > index}:
del self.seq[k]
self.seq[k + 1] = v
self[index] = value
When I try to insert an item into a MappingList, I get the following error:
File "C:\...\My Python Programs\free_time\mappinglist.py", line 103, in test_insert
self.li.insert(1, MappingList(["blah", 1, 5.8]))
File "C:\...\My Python Programs\free_time\mappinglist.py", line 85, in insert
for k, v in {k: v for k, v in self.seq.items() if k > index}:
TypeError: cannot unpack non-iterable int object
Why is this error happening? Does OrderedDict.items() return an integer?
The error doesn't happen due to that.
When you don't provide keys(), values(), items(), python iterates over the keys by default. You need to provide items() to tell python to get the keys and values.
for k, v in {k: v for k, v in self.seq.items() if k > index}.items():
How can I read a list inside a dictionary and try to change string numbers to digits? For example:
obj = {'azul':'4','rojo':[{'rojo_a':'1','rojo_b':'2'}],'amarillo':'xxx','naranja':[{'naranja_1':'1','naranja_2':'2'}]}
I use this to change dictionary number strings to integers:
{k:int(v) if v.isdigit() else v for k,v in obj.items()}
But it doesn't work, so I was trying something like this:
for objs in obj:
if objs.isdigit():
k:int(v)
else:
for k,v in objs.items():
print k
But this fails as well.
this seems like a good problem for recursion
obj = {'azul':'4','rojo':[{'rojo_a':'1','rojo_b':'2'}],'amarillo':'xxx','naranja':[{'naranja_1':'1','naranja_2':'2'}]}
def fix_ints(obj):
if isinstance(obj,basestring):
try:
return int(obj)
except ValueError:
print "I cant Make %r an int"%obj
return obj
elif isinstance(obj,(list,tuple)):
return [fix_ints(item) for item in obj]
elif isinstance(obj,dict):
return dict((key,fix_ints(value)) for key,value in obj.items())
else:
print "I have no idea what to do with %r"%obj
new_obj = fix_ints(obj)
print new_obj
note that python does not support tail recursion so if this data structure goes very deep (greater than 1k levels of nesting) then recursion may not be appropriate ...
of coarse you can also do silly string tricks with it
import json,re
new_obj = json.loads(re.sub("\"(\d+)\"","\\1",json.dumps(obj)))
(although really you should do it like i do in my first exzample ... this second method is really just for fun)
String to number:
def int_it(obj):
if obj.isdigit():
obj = int(obj)
return obj
Dict to number (regardless of the number of nested dicts or lists):
class Convert(object):
def __init__(self, obj):
self.obj = obj
if isinstance(obj, dict):
self.handle_dict(obj)
def handle_dict(self, obj):
for key, value in obj.items():
if isinstance(value, str) and value.isdigit():
self.obj[key] = int_it(value)
elif isinstance(obj[key], list):
ins = HandleList(obj[key])
self.obj[key] = ins.obj
elif isinstance(obj[key], dict):
ins = Convert(obj.items())
self.obj[key] = ins.obj
return obj
List to numbers, regardless of the number of nested lists or dicts.
class HandleList(object):
def __init__(self, obj):
self.obj = obj
self.handle_list(obj)
def handle_list(self, obj):
for index, item in enumerate(obj):
if isinstance(item, list):
obj.index(index, [HandleList(val).obj for val in item])
elif isinstance(item, str):
obj.index(index, int_it(item))
elif isinstance(item, dict):
Convert(item)
return obj
output = Convert(values)
print(output.obj)
Returns:
{
'amarillo': 'xxx',
'naranja': [{'naranja_1': 1, 'naranja_2': 2}],
'rojo': [{'rojo_b': 2, 'rojo_a': 1}],
'azul': 4
}
Given the input:
values = {
'azul':'4',
'rojo': [
{'rojo_a':'1',
'rojo_b':'2'
}
],
'amarillo':'xxx',
'naranja': [
{'naranja_1':'1',
'naranja_2':'2'
}
]
}
Try to filter a nested dictionary. My solution is clunky, was hoping to see if there is a better method something using comprehensions. Only interested in the dictionary and lists for this example.
_dict_key_filter() will filter the keys of a nested dictionary or a list of nested dictionaries. Anything not in the obj_filter will be ignored on all nested levels.
obj : can be a dictionary or a list of dictionaries.
obj_filter: has to be a list of filter values
def _dict_key_filter(self, obj, obj_filter):
if isinstance(obj, dict):
retdict = {}
for key, value in obj.iteritems():
if key in obj_filter:
retdict[key] = copy.deepcopy(value)
elif isinstance(value, (dict, list)):
child = self._dict_key_filter(value, obj_filter)
if child:
retdict[key] = child
return retdict if retdict else None
elif isinstance(obj, list):
retlist = []
for value in list:
child = self._dict_key_filter(value, obj_filter)
if child:
retlist.append(child)
return retlist if retlist else None
else:
return None
Example#
dict1 = {'test1': {'test2':[1,2]}, 'test3': [{'test6': 2},
{'test8': {'test9': 23}}], 'test4':{'test5': 5}}
filter = ['test5' , 'test9']
return = _dict_key_filter(dict1, filter)
return value would be {'test3': [{'test8': {'test9': 23}}], 'test4': {'test5': 5}}
It's a really old question. I came across a similar problem recently.
It maybe obvious, but you are dealing with a tree in which each node has an arbitray number of children. You want to cut the subtrees that do not contain some items as nodes (not leaves). To achieve this, you are using a custom DFS: the main function returns either a subtree or None. If the value is None then you "cut" the branch.
First of all, the function dict_key_filter returns a (non empty) dict, a (non empty) list or None if no filter key was not found in the branch.
To reduce complexity, you could return a sequence in every case: an empty sequence if no filter key was found, and a non empty sequence if you are still searching or you found the leaf of the tree. Your code would look like:
def dict_key_filter(obj, obj_filter):
if isinstance(obj, dict):
retdict = {}
...
return retdict # empty or not
elif isinstance(obj, list):
retlist = []
...
return retlist # empty or not
else:
return [] # obvioulsy empty
This was the easy part. Now we have to fill the dots.
The list case
Let's begin with the list case, since it is the easier to refactor:
retlist = []
for value in obj:
child = dict_key_filter0(value, obj_filter)
if child:
retlist.append(child)
We can translate this into a simple list comprehension:
retlist = [dict_key_filter(value, obj_filter) for value in obj if dict_key_filter(value, obj_filter)]
The drawback is that dict_key_filter is evaluated twice. We can avoid this with a little trick (see https://stackoverflow.com/a/15812866):
retlist = [subtree for subtree in (dict_key_filter(value, obj_filter) for value in obj) if subtree]
The inner expression (dict_key_filter(value, obj_filter) for value in obj) is a generator that calls dict_key_filter once per value. But we can even do better if we build a closure of dict_key_filter:
def dict_key_filter(obj, obj_filter):
def inner_dict_key_filter(obj): return dict_key_filter(obj, obj_filter)
...
retlist = list(filter(len, map(inner_dict_key_filter, obj)))
Now we are in the functional world: map applies inner_dict_key_filter to every element of the list and then the subtrees are filtered to exclude empty subtrees (len(subtree) is true iff subtree is not empty). Now, the code looks like:
def dict_key_filter(obj, obj_filter):
def inner_dict_key_filter(obj): return dict_key_filter(obj, obj_filter)
if isinstance(obj, dict):
retdict = {}
...
return retdict
elif isinstance(obj, list):
return list(filter(len, map(inner_dict_key_filter, obj)))
else:
return []
If you are familiar with functional programming, the list case is readable (not quite as readable as it would be in Haskell, but still readable).
The dict case
I do not forget the dictionary-comprehension tag in your question. The first idea is to create a function to return either a whole copy of the branch or the result of the rest of the DFS.
def build_subtree(key, value):
if key in obj_filter:
return copy.deepcopy(value) # keep the branch
elif isinstance(value, (dict, list)):
return inner_dict_key_filter(value) # continue to search
return [] # just an orphan value here
As in the list case, we do not refuse empty subtrees for now:
retdict = {}
for key, value in obj.items():
retdict[key] = build_subtree(key, value)
We have now a perfect case for dict comprehension:
retdict = {key: build_subtree(key, value) for key, value in obj.items() if build_subtree(key, value)}
Again, we use the little trick to avoid to compute a value twice:
retdict = {key:subtree for key, subtree in ((key, build_subtree(key, value)) for key, value in obj.items()) if subtree}
But we have a little problem here: the code above is not exaclty equivalent to the original code. What if the value is 0? In the original version, we have retdict[key] = copy.deepcopy(0) but in the new version we have nothing. The 0 value is evaluated as false and filtered. And then the dict may become empty and we cut the branch wrongfully. We need another test to be sure we want to remove a value: if it's an empty list or dict, then remove it, else keep it:
def to_keep(subtree): return not (isinstance(subtree, (dict, list)) or len(subtree) == 0)
That is:
def to_keep(subtree): return not isinstance(subtree, (dict, list)) or subtree
If you remember a bit of logic (https://en.wikipedia.org/wiki/Truth_table#Logical_implication) you can interpret this as: if subtree is a dict or a list, then it must not be empty.
Let's put the pieces together:
def dict_key_filter(obj, obj_filter):
def inner_dict_key_filter(obj): return dict_key_filter(obj, obj_filter)
def to_keep(subtree): return not isinstance(subtree, (dict, list)) or subtree
def build_subtree(key, value):
if key in obj_filter:
return copy.deepcopy(value) # keep the branch
elif isinstance(value, (dict, list)):
return inner_dict_key_filter(value) # continue to search
return [] # just an orphan value here
if isinstance(obj, dict):
key_subtree_pairs = ((key, build_subtree(key, value)) for key, value in obj.items())
return {key:subtree for key, subtree in key_subtree_pairs if to_keep(subtree)}
elif isinstance(obj, list):
return list(filter(to_keep, map(inner_dict_key_filter, obj)))
return []
I don't know if this is more pythonic, but it seems clearer to me.
dict1 = {
'test1': { 'test2':[1,2] },
'test3': [
{'test6': 2},
{
'test8': { 'test9': 23 }
}
],
'test4':{'test5': 0}
}
obj_filter = ['test5' , 'test9']
print (dict_key_filter(dict1, obj_filter))
# {'test3': [{'test8': {'test9': 23}}], 'test4': {'test5': 0}}
I'm currently in the need for a Python container class with similar functionality like the builtin dict type. Basically what I need is a dictionary, where an arbitrary number of keys beside a primary key, which map to the very same value. However when iterating over it, it should iterate only over the (primary_key, value) pairs and only the primary key if the list of keys is requested.
If this has already been implemented I'd rather not reinvent the wheel. So is there already a module providing such a container? If not, I'm going to implement it myself.
Here is a quick implementation:
class MultipleKeyDict(dict):
__slots__ = ["_primary_keys"]
def __init__(self, arg=None, **kwargs):
self._primary_keys = {}
self.update(arg, **kwargs)
def __setitem__(self, key, value):
super(MultipleKeyDict, self).__setitem__(key, value)
self._primary_keys.setdefault(value, key)
def __delitem__(self, key):
value = self[key]
super(MultipleKeyDict, self).__delitem__(key)
if self._primary_keys[value] == key:
del self._primary_keys[value]
for k, v in super(MultipleKeyDict, self).iteritems():
if v == value:
self._primary_keys[value] = k
break
def __iter__(self):
return self.iterkeys()
def update(self, arg=None, **kwargs):
if arg is not None:
if isinstance(arg, collections.Mapping):
for k in arg:
self[k] = arg[k]
else:
for k, v in arg:
self[k] = v
for k in kwargs:
self[k] = kwargs[k]
def clear(self):
super(MultipleKeyDict, self).clear()
self._primary_keys.clear()
def iteritems(self):
for v, k in self._primary_keys.iteritems():
yield k, v
def items(self):
return list(self.iteritems())
def itervalues(self):
return self._primary_keys.iterkeys()
def values(self):
return self._primary_keys.keys()
def iterkeys(self):
return self._primary_keys.itervalues()
def keys(self):
return self._primary_keys.values()
The only messy bit is that it has to search the whole dict in case a primary key gets deleted.
I omitted copy(), pop(), popitem() and setdefault(). If you need them, you'll have to implement them yourself.
The simplest and easiest solution would be to use two dictionaries, one of which maps secondary keys to a primary key. If for some reason you need a reverse mapping, that could be included in the primary dictionary.
sec = {'one': 'blue', 'two': 'red', 'three': 'blue', # seconary keys
'blue': 'blue', 'red': 'red'} # include identity mapping for primaries
dict = {'blue': ('doll', '$9.43', ('one', 'three')),
'red': ('truck', '$14.99', ('two',)) }
record = dict[sec['two']]
print('Toy=', record[0], 'Price=', record[1])
There is now a multiple key dictionary python package.
https://pypi.python.org/pypi/multi_key_dict/1.0.2
From the link:
from multi_key_dict import multi_key_dict
k = multi_key_dict()
k[1000, 'kilo', 'k'] = 'kilo (x1000)'
print k[1000] # will print 'kilo (x1000)'
print k['k'] # will also print 'kilo (x1000)'
# the same way objects can be updated, deleted:
# and if an object is updated using one key, the new value will
# be accessible using any other key, e.g. for example above:
k['kilo'] = 'kilo'
print k[1000] # will now print 'kilo' as value was updated