We Keep Coding

How to find Dictionary Key(s) from Value in a large nested dictionary of variable depth?

Say that I have a large dictionary full of nested values such as this:
large_dic ={
...
"key":{"sub-key1" :{"sub-key2": "Test"}},
"0key":{"0sub-key1": "0Test"},
"1key":{"1sub-key1":{"1sub-key2":{"1sub-key3":"1Test"}}}
...
}
What I would like to do is to be able to get for example from the final value:
"1Test"
the key(s) to access it, such as in this case:
large_dic["1key"]["1sub-key1"]["1sub-key2"]["1sub-key3"]
Thanks for the support.
Edit to add more infos: The dictionary trees I'm talking about are linear(YAML files converted into a python dictionary structure), there is never more than one key, the ending leaf values may not be unique.

Since OP is looking for hierarchical keys instead
I made this class :
class PointingSlice:
def __init__(self, obj, *slices) -> None:
self.obj = obj
self.slices = slices
def __str__(self):
return f"{str(self.obj)}{''.join(map(self._repr_slice, self.slices))}"
def _repr_slice(self, sliced: slice):
sqbrackets = "[{}]"
if not isinstance(sliced, slice):
return sqbrackets.format(repr(sliced))
items = [sliced.start, sliced.stop, sliced.step]
fn = lambda x: str() if x is None else str(x)
return sqbrackets.format(":".join(map(fn, items)))
def resolve(self):
obj = self.obj
for sliced in self.slices:
obj = obj.__getitem__(sliced)
return obj
and this function for instantiation :
def find_longest(mapping, key):
keys = [key]
value = mapping[key]
while isinstance(value, dict):
((k, value),) = value.items()
keys.append(k)
return PointingSlice(mapping, *keys)
Example use:
print(find_longest(large_dic, "1key"))
# output:
# {'key': {'sub-key1': {'sub-key2': 'Test'}}, '0key': {'0sub-key1': '0Test'}, '1key': {'1sub-key1': {'1sub-key2': {'1sub-key3': '1Test'}}}}['1key']['1sub-key1']['1sub-key2']['1sub-key3']
# do note that it is the same thing as large_dic['1key']['1sub-key1']['1sub-key2']['1sub-key3']
print(find_longest(large_dic, "1key").resolve()) # 1Test
So I made some changes and now it supports additional repr options matching your exact use case :
class PointingSlice:
def __init__(self, obj, *slices, object_name=None) -> None:
self.obj = obj
self.slices = slices
self.object_name = object_name
def __str__(self):
return f"{self.object_name or str(self.obj)}{''.join(map(self._repr_slice, self.slices))}"
def _repr_slice(self, sliced: slice):
sqbrackets = "[{}]"
if not isinstance(sliced, slice):
return sqbrackets.format(repr(sliced))
items = [sliced.start, sliced.stop, sliced.step]
fn = lambda x: str() if x is None else str(x)
return sqbrackets.format(":".join(map(fn, items)))
def resolve(self):
obj = self.obj
for sliced in self.slices:
obj = obj.__getitem__(sliced)
return obj
large_dic = {
"key": {"sub-key1": {"sub-key2": "Test"}},
"0key": {"0sub-key1": "0Test"},
"1key": {"1sub-key1": {"1sub-key2": {"1sub-key3": "1Test"}}},
}
def find_longest(mapping, key):
keys = [key]
value = mapping[key]
while isinstance(value, dict):
((k, value),) = value.items()
keys.append(k)
return PointingSlice(mapping, *keys)
f = find_longest(large_dic, "1key")
f.object_name = "large_dic" # for representational purposes, it works without this
print(f) # large_dic['1key']['1sub-key1']['1sub-key2']['1sub-key3']
print(f.resolve()) # 1Test

There are numerous ways to achieve this. You might want to look up "prefix tree traversal" (or "trie traversal").
A simple recursive solution with poor memory efficiency could look like this:
def find_trie_leaf_path(trie: dict, leaf_value, trie_path: list[str] = []):
for key, value in trie.items():
if isinstance(value, dict):
yield from find_trie_leaf_path(value, leaf_value, trie_path + [key])
elif value == leaf_value:
yield trie_path + [key]
large_dic = {
"key": {"sub-key1": {"sub-key2": "Test"}},
"0key": {"0sub-key1": "0Test"},
"1key": {"1sub-key1": {"1sub-key2": {"1sub-key3": "Test"}}},
}
first_match = next(find_trie_leaf_path(large_dic, "Test"))
all_matches = list(find_trie_leaf_path(large_dic, "Test"))
This should work even if your trie is very wide. If it is very high, I'd rather use an iterative algorithm.
I want to point out, though, that prefix trees are usually used the other way round. If you find yourself needing this search a lot, you should consider a different data structure.

Yes, it's totally possible. Here's the function to get the deeply nested value:
def get_final_value(mapping, key):
value = mapping[key]
while isinstance(value, dict):
(value,) = value.values()
return value
Example use:
>>> get_final_value(large_dic, "key")
'Test'
>>> get_final_value(large_dic, "0key")
'0Test'
>>> get_final_value(large_dic, "1key")
'1Test'
>>>

Can the parent keys be deduced from your final value in any way or is the tree structure rather random? If latter is the case then you'll probably just end up searching your tree until you find your value, what path search algorithm you choose for that again depends on the tree structure you have. As already asked in the comments, does each node only have one other node or is it binary or can it have many child nodes?

Is there a recursive version of the dict.get() built-in?

I have a nested dictionary object and I want to be able to retrieve values of keys with an arbitrary depth. I'm able to do this by subclassing dict:
>>> class MyDict(dict):
... def recursive_get(self, *args, **kwargs):
... default = kwargs.get('default')
... cursor = self
... for a in args:
... if cursor is default: break
... cursor = cursor.get(a, default)
... return cursor
...
>>> d = MyDict(foo={'bar': 'baz'})
>>> d
{'foo': {'bar': 'baz'}}
>>> d.get('foo')
{'bar': 'baz'}
>>> d.recursive_get('foo')
{'bar': 'baz'}
>>> d.recursive_get('foo', 'bar')
'baz'
>>> d.recursive_get('bogus key', default='nonexistent key')
'nonexistent key'
However, I don't want to have to subclass dict to get this behavior. Is there some built-in method that has equivalent or similar behavior? If not, are there any standard or external modules that provide this behavior?
I'm using Python 2.7 at the moment, though I would be curious to hear about 3.x solutions as well.

A very common pattern to do this is to use an empty dict as your default:
d.get('foo', {}).get('bar')
If you have more than a couple of keys, you could use reduce (note that in Python 3 reduce must be imported: from functools import reduce) to apply the operation multiple times
reduce(lambda c, k: c.get(k, {}), ['foo', 'bar'], d)
Of course, you should consider wrapping this into a function (or a method):
def recursive_get(d, *keys):
return reduce(lambda c, k: c.get(k, {}), keys, d)

#ThomasOrozco's solution is correct, but resorts to a lambda function, which is only necessary to avoid TypeError if an intermediary key does not exist. If this isn't a concern, you can use dict.get directly:
from functools import reduce
def get_from_dict(dataDict, mapList):
"""Iterate nested dictionary"""
return reduce(dict.get, mapList, dataDict)
Here's a demo:
a = {'Alice': {'Car': {'Color': 'Blue'}}}
path = ['Alice', 'Car', 'Color']
get_from_dict(a, path) # 'Blue'
If you wish to be more explicit than using lambda while still avoiding TypeError, you can wrap in a try / except clause:
def get_from_dict(dataDict, mapList):
"""Iterate nested dictionary"""
try:
return reduce(dict.get, mapList, dataDict)
except TypeError:
return None # or some other default value
Finally, if you wish to raise KeyError when a key does not exist at any level, use operator.getitem or dict.__getitem__:
from functools import reduce
from operator import getitem
def getitem_from_dict(dataDict, mapList):
"""Iterate nested dictionary"""
return reduce(getitem, mapList, dataDict)
# or reduce(dict.__getitem__, mapList, dataDict)
Note that [] is syntactic sugar for the __getitem__ method. So this relates precisely how you would ordinarily access a dictionary value. The operator module just provides a more readable means of accessing this method.

You can actually achieve this really neatly in Python 3, given its handling of default keyword arguments and tuple decomposition:
In [1]: def recursive_get(d, *args, default=None):
...: if not args:
...: return d
...: key, *args = args
...: return recursive_get(d.get(key, default), *args, default=default)
...:
Similar code will also work in python 2, but you'd need to revert to using **kwargs, as you did in your example. You'd also need to use indexing to decompose *args.
In any case, there's no need for a loop if you're going to make the function recursive anyway.
You can see that the above code demonstrates the same functionality as your existing method:
In [2]: d = {'foo': {'bar': 'baz'}}
In [3]: recursive_get(d, 'foo')
Out[3]: {'bar': 'baz'}
In [4]: recursive_get(d, 'foo', 'bar')
Out[4]: 'baz'
In [5]: recursive_get(d, 'bogus key', default='nonexistent key')
Out[5]: 'nonexistent key'

You can use a defaultdict to give you an empty dict on missing keys:
from collections import defaultdict
mydict = defaultdict(dict)
This only goes one level deep - mydict[missingkey] is an empty dict, mydict[missingkey][missing key] is a KeyError. You can add as many levels as needed by wrapping it in more defaultdicts, eg defaultdict(defaultdict(dict)). You could also have the innermost one as another defaultdict with a sensible factory function for your use case, eg
mydict = defaultdict(defaultdict(lambda: 'big summer blowout'))
If you need it to go to arbitrary depth, you can do that like so:
def insanity():
return defaultdict(insanity)
print(insanity()[0][0][0][0])

There is none that I am aware of. However, you don't need to subclass dict at all, you can just write a function that takes a dictionary, args and kwargs and does the same thing:
def recursive_get(d, *args, **kwargs):
default = kwargs.get('default')
cursor = d
for a in args:
if cursor is default: break
cursor = recursive_get(cursor, a, default)
return cursor
use it like this
recursive_get(d, 'foo', 'bar')

The OP requested the following behavior
>>> d.recursive_get('bogus key', default='nonexistent key')
'nonexistent key'
(As of June 15, 22022) none of the up-voted answers accomplish this, so I have modified #ThomasOrozco's solution to resolve this
from functools import reduce
def rget(d, *keys, default=None):
"""Use a sentinel to handle both missing keys AND alternate default values"""
sentinel = {}
v = reduce(lambda c, k: c.get(k, sentinel), keys, d)
if v is sentinel:
return default
return v
Below is a complete, unit-test-like demonstration of where the other answers have issues. I've named each approach according to its author. Note that this answer is the only one which passes all 4 test cases, namely
Basic retrieval when key-tree exists
Non-existent key-tree returns None
Option to specify a default aside from None
Values which are an empty dict should return as themselves rather than the default
from functools import reduce
def thomas_orozco(d, *keys):
return reduce(lambda c, k: c.get(k, {}), keys, d)
def jpp(dataDict, *mapList):
"""Same logic as thomas_orozco but exits at the first missing key instead of last"""
try:
return reduce(dict.get, *mapList, dataDict)
except TypeError:
return None
def sapi(d, *args, default=None):
if not args:
return d
key, *args = args
return sapi(d.get(key, default), *args, default=default)
def rget(d, *keys, default=None):
sentinel = {}
v = reduce(lambda c, k: c.get(k, sentinel), keys, d)
if v is sentinel:
return default
return v
def assert_rget_behavior(func):
"""Unit tests for desired behavior of recursive dict.get()"""
fail_count = 0
# Basic retrieval when key-tree exists
d = {'foo': {'bar': 'baz', 'empty': {}}}
try:
v = func(d, 'foo', 'bar')
assert v == 'baz', f'Unexpected value {v} retrieved'
except Exception as e:
print(f'Case 1: Failed basic retrieval with {repr(e)}')
fail_count += 1
# Non-existent key-tree returns None
try:
v = func(d, 'bogus', 'key')
assert v is None, f'Missing key retrieved as {v} instead of None'
except Exception as e:
print(f'Case 2: Failed missing retrieval with {repr(e)}')
fail_count += 1
# Option to specify a default aside from None
default = 'alternate'
try:
v = func(d, 'bogus', 'key', default=default)
assert v == default, f'Missing key retrieved as {v} instead of {default}'
except Exception as e:
print(f'Case 3: Failed default retrieval with {repr(e)}')
fail_count += 1
# Values which are an empty dict should return as themselves rather than the default
try:
v = func(d, 'foo', 'empty')
assert v == {}, f'Empty dict value retrieved as {v} instead of {{}}'
except Exception as e:
print(f'Case 4: Failed retrieval of empty dict value with {repr(e)}')
fail_count += 1
# Success only if all pass
if fail_count == 0:
print('Passed all tests!')
if __name__ == '__main__':
assert_rget_behavior(thomas_orozco) # Fails cases 2 and 3
assert_rget_behavior(jpp) # Fails cases 1, 3, and 4
assert_rget_behavior(sapi) # Fails cases 2 and 3
assert_rget_behavior(rget) # Only one to pass all 3

collections.default_dict will handle the providing of default values for nonexistent keys at least.

The Iterative Solution
def deep_get(d:dict, keys, default=None, create=True):
if not keys:
return default
for key in keys[:-1]:
if key in d:
d = d[key]
elif create:
d[key] = {}
d = d[key]
else:
return default
key = keys[-1]
if key in d:
return d[key]
elif create:
d[key] = default
return default
def deep_set(d:dict, keys, value, create=True):
assert(keys)
for key in keys[:-1]:
if key in d:
d = d[key]
elif create:
d[key] = {}
d = d[key]
d[keys[-1]] = value
return value
I am about to test it inside of a Django project with a line such as:
keys = ('options', 'style', 'body', 'name')
val = deep_set(d, keys, deep_get(s, keys, 'dotted'))

What is an elegant way to select all non-None elements from parameters and place them in a python dictionary?

def function(varone=None, vartwo=None, varthree=None):
values = {}
if var1 is not None:
values['var1'] = varone
if var2 is not None:
values['var2'] = vartwo
if var3 is not None:
values['var3'] = varthree
if not values:
raise Exception("No values provided")
Can someone suggest a more elegant, pythonic way to accomplish taking placing non-null named variables and placing them in a dictionary? I do not want the values to be passed in as a dictionary. The key names of "values" are important and must be as they are. The value of "varone" must go into var1, "vartwo" must go into var2 and so on; Thanks.

You could use kwargs:
def function(*args, **kwargs):
values = {}
for k in kwargs:
if kwargs[k] is not None:
values[k] = kwargs[k]
if not values:
raise Exception("No values provided")
return values
>>> function(varone=None, vartwo="fish", varthree=None)
{'vartwo': 'fish'}
With this syntax, Python removes the need to explicitly specify any argument list, and allows functions to handle any old keyword arguments they want.
If you're specifically looking for keys var1 etc instead of varone you just modify the function call:
>>> function(var1=None, var2="fish", var3=None)
{'var2': 'fish'}
If you want to be REALLY slick, you can use list comprehensions:
def function(**kwargs):
values = dict([i for i in kwargs.iteritems() if i[1] != None])
if not values:
raise Exception("foo")
return values
Again, you'll have to alter your parameter names to be consistent with your output keys.

Use **kwargs. Example:
def function(**kwargs):
if not kwargs:
raise Exception("No values provided")
for k, v in kwargs.items():
print("%s: %r") % (k, v)
If you really are going to call function with None arguments, you can strip them out:
def function(**kwargs):
for k, v in kwargs.items():
if v is None:
del kwargs[k]
if not kwargs:
raise Exception("No values provided")
for k, v in kwargs.items():
print("%s: %r") % (k, v)
Obviously you could call the dict values instead, but kwargs is the conventional name, and will make your code more intelligible to other people.

Well, you can pass all those values inside a keyword argument: -
def function(*nkwargs, **kwargs):
values = {}
for k in kwargs:
if kwargs[k] is not None:
values[k] = kwargs[k]
if not values:
raise Exception("No values")
print values
try:
function()
except Exception, e:
print e
function(varOne=123, varTwo=None)
function(varOne=123, varTwo=234)
OUTPUT: -
No values
{'varOne': 123}
{'varOne': 123, 'varTwo': 234}

Call your function as usual, but accept as **kwargs. Then filter them:
def fn(**kwargs):
items = {'var%s' % i: v for i, (k, v) in enumerate(items)}
fn(a=1, b=2, c=3)
if you need a specific set of names, then make a dict of names:
names = dict(zip('varOne varTwo varThree'.split(), range(1, 4)))
walk over this dict and check if the var is in kwargs:
items = {'var%s' % k: kwargs[v] for k, v in names.items() if v in kwargs}

Python StructuredProperty to dictionary

My models all have a method which converts the model to a dictionary:
def to_dict(model):
output = {}
SIMPLE_TYPES = (int, long, float, bool, dict, basestring, list)
for key, prop in model._properties.iteritems():
value = getattr(model, key)
if value is None:
continue
if isinstance(value, SIMPLE_TYPES):
output[key] = value
elif isinstance(value, datetime.date):
dateString = value.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
output[key] = dateString
elif isinstance(value, ndb.Model):
output[key] = to_dict(value)
else:
raise ValueError('cannot encode ' + repr(prop))
return output
Now, one of my models, X, has a LocalStructuredProperty:
metaData = ndb.LocalStructuredProperty(MetaData, repeated=True)
So, repeated=True means this will be a list of MetaData objects. MetaData is another model, and it also has the same to_dict method.
However, when I call json.dumps(xInstance.to_dict()), I get an exception:
raise TypeError(repr(o) + " is not JSON serializable")
TypeError: MetaData(count=0, date=datetime.datetime(2012, 9, 19, 2, 46, 56, 660000), unique_id=u'8E2C3B07A06547C78AB00DD73B574B8C') is not JSON serializable
How can I handle this?

If you want to handle this in to_dict() and before the level of serializing to JSON, you'll just need a few more cases in your to_dict(). Firstly, you said the to_dict definition above is a method. I would have it delegate to a function or staticmethod so you have something you can call on ints and such without checking the type first. The code will just come out better that way.
def coerce(value):
SIMPLE_TYPES = (int, long, float, bool, basestring)
if value is None or isinstance(value, SIMPLE_TYPES):
return value
elif isinstance(value, datetime.date):
return value.strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
elif hasattr(value, 'to_dict'): # hooray for duck typing!
return value.to_dict()
elif isinstance(value, dict):
return dict((coerce(k), coerce(v)) for (k, v) in value.items())
elif hasattr(value, '__iter__'): # iterable, not string
return map(coerce, value)
else:
raise ValueError('cannot encode %r' % value)
Then just plug that into your to_dict method itself:
def to_dict(model):
output = {}
for key, prop in model._properties.iteritems():
value = coerce(getattr(model, key))
if value is not None:
output[key] = value
return output

All you need to do to serialize is to implement a function
def default_encode(obj):
return obj.to_dict()
and then encode your JSON with
json.dumps(X.to_dict(), default=default_encode)

I figured out how to solve the issue: in the X class, add this to the to_dict() method:
...
if value is None:
continue
if key == 'metaData':
array = list()
for data in value:
array.append(data.to_dict())
output[key] = array
elif isinstance(value, SIMPLE_TYPES):
output[key] = value
...
Though I'm not really sure how to automate this case where it's not based off key, but rather whenever it encounters a list of custom objects, it first converts each object in the list to_dict() first.

How to recursively remove certain keys from a multi-dimensional(depth not known) python dictionary?

I'm using kendoUI Grid in one of my projects. I retrieved a piece of data using their api and found that it added some "unwanted" data to my json/dictionary. After passing the json back to my Pyramid backend, I need to remove these keys. The problem is, the dictionary can be of whatever depth and I don't know the depth in advance.
Example:
product = {
id: "PR_12"
name: "Blue shirt",
description: "Flowery shirt for boys above 2 years old",
_event: {<some unwanted data here>},
length: <some unwanted data>,
items: [{_event: {<some rubbish data>}, length: <more rubbish>, price: 23.30, quantity: 34, color: "Red", size: "Large"}, {_event: {<some more rubbish data>}, length: <even more rubbish>, price: 34.50, quantity: 20, color: "Blue", size: "Large"} ....]
}
I want to remove two keys in particular: "_event" & "length". I tried writing a recursive function to remove the data but I can't seem to get it right. Can someone please help?
Here's what I have:
def remove_specific_key(the_dict, rubbish):
for key in the_dict:
if key == rubbish:
the_dict.pop(key)
else:
# check for rubbish in sub dict
if isinstance(the_dict[key], dict):
remove_specific_key(the_dict[key], rubbish)
# check for existence of rubbish in lists
elif isinstance(the_dict[key], list):
for item in the_dict[key]:
if item == rubbish:
the_dict[key].remove(item)
return the_dict

If you allow remove_specific_key (renamed remove_keys) to accept any object as its first argument, then you can simplify the code:
def remove_keys(obj, rubbish):
if isinstance(obj, dict):
obj = {
key: remove_keys(value, rubbish)
for key, value in obj.iteritems()
if key not in rubbish}
elif isinstance(obj, list):
obj = [remove_keys(item, rubbish)
for item in obj
if item not in rubbish]
return obj
Since you wish to remove more than one key, you might as well let rubbish be a set instead of one particular key.
With the above code, you'd remove '_event' and 'length' keys with
product = remove_keys(product, set(['_event', 'length']))
Edit: remove_key uses dict comprehension, introduced in Python2.7. For older version of Python, the equivalent would be
obj = dict((key, remove_keys(value, rubbish))
for key, value in obj.iteritems()
if key not in rubbish)

Modifying a dict as you iterate it bad, an unnecessary, since you know exactly what key you are looking for. Also, your list of dicts aren't being handled right:
def remove_specific_key(the_dict, rubbish):
if rubbish in the_dict:
del the_dict[rubbish]
for key, value in the_dict.items():
# check for rubbish in sub dict
if isinstance(value, dict):
remove_specific_key(value, rubbish)
# check for existence of rubbish in lists
elif isinstance(value, list):
for item in value:
if isinstance(item, dict):
remove_specific_key(item, rubbish)

dict or list can not be delete while iteratering, so replace the iterator with a test function.
def remove_specific_key(the_dict, rubbish):
if the_dict.has_key(rubbish):
the_dict.pop(rubbish)
else:
for key in the_dict:
if isinstance(the_dict[key], dict):
remove_specific_key(the_dict[key], rubbish)
elif isinstance(the_dict[key], list):
if the_dict[key].count(rubbish):
the_dict[key].remove(rubbish)
return the_dict
d = {"a": {"aa": "foobar"}}
remove_specific_key(d, "aa")
print d
d = {"a": ["aa", "foobar"]}
remove_specific_key(d, "aa")
print d