I have a dictionary of this kind where the values are dictionaries as well the dictionaries can have nested dictionaries in them. like this:
data = {'key1': {
'keya':{
'keyc': None
}
'keyb': None
}
'key2': {
'keyi':None,
'keyii': None
}
}
The dictionaries can be many (we don't know how many dictionaries can be there inside the values). How can I get all keys in all values like this?
['key1', 'key2', 'keya', 'keyb', 'keyi', 'keyii']
you could get all the keys using recursion
def get_all_keys_rec(dic):
keys = [key for key in dic]
for val in dic.values():
if type(val)==dict:
inner_keys = get_all_keys_rec(val)
keys.extend(inner_keys)
return keys
print(get_all_keys_rec(data))
output:
['key1', 'key2', 'keya', 'keyb', 'keyc', 'keyi', 'keyii']
keys = []
for key, val in data.items():
keys.append(key)
if isinstance(val, dict):
item = val
while True:
for k, v in item.items():
keys.append(k)
if isinstance(v, dict):
item = v
break
else:
break
print(keys)
This outputs:
['key1', 'keya', 'keyc', 'key2', 'keyi', 'keyii']
Recursive generation, yield from should be your partner:
>>> data = {'key1': {
... 'keya': {
... 'keyc': None
... },
... 'keyb': None
...
... },
... 'key2': {
... 'keyi': None,
... 'keyii': None
... }
... }
>>> def get_all_keys(dct):
... def gen_all_keys(d):
... if isinstance(d, dict):
... yield from d
... for v in d.values():
... yield from gen_all_keys(v)
... return list(gen_all_keys(dct))
...
>>> get_all_keys(data)
['key1', 'key2', 'keya', 'keyb', 'keyc', 'keyi', 'keyii']
Related
How can I pass a dictionary path as an argument? I thought maybe it had something to do with *args or **kwargs but I didn't understand how to use them for this
dictionary = {
'key1': {
'attribute1': 'green',
},
'attribute2': 5
}
def SaveToFile(target, value):
with open('savefile.json', 'r') as savefile:
dictionary = json.load(savefile)
dictionary[target] = value
with open('savefile.json', 'w') as savefile:
json.dump(dictionary, savefile)
SaveToFile('["key1"]["attribute1"]', 'blue')
SaveToFile('["attribute2"]', 10)
print(dictionary)
desired output:
{
'key1': {
'attribute1': 'blue'
},
'attribute2': 10
}
use regex and recursion to solve this
dictionary = {
'key1': {
'attribute1': 'green',
},
'attribute2': 5
}
import re
def update_dict(d_, val, *keys):
if not keys:
return {}
key = keys[0]
if isinstance(d_[key], dict):
d_[key].update(update_dict(d_[key], val, *keys[1:]))
else:
if key in d_:
d_[key]= val
return d_
def ChangeValue(target, value):
keys = filter(lambda x: bool(x), re.split('\[\"(.*?)\"\]', target))
update_dict(dictionary, value, *keys)
ChangeValue('["key1"]["attribute1"]', 'blue')
ChangeValue('["attribute2"]', 10)
dictionary
# output {'key1': {'attribute1': 'blue'}, 'attribute2': 10}
I have sample data that looks like this (nested and is dynamic so it does change on the fly):
...
"counts":{
"1":{
"1":21082,
"2":14999
},
"2":{
"1":9180,
"2":10023
}
},
...
I need to recursively go through and replace all the values with -1, so the result will be something like:
...
"counts":{
"1":{
"1":-1,
"2":-1
},
"2":{
"1":-1,
"2":-1
}
},
...
I can do it if the dictionary is only one level deep like so:
result['counts'] = {x: '-1' for x in result['counts']}
How do I do it for 2 or more levels on the fly?
Also sometimes the key's can be like 1(1, 2)` or other things like so:
...
"counts":{
"(1, 2)":{
"1":21082,
"2":14999
},
"(2, 1)":{
"1":9180,
"2":10023
}
},
...
Is there any easy way to do this?
Use a recursive function that calls itself on each nested dict:
>>> def replace_value(d, new_val):
... return {
... k: replace_value(v, new_val) if isinstance(v, dict) else new_val
... for k, v in d.items()
... }
...
>>> replace_value({"foo": {"foo": "bar"}}, -1)
{'foo': {'foo': -1}}
Alternate version with the base case outside the comprehension:
>>> def replace_value(d, new_val):
... if not isinstance(d, dict):
... return new_val
... return {k: replace_value(v, new_val) for k, v in d.items()}
...
>>> replace_value({"foo": {"foo": "bar"}}, -1)
{'foo': {'foo': -1}}
My choice for similar tasks is remap from boltons
>>> def visitor(path, key, value):
... if not isinstance(value, dict):
... return key, -1
... return True
...
>>> from boltons.iterutils import remap # pip install boltons
>>> d
{'counts': {'1': {'1': 21082, '2': 14999}, '2': {'1': 9180, '2': 10023}}}
>>> remap(d, visit=visitor)
{'counts': {'1': {'1': -1, '2': -1}, '2': {'1': -1, '2': -1}}}
The visit callable accepts a path, key, and value, and should return the new key and value (or return True as a shorthand to keep old item unmodified).
I prefer this to a recursive function, because it uses a stack-based iterative approach. It is too easy to blow past the recursion limit in Python when dealing with nested data structures.
My json data would look like this:
{
"a":1,
"b":[
{
"c":2,
"d":{
"e":3
},
"f":{
"g":4
},
"h":[
{
"i":5
},
{
"j":6
}
]
}
]
}
Is there a way I can get values for certain fields in the response along with their keys. So from this response, the fields for which I expect values are a, c,e,g,i,j along with the respective keys.
Eg: [a:1,c:2,e:3,g:4,i:5,j:6]. Could this be done?
My response contained something like:
{
"a":1,
"b":[
{
"c":2,
"d":{
"e":3
},
"f":{
"g":4,
"k":[
"l","m"]
},
"h":[
{
"i":5
},
{
"j":6
}
]
}
]
}
Which resulted in the error. I have made the following fix for it.
def get_key_value(dct, res_dct, lst):
for k,v in dct.items():
if isinstance(v, list):
for d in v:
if isinstance(d,dict):
get_key_value(d, res_dct, lst)
else:
lst.append(f'{k}:{v}')
elif isinstance(v, dict):
get_key_value(v, res_dct, lst)
else:
res_dct[k] = v
# If you want to store in 'list' you can store as string
lst.append(f'{k}:{v}')
res_dct = {}
lst = []
get_key_value(staging_dict, res_dct, lst)
You can use a recursive function and store key & value if only value not list or dict.
def get_key_value(dct, res_dct, lst):
for k,v in dct.items():
if isinstance(v, list):
for d in v:
get_key_value(d, res_dct, lst)
elif isinstance(v, dict):
get_key_value(v, res_dct, lst)
else:
res_dct[k] = v
# If you want to store in 'list' you can store as string
lst.append(f'{k}:{v}')
res_dct = {}
lst = []
get_key_value(dct, res_dct, lst)
print(res_dct)
print(lst)
Output:
# res_dct
{'a': 1, 'c': 2, 'e': 3, 'g': 4, 'i': 5, 'j': 6}
# lst
['a:1', 'c:2', 'e:3', 'g:4', 'i:5', 'j:6']
I get JSON Data back from an API. The dataset is large and nested. I can access the Datenreihen key like this:
jsondata.get("Ergebnis")[0].get("Kontakte").get("Datenreihen")
As you can see, this is a mix of dictionaries and lists.
I tried the following, but with lists it does not work :-(.
def recursive_lookup(k, d):
if k in d:
return d[k]
for v in d.values():
if isinstance(v, dict):
return recursive_lookup(k, v)
return None
# Works
recursive_lookup("Ergebnis", jsondata)
# Returns None
recursive_lookup("Datenreihen", jsondata)
Is there an easy way to access and key in my dictionary, no matter how deeply my object is nested?
This is exampledata:
{
"Success":true,
"Ergebnis":[
{
"ErgA1a: KPI Zeitreihe":{
"Message":"",
"MitZielgruppe":true,
"Beschriftung":[
"2019 KW 27",
"2019 KW 28",
"2019 KW 29"
],
"Datenreihen":{
"Gesamt":{
"Name":"Sympathie [#4]\n(Sehr sympathisch, Sympathisch)",
"Werte":[
39.922142815641145,
37.751410794385762,
38.35504885993484
]
}
}
}
}
],
"rest":[
{
"test":"bla"
}
]
}
data.get("ErgebnisseAnalyse")[0].get("ErgA1a: KPI Zeitreihe")
recursive_lookup("ErgA1a: KPI Zeitreihe", data)
Recursive function to find value in nested dictionary based upon key field
Code
def find_item(obj, field):
"""
Takes a dict with nested lists and dicts,
and searches all dicts for a key of the field
provided.
"""
if isinstance(obj, dict):
for k, v in obj.items():
if k == field:
yield v
elif isinstance(v, dict) or isinstance(v, list):
yield from find_item(v, field)
elif isinstance(obj, list):
for v in obj:
yield from find_item(v, field)
Usage
value = next(find_item(dictionary_object, field), None)
Test
# Nested dictionary
dic = {
"a": [{"b": {"c": 1}},
{"d": 2}],
"e": 3}
# Values form various fields
print(next(find_item(dic, "a"), None)) # Output: [{'b': {'c': 1}}, {'d': 2}]
print(next(find_item(dic, "b"), None)) # Output: {'c': 1}
print(next(find_item(dic, "c"), None)) # Output: 1
print(next(find_item(dic, "d"), None)) # Output: 2
print(next(find_item(dic, "e"), None)) # Output: 3
print(next(find_item(dic, "h"), None)) # Output: None
I have a huge json in the format something like :
{
"Name1": {
"NNum": "11",
"Node1": {
"SubNodeA": "Thomas",
"SubNodeB": "27"
},
"Node2": {
"SubNodeA": "ZZZ",
"SubNodeD": "XXX",
"SubNodeE": "yy"
},
"Node3": {
"child1": 11,
"child2": {
"grandchild": {
"greatgrandchild1": "Rita",
"greatgrandchild2": "US"
}
}
}
}
}
The format or keys are not defined and can go to any depth
I would like to get the list of keys like
keyList= ["Name1.NNum","Name1.Node1.SubNodeA",""Name1.Node1.SubNodeB","Name1.Node2.SubNodeA","Name1.Node2.SubNodeD","Name1.Node2.SubNodeE","Name1.Node3.child1","Name1.Node3.child2.grandchild.greatgrandchild1","Name1.Node3.child2.grandchild.greatgrandchild2"]
A snapshot of the code
def extract_values(obj):
"""Pull all values of specified key from nested JSON."""
arr = []
key_list = []
parent = ""
def extract(obj, arr,parent):
"""Recursively search for values of key in JSON tree."""
if isinstance(obj, dict):
grandparent = ""
for k, v in obj.items():
print ("k ............",k)
parent = grandparent
temp_parent = k
print ("parent >>>>> ",parent)
if isinstance(v, (dict, list)):
parent = temp_parent
print ("IF VALUE DICT .. parent ", parent)
extract(v, arr,parent)
else:
grandparent = parent
parent = parent + "_" + temp_parent
print ("!!!! NOT DICT :).... **** parent ... ", parent)
arr.append(parent)
elif isinstance(obj, list):
for item in obj:
extract(item, arr)
#print ("arr >>>>>>>>>> ", arr)
time.sleep(5)
return arr
results = extract(obj, arr,parent)
return results
but this does not give the expected output.
Expected Output:
keyList= ["Name1.NNum","Name1.Node1.SubNodeA",""Name1.Node1.SubNodeB","Name1.Node2.SubNodeA","Name1.Node2.SubNodeD","Name1.Node2.SubNodeE","Name1.Node3.child1","Name1.Node3.child2.grandchild.greatgrandchild1","Name1.Node3.child2.grandchild.greatgrandchild2"]
Can anybody help me with this.
Thanks in advance
You can use recursion:
d = {'Name1': {'NNum': '11', 'Node1': {'SubNodeA': 'Thomas', 'SubNodeB': '27'}, 'Node2': {'SubNodeA': 'ZZZ', 'SubNodeD': 'XXX', 'SubNodeE': 'yy'}, 'Node3': {'child1': 11, 'child2': {'grandchild': {'greatgrandchild1': 'Rita', 'greatgrandchild2': 'US'}}}}}
def keys(d, c = []):
return [i for a, b in d.items() for i in ([c+[a]] if not isinstance(b, dict) else keys(b, c+[a]))]
result = list(map('.'.join, keys(d)))
Output:
['Name1.NNum', 'Name1.Node1.SubNodeA', 'Name1.Node1.SubNodeB', 'Name1.Node2.SubNodeA', 'Name1.Node2.SubNodeD', 'Name1.Node2.SubNodeE', 'Name1.Node3.child1', 'Name1.Node3.child2.grandchild.greatgrandchild1', 'Name1.Node3.child2.grandchild.greatgrandchild2']
def getKeys(object, prev_key = None, keys = []):
if type(object) != type({}):
keys.append(prev_key)
return keys
new_keys = []
for k, v in object.items():
if prev_key != None:
new_key = "{}.{}".format(prev_key, k)
else:
new_key = k
new_keys.extend(getKeys(v, new_key, []))
return new_keys
This solution assumes that the inner types that might have children are dictionaries.
You can do simple recursion:
d = {
"Name1": {
"NNum": "11",
"Node1": {
"SubNodeA": "Thomas",
"SubNodeB": "27"
},
"Node2": {
"SubNodeA": "ZZZ",
"SubNodeD": "XXX",
"SubNodeE": "yy"
},
"Node3": {
"child1": 11,
"child2": {
"grandchild": {
"greatgrandchild1": "Rita",
"greatgrandchild2": "US"
}
}
}
}
}
def get_keys(d, curr_key=[]):
for k, v in d.items():
if isinstance(v, dict):
yield from get_keys(v, curr_key + [k])
elif isinstance(v, list):
for i in v:
yield from get_keys(i, curr_key + [k])
else:
yield '.'.join(curr_key + [k])
print([*get_keys(d)])
Prints:
['Name1.NNum', 'Name1.Node1.SubNodeA', 'Name1.Node1.SubNodeB', 'Name1.Node2.SubNodeA', 'Name1.Node2.SubNodeD', 'Name1.Node2.SubNodeE', 'Name1.Node3.child1', 'Name1.Node3.child2.grandchild.greatgrandchild1', 'Name1.Node3.child2.grandchild.greatgrandchild2']
What about this?
from collections import Mapping
def extract_paths(base_path, dd):
new_paths = []
for key, value in dd.items():
new_path = base_path + ('.' if base_path else '') + key
if isinstance(value, Mapping):
new_paths.extend(extract_paths(new_path, value))
else:
new_paths.append(new_path)
return new_paths
extract_paths('', your_dict)
Use isinstance to check the dict or not called by function recursively. If dict append to path recursively else print the path
def print_nested_keys(dic,path=''):
for k,v in dic.items():
if isinstance(v,dict):
path+=k+"."
yield from print_nested_keys(v,path)
else:
path+=k
yield path
Output:
>>> [*print_nested_keys(d)] # Here, d is your nested dictionary
['Name1.NNum',
'Name1.NNumNode1.SubNodeA',
'Name1.NNumNode1.SubNodeASubNodeB',
'Name1.NNumNode1.Node2.SubNodeA',
'Name1.NNumNode1.Node2.SubNodeASubNodeD',
'Name1.NNumNode1.Node2.SubNodeASubNodeDSubNodeE',
'Name1.NNumNode1.Node2.Node3.child1',
'Name1.NNumNode1.Node2.Node3.child1child2.grandchild.greatgrandchild1',
'Name1.NNumNode1.Node2.Node3.child1child2.grandchild.greatgrandchild1greatgrandchild2']