My json data would look like this:
{
"a":1,
"b":[
{
"c":2,
"d":{
"e":3
},
"f":{
"g":4
},
"h":[
{
"i":5
},
{
"j":6
}
]
}
]
}
Is there a way I can get values for certain fields in the response along with their keys. So from this response, the fields for which I expect values are a, c,e,g,i,j along with the respective keys.
Eg: [a:1,c:2,e:3,g:4,i:5,j:6]. Could this be done?
My response contained something like:
{
"a":1,
"b":[
{
"c":2,
"d":{
"e":3
},
"f":{
"g":4,
"k":[
"l","m"]
},
"h":[
{
"i":5
},
{
"j":6
}
]
}
]
}
Which resulted in the error. I have made the following fix for it.
def get_key_value(dct, res_dct, lst):
for k,v in dct.items():
if isinstance(v, list):
for d in v:
if isinstance(d,dict):
get_key_value(d, res_dct, lst)
else:
lst.append(f'{k}:{v}')
elif isinstance(v, dict):
get_key_value(v, res_dct, lst)
else:
res_dct[k] = v
# If you want to store in 'list' you can store as string
lst.append(f'{k}:{v}')
res_dct = {}
lst = []
get_key_value(staging_dict, res_dct, lst)
You can use a recursive function and store key & value if only value not list or dict.
def get_key_value(dct, res_dct, lst):
for k,v in dct.items():
if isinstance(v, list):
for d in v:
get_key_value(d, res_dct, lst)
elif isinstance(v, dict):
get_key_value(v, res_dct, lst)
else:
res_dct[k] = v
# If you want to store in 'list' you can store as string
lst.append(f'{k}:{v}')
res_dct = {}
lst = []
get_key_value(dct, res_dct, lst)
print(res_dct)
print(lst)
Output:
# res_dct
{'a': 1, 'c': 2, 'e': 3, 'g': 4, 'i': 5, 'j': 6}
# lst
['a:1', 'c:2', 'e:3', 'g:4', 'i:5', 'j:6']
Related
This question already has answers here:
How to find a particular JSON value by key?
(9 answers)
Closed 2 years ago.
for the most part of my day i've been trying to find a way to solve this. I'm trying to find a way to find all keys in my json code that have the key "price" and if they do, populate a dictionary or something with the price and the name of each item that has a price. This is the simplified json, please note that the "price" keys can also be further nested. I'm trying to search the whole json code for the key:
{
"status": "success",
"data": {
"top_products": {
"products": [
{
"price": 3,
"name": "Apple"
},
{
"price": 2,
"name": "Banana"
}
]
},
"products": {
"fruits": {
"list": [
{
"price": 4,
"name": "Pear"
},
{
"name": "Kiwi"
},
{
"price": 4,
"name": "Pineapple"
},
{
"name": "Cherry"
}
]
},
"veggies": {
"list": [
{
"price": 3,
"name": "cucumber"
},
{
"name": "tomato"
},
{
"price": 2,
"name": "onion"
},
{
"name": "green pepper"
}
]
}
}
}
}
Here is what i've managed to get working so far (didnt come up with this, found it in some other response):
def findkeys(node, kv):
if isinstance(node, list):
for i in node:
for x in findkeys(i, kv):
yield x
elif isinstance(node, dict):
if kv in node:
yield node[kv]
for j in node.values():
for x in findkeys(j, kv):
yield x
print(list(findkeys(jsonResponse, 'price')))
The first part works, it returns all the keys that have a price. I'm trying to figure out a way to also write the "name" key for all the prices, preferably into a dictionary. Whats the best approach to do this?
Thanks,
Rob
Use the following code, if there are only unique items:
def create_db(data, find, other):
db = {}
def recurse(data):
if isinstance(data, list):
for elem in data:
recurse(elem)
elif isinstance(data, dict):
if find in data:
db[data[other]] = data[find]
for k, v in data.items():
recurse(v)
recurse(data)
return db
>>> create_db(data, 'price', 'name')
{'Apple': 3, 'Banana': 2, 'Pear': 4, 'Pineapple': 4, 'cucumber': 3, 'onion': 2}
Else:
def create_db(data, find, other):
db = {}
ctr = {}
def recurse(data):
if isinstance(data, list):
for elem in data:
recurse(elem)
elif isinstance(data, dict):
if find in data:
if data[other] in ctr:
ctr[data[other]] = str(int(ctr[data[other]] or '1') + 1)
else:
ctr[data[other]] = ''
key = data[other] + ctr[data[other]]
db[key] = data[find]
for k, v in data.items():
recurse(v)
recurse(data)
return db
For example, if data had two Apples:
data = {'status': 'success',
'data': {'top_products': {'products': [{'price': 3, 'name': 'Apple'},
{'price': 4, 'name': 'Apple'},
{'price': 2, 'name': 'Banana'}]}}}
# Second approach will add a serial number to each duplicate item
>>> create_db(data, 'price', 'name')
{'Apple': 3, 'Apple2': 4, 'Banana': 2}
For easier access in case of duplicates, you can create a nested dict:
def create_db(data, find, other):
db = {}
def recurse(data):
if isinstance(data, list):
for elem in data:
recurse(elem)
elif isinstance(data, dict):
if find in data:
if data[other] in db:
if isinstance(db[data[other]], dict):
db[data[other]][len(db[data[other]]) + 1] = data[other]
else:
db[data[other]] = {0: db.pop(data[other]), 1: data[find]}
else:
db[data[other]] = data[find]
for k, v in data.items():
recurse(v)
recurse(data)
return db
# For the data in above approach:
>>> create_db(data, 'price', 'name')
{'Apple': {0: 3, 1: 4}, 'Banana': 2}
Having the following dict, where some of the values can be list of dictionaries:
{
"A": [
{
"B": {
"C": "D",
"X": "CHNAGE ME"
}
},
{
"E": "F"
}
],
"G": {
"Y": "CHANGE ME"
}
}
I would like to recursively iterate over the items and change the pairs of key values where the value is "CHANGE ME", so the result would be:
{
"A": [
{
"B": {
"C": "D",
"X.CHANGED": "CHANGED"
}
},
{
"E": "F"
}
],
"G": {
"Y.CHANGED": "CHANGED"
}
}
Solutions I've found were not handling a case where the value is a list, for example:
import collections
def nested_dict_iter(nested):
for key, value in nested.iteritems():
if isinstance(value, collections.Mapping):
for inner_key, inner_value in nested_dict_iter(value):
yield inner_key, inner_value
else:
yield key, value
How can I achieve my goal?
Using recursion
Ex:
def update(data):
for k, v in data.copy().items():
if isinstance(v, dict): # For DICT
data[k] = update(v)
elif isinstance(v, list): # For LIST
data[k] = [update(i) for i in v]
elif v == 'CHANGE ME': # Update Key-Value
# data.pop(k)
# OR
del data[k]
data[f"{k}.CHANGED"] = 'CHANGED'
return data
print(update(data))
Output:
{
'A':[{'B': {'C': 'D', 'X.CHANGED': 'CHANGED'}}, {'E': 'F'}],
'G':{'Y.CHANGED': 'CHANGED'}
}
Note: I have not tested all corner cases
I get JSON Data back from an API. The dataset is large and nested. I can access the Datenreihen key like this:
jsondata.get("Ergebnis")[0].get("Kontakte").get("Datenreihen")
As you can see, this is a mix of dictionaries and lists.
I tried the following, but with lists it does not work :-(.
def recursive_lookup(k, d):
if k in d:
return d[k]
for v in d.values():
if isinstance(v, dict):
return recursive_lookup(k, v)
return None
# Works
recursive_lookup("Ergebnis", jsondata)
# Returns None
recursive_lookup("Datenreihen", jsondata)
Is there an easy way to access and key in my dictionary, no matter how deeply my object is nested?
This is exampledata:
{
"Success":true,
"Ergebnis":[
{
"ErgA1a: KPI Zeitreihe":{
"Message":"",
"MitZielgruppe":true,
"Beschriftung":[
"2019 KW 27",
"2019 KW 28",
"2019 KW 29"
],
"Datenreihen":{
"Gesamt":{
"Name":"Sympathie [#4]\n(Sehr sympathisch, Sympathisch)",
"Werte":[
39.922142815641145,
37.751410794385762,
38.35504885993484
]
}
}
}
}
],
"rest":[
{
"test":"bla"
}
]
}
data.get("ErgebnisseAnalyse")[0].get("ErgA1a: KPI Zeitreihe")
recursive_lookup("ErgA1a: KPI Zeitreihe", data)
Recursive function to find value in nested dictionary based upon key field
Code
def find_item(obj, field):
"""
Takes a dict with nested lists and dicts,
and searches all dicts for a key of the field
provided.
"""
if isinstance(obj, dict):
for k, v in obj.items():
if k == field:
yield v
elif isinstance(v, dict) or isinstance(v, list):
yield from find_item(v, field)
elif isinstance(obj, list):
for v in obj:
yield from find_item(v, field)
Usage
value = next(find_item(dictionary_object, field), None)
Test
# Nested dictionary
dic = {
"a": [{"b": {"c": 1}},
{"d": 2}],
"e": 3}
# Values form various fields
print(next(find_item(dic, "a"), None)) # Output: [{'b': {'c': 1}}, {'d': 2}]
print(next(find_item(dic, "b"), None)) # Output: {'c': 1}
print(next(find_item(dic, "c"), None)) # Output: 1
print(next(find_item(dic, "d"), None)) # Output: 2
print(next(find_item(dic, "e"), None)) # Output: 3
print(next(find_item(dic, "h"), None)) # Output: None
I need help with a function to flatten a nested dictionary in the following format:
dict_test = {
"id" : "5d4c2c0fd89234260ec81",
"Reference Number" : "JA-L800D-191",
"entities_discovered" : {
"OTHER_ID" : [
"L800DFAG02191"
],
"CODE_ID" : [
"160472708",
"276954773"
]
},
"label_field" : [
"ELECTRONICS",
"HDMI"
],
"numeric_field" : [
491,
492
],
}
The function I was working with, flattens the dictionary to one dimension (key:value) as I want, but doesn´t join the values within the same key iteration.
def flatten(d):
agg = {}
def _flatten(d, prev_key=''):
if isinstance(d, list):
for i, item in enumerate(d):
new_k = '%s.%s' % (prev_key, i) if prev_key else i
_flatten(item, prev_key=new_k)
elif isinstance(d, dict):
for k, v in d.items():
new_k = '%s.%s' % (prev_key, k) if prev_key else k
_flatten(v, prev_key=new_k)
else:
agg[prev_key] = d
_flatten(d)
return agg
My current output is:
{
"id" : "5d4c2c0fd89234260ec81",
"Reference Number" : "JA-L800D-191",
"entities_discovered.OTHER_ID.0" : "L800DFAG02191",
"entities_discovered.CODE_ID.0" : "160472708",
"entities_discovered.CODE_ID.1" : "276954773",
"label_field.0" : "ELECTRONICS",
"label_field.1" : "HDMI",
"numeric_field.0" : 491,
"numeric_field.1" : 492
}
But actually I´m looking for something like (joining the values into the same string and separated by , or |):
{
"id" : "5d4c2c0fd89234260ec81",
"Reference Number" : "JA-L800D-191",
"OTHER_ID" : "L800DFAG02191",
"CODE_ID" : "160472708, 276954773",
"label_field" : "ELECTRONICS, HDMI",
"numeric_field" : ¨491, 492¨
}
You can use join() built-in method to join values together.
def do():
dict_test = {
"id": "5d4c2c0fd89234260ec81",
"Reference Number": "JA-L800D-191",
"entities_discovered": {
"OTHER_ID": [
"L800DFAG02191"
],
"CODE_ID": [
"160472708",
"276954773"
]
},
"label_field": [
"ELECTRONICS",
"HDMI"
],
"numeric_field": [
491,
492
],
}
new_dict = {}
for key, value in dict_test.items():
if isinstance(value, dict):
for _key, _value in value.items():
if isinstance(_value, list):
new_dict.update({_key: ', '.join([str(item) for item in _value])})
elif isinstance(value, list):
new_dict.update({key: ', '.join([str(item) for item in value])})
else:
new_dict.update({key: value})
return new_dict
if __name__ == '__main__':
print(do())
Output:
{
'id': '5d4c2c0fd89234260ec81',
'Reference Number': 'JA-L800D-191',
'OTHER_ID': 'L800DFAG02191',
'CODE_ID': '160472708, 276954773',
'label_field': 'ELECTRONICS, HDMI',
'numeric_field': '491, 492'
}
def recursive_flatten_dict(tmp, dict_test):
for i,v in dict_test.items():
if type(v) == type({}):
recursive_flatten_dict(tmp,v)
else:
tmp[i] = v
return tmp
recursive_flatten_dict({},dict_test)
Simple recursion using a generator:
def flatten(d):
for a, b in d.items():
if isinstance(b, dict):
yield from flatten(b)
else:
yield (a, b if not isinstance(b, list) else ', '.join(map(str, b)))
print(dict(flatten(dict_test)))
Output:
{
'id': '5d4c2c0fd89234260ec81',
'Reference Number': 'JA-L800D-191',
'OTHER_ID': 'L800DFAG02191',
'CODE_ID': '160472708, 276954773',
'label_field': 'ELECTRONICS, HDMI',
'numeric_field': '491, 492'
}
def flatten(dict_test):
for key in ['label_field', 'numeric_field']:
dict_test[key]= ', '.join([str(c) for c in dict_test[key]])
for c in dict_test['entities_discovered'].keys():
dict_test[c]= ', '.join(dict_test['entities_discovered'][c])
return dict_test
The above function does the job. I hope this what you are looking for?
I have a huge json in the format something like :
{
"Name1": {
"NNum": "11",
"Node1": {
"SubNodeA": "Thomas",
"SubNodeB": "27"
},
"Node2": {
"SubNodeA": "ZZZ",
"SubNodeD": "XXX",
"SubNodeE": "yy"
},
"Node3": {
"child1": 11,
"child2": {
"grandchild": {
"greatgrandchild1": "Rita",
"greatgrandchild2": "US"
}
}
}
}
}
The format or keys are not defined and can go to any depth
I would like to get the list of keys like
keyList= ["Name1.NNum","Name1.Node1.SubNodeA",""Name1.Node1.SubNodeB","Name1.Node2.SubNodeA","Name1.Node2.SubNodeD","Name1.Node2.SubNodeE","Name1.Node3.child1","Name1.Node3.child2.grandchild.greatgrandchild1","Name1.Node3.child2.grandchild.greatgrandchild2"]
A snapshot of the code
def extract_values(obj):
"""Pull all values of specified key from nested JSON."""
arr = []
key_list = []
parent = ""
def extract(obj, arr,parent):
"""Recursively search for values of key in JSON tree."""
if isinstance(obj, dict):
grandparent = ""
for k, v in obj.items():
print ("k ............",k)
parent = grandparent
temp_parent = k
print ("parent >>>>> ",parent)
if isinstance(v, (dict, list)):
parent = temp_parent
print ("IF VALUE DICT .. parent ", parent)
extract(v, arr,parent)
else:
grandparent = parent
parent = parent + "_" + temp_parent
print ("!!!! NOT DICT :).... **** parent ... ", parent)
arr.append(parent)
elif isinstance(obj, list):
for item in obj:
extract(item, arr)
#print ("arr >>>>>>>>>> ", arr)
time.sleep(5)
return arr
results = extract(obj, arr,parent)
return results
but this does not give the expected output.
Expected Output:
keyList= ["Name1.NNum","Name1.Node1.SubNodeA",""Name1.Node1.SubNodeB","Name1.Node2.SubNodeA","Name1.Node2.SubNodeD","Name1.Node2.SubNodeE","Name1.Node3.child1","Name1.Node3.child2.grandchild.greatgrandchild1","Name1.Node3.child2.grandchild.greatgrandchild2"]
Can anybody help me with this.
Thanks in advance
You can use recursion:
d = {'Name1': {'NNum': '11', 'Node1': {'SubNodeA': 'Thomas', 'SubNodeB': '27'}, 'Node2': {'SubNodeA': 'ZZZ', 'SubNodeD': 'XXX', 'SubNodeE': 'yy'}, 'Node3': {'child1': 11, 'child2': {'grandchild': {'greatgrandchild1': 'Rita', 'greatgrandchild2': 'US'}}}}}
def keys(d, c = []):
return [i for a, b in d.items() for i in ([c+[a]] if not isinstance(b, dict) else keys(b, c+[a]))]
result = list(map('.'.join, keys(d)))
Output:
['Name1.NNum', 'Name1.Node1.SubNodeA', 'Name1.Node1.SubNodeB', 'Name1.Node2.SubNodeA', 'Name1.Node2.SubNodeD', 'Name1.Node2.SubNodeE', 'Name1.Node3.child1', 'Name1.Node3.child2.grandchild.greatgrandchild1', 'Name1.Node3.child2.grandchild.greatgrandchild2']
def getKeys(object, prev_key = None, keys = []):
if type(object) != type({}):
keys.append(prev_key)
return keys
new_keys = []
for k, v in object.items():
if prev_key != None:
new_key = "{}.{}".format(prev_key, k)
else:
new_key = k
new_keys.extend(getKeys(v, new_key, []))
return new_keys
This solution assumes that the inner types that might have children are dictionaries.
You can do simple recursion:
d = {
"Name1": {
"NNum": "11",
"Node1": {
"SubNodeA": "Thomas",
"SubNodeB": "27"
},
"Node2": {
"SubNodeA": "ZZZ",
"SubNodeD": "XXX",
"SubNodeE": "yy"
},
"Node3": {
"child1": 11,
"child2": {
"grandchild": {
"greatgrandchild1": "Rita",
"greatgrandchild2": "US"
}
}
}
}
}
def get_keys(d, curr_key=[]):
for k, v in d.items():
if isinstance(v, dict):
yield from get_keys(v, curr_key + [k])
elif isinstance(v, list):
for i in v:
yield from get_keys(i, curr_key + [k])
else:
yield '.'.join(curr_key + [k])
print([*get_keys(d)])
Prints:
['Name1.NNum', 'Name1.Node1.SubNodeA', 'Name1.Node1.SubNodeB', 'Name1.Node2.SubNodeA', 'Name1.Node2.SubNodeD', 'Name1.Node2.SubNodeE', 'Name1.Node3.child1', 'Name1.Node3.child2.grandchild.greatgrandchild1', 'Name1.Node3.child2.grandchild.greatgrandchild2']
What about this?
from collections import Mapping
def extract_paths(base_path, dd):
new_paths = []
for key, value in dd.items():
new_path = base_path + ('.' if base_path else '') + key
if isinstance(value, Mapping):
new_paths.extend(extract_paths(new_path, value))
else:
new_paths.append(new_path)
return new_paths
extract_paths('', your_dict)
Use isinstance to check the dict or not called by function recursively. If dict append to path recursively else print the path
def print_nested_keys(dic,path=''):
for k,v in dic.items():
if isinstance(v,dict):
path+=k+"."
yield from print_nested_keys(v,path)
else:
path+=k
yield path
Output:
>>> [*print_nested_keys(d)] # Here, d is your nested dictionary
['Name1.NNum',
'Name1.NNumNode1.SubNodeA',
'Name1.NNumNode1.SubNodeASubNodeB',
'Name1.NNumNode1.Node2.SubNodeA',
'Name1.NNumNode1.Node2.SubNodeASubNodeD',
'Name1.NNumNode1.Node2.SubNodeASubNodeDSubNodeE',
'Name1.NNumNode1.Node2.Node3.child1',
'Name1.NNumNode1.Node2.Node3.child1child2.grandchild.greatgrandchild1',
'Name1.NNumNode1.Node2.Node3.child1child2.grandchild.greatgrandchild1greatgrandchild2']