We Keep Coding

How to replace None values in JSON response?

I'm fetching financial information from an api endpoint and when I get a 200 response through
r = requests.get(url)
data = r.json()
It'll return None for all null values. How do I convert all null/None values to 0? Since it's financial data, the JSON is usually quite massive (300k-400k lines, some with deep nested nulls) so I can't do a try/except block on each TypeError.
An extract of the json response looks something like this:
{'0':
'Highlights': {'QuarterlyRevenueGrowthYOY': 0.671, 'GrossProfitTTM': 3750684, 'DilutedEpsTTM': 0.2, 'QuarterlyEarningsGrowthYOY': 0.95
5}, 'Valuation': {'TrailingPE': 60.75, 'ForwardPE': 0, 'PriceSalesTTM': 2.0817, 'PriceBookMRQ': 4.207, 'EnterpriseValueRevenue': 1.
806, 'EnterpriseValueEbitda': 0.0952}, 'Technicals': {'Beta': None, '52WeekHigh': 12.35, '52WeekLow': 7.84, '50DayMA': 11.0197, '20
0DayMA': 10.2209, 'SharesShort': 0, 'SharesShortPriorMonth': 0, 'ShortRatio': 0, 'ShortPercent': 0}, 'SplitsDividends': {'ForwardAn
nualDividendRate': 0.18, 'ForwardAnnualDividendYield': 0.0151, 'PayoutRatio': 0.9, 'DividendDate': '0000-00-00', 'ExDividendDate':
'2020-06-11', 'LastSplitFactor': '', 'LastSplitDate': '0000-00-00'}, 'Earnings': {'Last_0': {'date': '2020-06-30', 'epsActual': 0.1
9, 'epsEstimate': None, 'epsDifference': None, 'surprisePercent': None}, 'Last_1': {'date': '2019-12-31', 'epsActual': 1.86, 'epsEs
timate': None, 'epsDifference': None, 'surprisePercent': None}, 'Last_2': {'date': '2019-06-30', 'epsActual': -0.82, 'epsEstimate':
None, 'epsDifference': None, 'surprisePercent': None}, 'Last_3': {'date': '0000-00-00', 'epsActual': 0, 'epsEstimate': 0, 'epsDiff
erence': 0, 'surprisePercent': 0}}, 'Financials': {'Balance_Sheet': {'currency_symbol': 'EUR', 'quarterly_last_0': {'date': '2020-0
6-30', 'filing_date': None, 'totalAssets': '12810000.00', 'intangibleAssets': '281000.00', 'otherCurrentAssets': '60000.00', 'total
Liab': '4225000.00', 'totalStockholderEquity': '8585000.00', 'deferredLongTermLiab': '74000.00', 'otherCurrentLiab': '1274000.00',
'commonStock': '80000.00', 'retainedEarnings': '311000.00', 'otherLiab': '200000.00', 'goodWill': '3381000.00', 'otherAssets': '730
00.00', 'cash': '4983000.00', 'totalCurrentLiabilities': '4025000.00', 'shortLongTermDebt': None,
...
}
Yeah you get the point.. a ton of None all over the place. Any quick fixes for this?

def recursive_replace(obj, findVal, replaceVal):
for k, v in obj.items():
if v == findVal:
obj[k] = replaceVal
elif isinstance(v, dict):
obj[k] = recursive_replace(obj[k], findVal, replaceVal)
return obj
result = recursive_replace(json.loads(yourdata), None, 0)

Found a way to do it, #Charles Duffy, thanks for the inspiration - borrowed some but couldn't get it quite to work. The final code looks like this if anyone would need it in the future
from collections.abc import Mapping, Iterable
def replace_none_values(noneVal, replaceVal='0.00'): # not sure if this is bad practice
if noneVal is None:
return replaceVal
if isinstance(noneVal, Mapping):
return {k: replace_none_values(v, replaceVal) for k, v in noneVal.items()}
elif not isinstance(noneVal, str) and isinstance(noneVal, Iterable):
return [replace_none_values(v, replaceVal) for v in noneVal]
return noneVal

How can I check that '0,1,2' exists in a.keys() and if does not exist then add '0,1,2'?

I am having some trouble confronting this problem.
a = {
None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None}
}
I would like to verify that a's values which are dictionaries have a value which exists in a.keys(), if it does not exist then add it as a key with a dict() value.
In this case, None values exist in a.keys() but '0,1,2'does not so I just add it to a.keys() and would have something like this.
a = {
None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None},
'0,1,2': {}
}
After some other operations, the empty dictionary will have some keys and values.
For example,
a = {
None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None},
'0,1,2': {0: None, 1: '0,1'}
}
Now '0,1' does not exist in a.keys() so I will add it.
So I need to verify over and over again that a.values() which are dictionaries, have their values in a.keys().

You can create a list of all missing values by iterating through all values of your "value dictionaries" by checking if they are not present as a key in the original dictionary
def add_keys(d):
missing_values = [v for vdict in d.values() for v in vdict.values() if v not in d]
for k in missing_values:
d[k] = {}
Note that generating missing_values takes time proportional to the total number of values present in all the "value dictionaries" of d:
So now if you call add_keys on a any time, it should update a with the missing keys:
a = {None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None}}
add_keys(a)
print(a)
# {'1,2': {0: '0,1,2', 1: None}, '0,1,2': {}, None: {0: None, 1: None}}
a = {None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None},
'0,1,2': {0: None, 1: '0,1'}}
add_keys(a)
print(a)
# {'1,2': {0: '0,1,2', 1: None}, '0,1': {}, '0,1,2': {0: None, 1: '0,1'}, None: {0: None, 1: None}}

Compare key and values of one nested dictionary in other nested dictionary

I am trying to recursively compare below two python dictionaries:
expectededr = {'uid': 'e579b8cb-7d9f-4c0b-97de-a03bb52a1ec3', 'attempted': {'smpp': {'registeredDelivery': 0}, 'status': 'success', 'OATON': 1, 'OANPI': 1, 'DATON': 1, 'DANPI': 1, 'OA': '12149921220', 'DA': '1514525404'}, 'customerID': 'customer01', 'productID': 'product'}
edr = {'Category': 'NO', 'Type': 'mt', 'uid': 'e579b8cb-7d9f-4c0b-97de-a03bb52a1ec3', 'protocolID': 'smpp', 'direction': 'attempted', 'attempted': {'status': 'success', 'OANPI': 1, 'DATON': 1, 't2': 1512549691602, 'DANPI': 1, 'OA': '12149921220', 'DA': '1514525404', 'smpp': {'fragmented': False, 'sequenceID': 1, 'registeredDelivery': 0, 'messageID': '4e7b48ad-b39e-4e91-a7bb-2de463e4a6ee', 'srcPort': 39417, 'messageType': 4, 'Status': 0, 'ESMClass': 0, 'dstPort': 0, 'size': 0}, 'OATON': 1, 'PID': 0, 't1': 1512549691602}, 'customerID': 'customer01', 'productID': 'product'}
I am trying to compare the in a way that find and compare the key and value of first dictionary in second and if matching then print PASS else print FAIL.
for key in expectededr:
if expectededr[key] == edr[key]:
print("PASS")
else:
print("FAIL")
Output:
FAIL
PASS
PASS
PASS
Above code is not able to compare all the keys and values as these are nested dictionaries.
As you can see below, if i print key and values above i see that its not going in sub dictionary and missing their keys:
for key in expectededr:
if expectededr[key] == edr[key]:
print(expectededr[key])
print(edr[key])
Output:
customer01
customer01
e579b8cb-7d9f-4c0b-97de-a03bb52a1ec3
e579b8cb-7d9f-4c0b-97de-a03bb52a1ec3
product
product
Could someone help to update this code so that I can do the comparision in these nested dictionaries ?

One way is to flatten the dictionaries and then compare if the keys match.
So Lets initialiaze your dicts first:
In [23]: expectededr = {'uid': 'e579b8cb-7d9f-4c0b-97de-a03bb52a1ec3', 'attempted': {'smpp': {'registeredDelivery': 0}, 'status': 'success', 'OATON': 1, 'OANP
...: I': 1, 'DATON': 1, 'DANPI': 1, 'OA': '12149921220', 'DA': '1514525404'}, 'customerID': 'customer01', 'productID': 'product'}
...:
...: edr = {'Category': 'NO', 'Type': 'mt', 'uid': 'e579b8cb-7d9f-4c0b-97de-a03bb52a1ec3', 'protocolID': 'smpp', 'direction': 'attempted', 'attempted': {'
...: status': 'success', 'OANPI': 1, 'DATON': 1, 't2': 1512549691602, 'DANPI': 1, 'OA': '12149921220', 'DA': '1514525404', 'smpp': {'fragmented': False, '
...: sequenceID': 1, 'registeredDelivery': 0, 'messageID': '4e7b48ad-b39e-4e91-a7bb-2de463e4a6ee', 'srcPort': 39417, 'messageType': 4, 'Status': 0, 'ESMCl
...: ass': 0, 'dstPort': 0, 'size': 0}, 'OATON': 1, 'PID': 0, 't1': 1512549691602}, 'customerID': 'customer01', 'productID': 'product'}
...:
For flattening your dictionaries, we can use the approach suggested in Flatten nested Python dictionaries, compressing keys:
In [24]: import collections
...:
...: def flatten(d, parent_key='', sep='_'):
...: items = []
...: for k, v in d.items():
...: new_key = parent_key + sep + k if parent_key else k
...: if isinstance(v, collections.MutableMapping):
...: items.extend(flatten(v, new_key, sep=sep).items())
...: else:
...: items.append((new_key, v))
...: return dict(items)
...:
And generated flattened dicts
In [25]: flat_expectededr = flatten(expectededr)
In [26]: flat_edr = flatten(edr)
Now its a simple comparison:
In [27]: for key in flat_expectededr:
...: if flat_edr.get(key) == flat_expectededr[key]:
...: print "PASS"
...: else:
...: print "FAIL"
PASS
PASS
PASS
PASS
PASS
PASS
PASS
PASS
PASS
PASS
PASS

Simple way :
for i in edr.keys():
if i in expectededr.keys():
print 'true : key'+i
else:
print 'fail : key'+ i

Replace keys in a nested dictionary

I have a nested dictionary {1: {2: {3: None}}} and a dictionary that maps keys of the nested dictionary to a set of values such as {1: x, 2: y, 3: z}. I want to transform the nested dictionary to this form {x: {y: {z: None}}}. I have tried a couple of recursive functions but I keep going in circles and confusing myself. What is the best way to achieve this?
The level of nesting is arbitrary. The above is a simple example.

You need to recurse through the dictionary while building a new one with new keys. Note that if you have a list or tuple in there somewhere that has other dictionaries in it, they won't be processed - you'd have to add some code to do that. You can actually do this without building a new dictionary, but I think this way is simpler.
od = { 1: { 2: { 3: None }}}
kd = { 1: 'x', 2: 'y', 3: 'z' }
def replace_keys(old_dict, key_dict):
new_dict = { }
for key in old_dict.keys():
new_key = key_dict.get(key, key)
if isinstance(old_dict[key], dict):
new_dict[new_key] = replace_keys(old_dict[key], key_dict)
else:
new_dict[new_key] = old_dict[key]
return new_dict
nd = replace_keys(od, kd)
print nd
outputs:
{'x': {'y': {'z': None}}}

The accepted answer will not support dict of list, adding the full feature
#bilentor,
od = {'name': 'John', '1': [{'name': 'innername'}]}
kd = { 'name': 'cname', '1': '2', 3: 'z' }
def replace_keys(data_dict, key_dict):
new_dict = { }
if isinstance(data_dict, list):
dict_value_list = list()
for inner_dict in data_dict:
dict_value_list.append(replace_keys(inner_dict, key_dict))
return dict_value_list
else:
for key in data_dict.keys():
value = data_dict[key]
new_key = key_dict.get(key, key)
if isinstance(value, dict) or isinstance(value, list):
new_dict[new_key] = replace_keys(value, key_dict)
else:
new_dict[new_key] = value
return new_dict
nd = replace_keys(od, kd)
print(nd)

You can use a NestedDict
from ndicts import NestedDict
d = {1: {2: {3: None}}}
replace = {1: 'x', 2: 'y', 3: 'z'}
def ndict_replace(ndict: dict, map: dict):
nd = NestedDict(nd)
new_nd = NestedDict()
for key, value in nd.items():
new_key = tuple(replace.get(k, k) for k in key)
new_nd[new_key] = value
return new_nd.to_dict()
>>> ndict_replace(d, replace)
{'x': {'y': {'z': None}}}
The solution is robust and works with any nested dictionary
>>> d = {
1: {2: {3: None}},
3: {4: None},
5: None
}
>>> ndict_replace(d, replace)
{'x': {'y': {'z': None}}, 'z': {4: None}, 4: None}}
To install ndicts pip install ndicts

Replace None in a python dictionary

I happen to have a complex dictionary (having lists, dicts within lists etc). The values for some of the keys are set as None
Is there a way I can replace this None with some default value of my own irrespective of the complex structure of the dictionary?

You can do it using object_pairs_hook from json module:
def dict_clean(items):
result = {}
for key, value in items:
if value is None:
value = 'default'
result[key] = value
return result
dict_str = json.dumps(my_dict)
my_dict = json.loads(dict_str, object_pairs_hook=dict_clean)

# replace_none_with_empty_str_in_dict.py
raw = {'place': 'coffee shop', 'time': 'noon', 'day': None}
def replace_none_with_empty_str(some_dict):
return { k: ('' if v is None else v) for k, v in some_dict.items() }
print(replace_none_with_empty_str(raw))

Here's a recursive solution that also replaces Nones inside lists.
First we define a simple class, Null, to act as the replacement for None.
class Null(object):
def __repr__(self):
return 'Null'
NULL = Null()
def replace_none(data):
for k, v in data.items() if isinstance(data, dict) else enumerate(data):
if v is None:
data[k] = NULL
elif isinstance(v, (dict, list)):
replace_none(v)
# Test
data = {
1: 'one',
2: ['two', 2, None],
3: None,
4: {4: None, 44: 'four'},
5: {
5: [55, 56, None],
6: {66: None, 67: None},
8: [88, {9:'nine', 99:None}, 100]
}
}
print(data)
replace_none(data)
print(data)
output
{1: 'one', 2: ['two', 2, None], 3: None, 4: {44: 'four', 4: None}, 5: {8: [88, {9: 'nine', 99: None}, 100], 5: [55, 56, None], 6: {66: None, 67: None}}}
{1: 'one', 2: ['two', 2, Null], 3: Null, 4: {44: 'four', 4: Null}, 5: {8: [88, {9: 'nine', 99: Null}, 100], 5: [55, 56, Null], 6: {66: Null, 67: Null}}}

for k, v in my_dict.items():
if v is None:
my_dict[k] = "my default value"

Recursive solution from Lutz:
def replace(any_dict):
for k, v in any_dict.items():
if v is None:
any_dict[k] = "my default value"
elif type(v) == type(any_dict):
replace(v)
replace(my_dict)

You could do it with recursive function that iterates over all dicts and lists:
def convert(obj):
if type(obj) == list:
for x in obj:
convert(x)
elif type(obj) == dict:
for k, v in obj.iteritems():
if v is None:
obj[k] = 'DEFAULT'
else:
convert(v)
data = {1: 'foo', 2: None, 3: [{1: 'foo', 2: None}]}
convert(data)
print data # -> {1: 'foo', 2: 'DEFAULT', 3: [{1: 'foo', 2: 'DEFAULT'}]}