We Keep Coding

Best way to replace `None` values with 0 in a dictionary

I have a dictionary -
{
'buy': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None},
'sell': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None}
}
What would be the best approach to replace the None values with 0.
Note - Not every time the values for these keys are None.

You can use dictionary comprehension approach, and change the value to 0 if it returns a falsely value (such as None, False, '' or 0).
d = {'a': None, 'b': 1}
d1 = {k: v or 0 for (k, v) in d.items()}
# {'a': 0, 'b': 1}

You can recursively replace all None with 0 in the dict by checking the type of the values to see it's a nested dict or not.
test_dict = {
'buy': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None},
'sell': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None}
}
def replace_none_with(d, replacement=0):
retval = {}
for key, val in d.items():
if val is None:
retval[key] = replacement
elif isinstance(val, dict):
retval[key] = replace_none_with(val, replacement)
else:
retval[key] = val
return retval
print(replace_none_with(test_dict))
Output:
{'buy': {'trade_transaction_amount__sum': 0, 'tax__sum': 0, 'trade_fee__sum': 0}, 'sell': {'trade_transaction_amount__sum': 0, 'tax__sum': 0, 'trade_fee__sum': 0}}

Here is the recursive approach which works with any levels of nesting dictionaries :
d = {
'buy': {'trade_transaction_amount__sum': None, 'tax__sum': None,
'trade_fee__sum': None},
'sell': {'trade_transaction_amount__sum': None, 'tax__sum': None,
'trade_fee__sum': None}
}
def replacer(dictionary):
for k, v in dictionary.items():
if isinstance(v, dict):
replacer(v)
elif v is None:
dictionary[k] = 0
replacer(d)
print(d)

How to replace None values in JSON response?

I'm fetching financial information from an api endpoint and when I get a 200 response through
r = requests.get(url)
data = r.json()
It'll return None for all null values. How do I convert all null/None values to 0? Since it's financial data, the JSON is usually quite massive (300k-400k lines, some with deep nested nulls) so I can't do a try/except block on each TypeError.
An extract of the json response looks something like this:
{'0':
'Highlights': {'QuarterlyRevenueGrowthYOY': 0.671, 'GrossProfitTTM': 3750684, 'DilutedEpsTTM': 0.2, 'QuarterlyEarningsGrowthYOY': 0.95
5}, 'Valuation': {'TrailingPE': 60.75, 'ForwardPE': 0, 'PriceSalesTTM': 2.0817, 'PriceBookMRQ': 4.207, 'EnterpriseValueRevenue': 1.
806, 'EnterpriseValueEbitda': 0.0952}, 'Technicals': {'Beta': None, '52WeekHigh': 12.35, '52WeekLow': 7.84, '50DayMA': 11.0197, '20
0DayMA': 10.2209, 'SharesShort': 0, 'SharesShortPriorMonth': 0, 'ShortRatio': 0, 'ShortPercent': 0}, 'SplitsDividends': {'ForwardAn
nualDividendRate': 0.18, 'ForwardAnnualDividendYield': 0.0151, 'PayoutRatio': 0.9, 'DividendDate': '0000-00-00', 'ExDividendDate':
'2020-06-11', 'LastSplitFactor': '', 'LastSplitDate': '0000-00-00'}, 'Earnings': {'Last_0': {'date': '2020-06-30', 'epsActual': 0.1
9, 'epsEstimate': None, 'epsDifference': None, 'surprisePercent': None}, 'Last_1': {'date': '2019-12-31', 'epsActual': 1.86, 'epsEs
timate': None, 'epsDifference': None, 'surprisePercent': None}, 'Last_2': {'date': '2019-06-30', 'epsActual': -0.82, 'epsEstimate':
None, 'epsDifference': None, 'surprisePercent': None}, 'Last_3': {'date': '0000-00-00', 'epsActual': 0, 'epsEstimate': 0, 'epsDiff
erence': 0, 'surprisePercent': 0}}, 'Financials': {'Balance_Sheet': {'currency_symbol': 'EUR', 'quarterly_last_0': {'date': '2020-0
6-30', 'filing_date': None, 'totalAssets': '12810000.00', 'intangibleAssets': '281000.00', 'otherCurrentAssets': '60000.00', 'total
Liab': '4225000.00', 'totalStockholderEquity': '8585000.00', 'deferredLongTermLiab': '74000.00', 'otherCurrentLiab': '1274000.00',
'commonStock': '80000.00', 'retainedEarnings': '311000.00', 'otherLiab': '200000.00', 'goodWill': '3381000.00', 'otherAssets': '730
00.00', 'cash': '4983000.00', 'totalCurrentLiabilities': '4025000.00', 'shortLongTermDebt': None,
...
}
Yeah you get the point.. a ton of None all over the place. Any quick fixes for this?

def recursive_replace(obj, findVal, replaceVal):
for k, v in obj.items():
if v == findVal:
obj[k] = replaceVal
elif isinstance(v, dict):
obj[k] = recursive_replace(obj[k], findVal, replaceVal)
return obj
result = recursive_replace(json.loads(yourdata), None, 0)

Found a way to do it, #Charles Duffy, thanks for the inspiration - borrowed some but couldn't get it quite to work. The final code looks like this if anyone would need it in the future
from collections.abc import Mapping, Iterable
def replace_none_values(noneVal, replaceVal='0.00'): # not sure if this is bad practice
if noneVal is None:
return replaceVal
if isinstance(noneVal, Mapping):
return {k: replace_none_values(v, replaceVal) for k, v in noneVal.items()}
elif not isinstance(noneVal, str) and isinstance(noneVal, Iterable):
return [replace_none_values(v, replaceVal) for v in noneVal]
return noneVal

How to flatten a list of dicts with nested dicts

I want to flatten a list of dict but having issues,
let's say i have a list of dict as,
d = [{'val': 454,'c': {'name': 'ss'}, 'r': {'name1': 'ff'}},{'val': 'ss', 'c': {'name': 'ww'}, 'r': {'name1': 'ff'}}, {'val': 22,'c': {'name': 'dd'}, 'r': {'name1': 'aa'}}]
And the output I'm trying to get is,
d = [{'val': 454,'name': 'ss', 'name1': 'ff'},{'val': 'ss','name': 'ww', 'name1': 'ff'},{'val': 22, 'name': 'dd', 'name1': 'aa'}]
For which I'm using the following function,
def flatten(structure, key="", flattened=None):
if flattened is None:
flattened = {}
if type(structure) not in(dict, list):
flattened[key] = structure
elif isinstance(structure, list):
for i, item in enumerate(structure):
flatten(item, "%d" % i, flattened)
else:
for new_key, value in structure.items():
flatten(value, new_key, flattened)
return flattened
Now, the issue I have is, it's only generating the first element in the dict

You are probably initializing something in the wrong place. Take a look at the code below:
d = [{'val': 454, 'c': {'name': 'ss'}, 'r': {'name1': 'ff'}}, {'val': 55, 'c': {'name': 'ww'}, 'r': {'name1': 'ff'}}, {'val': 22, 'c': {'name': 'dd'}, 'r': {'name1': 'aa'}}]
# ^ typo here
def flatten(my_dict):
res = []
for sub in my_dict:
print(sub)
dict_ = {}
for k, v in sub.items():
if isinstance(v, dict):
for k_new, v_new in v.items():
dict_[k_new] = v_new
else:
dict_[k] = v
res.append(dict_)
return res
result = flatten(d)
print(result) # [{'name': 'ss', 'name1': 'ff', 'val': 454}, {'name': 'ww', 'name1': 'ff', 'val': 55}, {'name': 'dd', 'name1': 'aa', 'val': 22}]

You should initialize flattened to the same type as structure if it's None, and pass None when recursing at the list case:
def flatten_2(structure, key="", flattened=None):
if flattened is None:
flattened = {} if isinstance(structure, dict) else []
if type(structure) not in(dict, list):
flattened[key] = structure
elif isinstance(structure, list):
for i, item in enumerate(structure):
flattened.append(flatten(item, "%d" % i))
else:
for new_key, value in structure.items():
flatten(value, new_key, flattened)
return flattened
In [13]: flatten_2(d)
Out[13]:
[{'name': 'ss', 'name1': 'ff', 'val': 454},
{'name': 'ww', 'name1': 'ff', 'val': 'ss'},
{'name': 'dd', 'name1': 'aa', 'val': 22}]
This of course only works for a limited type of data.

Python sort a list by value of a dict inside the list

I have a List and inside the list i got a dict and i want to sort the list by a value of the dict.
How does this work?
[{'id': 0, 'thread': 'First',
'post': [
{'id': 0, 'title': 'MyPost', 'time': '2015-11-07 01:06:08.939687'}]
},
{'id': 1, 'thread': 'Second',
'post': [
{'id': 0, 'title': 'MyPost', 'time': '2015-11-07 01:06:42.933263'}]},
{'id': 2, 'name': 'NoPosts', 'post': []}]
I would like to sort my Threadlist by time of the first post, is that possible?

You can pass sort or sorted a key function:
In [11]: def key(x):
try:
return x["post"][0]["time"] # return ISO date string
except IndexError:
return "Not a date string" # any letter > all date strings
In [12]: sorted(d, key=key)
Out[12]:
[{'id': 0,
'post': [{'id': 0, 'time': '2015-11-07 01:06:08.939687', 'title': 'MyPost'}],
'thread': 'First'},
{'id': 1,
'post': [{'id': 0, 'time': '2015-11-07 01:06:42.933263', 'title': 'MyPost'}],
'thread': 'Second'},
{'id': 2, 'name': 'NoPosts', 'post': []}]

Filter nested dictionary in python based on key values

How do I filter a nested dictionary in python based on key values:
d = {'data': {'country': 'US', 'city': 'New York', 'state': None},
'tags': ['US', 'New York'],
'type': 'country_info',
'growth_rate': None
}
I want to filter this dictionary to eliminate NoneType values so the resulting dict should be:
d = {'data': {'country': 'US', 'city': 'New York'},
'tags': ['US', 'New York'],
'type': 'country_info',
}
Also, the dict can have multiple levels of nesting. I want to remove all NoneType values from the dict.

You can define this recursively pretty easily with a dict comprehension.
def remove_keys_with_none_values(item):
if not hasattr(item, 'items'):
return item
else:
return {key: remove_keys_with_none_values(value) for key, value in item.items() if value is not None}
Recursion isn't too optimised in Python, but given the relatively small number of nestings that are likely, I wouldn't worry.
Looking before we leap isn't too Pythonic, I think it is a better option than catching the exception - as it's likely that the value will not be a dict most of the time (it is likely we have more leaves than branches).
Also note that in Python 2.x, you probably want to swap in iteritems() for items().

I really appreciate the answer by #Lattyware. It helped me filter out a nested object and remove empty values regardless of type being dict, list, or str.
Here is what I came up with:
remove-keys-with-empty-values.py
# remove-keys-with-empty-values.py
from pprint import pprint
def remove_keys_with_empty_values(item):
if hasattr(item, 'items'):
return {key: remove_keys_with_empty_values(value) for key, value in item.items() if value==0 or value}
elif isinstance(item, list):
return [remove_keys_with_empty_values(value) for value in item if value==0 or value]
else:
return item
d = {
'string': 'value',
'integer': 10,
'float': 0.5,
'zero': 0,
'empty_list': [],
'empty_dict': {},
'empty_string': '',
'none': None,
}
d['nested_dict'] = d.copy()
l = d.values()
d['nested_list'] = l
pprint({
"DICT FILTERED": remove_keys_with_empty_values(d),
"DICT ORIGINAL": d,
"LIST FILTERED": remove_keys_with_empty_values(l),
"LIST ORIGINAL": l,
})
execution
python remove-keys-with-empty-values.py
{'DICT FILTERED': {'float': 0.5,
'integer': 10,
'nested_dict': {'float': 0.5,
'integer': 10,
'string': 'value',
'zero': 0},
'nested_list': [0,
'value',
10,
0.5,
{'float': 0.5,
'integer': 10,
'string': 'value',
'zero': 0}],
'string': 'value',
'zero': 0},
'DICT ORIGINAL': {'empty_dict': {},
'empty_list': [],
'empty_string': '',
'float': 0.5,
'integer': 10,
'nested_dict': {'empty_dict': {},
'empty_list': [],
'empty_string': '',
'float': 0.5,
'integer': 10,
'none': None,
'string': 'value',
'zero': 0},
'nested_list': [{},
0,
'value',
None,
[],
10,
0.5,
'',
{'empty_dict': {},
'empty_list': [],
'empty_string': '',
'float': 0.5,
'integer': 10,
'none': None,
'string': 'value',
'zero': 0}],
'none': None,
'string': 'value',
'zero': 0},
'LIST FILTERED': [0,
'value',
10,
0.5,
{'float': 0.5,
'integer': 10,
'string': 'value',
'zero': 0}],
'LIST ORIGINAL': [{},
0,
'value',
None,
[],
10,
0.5,
'',
{'empty_dict': {},
'empty_list': [],
'empty_string': '',
'float': 0.5,
'integer': 10,
'none': None,
'string': 'value',
'zero': 0}]}