I happen to have a complex dictionary (having lists, dicts within lists etc). The values for some of the keys are set as None
Is there a way I can replace this None with some default value of my own irrespective of the complex structure of the dictionary?
You can do it using object_pairs_hook from json module:
def dict_clean(items):
result = {}
for key, value in items:
if value is None:
value = 'default'
result[key] = value
return result
dict_str = json.dumps(my_dict)
my_dict = json.loads(dict_str, object_pairs_hook=dict_clean)
# replace_none_with_empty_str_in_dict.py
raw = {'place': 'coffee shop', 'time': 'noon', 'day': None}
def replace_none_with_empty_str(some_dict):
return { k: ('' if v is None else v) for k, v in some_dict.items() }
print(replace_none_with_empty_str(raw))
Here's a recursive solution that also replaces Nones inside lists.
First we define a simple class, Null, to act as the replacement for None.
class Null(object):
def __repr__(self):
return 'Null'
NULL = Null()
def replace_none(data):
for k, v in data.items() if isinstance(data, dict) else enumerate(data):
if v is None:
data[k] = NULL
elif isinstance(v, (dict, list)):
replace_none(v)
# Test
data = {
1: 'one',
2: ['two', 2, None],
3: None,
4: {4: None, 44: 'four'},
5: {
5: [55, 56, None],
6: {66: None, 67: None},
8: [88, {9:'nine', 99:None}, 100]
}
}
print(data)
replace_none(data)
print(data)
output
{1: 'one', 2: ['two', 2, None], 3: None, 4: {44: 'four', 4: None}, 5: {8: [88, {9: 'nine', 99: None}, 100], 5: [55, 56, None], 6: {66: None, 67: None}}}
{1: 'one', 2: ['two', 2, Null], 3: Null, 4: {44: 'four', 4: Null}, 5: {8: [88, {9: 'nine', 99: Null}, 100], 5: [55, 56, Null], 6: {66: Null, 67: Null}}}
for k, v in my_dict.items():
if v is None:
my_dict[k] = "my default value"
Recursive solution from Lutz:
def replace(any_dict):
for k, v in any_dict.items():
if v is None:
any_dict[k] = "my default value"
elif type(v) == type(any_dict):
replace(v)
replace(my_dict)
You could do it with recursive function that iterates over all dicts and lists:
def convert(obj):
if type(obj) == list:
for x in obj:
convert(x)
elif type(obj) == dict:
for k, v in obj.iteritems():
if v is None:
obj[k] = 'DEFAULT'
else:
convert(v)
data = {1: 'foo', 2: None, 3: [{1: 'foo', 2: None}]}
convert(data)
print data # -> {1: 'foo', 2: 'DEFAULT', 3: [{1: 'foo', 2: 'DEFAULT'}]}
Related
I have a dictionary -
{
'buy': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None},
'sell': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None}
}
What would be the best approach to replace the None values with 0.
Note - Not every time the values for these keys are None.
You can use dictionary comprehension approach, and change the value to 0 if it returns a falsely value (such as None, False, '' or 0).
d = {'a': None, 'b': 1}
d1 = {k: v or 0 for (k, v) in d.items()}
# {'a': 0, 'b': 1}
You can recursively replace all None with 0 in the dict by checking the type of the values to see it's a nested dict or not.
test_dict = {
'buy': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None},
'sell': {'trade_transaction_amount__sum': None, 'tax__sum': None, 'trade_fee__sum': None}
}
def replace_none_with(d, replacement=0):
retval = {}
for key, val in d.items():
if val is None:
retval[key] = replacement
elif isinstance(val, dict):
retval[key] = replace_none_with(val, replacement)
else:
retval[key] = val
return retval
print(replace_none_with(test_dict))
Output:
{'buy': {'trade_transaction_amount__sum': 0, 'tax__sum': 0, 'trade_fee__sum': 0}, 'sell': {'trade_transaction_amount__sum': 0, 'tax__sum': 0, 'trade_fee__sum': 0}}
Here is the recursive approach which works with any levels of nesting dictionaries :
d = {
'buy': {'trade_transaction_amount__sum': None, 'tax__sum': None,
'trade_fee__sum': None},
'sell': {'trade_transaction_amount__sum': None, 'tax__sum': None,
'trade_fee__sum': None}
}
def replacer(dictionary):
for k, v in dictionary.items():
if isinstance(v, dict):
replacer(v)
elif v is None:
dictionary[k] = 0
replacer(d)
print(d)
I want to replace the values (formated as strings) with the same values as integers, whenever the key is 'current_values'.
d = {'id': '10', 'datastreams': [{'current_value': '5'}, {'current_value': '4'}]}
Desired Output:
d = {'id': '10', 'datastreams': [{'current_value': 5}, {'current_value': 4}]}
The following piece of code replaces (substrings of) values in a dictionary. It works for nested json structures and copes with json, list and string types. You can easily add other types if needed.
def dict_replace_value(d: dict, old: str, new: str) -> dict:
x = {}
for k, v in d.items():
if isinstance(v, dict):
v = dict_replace_value(v, old, new)
elif isinstance(v, list):
v = list_replace_value(v, old, new)
elif isinstance(v, str):
v = v.replace(old, new)
x[k] = v
return x
def list_replace_value(l: list, old: str, new: str) -> list:
x = []
for e in l:
if isinstance(e, list):
e = list_replace_value(e, old, new)
elif isinstance(e, dict):
e = dict_replace_value(e, old, new)
elif isinstance(e, str):
e = e.replace(old, new)
x.append(e)
return x
# See input and output below
output = dict_replace_value(input, 'string', 'something')
Input:
input = {
'key1': 'a string',
'key2': 'another string',
'key3': [
'a string',
'another string',
[1, 2, 3],
{
'key1': 'a string',
'key2': 'another string'
}
],
'key4': {
'key1': 'a string',
'key2': 'another string',
'key3': [
'a string',
'another string',
500,
1000
]
},
'key5': {
'key1': [
{
'key1': 'a string'
}
]
}
}
Output:
print(output)
{
"key1":"a something",
"key2":"another something",
"key3":[
"a something",
"another something",
[
1,
2,
3
],
{
"key1":"a something",
"key2":"another something"
}
],
"key4":{
"key1":"a something",
"key2":"another something",
"key3":[
"a something",
"another something",
500,
1000
]
},
"key5":{
"key1":[
{
"key1":"a something"
}
]
}
}
d = {'id': '10', 'datastreams': [{'current_value': '5'}, {'current_value': '4'}]}
for elem in d['datastreams']: # for each elem in the list datastreams
for k,v in elem.items(): # for key,val in the elem of the list
if 'current_value' in k: # if current_value is in the key
elem[k] = int(v) # Cast it to int
print(d)
OUTPUT:
{'id': '10', 'datastreams': [{'current_value': 5}, {'current_value': 4}]}
A general approach (assuming you don't know in advance which key of the dict is pointing to a list) would be to iterate over the dict and check the type of its values and then iterate again into each value if needed.
In your case, your dictionary may contain a list of dictionaries as values, so it is enough to check if a value is of type list, if so, iterate over the list and change the dicts you need.
It can be done recursively with a function like the following:
def f(d):
for k,v in d.items():
if k == 'current_value':
d[k] = int(v)
elif type(v) is list:
for item in v:
if type(item) is dict:
f(item)
>>> d = {'id': '10', 'datastreams': [{'current_value': '5'}, {'current_value': '4'}]}
>>> f(d)
>>> d
{'id': '10', 'datastreams': [{'current_value': 5}, {'current_value': 4}]}
Can be done with list comprehension:
d['datastreams'] = [{'current_value': int(ds['current_value'])} if ('current_value' in ds) else ds for ds in d['datastreams']]
You can use ast.literal_eval to evaluate the underlying value for items with current_value key in the d['datastreams'] list. Then check whether the type is an int using isinstance for such values. Finally, type cast such values to int.
import ast
d = {'id': '10', 'datastreams': [{'current_value': '5'}, {'current_value': '4'}]}
for i in d['datastreams']:
for k,v in i.items():
if 'current_value' in k and isinstance(ast.literal_eval(v),int):
i[k] = int(v)
#Output:
print(d)
{'id': '10', 'datastreams': [{'current_value': 5}, {'current_value': 4}]}
You could use this method
which would loop through checks for current_value in list and change it to integer by passing the value through int() function:
for value in d.values():
for element in value:
if 'current_value' in element:
element['current_value'] = int(element['current_value'])
Taking alec_djinn's solution little farther to handle also nested dicts:
def f(d):
for k,v in d.items():
if k == 'current_value':
d[k] = int(v)
elif type(v) is list:
for item in v:
if type(item) is dict:
f(item)
if type(v) is dict:
f(v)
I am having some trouble confronting this problem.
a = {
None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None}
}
I would like to verify that a's values which are dictionaries have a value which exists in a.keys(), if it does not exist then add it as a key with a dict() value.
In this case, None values exist in a.keys() but '0,1,2'does not so I just add it to a.keys() and would have something like this.
a = {
None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None},
'0,1,2': {}
}
After some other operations, the empty dictionary will have some keys and values.
For example,
a = {
None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None},
'0,1,2': {0: None, 1: '0,1'}
}
Now '0,1' does not exist in a.keys() so I will add it.
So I need to verify over and over again that a.values() which are dictionaries, have their values in a.keys().
You can create a list of all missing values by iterating through all values of your "value dictionaries" by checking if they are not present as a key in the original dictionary
def add_keys(d):
missing_values = [v for vdict in d.values() for v in vdict.values() if v not in d]
for k in missing_values:
d[k] = {}
Note that generating missing_values takes time proportional to the total number of values present in all the "value dictionaries" of d:
So now if you call add_keys on a any time, it should update a with the missing keys:
a = {None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None}}
add_keys(a)
print(a)
# {'1,2': {0: '0,1,2', 1: None}, '0,1,2': {}, None: {0: None, 1: None}}
a = {None: {0: None, 1: None},
'1,2': {0: '0,1,2', 1: None},
'0,1,2': {0: None, 1: '0,1'}}
add_keys(a)
print(a)
# {'1,2': {0: '0,1,2', 1: None}, '0,1': {}, '0,1,2': {0: None, 1: '0,1'}, None: {0: None, 1: None}}
I want to flatten a list of dict but having issues,
let's say i have a list of dict as,
d = [{'val': 454,'c': {'name': 'ss'}, 'r': {'name1': 'ff'}},{'val': 'ss', 'c': {'name': 'ww'}, 'r': {'name1': 'ff'}}, {'val': 22,'c': {'name': 'dd'}, 'r': {'name1': 'aa'}}]
And the output I'm trying to get is,
d = [{'val': 454,'name': 'ss', 'name1': 'ff'},{'val': 'ss','name': 'ww', 'name1': 'ff'},{'val': 22, 'name': 'dd', 'name1': 'aa'}]
For which I'm using the following function,
def flatten(structure, key="", flattened=None):
if flattened is None:
flattened = {}
if type(structure) not in(dict, list):
flattened[key] = structure
elif isinstance(structure, list):
for i, item in enumerate(structure):
flatten(item, "%d" % i, flattened)
else:
for new_key, value in structure.items():
flatten(value, new_key, flattened)
return flattened
Now, the issue I have is, it's only generating the first element in the dict
You are probably initializing something in the wrong place. Take a look at the code below:
d = [{'val': 454, 'c': {'name': 'ss'}, 'r': {'name1': 'ff'}}, {'val': 55, 'c': {'name': 'ww'}, 'r': {'name1': 'ff'}}, {'val': 22, 'c': {'name': 'dd'}, 'r': {'name1': 'aa'}}]
# ^ typo here
def flatten(my_dict):
res = []
for sub in my_dict:
print(sub)
dict_ = {}
for k, v in sub.items():
if isinstance(v, dict):
for k_new, v_new in v.items():
dict_[k_new] = v_new
else:
dict_[k] = v
res.append(dict_)
return res
result = flatten(d)
print(result) # [{'name': 'ss', 'name1': 'ff', 'val': 454}, {'name': 'ww', 'name1': 'ff', 'val': 55}, {'name': 'dd', 'name1': 'aa', 'val': 22}]
You should initialize flattened to the same type as structure if it's None, and pass None when recursing at the list case:
def flatten_2(structure, key="", flattened=None):
if flattened is None:
flattened = {} if isinstance(structure, dict) else []
if type(structure) not in(dict, list):
flattened[key] = structure
elif isinstance(structure, list):
for i, item in enumerate(structure):
flattened.append(flatten(item, "%d" % i))
else:
for new_key, value in structure.items():
flatten(value, new_key, flattened)
return flattened
In [13]: flatten_2(d)
Out[13]:
[{'name': 'ss', 'name1': 'ff', 'val': 454},
{'name': 'ww', 'name1': 'ff', 'val': 'ss'},
{'name': 'dd', 'name1': 'aa', 'val': 22}]
This of course only works for a limited type of data.
I have a nested dictionary {1: {2: {3: None}}} and a dictionary that maps keys of the nested dictionary to a set of values such as {1: x, 2: y, 3: z}. I want to transform the nested dictionary to this form {x: {y: {z: None}}}. I have tried a couple of recursive functions but I keep going in circles and confusing myself. What is the best way to achieve this?
The level of nesting is arbitrary. The above is a simple example.
You need to recurse through the dictionary while building a new one with new keys. Note that if you have a list or tuple in there somewhere that has other dictionaries in it, they won't be processed - you'd have to add some code to do that. You can actually do this without building a new dictionary, but I think this way is simpler.
od = { 1: { 2: { 3: None }}}
kd = { 1: 'x', 2: 'y', 3: 'z' }
def replace_keys(old_dict, key_dict):
new_dict = { }
for key in old_dict.keys():
new_key = key_dict.get(key, key)
if isinstance(old_dict[key], dict):
new_dict[new_key] = replace_keys(old_dict[key], key_dict)
else:
new_dict[new_key] = old_dict[key]
return new_dict
nd = replace_keys(od, kd)
print nd
outputs:
{'x': {'y': {'z': None}}}
The accepted answer will not support dict of list, adding the full feature
#bilentor,
od = {'name': 'John', '1': [{'name': 'innername'}]}
kd = { 'name': 'cname', '1': '2', 3: 'z' }
def replace_keys(data_dict, key_dict):
new_dict = { }
if isinstance(data_dict, list):
dict_value_list = list()
for inner_dict in data_dict:
dict_value_list.append(replace_keys(inner_dict, key_dict))
return dict_value_list
else:
for key in data_dict.keys():
value = data_dict[key]
new_key = key_dict.get(key, key)
if isinstance(value, dict) or isinstance(value, list):
new_dict[new_key] = replace_keys(value, key_dict)
else:
new_dict[new_key] = value
return new_dict
nd = replace_keys(od, kd)
print(nd)
You can use a NestedDict
from ndicts import NestedDict
d = {1: {2: {3: None}}}
replace = {1: 'x', 2: 'y', 3: 'z'}
def ndict_replace(ndict: dict, map: dict):
nd = NestedDict(nd)
new_nd = NestedDict()
for key, value in nd.items():
new_key = tuple(replace.get(k, k) for k in key)
new_nd[new_key] = value
return new_nd.to_dict()
>>> ndict_replace(d, replace)
{'x': {'y': {'z': None}}}
The solution is robust and works with any nested dictionary
>>> d = {
1: {2: {3: None}},
3: {4: None},
5: None
}
>>> ndict_replace(d, replace)
{'x': {'y': {'z': None}}, 'z': {4: None}, 4: None}}
To install ndicts pip install ndicts