I have dicts in dict and need to change where the value of inner dicts is None to 'None' so it could show in csv file, not to be a blank cell.
Example:
{Object1 : {cellload : 1, cellthr : None, cellcap : 40}
Object2 : {cellload : None, cellthr : 2, cellcap : 40}
...
ObjectN : {cellload : None, cellthr : 5, cellcap : 50}}
So, I don't need None or NaN values, that can't be recognized in csv file, I need 'None' as a string.
I've tried this:
def replace(main_dif):
for k, v in main_dif.items():
for ki, ve in v.items():
if ve is None:
ve = "None"
elif type(ve) == type(main_dif):
replace(ve)
replace(main_dif)
And this:
outer = {}
for k, v in main_dif.items():
inner = {}
for ki, ve in v.items():
if ve is None:
ve == 'None'
else:
ve = ve
inner.update({ki:ve})
outer.update({k:inner})
And None is there just like always, like I did nothing.
One option is just to brute-force through all the values of all the objects and replace the Nones.
for obj in objects:
for k,v in obj.items():
if v is None:
obj[k] = 'None'
EDIT It's best to use the python csv module where possible.
In this case we can make a custom DictWriter to handle the None situations for us (it might need tweaking depending on the data).
import csv
class NoneSafeWriter(csv.DictWriter):
def _extract_val(self, rowdict, key):
v = rowdict.get(key)
return 'None' if v == None else v
def _dict_to_list(self, rowdict):
return [self._extract_val(rowdict, key) for key in self.fieldnames]
with open('output.csv', 'w') as csvfile:
fieldnames = ['cellload', 'cellthr', 'cellcap']
writer = NoneSafeWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
writer.writerows(objects.values())
Related
How do I convert below comma delimited records -
COL1,COL2,COL3,COL4
A101,P501,U901,US_A
A101,P501,U902,US_B
A101,P502,U901,US_A
A102,P501,U901,US_A
A102,P502,U902,US_B
into python dictionary -
result = {
"A101": {
"P501": {"U901": "US_A", "U902": "US_B"},
"P502": {"U901": "US_A"}
},
"A102": {
"P501": {"U901": "US_A"},
"P502": {"U902": "US_B"}
}
}
Thank you for your help!
Approach
We can process the rows of the CSV file as follows:
Convert each row in CSV file from a list to a nested dictionary using Convert a list to nested dictionary i.e. line reduce(lambda x, y: {y: x}, reversed(row))) in code below.
Merge the nested dictionaries using
Merge nested dictionaries in Python using merge_dict function below
Code
import csv
def csv_to_nested_dict(filenm):
' CSV file to nested dictionary '
with open(filenm, 'r') as csvfile:
csv_reader = csv.reader(csvfile, delimiter=',')
next(csv_reader) # skip header row
result = {}
for row in csv_reader:
# Convert row to nested dictionary and
# Merge into result
result = merge_dict(result,
reduce(lambda x, y: {y: x}, reversed(row))) # row to nested dictionary
return result
def merge_dict(dict1, dict2):
' Merges nested dictionaries '
for key, val in dict1.items():
if type(val) == dict:
if key in dict2 and type(dict2[key] == dict):
merge_dict(dict1[key], dict2[key])
else:
if key in dict2:
dict1[key] = dict2[key]
for key, val in dict2.items():
if not key in dict1:
dict1[key] = val
return dict1
Test
Usage:
res = csv_to_nested_dict('test.txt') # result
# Use json to pretty print nested dictionary res
import json
print(json.dumps(res, indent = 4))
Input File test.txt
COL1,COL2,COL3,COL4
A101,P501,U901,US_A
A101,P501,U902,US_B
A101,P502,U901,US_A
A102,P501,U901,US_A
A102,P502,U902,US_B
Output
{
"A101": {
"P501": {
"U901": "US_A",
"U902": "US_B"
},
"P502": {
"U901": "US_A"
}
},
"A102": {
"P501": {
"U901": "US_A"
},
"P502": {
"U902": "US_B"
}
}
}
here is simple version of solution -
def dict_reader(file_name):
with open(file_name, 'r') as csvfile:
reader = csv.DictReader(csvfile)
try:
data = dict()
for row in reader:
col1, col2, col3, col4 = (row["col1"], row["col2"], row["col3"], row["col4"])
if col1 in data:
if col2 in data[col1]:
data[col1][col2].update({col3: col4})
else:
data[col1][col2] = {col3: col4}
else:
data[col1] = {col2: {col3: col4}}
except csv.Error as e:
sys.exit('file {}, line {}: {}'.format(file_name, reader.line_num, e))
finally:
return data
it is not very elegant solution but works.
I am trying to update the dictionary "menu" values by getting a value from an appropriate dictionary. I try the following code and get the desired result. But I expect some other alternate ways and a more generic solution in a pythonic way.
Code
language = {"english" : {"lbl01":"File" ,"lbl02":"Accounts"},
"tamil" : {"lbl01":"கோப்பு" ,"lbl02":"கணக்கியல்"},
"hindi" : {"lbl01":"Hindi_File","lbl02":"Hindi_accounts"}}
scut = {"user1": {"lbl01":"Alt+F","lbl02":"F5"},
"user2": {"lbl01":"Ctrl+F","lbl02":"Shift+F5"}}
menu = {"lbl01" :{"id":"file" ,"lan1":"","lan2":"","scut":""},
"lbl02" :{"id":"accounts","lan1":"","lan2":"","scut":""}}
user = ["user2"]
selected_lan = ["tamil","hindi"]
menukey_lst,submenukey_lst =[],[]
for menukey,menuvalue in menu.items():
if menukey not in menukey_lst: menukey_lst.append(menukey)
for submenukey,submenuvalue in menuvalue.items():
if submenukey not in submenukey_lst: submenukey_lst.append(submenukey)
for index,mk in enumerate(menu.keys()):
for item in submenukey_lst:
menu[menukey_lst[index]]["lan1"] = language[selected_lan[0]][menukey_lst[index]]
menu[menukey_lst[index]]["lan2"] = language[selected_lan[1]][menukey_lst[index]]
menu[menukey_lst[index]]["scut"] = (scut[user[0]][mk])
print(menu)
Output
{'lbl01': {'id': 'file', 'lan1': 'கோப்பு', 'lan2': 'Hindi_File', 'scut': 'Ctrl+F'}, `'lbl02': {'id': 'accounts', 'lan1': 'கணக்கியல்', 'lan2': 'Hindi_accounts', 'scut': 'Shift+F5'}}`
This should work.
for key, value in menu.items():
for sub_key in value.keys():
if sub_key == 'lan1':
value[sub_key] = language[selected_lan[0]][key]
elif sub_key == 'lan2':
value[sub_key] = language[selected_lan[1]][key]
elif sub_key == 'scut':
value[sub_key] = scut[user[0]][key]
I receive data from the Loggly service in dot notation, but to put data back in, it must be in JSON.
Hence, I need to convert:
{'json.message.status.time':50, 'json.message.code.response':80, 'json.time':100}
Into:
{'message': {'code': {'response': 80}, 'status': {'time': 50}}, 'time': 100}
I have put together a function to do so, but I wonder if there is a more direct and simpler way to accomplish the same result.
def dot_to_json(a):
# Create root for JSON tree structure
resp = {}
for k,v in a.items():
# eliminate json. (if metric comes from another type, it will keep its root)
k = re.sub(r'\bjson.\b','',k)
if '.' in k:
# Field has a dot
r = resp
s = ''
k2 = k.split('.')
l = len(k2)
count = 0
t = {}
for f in k2:
count += 1
if f not in resp.keys():
r[f]={}
r = r[f]
if count < l:
s += "['" + f + "']"
else:
s = "resp%s" % s
t = eval(s)
# Assign value to the last branch
t[f] = v
else:
r2 = resp
if k not in resp.keys():
r2[k] = {}
r2[k] = v
return resp
You can turn the path into dictionary access with:
def dot_to_json(a):
output = {}
for key, value in a.iteritems():
path = key.split('.')
if path[0] == 'json':
path = path[1:]
target = reduce(lambda d, k: d.setdefault(k, {}), path[:-1], output)
target[path[-1]] = value
return output
This takes the key as a path, ignoring the first json part. With reduce() you can walk the elements of path (except for the last one) and fetch the nested dictionary with it.
Essentially you start at output and for each element in path fetch the value and use that value as the input for the next iteration. Here dict.setdefault() is used to default to a new empty dictionary each time a key doesn't yet exist. For a path ['foo', 'bar', 'baz'] this comes down to the call output.setdefault('foo', {}).setdefault('bar', {}).setdefault('baz', {}), only more compact and supporting arbitrary length paths.
The innermost dictionary is then used to set the value with the last element of the path as the key.
Demo:
>>> def dot_to_json(a):
... output = {}
... for key, value in a.iteritems():
... path = key.split('.')[1:] # ignore the json. prefix
... target = reduce(lambda d, k: d.setdefault(k, {}), path[:-1], output)
... target[path[-1]] = value
... return output
...
>>> dot_to_json({'json.message.status.time':50, 'json.message.code.response':80, 'json.time':100}))
{'message': {'status': {'time': 50}, 'code': {'response': 80}}, 'time': 100}
I am trying to create a nested dictionary from a mysql query but I am getting a key error
result = {}
for i, q in enumerate(query):
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email
error
KeyError: 'data'
desired result
result = {
'data': {
0: {'firstName': ''...}
1: {'firstName': ''...}
2: {'firstName': ''...}
}
}
You wanted to create a nested dictionary
result = {} will create an assignment for a flat dictionary, whose items can have any values like "string", "int", "list" or "dict"
For this flat assignment
python knows what to do for result["first"]
If you want "first" also to be another dictionary you need to tell Python by an assingment
result['first'] = {}.
otherwise, Python raises "KeyError"
I think you are looking for this :)
>>> from collections import defaultdict
>>> mydict = lambda: defaultdict(mydict)
>>> result = mydict()
>>> result['Python']['rules']['the world'] = "Yes I Agree"
>>> result['Python']['rules']['the world']
'Yes I Agree'
result = {}
result['data'] = {}
for i, q in enumerate(query):
result['data']['i'] = {}
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email
Alternatively, you can use you own class which adds the extra dicts automatically
class AutoDict(dict):
def __missing__(self, k):
self[k] = AutoDict()
return self[k]
result = AutoDict()
for i, q in enumerate(query):
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email
result['data'] does exist. So you cannot add data to it.
Try this out at the start:
result = {'data': []};
You have to create the key data first:
result = {}
result['data'] = {}
for i, q in enumerate(query):
result['data'][i] = {}
result['data'][i]['firstName'] = q.first_name
result['data'][i]['lastName'] = q.last_name
result['data'][i]['email'] = q.email
I have a question related to python code.
I need to aggregate if the key = kv1, how can I do that?
input='num=123-456-7890&kv=1&kv2=12&kv3=0'
result={}
for pair in input.split('&'):
(key,value) = pair.split('=')
if key in 'kv1':
print value
result[key] += int(value)
print result['kv1']
Thanks a lot!!
I'm assuming you meant key == 'kv1' and also the kv within input was meant to be kv1 and that result is an empty dict that doesn't need result[key] += int(value) just result[key] = int(value)
input = 'num=123-456-7890&kv1=1&kv2=12&kv3=0'
keys = {k: v for k, v in [i.split('=') for i in input.split('&')]}
print keys # {'num': '123-456-7890', 'kv2': '12', 'kv1': '1', 'kv3': '0'}
result = {}
for key, value in keys.items():
if key == 'kv1':
# if you need to increase result['kv1']
_value = result[key] + int(value) if key in result else int(value)
result[key] = _value
# if you need to set result['kv1']
result[key] = int(value)
print result # {'kv1': 1}
Assuming you have multiple lines with data like:
num=123-456-7890&kv1=2&kv2=12&kv3=0
num=123-456-7891&kv1=1&kv2=12&kv3=0
num=123-456-7892&kv1=4&kv2=12&kv3=0
Reading line-by-line in a file:
def get_key(data, key):
keys = {k: v for k, v in [i.split('=') for i in data.split('&')]}
for k, v in keys.items():
if k == key: return int(v)
return None
results = []
for line in [line.strip() for line in open('filename', 'r')]:
value = get_key(line, 'kv1')
if value:
results.append({'kv1': value})
print results # could be [{'kv1': 2}, {'kv1': 1}, {'kv1': 4}]
Or just one string:
with open('filename', 'r') as f: data = f.read()
keys = {k: v for k, v in [i.split('=') for i in data.split('&')]}
result = {}
for key, value in keys.items():
if key == 'kv1':
result[key] = int(value)
Console i/o:
c:\nathan\python\bnutils>python
Python 2.7.5 (default, May 15 2013, 22:44:16) [MSC v.1500 64 bit (AMD64)] on win32
Type "help", "copyright", "credits" or "license" for more information.
>>> def get_key(data, key):
... keys = {k: v for k, v in [i.split('=') for i in data.split('&')]}
... for k, v in keys.items():
... if k == key: return int(v)
... return None
...
>>> results = []
>>> for line in [line.strip() for line in open('test.txt', 'r')]:
... value = get_key(line, 'kv1')
... if value:
... results.append({'kv1': value})
...
>>> print results
[{'kv1': 2}, {'kv1': 1}, {'kv1': 4}]
>>>
test.txt:
num=123-456-7890&kv1=2&kv2=12&kv3=0
num=123-456-7891&kv1=1&kv2=12&kv3=0
num=123-456-7892&kv1=4&kv2=12&kv3=0
import urlparse
urlparse.parse_qs(input)
results in: {'num': ['123-456-7890'], 'kv2': ['12'], 'kv': ['1'], 'kv3': ['0']}
The keys are aggregated for you.
You could do it this way, so basically just add an extra if else block dealing with the empty case for key
input='num=123-456-7890&kv=1&kv2=12&kv3=0'
result={}
for pair in input.split('&'):
temp = pair.split('=')
key = temp[0]
value = [1]
if key in 'kv1':
if key in p:
print value //do you really want to output this?
result[key] += int(value)
else:
print value //do you really want to output this?
result[key] = int(value)
print result['kv1']