I have a dictionary returned by a response in AWS Lambdas which looks like below:
data = {
"id": 1,
"name": "Hey",
"category": null,
"salary": 1222,
"Roles": [
{
"attributes1": {
"key": "sum_assured",
"value": 192000
},
"attributes2": {
"key": "cash_value",
"value": 48576
},
"attributes3": {
"key": "expiry_date",
"value": null
}
}
]
}
Now, I want to format a few numeric columns like salary to 1222.00 and convert other non-string columns to strings for "ID" I should get "1" as a string and not an Int. Here, null is the actual NaN returned by the response.
Below is the code which I tried:
new_dict = {}
for i, j in data.items():
print(i)
if (i == "salary"):
new_dict[i] = "%0.2f" % j
if isinstance(j, str):
new_dict[i] = j
elif j is None:
new_dict[i] = ""
else:
new_dict[i] = str(j)
I know that I could maintain a list for such amount related columns and then I can do some kind of comparisons and replace it. When I test the value for salary inside the loop it gives me decimal places but in the response, it doesn't show up.
Output:
{
"id": "1",
"name": "Hey",
"category": "",
"salary": "1222",
"Roles": "[{'attributes4': {'key': 'sum_assured', 'value': 192000}, 'attributes5': {'key': 'cash_value', 'value': 48576}, 'attributes6': {'key': 'expiry_date', 'value': 'null'}}]"
}
Expected:
{
"id": "1",
"name": "Hey",
"category": "",
"salary": "1222",
"Roles": "[{"attributes4": {"key": "sum_assured", "value": "192000.00"}, "attributes5": {"key": "cash_value", "value": "48576.00"}, "attributes6": {"key": "expiry_date", "value": ""}}]"
}
Also, how could I format the records of the Roles array with some kind of indent in it? I wasn't able to do that.
Update 1:
I tried the below code for accessing the nested attributes as per my JSON but it gives me a key error:
for i, j in formatted_json.items():
print(type(j))
if isinstance(j, list):
for k in j:
for key, value in k.items():
for l, m in value.items():
if (m == "salary" or m == 'value'):
new_dict[key][l] = "%0.2f" % m
elif isinstance(m, str):
new_dict[key][l] = m
elif m is None:
new_dict[key][l] = ""
else:
new_dict[key][l] = str(m)
elif (i == "salary" or i == 'value'):
new_dict[i] = "%0.2f" % j
elif isinstance(j, str):
new_dict[i] = j
elif j is None:
new_dict[i] = ""
else:
new_dict[i] = str(j)
Traceback (most recent call last):
File "/home/.../Downloads/pyspark/src/sample/json_to_string.py", line 392, in <module>
new_dict[key][l] = m
KeyError: 'attributes4'
Update 2:
The actual dataset is as below:
json_data = '''{
"policy_no": "",
"paid_to_date": "2016-09-28",
"proposal_no": null,
"agent_code": "1234",
"policy_status": "Death",
"policy_term": "15",
"policy_premium_payment_term": "7",
"policy_effective_date": "2013-09-28",
"last_premium_paid": 13790.85,
"last_premium_paid_date": "2016-06-17",
"pivc_flag_status": null,
"product_code": "",
"product_name": "Monthly Income Plan",
"application_date": "2013-09-16",
"underwriting_decesion_date": "2013-09-26",
"issuance_date": "2013-09-30",
"ols_policy": "N",
"next_premium_due_date": "2016-09-28",
"policy_year": null,
"premium_holiday_from_date": null,
"premium_holiday_to_date": null,
"product_type": "Non-Linked",
"proposer_client_code": "",
"life_insured_client_code": "",
"ppt_completion_date": "2020-09-28",
"ppt_left": 4,
"auto_termination_date": null,
"fund_value": null,
"loan_amt": 0,
"welcome_kit_current_status": null,
"dispatch_date": "2013-10-25",
"status_date": "2013-10-25",
"updated_on": "2013-10-26 19:51:51.983",
"surrender_value": null,
"loan_eligiability": null,
"courier_name": null,
"awb_no": "",
"status_id": 5,
"claim_registration_date": "2021-12-15",
"claim_approval_reject_date": "2021-12-15",
"claim_requirement_document": "HEART ATTACK",
"claim_requirement_status": "P",
"requirement_raised_on": "2021-12-15",
"requirement_closed_on": "2021-12-15",
"claim_status": "P",
"status_for_death_claims": "P",
"date_of_transaction": "2021-12-15",
"ecs_si_registration_date": null,
"ecs_si_status": "inactive",
"ecs_si_failure_reasons": null,
"ecs_si_status_as_per_rls": null,
"ecs_day": "0",
"payment_mode": "S",
"payment_mode_desc": "Semi Anually",
"payment_method": "S",
"payment_method_desc": "Cash/Cheque",
"payment_date": "2021-12-15",
"payment_amount": 13790.85,
"premium_base_amt": null,
"premium_tax_amt": null,
"due_premium": null,
"total_premium": 13790.85,
"advance_premium": null,
"assignee": [
{
"assignee_name": "",
"attributes1": {
"key": "assignee_client_id",
"value": ""
},
"attributes2": {
"key": "assignee_type",
"value": ""
}
}
],
"agents": [
{
"attributes1": {
"key": "policy_no",
"value": ""
},
"agent_name": "",
"attributes2": {
"key": "agent_code",
"value": "1234"
},
"attributes3": {
"key": "email",
"value": ""
},
"attributes4": {
"key": "channel",
"value": "AGENCY"
},
"attributes5": {
"key": "subchannel",
"value": null
},
"attributes6": {
"key": "branch_name",
"value": ""
},
"attributes7": {
"key": "contact_no",
"value": ""
},
"attributes8": {
"key": "status",
"value": "T"
}
}
],
"bank_accounts": {
"attributes1": {
"key": "accountnumber",
"value": ""
},
"attributes2": {
"key": "accountholdername",
"value": ""
},
"attributes3": {
"key": "ifsccode",
"value": ""
},
"attributes4": {
"key": "micrcode",
"value": ""
},
"bankname": "",
"attributes5": {
"key": "branchname",
"value": ""
},
"attributes6": {
"key": "bank_address",
"value": ""
}
},
"Funds": [
{
"attributes1": {
"key": "policy_no",
"value": ""
},
"attributes2": {
"key": "allocation_mode",
"value": "percentage"
},
"attributes3": {
"key": "allocation_value",
"value": null
},
"attributes4": {
"key": "fund_code",
"value": null
},
"fund_name": null,
"investment_strategy_name": null
}
],
"Investment_Strategies": [
{
"attributes1": {
"key": "policy_no",
"value": ""
},
"attributes2": {
"key": "allocation_mode",
"value": "percentage"
},
"attributes3": {
"key": "allocation_value",
"value": null
},
"attributes4": {
"key": "fund_code",
"value": null
},
"fund_name": null,
"investment_strategy_name": null
}
],
"Roles": [
{
"attributes1": {
"key": "policy_no",
"value": ""
},
"attributes2": {
"key": "beneficiary_percentage",
"value": "100.00"
},
"attributes3": {
"key": "relationship",
"value": "SPOUSE"
},
"role_name": "Nominee",
"attributes4": {
"key": "age",
"value": "NaN"
},
"attributes5": {
"key": "party_name",
"value": ""
},
"attributes6": {
"key": "gender",
"value": "F"
},
"attributes7": {
"key": "date_of_birth",
"value": "1953-03-20"
},
"attributes8": {
"key": "email",
"value": null
},
"attributes9": {
"key": "registred_mobile",
"value": null
}
},
{
"attributes1": {
"key": "policy_no",
"value": ""
},
"attributes2": {
"key": "beneficiary_percentage",
"value": null
},
"attributes3": {
"key": "relationship",
"value": null
},
"role_name": "Insured",
"attributes4": {
"key": "age",
"value": 63
},
"attributes5": {
"key": "party_name",
"value": ""
},
"attributes6": {
"key": "gender",
"value": "M"
},
"attributes7": {
"key": "date_of_birth",
"value": "1950-06-20"
},
"attributes8": {
"key": "email",
"value": null
},
"attributes9": {
"key": "registred_mobile",
"value": null
}
},
{
"attributes1": {
"key": "policy_no",
"value": ""
},
"attributes2": {
"key": "beneficiary_percentage",
"value": null
},
"attributes3": {
"key": "relationship",
"value": null
},
"role_name": "Owner",
"attributes4": {
"key": "age",
"value": 63
},
"attributes5": {
"key": "party_name",
"value": ""
},
"attributes6": {
"key": "gender",
"value": "M"
},
"attributes7": {
"key": "date_of_birth",
"value": "1950-06-20"
},
"attributes8": {
"key": "email",
"value": ""
},
"attributes9": {
"key": "registred_mobile",
"value": ""
}
}
],
"Benefits": [
{
"attributes1": {
"key": "policy_no",
"value": ""
},
"benefit_name": "Miii",
"attributes2": {
"key": "benefit_term",
"value": "7"
},
"attributes3": {
"key": "modal_premium",
"value": null
},
"attributes4": {
"key": "sum_assured",
"value": 192000
},
"attributes5": {
"key": "cash_value",
"value": 48576
},
"attributes6": {
"key": "expiry_date",
"value": null
},
"attributes7": {
"key": "status",
"value": "Death"
},
"attributes8": {
"key": "benefit_code",
"value": ""
},
"attributes9": {
"key": "benefit_type",
"value": "base"
}
}
]
}
'''
When you have a strong model in your data, and you want to process/validate the data you receive, its better to be done in a object oriented fashion. If you can consider your data value as an object with all logic written inside for validating or correcting, it becomes simple.
See an example approach below:
(Assumed your null as None)
import json
# Your data object..
data = '''{
"id": 1,
"name": "Hey",
"category": null,
"salary": 1222,
"Roles": [
{
"attributes1": {
"key": "sum_assured",
"value": 192000
},
"attributes2": {
"key": "cash_value",
"value": 48576
},
"attributes3": {
"key": "expiry_date",
"value": null
}
}
]
}'''
# Create simple object models to represent the data.
class Role:
def __init__(self, key, value):
self.key = key
self.value = value or "" # fallback to empty string
class Policy:
def __init__(self, id, name, category, salary, Roles):
# write all your pre/post processing here
self.id = id
self.name = name
self.category = category or ""
self.salary = format(salary, '0.2f') if salary else ""
self.Roles = [Role(**role) for role in Roles[0].values()] if Roles else []
# a json serialization helper function
def serial_default(obj):
if isinstance(obj, (Policy, Role)):
return obj.__dict__
else:
return obj
# You just create the object with your data. Now all data is in the object.
data = json.loads(data)
p = Policy(**data)
# Serialize the object to json in the proper format
print(json.dumps(p, default=serial_default))
The print generates a valid version of your object as JSON like
{"id": 1, "name": "Hey", "category": "", "salary": "1222.00", "Roles": [{"key": "sum_assured", "value": 192000}, {"key": "cash_value", "value": 48576}, {"key": "expiry_date", "value": ""}]}
This is just an example to the approach. You can modify or improvise a similar approach and make things simpler.
Firstly i have to agree with Kris answer, this kind of conversion looks now a bit ugly and with every added special case Object-Oriented-style makes more sense.
A few points:
null, None, ""
It seems that null won't be seen as None, so your not getting into the if clause.
Check JSON to Dict for that
How can JSON data with null value be converted to a dictionary
Salary
According to your question you would expect
"salary": "1222.00"
correct?
If so you are currently overwriting your previously formatted number with
if isinstance(j, str):
new_dict[i] = j
Just change it to elif so you wont overwrite your conversion.
Roles
The objects within the array of Roles are currently not iterated over. Hence the number are not being formatted. For that you would need another loop on the Roles array.
Following a recursion attempt to cover multiple nested collections.
import json
from collections import abc
dataRaw = '''
{
"id": 1,
"name": "Hey",
"category": null,
"salary": 1222,
"Roles": [
{
"attributes1": {
"key": "sum_assured",
"value": 192000
},
"attributes2": {
"key": "cash_value",
"value": 48576
},
"attributes3": {
"key": "expiry_date",
"value": null
}
}
]
}
'''
data = json.loads(dataRaw)
def processNested(nested):
# Check for list
if (isinstance(nested, list)):
# for each element in list processNested
for element in nested:
element = processNested(element)
else:
for key, value in nested.items():
# If nested Collection
if isinstance(value, abc.Mapping):
# Allow special "treatment" for {"key": "...", "value": ... } like objects
if "key" in value and "value" in value:
if ((value["key"] == "cash_value" or value["key"] == "sum_assured") and value["value"] != "" and
value["value"] != None):
nested[key]["value"] = str("%0.2f" % int(value["value"]))
else:
nested[key] = processNested(value)
else:
if (key == "value" and value != None and value != ""):
nested[key] = str(nested["value"])
return nested
new_dict = data
for i, j in new_dict.items():
if (i == "salary"):
new_dict[i] = "%0.2f" % j
elif isinstance(j, str):
new_dict[i] = j
elif j is None:
new_dict[i] = ""
elif i == "Roles":
new_dict[i] = processNested(j)
else:
new_dict[i] = str(j)
print(new_dict)
I have this sample.json file with me:
{
"details":[
{
"name": "",
"class": "4",
"marks": "72.6"
},
{
"name": "David",
"class": "",
"marks": "78.2"
},
{
"name": "Emily",
"class": "4",
"marks": ""
}
]
}
As you can see for the first one; "name" is string datatype is actually empty.
For the second one; "class" with integer datatype is empty.
And for the third one; "marks" with float datatype is empty.
Now my task is;
to find the fields which are empty, if string is empty replace it with "BLANK", if integer is empty replace it with 0, and if float is empty replace it with 0.0
P.S: I'm doing this with Python like this:
import json
path = open('D:\github repo\python\sample.json')
df = json.load(path)
for i in df["details"]:
print(i["name"])
Also make sure that I don't want to hard-code the values. Coz here if we see there are only 3 fields(name, class, marks) but what if I have more that 3. Then what? How will I find which fields are empty or not?
Like you see here:
{
"code": "AAA",
"lat": "-17.3595",
"lon": "-145.494",
"name": "Anaa Airport",
"city": "Anaa",
"state": "Tuamotu-Gambier",
"country": "French Polynesia",
"woeid": "12512819",
"tz": "Pacific\/Midway",
"phone": "",
"type": "Airports",
"email": "",
"url": "",
"runway_length": "4921",
"elev": "7",
"icao": "NTGA",
"direct_flights": "2",
"carriers": "1"
},
This is just one block, I've N-number of blocks like this. That's why I can't hard_code the values right?
Can anybody help me with it!
Thank You so much!
Since the type info isn't available anywhere programmatically, and there seem to be only three hard-coded fields, I'd just check each of them explicitly.
Short-circuiting with the or operator would even allow you to achieve this fairly elegantly:
for d in df['details']:
d['name'] = d['name'] or 'BLANK'
d['class'] = d['class'] or '0'
d['marks'] = d['marks'] or '0.0'
You could check whether the string is empty with a simple if statement like so.
if not i['name'] == ""
Alternatively, you could also do
if not i['name']
The second if statement makes use of falsy and truthy values in Python. Here's a link to read more about it
You could create a dictionary empty_replacements mapping each key to its corresponding desired empty value:
import json
sample_json = {
"details": [
{
"name": "",
"class": "4",
"marks": "72.6"
},
{
"name": "David",
"class": "",
"marks": "78.2"
},
{
"name": "Emily",
"class": "4",
"marks": ""
}
]
}
empty_replacements = {"name": "BLANK", "class": "0", "marks": "0.0"}
sample_json["details"] = [{
k: v if v else empty_replacements[k]
for k, v in d.items()
} for d in sample_json["details"]]
print('sample_json after replacements: ')
print(json.dumps(
sample_json,
sort_keys=False,
indent=4,
))
Output:
sample_json after replacements:
{
"details": [
{
"name": "BLANK",
"class": "4",
"marks": "72.6"
},
{
"name": "David",
"class": "0",
"marks": "78.2"
},
{
"name": "Emily",
"class": "4",
"marks": "0.0"
}
]
}
I 'm assuming by the dictionary which you provided that marks & class are stored as String.
li=[]
for d in df["details"]:
for k,v in d.items():
if (v==''):
if (k=='name'):
d[k]="BLANK"
elif (k=='class') :
d[k]='0'
elif (k=='marks'):
d[k]='0.0'
li.append(d)
df['details']=li
How can I call a specific list with items in a dictionary? I want to use the key name, and output Richmond, in the station list. A tutorial I was using is outdated, so the best I could manage was this loop that printed the keys and items:
for key, value in data.items():
print(key, value)
That revealed the two outermost keys (?xml and root), but there are nested dictionaries I would like to access.
What I would like:
for item in data['station']
print(item['name'])
>>> Richmond
Instead I get KeyError with station. Seeing as how ?xml and root are the keys identified with the loop, I'm presuming I need a nested loop, first going through the root key, and then accessing station, and then using the key name to print Richmond.
The API result:
{
"?xml": {
"#version": "1.0",
"#encoding": "utf-8"
},
"root": {
"#id": "1",
"uri": {
"#cdata-section": "http://api.bart.gov/api/etd.aspx?cmd=etd&orig=RICH&json=y"
},
"date": "10/14/2017",
"time": "07:50:17 PM PDT",
"station": [{
"name": "Richmond",
"abbr": "RICH",
"etd": [{
"destination": "Warm Springs",
"abbreviation": "WARM",
"limited": "0",
"estimate": [{
"minutes": "4",
"platform": "2",
"direction": "South",
"length": "6",
"color": "ORANGE",
"hexcolor": "#ff9933",
"bikeflag": "1",
"delay": "0"
}, {
"minutes": "24",
"platform": "2",
"direction": "South",
"length": "6",
"color": "ORANGE",
"hexcolor": "#ff9933",
"bikeflag": "1",
"delay": "0"
}]
}]
}],
"message": ""
}
}
I've been trying to figure out how I can iterate over a json like object, so I could get a user id by its name.
json
{
"ApiSearchResult": [
{
"totalNumberResults": 55,
"type": "User",
"searchResults": [
{
"firstName": "shashank",
"name": "0o_shashank._o0",
"uid": 81097836
},
{
"firstName": "Shahnawaz",
"name": "0shahnawaz.0",
"uid": 83697589
},
{
"firstName": "Ashu",
"name": "ashu.-3",
"uid": 83646061
},
{
"bgImage": "photoalbum_491396460_user82597906-1-jpeg.jpg",
"firstName": "Garfield",
"name": "beast.boy",
"uid": 82597906
},
{
"firstName": "Bharath",
"name": "bharath_mohan69",
"uid": 80197615
},
{
"bgImage": "photoalbum_481041410_user79819261-1-jpg.jpg",
"firstName": "Wille-ICE",
"name": "blowhole",
"uid": 79819261
}
]
}
]
}
Python
def getidbyname(name):
event = response['ApiSearchResult'][0]['searchResults'][0]
for key, value in event.iteritems():
if value == name: continue
elif key == "uid":
return value
But, this won't work, I've never really worked with this many nested elements.
def getidbyname(name):
for i in data['ApiSearchResult'][0]['searchResults']:
if i['name'] == name:
return i['uid']
This might work if your response is already a python dictionary:
def getidbyname(name):
for event in data["ApiSearchResult"][0]["searchResults"]:
if event["name"] == name:
return event["uid"]
If your input is a text value, you need to use json.loads(response) to get a python dictionary out of it.
I am writing a class to take a Chrome bookmarks file (see example below):
{
"checksum": "452bebcad611a3faffb2c009099139e5",
"roots": {
"bookmark_bar": {
"children": [ {
"date_added": "13028719861473329",
"id": "4",
"name": "first bookmark",
"type": "url",
"url": "chrome://newtab/"
}, {
"children": [ {
"children": [ {
"date_added": "13026904508000000",
"id": "7",
"name": "Getting Started",
"type": "url",
"url": "https://www.mozilla.org/en-GB/firefox/central/"
} ],
"date_added": "13028740260032410",
"date_modified": "0",
"id": "6",
"name": "Bookmarks Toolbar",
"type": "folder"
}, {
"children": [ {
"date_added": "13026904508000000",
"id": "9",
"name": "Help and Tutorials",
"type": "url",
"url": "https://www.mozilla.org/en-GB/firefox/help/"
}, {
"date_added": "13026904508000000",
"id": "10",
"name": "Customise Firefox",
"type": "url",
"url": "https://www.mozilla.org/en-GB/firefox/customize/"
}, {
"date_added": "13026904508000000",
"id": "11",
"name": "Get Involved",
"type": "url",
"url": "https://www.mozilla.org/en-GB/contribute/"
}, {
"date_added": "13026904508000000",
"id": "12",
"name": "About Us",
"type": "url",
"url": "https://www.mozilla.org/en-GB/about/"
} ],
"date_added": "13028740260032410",
"date_modified": "0",
"id": "8",
"name": "Mozilla Firefox",
"type": "folder"
}, {
"date_added": "13026904551000000",
"id": "13",
"name": "Welcome to Firefox",
"type": "url",
"url": "http://www.mozilla.org/en-US/firefox/24.0/firstrun/"
} ],
"date_added": "13028740260004410",
"date_modified": "0",
"id": "5",
"name": "Imported From Firefox",
"type": "folder"
} ],
"date_added": "13028719626916276",
"date_modified": "13028719861473329",
"id": "1",
"name": "Bookmarks bar",
"type": "folder"
},
"other": {
"children": [ ],
"date_added": "13028719626916276",
"date_modified": "0",
"id": "2",
"name": "Other bookmarks",
"type": "folder"
},
"synced": {
"children": [ ],
"date_added": "13028719626916276",
"date_modified": "0",
"id": "3",
"name": "Mobile bookmarks",
"type": "folder"
}
},
"version": 1
}
I convert from JSON to a nested dictionary then extract the bookmark urls under each relevant bookmark folder in my write_data method.
As there can be any number of bookmark folders and/or bookmarks nested within each folder, I want to call the write_data method within itself so that it keeps on extracting child data every time it finds a nested folder. I just can't work out how to pass the relevant child dictionaries into the same method.
I've tried building up the dictionary path with a string. I think I need to pass in a tuple or list of keys to loop through and dynamically build up the path but I can't get it working and my poor head is wrecked!
There is a similar question but the answer uses yield which has confused me totally and was not a totally working solution anyway. Please help!
import json
import sys
import codecs
class FileExtractor(object):
def __init__(self, input_file):
self.infile = codecs.open(input_file, encoding='utf-8')
self.bookmark_data = json.load(self.infile)
def write_data(self, my_key):
for key, value in self.bookmark_data[my_key].iteritems():
if type(self.bookmark_data[my_key][key]) is dict:
print self.bookmark_data[my_key][key]['name']
for subkey, subvalue in self.bookmark_data[my_key][key].iteritems():
if subkey == "children" and len(self.bookmark_data[my_key][key][subkey]) <> 0:
print "this is a child. I can't figure out how to use write_data with this"
#self.write_data('[my_key][key][subkey]')
else:
print subkey, ": ", self.bookmark_data[my_key][key][subkey]
if(__name__=="__main__"):
stuff= FileExtractor(sys.argv[1])
stuff.write_data(('roots'))
Not sure if this is what you are getting at but if you pass in the dictionary object itself to write_data rather than the key you can recurse as far down as the dictionary goes. WARNING I haven't tested this, it's just to give you an idea.
def write_data(self, my_dict=None):
my_dict = my_dict or self.bookmark_data['roots']
for key, value in my_dict.items():
if type(my_dict[key]) is dict:
print my_dict[key]['name']
for subkey, subvalue in my_dict[key].items():
if subkey == "children" and len(my_dict[key][subkey]) <> 0:
for child in my_dict[key][subkey]:
self.write_data(child)
else:
print subkey, ": ", my_dict[key][subkey]
better version:
def write_data(self, my_dict=None):
my_dict = my_dict or self.bookmark_data['roots']
if 'name' in my_dict:
print my_dict['name']
for key, value in my_dict.items():
if type(my_dict[key]) is dict:
self.write_data(my_dict[key])
elif type(my_dict[key]) is list:
for item in my_dict[key]:
self.write_data(item)
else:
print key, ": ", my_dict[key]