We Keep Coding

Expecting property name enclosed in double quotes: line 1 column 2 (char 1). Getting this error even after all the properties are in double quotes

I am getting the json decode error when I try to hit the following json file for a post request. Please help
API Code:
import requests
from utilities.configurations import *
from utilities.resources import *
#from PayLoads.plans.payload1A1C import *
from PayLoads.plans.testpayload import *
config = configparser.ConfigParser()
config.read('utilities/properties.ini')
url = getCOnfigurations()['API']['URL']+APIResources.endpoints_plans
response = requests.post(url, json= plan1A1C(),)
from behave.formatter import null
import json
def plan1A1C():
body = {
"payload": {
"data": {
"organization_code": "TEST",
"quote_id": 71728,
"quote_token": "odQ53i3SxCANUR68Lry3",
"session_token": "odQ53i3SxCANUR68Lry3",
"partner_code": "TEST",
"category_code": "HEALTH_RETAIL",
"insured": [
{
"person_id": 45820,
"name": null,
"occupation": null,
"annual_income": null,
"phone_number": "8789538796",
"email": null,
"age": 40,
"dob": "07-09-1981",
"gender": null,
"nominee_name": " ",
"nominee_dob": null,
"nominee_age": null,
"nominee_gender": null,
"appointee_name": " ",
"appointee_dob": null,
"relationship_with_appointee": null,
"appointee_gender": null,
"relationship_with_nominee": null,
"relation_with_primary_member": "self",
"height": null,
"weight": null,
"salutation": null,
"marital_status": null
},
{
"person_id": 45821,
"name": null,
"occupation": null,
"annual_income": null,
"phone_number": "8789538796",
"email": null,
"age": 10,
"dob": "07-09-2011",
"gender": null,
"nominee_name": " ",
"nominee_dob": null,
"nominee_age": null,
"nominee_gender": null,
"appointee_name": " ",
"appointee_dob": null,
"relationship_with_appointee": null,
"appointee_gender": null,
"relationship_with_nominee": null,
"relation_with_primary_member": "kid-1",
"height": null,
"weight": null,
"salutation": null,
"marital_status": null
}
],
"primary_member_age": 40,
"sum_insured": 100000,
"pin_code": "751008",
"payment_frequency": "SINGLE",
"payment_option": null,
"limited_pay": null,
"policy_paying_term": null,
"tobacco_habit": null,
"risk_profile": null,
"maturity_benefit_type": null,
"investment_goal": null,
"investment_amount": null,
"cover_list": null,
"insurance_cover_list": null,
"posp": {
"name": "abcd",
"user_id": "9991112223",
"pan": null,
"aadhar": null,
"level": {
"id": 41,
"partner_id": 52,
"role": "ANAND_RATHI_staff",
"code": "ANAND_RATHI_POSP",
"superior_role_id": null,
"path": "//41",
"created_at": "2021-06-11T17:48:01.000+05:30",
"updated_at": "2021-06-12T16:42:40.000+05:30"
},
"staf_type": "POSP"
},
"product_code": "TEST",
"insurance_code": "TEST",
"tenure": 1
},
"config": {
"coi_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PolicySchedule",
"password": "in5uR#nceAnan0",
"plan_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PremiumComputation",
"username": "ptrn_anandinsurance",
"token_url": "https://developer.TEST.com/endpoint/token",
"grant_type": "password",
"policy_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PolicyGeneration",
"proposal_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/ProposalSave",
"PaymentOption": "A",
"header_password": "TEST",
"header_username": "TEST",
"IntermediaryCode": "2002954741760001",
"UniqueTransactionID": "8765456711111111112232"
}
},
"insurance_company_code": "TEST",
"ic": "TEST",
"insurance_code": "TEST",
"product_code": "TEST",
"category_code": "TEST",
"callbackUrl": "https://api.iifl.TEST.com/api_gateway_callbacks/plans"
}
s1 = json.loads(str(body))
return s1
For the above code, I am getting the follpowing error as "json.decoder.JSONDecodeError: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)". I have also checked that the property values in the json are inb double quotes only

Try the below (note that body str is defined as a string)
import json
def foo():
body_str = '''{
"payload": {
"data": {
"organization_code": "TEST",
"quote_id": 71728,
"quote_token": "odQ53i3SxCANUR68Lry3",
"session_token": "odQ53i3SxCANUR68Lry3",
"partner_code": "TEST",
"category_code": "HEALTH_RETAIL",
"insured": [
{
"person_id": 45820,
"name": null,
"occupation": null,
"annual_income": null,
"phone_number": "8789538796",
"email": null,
"age": 40,
"dob": "07-09-1981",
"gender": null,
"nominee_name": " ",
"nominee_dob": null,
"nominee_age": null,
"nominee_gender": null,
"appointee_name": " ",
"appointee_dob": null,
"relationship_with_appointee": null,
"appointee_gender": null,
"relationship_with_nominee": null,
"relation_with_primary_member": "self",
"height": null,
"weight": null,
"salutation": null,
"marital_status": null
},
{
"person_id": 45821,
"name": null,
"occupation": null,
"annual_income": null,
"phone_number": "8789538796",
"email": null,
"age": 10,
"dob": "07-09-2011",
"gender": null,
"nominee_name": " ",
"nominee_dob": null,
"nominee_age": null,
"nominee_gender": null,
"appointee_name": " ",
"appointee_dob": null,
"relationship_with_appointee": null,
"appointee_gender": null,
"relationship_with_nominee": null,
"relation_with_primary_member": "kid-1",
"height": null,
"weight": null,
"salutation": null,
"marital_status": null
}
],
"primary_member_age": 40,
"sum_insured": 100000,
"pin_code": "751008",
"payment_frequency": "SINGLE",
"payment_option": null,
"limited_pay": null,
"policy_paying_term": null,
"tobacco_habit": null,
"risk_profile": null,
"maturity_benefit_type": null,
"investment_goal": null,
"investment_amount": null,
"cover_list": null,
"insurance_cover_list": null,
"posp": {
"name": "abcd",
"user_id": "9991112223",
"pan": null,
"aadhar": null,
"level": {
"id": 41,
"partner_id": 52,
"role": "ANAND_RATHI_staff",
"code": "ANAND_RATHI_POSP",
"superior_role_id": null,
"path": "//41",
"created_at": "2021-06-11T17:48:01.000+05:30",
"updated_at": "2021-06-12T16:42:40.000+05:30"
},
"staf_type": "POSP"
},
"product_code": "TEST",
"insurance_code": "TEST",
"tenure": 1
},
"config": {
"coi_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PolicySchedule",
"password": "in5uR#nceAnan0",
"plan_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PremiumComputation",
"username": "ptrn_anandinsurance",
"token_url": "https://developer.TEST.com/endpoint/token",
"grant_type": "password",
"policy_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PolicyGeneration",
"proposal_url": "https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/ProposalSave",
"PaymentOption": "A",
"header_password": "TEST",
"header_username": "TEST",
"IntermediaryCode": "2002954741760001",
"UniqueTransactionID": "8765456711111111112232"
}
},
"insurance_company_code": "TEST",
"ic": "TEST",
"insurance_code": "TEST",
"product_code": "TEST",
"category_code": "TEST",
"callbackUrl": "https://api.iifl.TEST.com/api_gateway_callbacks/plans"
}'''
s1 = json.loads(body_str)
return s1
print(foo())
output
{'payload': {'data': {'organization_code': 'TEST', 'quote_id': 71728, 'quote_token': 'odQ53i3SxCANUR68Lry3', 'session_token': 'odQ53i3SxCANUR68Lry3', 'partner_code': 'TEST', 'category_code': 'HEALTH_RETAIL', 'insured': [{'person_id': 45820, 'name': None, 'occupation': None, 'annual_income': None, 'phone_number': '8789538796', 'email': None, 'age': 40, 'dob': '07-09-1981', 'gender': None, 'nominee_name': ' ', 'nominee_dob': None, 'nominee_age': None, 'nominee_gender': None, 'appointee_name': ' ', 'appointee_dob': None, 'relationship_with_appointee': None, 'appointee_gender': None, 'relationship_with_nominee': None, 'relation_with_primary_member': 'self', 'height': None, 'weight': None, 'salutation': None, 'marital_status': None}, {'person_id': 45821, 'name': None, 'occupation': None, 'annual_income': None, 'phone_number': '8789538796', 'email': None, 'age': 10, 'dob': '07-09-2011', 'gender': None, 'nominee_name': ' ', 'nominee_dob': None, 'nominee_age': None, 'nominee_gender': None, 'appointee_name': ' ', 'appointee_dob': None, 'relationship_with_appointee': None, 'appointee_gender': None, 'relationship_with_nominee': None, 'relation_with_primary_member': 'kid-1', 'height': None, 'weight': None, 'salutation': None, 'marital_status': None}], 'primary_member_age': 40, 'sum_insured': 100000, 'pin_code': '751008', 'payment_frequency': 'SINGLE', 'payment_option': None, 'limited_pay': None, 'policy_paying_term': None, 'tobacco_habit': None, 'risk_profile': None, 'maturity_benefit_type': None, 'investment_goal': None, 'investment_amount': None, 'cover_list': None, 'insurance_cover_list': None, 'posp': {'name': 'abcd', 'user_id': '9991112223', 'pan': None, 'aadhar': None, 'level': {'id': 41, 'partner_id': 52, 'role': 'ANAND_RATHI_staff', 'code': 'ANAND_RATHI_POSP', 'superior_role_id': None, 'path': '//41', 'created_at': '2021-06-11T17:48:01.000+05:30', 'updated_at': '2021-06-12T16:42:40.000+05:30'}, 'staf_type': 'POSP'}, 'product_code': 'TEST', 'insurance_code': 'TEST', 'tenure': 1}, 'config': {'coi_url': 'https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PolicySchedule', 'password': 'in5uR#nceAnan0', 'plan_url': 'https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PremiumComputation', 'username': 'ptrn_anandinsurance', 'token_url': 'https://developer.TEST.com/endpoint/token', 'grant_type': 'password', 'policy_url': 'https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/PolicyGeneration', 'proposal_url': 'https://developer.TEST.com/endpoint/Health-flexiretail/v1.0.0/ProposalSave', 'PaymentOption': 'A', 'header_password': 'TEST', 'header_username': 'TEST', 'IntermediaryCode': '2002954741760001', 'UniqueTransactionID': '8765456711111111112232'}}, 'insurance_company_code': 'TEST', 'ic': 'TEST', 'insurance_code': 'TEST', 'product_code': 'TEST', 'category_code': 'TEST', 'callbackUrl': 'https://api.iifl.TEST.com/api_gateway_callbacks/plans'}

The issue here seems to be the way Python converts dictionaries to strings. Python by default uses single quotes to enclose strings in dictionaries when converted to strings by str().
import json
data = {"foo":"bar"}
str(data)
#"{'foo': 'bar'}"
json.loads(str(data))
# raises json.decoder.JSONDecodeError: Expecting property name enclosed in double quotes: line 1 column 2 (char 1)
This becomes problematic for json.loads() as single quotes are not acceptable JSON format. Also refer to this SO question: Python/Json:Expecting property name enclosed in double quotes
Instead, you should use json.dumps(), which serialises Python objects into JSON formatted string:
import json
data = {"foo":"bar"}
json.dumps(data)
#'{"foo": "bar"}'
json.loads(json.dumps(data))
#{'foo': 'bar'}

Retrieving data from a json object

I am writing a parser to extract the list of ads
response = requests.get(url).json()
items = response['data']
iter1 = []
for item in items:
iter1.append({
'name': item.get('name', 'NA'),
'owner': item.get('owner', 'NA'),
'date_published': item.get('date_published', 'NA'),
'images': item.get('images', 'NA'),
'short_url': item.get('short_url', 'NA')
})
At the moment, I get the following output. I need to make my conclusion shorter.
[
{
"name": "Announcement name",
"owner": {
"id": "58f84949700743"
},
"date_published": 1627666233,
"images": [
{
"id": "58fb7032ca5544fb5a2",
"num": 1,
"url": "https://cache3.com/images/orig/58/fb/58fb70f2132a554804fb5a2.jpg",
"width": 1936,
"height": 2581
},
{
"id": "58fb70f29e94ba0384507554",
"num": 2,
"url": "https://cache3.com/images/orig/58/fb/58fb70f29e94b384507554.jpg",
"width": 750,
"height": 1334
},
{
"id": "58fb70f2f8efdc109d76c2e5",
"num": 3,
"url": "https://cache3.com/images/orig/58/fb/58fb70f2fdc109d76c2e5.jpg",
"width": 750,
"height": 1334
}
],
"short_url": "https://short.com/p58gb7b9a4c80320f03"
}
]
I would like to bring to the form:
"name": "Announcement name", #Name
"id": "58f84949700743" #Owner ID
"date_published": 1627666233, #Date
"url": "https://cache3.com/images/orig/58/fb/58fb70f2132a554804fb5a2.jpg",#Url-img
"short_url": "https://short.com/p58gb7b9a4c80320f03" #Announcement url
How can I extract information from owner{.id} and images[.url] ?

dict=[ { "name": "Announcement name", "owner": { "id": "58f84949700743" }, "date_published": 1627666233, "images": [ { "id": "58fb7032ca5544fb5a2", "num": 1, "url": "https://cache3.com/images/orig/58/fb/58fb70f2132a554804fb5a2.jpg", "width": 1936, "height": 2581 }, { "id": "58fb70f29e94ba0384507554", "num": 2, "url": "https://cache3.com/images/orig/58/fb/58fb70f29e94b384507554.jpg", "width": 750, "height": 1334 }, { "id": "58fb70f2f8efdc109d76c2e5", "num": 3, "url": "https://cache3.com/images/orig/58/fb/58fb70f2fdc109d76c2e5.jpg", "width": 750, "height": 1334 } ], "short_url": "https://short.com/p58gb7b9a4c80320f03" } ]
result = {}
result["name"] = dict[0].get("name", 'NA')
result["id"] = dict[0].get('owner', {}).get('id', 'NA')
result["date_published"] = dict[0].get("date_published", 'NA')
result["url"] = []
result["short_url"] = dict[0].get("short_url", 'NA')
for img in dict[0].get("images", []):
if "url" in img:
result["url"].append(img["url"])
print(result)

You could replace:-
'owner': item.get('owner', 'NA'),
...with...
'id': item.get('owner', {}).get('id', 'NA'),

You could do it by only extracting the information you want:
items = response['data']
iter1 = []
for item in items:
iter1.append({
'name': item.get('name', 'NA'),
'id': item.get('owner', {}).get('id', 'NA'),
'date_published': item.get('date_published', 'NA'),
'urls': [entry.get('url', 'NA') for entry in item.get('images', [])],
'short_url': item.get('short_url', 'NA')
})
Result:
[{'name': 'Announcement name',
'id': '58f84949700743',
'date_published': 1627666233,
'urls': ['https://cache3.com/images/orig/58/fb/58fb70f2132a554804fb5a2.jpg',
'https://cache3.com/images/orig/58/fb/58fb70f29e94b384507554.jpg',
'https://cache3.com/images/orig/58/fb/58fb70f2fdc109d76c2e5.jpg'],
'short_url': 'https://short.com/p58gb7b9a4c80320f03'}]

Convert json response object to pandas data frame

I have a JSON response that I want to convert to a Pandas dataframe for further analysis.
{
"isCurrentUserFollowing": False,
"id": 1,
"contactType": "Household",
"isPrivate": False,
"name": "Mrs Mattie Mahoney",
"informalName": "Mattie",
"description": None,
"website": None,
"maritalStatus": None,
"anniversaryMonth": None,
"anniversaryDay": None,
"anniversaryYear": None,
"mergedIntoContactId": None,
"address": {
"id": 1,
"label": "Primary",
"address1": "xyz",
"address2": None,
"city": "abc",
"state": "aa",
"postal": "11111",
"country": "United States",
"isPrimary": True,
"canBePrimary": False,
"startMonth": 1,
"startDay": 1,
"endMonth": 12,
"endDay": 31
},
"giftAskAmount": "$250",
"giftAskType": "Recurring Gift",
"lifeToDateGiving": "$225",
"yearToDateGiving": "$225",
"lastGiftAmount": "$225",
"lastGiftDate": "7/2/2021",
"contactIndividuals": [
{
"id": 1,
"contactId": 1,
"prefix": "Mrs",
"firstName": "Joan",
"middleName": None,
"lastName": "Arc",
"suffix": None,
"gender": "Female",
"isPrimary": True,
"canBePrimary": False,
"isSecondary": False,
"canBeSecondary": False,
"birthMonth": 1,
"birthDay": 1,
"birthYear": 1900,
"birthDate": "April 1",
"approximateAge": 100,
"isDeceased": False,
"passion": None,
"avatarUrl": "www.abc.com",
"contactMethods": [
{
"id": 1,
"type": "Home Email",
"value": "abc#hotmail.com",
"isOptedIn": False,
"isPrimary": True,
"canBePrimary": False
},
{
"id": 2,
"type": "Mobile Phone",
"value": "(111) 1111111",
"isOptedIn": False,
"isPrimary": True,
"canBePrimary": False
}
],
"createDateTimeUtc": "2021-04-15T17:41:31.2166667",
"modifiedDateTimeUtc": "2021-04-15T17:41:31.2166667",
"customFields": [],
"customCollections": []
},
{
"id": 2,
"contactId": 1,
"prefix": None,
"firstName": "John",
"middleName": None,
"lastName": "Smith",
"suffix": None,
"gender": "Female",
"isPrimary": False,
"canBePrimary": True,
"isSecondary": True,
"canBeSecondary": False,
"birthMonth": 1,
"birthDay": 11,
"birthYear": 1990,
"birthDate": "July 1",
"approximateAge": 100,
"isDeceased": False,
"passion": None,
"avatarUrl": "www.google.com",
"contactMethods": [
{
"id": 3,
"type": "Home Email",
"value": "mno#gmail.com",
"isOptedIn": False,
"isPrimary": True,
"canBePrimary": False
}
],
"createDateTimeUtc": "2021-04-15T17:41:31.2166667",
"modifiedDateTimeUtc": "2021-04-15T17:41:31.2166667",
"customFields": [],
"customCollections": []
}
],
"contactGiftsUrl": "/api/Gift/ByContact/1",
"contactPassthroughGiftsUrl": "/api/Gift/Passthrough/ByContact/1",
"contactPlannedGiftsUrl": "/api/PlannedGift/ByContact/1",
"contactRecurringGiftsUrl": "/api/RecurringGift/ByContact/1",
"contactImportantNotesUrl": "/api/ContactNote/Important/ByContact/1",
"contactNotesUrl": "/api/ContactNote/ByContact/1",
"contactTagsUrl": "/api/ContactTag/ByContact/1",
"contactRelationshipsUrl": "/api/Relationship/ByContact/1",
"primaryAvatarUrl": "www.apple.com",
"contactReferences": [],
"originSegmentId": None,
"originSegment": None,
"createDateTimeUtc": "2021-04-15T17:41:31.217",
"modifiedDateTimeUtc": "2021-04-15T17:41:31.217",
"customFields": [],
"customCollections": []
}
I am struggling with the code I need to write to get the relevant fields out
I am able to get some fields but I cannot get the gender and emails to be retrieved. Can someone assist ?
df = pd.DataFrame(
columns=[
"contactcustomerid",
"contactType",
"contactName",
"contactaddress",
"contactcity",
"contactstate",
"contactzip",
"contactcountry",
"giftAskAmount",
"giftAskType",
"lifetodateGiving",
"yeartoDateGiving",
"lastGiftAmount",
"lastGiftDate",
"contactGender",
"contactEmail",
]
)
if json_data:
row = {
"contactcustomerid": json_data.get("id"),
"contactType": json_data.get("contactType"),
"contactName": json_data.get("name"),
"contactaddress": json_data.get("address").get("address1"),
"contactcity": json_data.get("address").get("city"),
"contactstate": json_data.get("address").get("state"),
"contactzip": json_data.get("address").get("postal"),
"contactcountry": json_data.get("address").get("country"),
"giftAskAmount": json_data.get("giftAskAmount"),
"giftAskType": json_data.get("giftAskType"),
"lifetodateGiving": json_data.get("lifeToDateGiving"),
"yeartoDateGiving": json_data.get("yearToDateGiving"),
"lastGiftAmount": json_data.get("lastGiftAmount"),
"lastGiftDate": json_data.get("lastGiftDate"),
}
df = df.append(row, ignore_index="True")
df = df.fillna(0)

How to iterate through nested Dictionary in python boto3

Hi want to find only stopped instance and their ids below is the below-nested JSON
{
"Reservations": [
{
"Groups": [],
"Instances": [
{
"AmiLaunchIndex": 0,
"Architecture": "x86_64",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"AttachTime": "2021-03-11 09:43:59+00:00",
"DeleteOnTermination": true,
"Status": "attached",
"VolumeId": "vol-0c49ff5c979904337"
}
}
],
"CapacityReservationSpecification": {
"CapacityReservationPreference": "open"
},
"ClientToken": "",
"CpuOptions": {
"CoreCount": 1,
"ThreadsPerCore": 1
},
"EbsOptimized": false,
"EnaSupport": true,
"EnclaveOptions": {
"Enabled": false
},
"HibernationOptions": {
"Configured": false
},
"Hypervisor": "xen",
"ImageId": "ami-038f1ca1bd58a5790",
"InstanceId": "i-041fb789f1554b7d5",
"InstanceType": "t2.micro",
"KeyName": "HadoopLearning",
"LaunchTime": "2021-03-11 09:43:58+00:00",
"MetadataOptions": {
"HttpEndpoint": "enabled",
"HttpPutResponseHopLimit": 1,
"HttpTokens": "optional",
"State": "applied"
},
"Monitoring": {
"State": "disabled"
},
"NetworkInterfaces": [
{
"Attachment": {
"AttachTime": "2021-03-11 09:43:58+00:00",
"AttachmentId": "eni-attach-022ac34489909cc78",
"DeleteOnTermination": true,
"DeviceIndex": 0,
"NetworkCardIndex": 0,
"Status": "attached"
},
"Description": "",
"Groups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"InterfaceType": "interface",
"Ipv6Addresses": [],
"MacAddress": "0a:15:95:72:23:6b",
"NetworkInterfaceId": "eni-0d356de72ae7e6bfc",
"OwnerId": "216808211954",
"PrivateDnsName": "ip-172-31-29-51.ec2.internal",
"PrivateIpAddress": "172.31.29.51",
"PrivateIpAddresses": [
{
"Primary": true,
"PrivateDnsName": "ip-172-31-29-51.ec2.internal",
"PrivateIpAddress": "172.31.29.51"
}
],
"SourceDestCheck": true,
"Status": "in-use",
"SubnetId": "subnet-e705ccaa",
"VpcId": "vpc-4401543e"
}
],
"Placement": {
"AvailabilityZone": "us-east-1d",
"GroupName": "",
"Tenancy": "default"
},
"PrivateDnsName": "ip-172-31-29-51.ec2.internal",
"PrivateIpAddress": "172.31.29.51",
"ProductCodes": [],
"PublicDnsName": "",
"RootDeviceName": "/dev/xvda",
"RootDeviceType": "ebs",
"SecurityGroups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"SourceDestCheck": true,
"State": {
"Code": 80,
"Name": "stopped" ####this one only which is stopped ###
},
"StateReason": {
"Code": "Client.UserInitiatedShutdown",
"Message": "Client.UserInitiatedShutdown: User initiated shutdown"
},
"StateTransitionReason": "User initiated (2021-03-11 10:13:00 GMT)",
"SubnetId": "subnet-e705ccaa",
"Tags": [
{
"Key": "Name",
"Value": "aviral"
}
],
"VirtualizationType": "hvm",
"VpcId": "vpc-4401543e"
},
{
"AmiLaunchIndex": 1,
"Architecture": "x86_64",
"BlockDeviceMappings": [
{
"DeviceName": "/dev/xvda",
"Ebs": {
"AttachTime": "2021-03-11 09:43:59+00:00",
"DeleteOnTermination": true,
"Status": "attached",
"VolumeId": "vol-069a9ca7f860fefd8"
}
}
],
"CapacityReservationSpecification": {
"CapacityReservationPreference": "open"
},
"ClientToken": "",
"CpuOptions": {
"CoreCount": 1,
"ThreadsPerCore": 1
},
"EbsOptimized": false,
"EnaSupport": true,
"EnclaveOptions": {
"Enabled": false
},
"HibernationOptions": {
"Configured": false
},
"Hypervisor": "xen",
"ImageId": "ami-038f1ca1bd58a5790",
"InstanceId": "i-0d0c876682eef71ae",
"InstanceType": "t2.micro",
"KeyName": "HadoopLearning",
"LaunchTime": "2021-03-11 09:43:58+00:00",
"MetadataOptions": {
"HttpEndpoint": "enabled",
"HttpPutResponseHopLimit": 1,
"HttpTokens": "optional",
"State": "applied"
},
"Monitoring": {
"State": "disabled"
},
"NetworkInterfaces": [
{
"Attachment": {
"AttachTime": "2021-03-11 09:43:58+00:00",
"AttachmentId": "eni-attach-0b0eba3fb5b3c3205",
"DeleteOnTermination": true,
"DeviceIndex": 0,
"NetworkCardIndex": 0,
"Status": "attached"
},
"Description": "",
"Groups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"InterfaceType": "interface",
"Ipv6Addresses": [],
"MacAddress": "0a:3d:3e:bc:96:e3",
"NetworkInterfaceId": "eni-0688951c8bd226676",
"OwnerId": "216808211954",
"PrivateDnsName": "ip-172-31-19-102.ec2.internal",
"PrivateIpAddress": "172.31.19.102",
"PrivateIpAddresses": [
{
"Primary": true,
"PrivateDnsName": "ip-172-31-19-102.ec2.internal",
"PrivateIpAddress": "172.31.19.102"
}
],
"SourceDestCheck": true,
"Status": "in-use",
"SubnetId": "subnet-e705ccaa",
"VpcId": "vpc-4401543e"
}
],
"Placement": {
"AvailabilityZone": "us-east-1d",
"GroupName": "",
"Tenancy": "default"
},
"PrivateDnsName": "ip-172-31-19-102.ec2.internal",
"PrivateIpAddress": "172.31.19.102",
"ProductCodes": [],
"PublicDnsName": "",
"RootDeviceName": "/dev/xvda",
"RootDeviceType": "ebs",
"SecurityGroups": [
{
"GroupId": "sg-5878120b",
"GroupName": "default"
}
],
"SourceDestCheck": true,
"State": {
"Code": 80,
"Name": "running" ###Not this one #####
},
"StateReason": {
"Code": "Client.UserInitiatedShutdown",
"Message": "Client.UserInitiatedShutdown: User initiated shutdown"
},
"StateTransitionReason": "User initiated (2021-03-11 10:13:00 GMT)",
"SubnetId": "subnet-e705ccaa",
"Tags": [
{
"Key": "Name",
"Value": "avinash"
}
],
"VirtualizationType": "hvm",
"VpcId": "vpc-4401543e"
}
],
"OwnerId": "216808211954",
"ReservationId": "r-0d646190512fec9f8"
}
],
"ResponseMetadata": {
"HTTPHeaders": {
"cache-control": "no-cache, no-store",
"content-type": "text/xml;charset=UTF-8",
"date": "Thu, 11 Mar 2021 10:14:39 GMT",
"server": "AmazonEC2",
"strict-transport-security": "max-age=31536000; includeSubDomains",
"transfer-encoding": "chunked",
"vary": "accept-encoding",
"x-amzn-requestid": "58065d1d-76fc-4f7a-9229-18584ee1031e"
},
"HTTPStatusCode": 200,
"RequestId": "58065d1d-76fc-4f7a-9229-18584ee1031e",
"RetryAttempts": 0
}
}
I only want an instance of those who are stopped and then I want to start them using their instance ID using boto3
I am using the below code
import boto3
import json
access_key = "XXXXXXXXXX"
secret_key = "XXXXXXXXXXXXXXXXXXXXX"
client = boto3.client('ec2', aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name='us-east-1')
ec2_result = client.describe_instances(
Filters=[
{
'Name': 'tag:Name',
'Values': ['?avi*']
}
]
)
#ids= [ec2_result['Reservations'][0]['Instances'][i]['InstanceId'] for i in range(len(ec2_result['Reservations'][0]['Instances']))]
#print(ids)
print(ec2_result)
using the above code I am getting first block JSON but I only want an instance which is stopped, as I have 4000 instances in my AWS

I think the following should work:
instances_stopped = []
for reservation in ec2_result['Reservations']:
for instance in reservation['Instances']:
instance_id = instance['InstanceId']
if 'State' not in instance:
print(f"Missing state for {instance_id}")
else:
state = instance['State']['Name']
if state == 'stopped':
instances_stopped.append(instance_id)
#print(instance_id, state)
print(instances_stopped)

Parse API output in Python

I'm new to python and I've been trying to parse the output of from an F5 API request and I'm struggling a bit. I would like to extract all the "name" and "destination" values and save those to a file. This should be a name and the corresponding IP address. From the output below, it would be:
test-account 10.1.1.1
test-account_443 10.1.1.5
I can extract the individual elements, but I'm struggling to figure out how to iterate over the output file so I can get all of the key values I want. I've cut the output short to make it easier to read.
parsed_json = json.loads(data)
destination = parsed_json['items'][0]['destination']
name = parsed_json['items'][0]['name']
This only extracts one key value from the whole output file. Many thanks in advance.
{
"kind": "tm:ltm:virtual:virtualcollectionstate",
"items": [
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account",
"partition": "Common",
"fullPath": "/Common/test-account",
"generation": 1,
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.1:80",
"enabled": true,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": true,
"vsIndex": 2,
"rules": [
"/Common/redirect-http-to-https"
],
"rulesReference": [
{
"link":
}
],
"policiesReference": {
"link":
"isSubcollection": true
},
"profilesReference": {
"link":
"isSubcollection": true
}
},
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account_443",
"partition": "Common",
"fullPath": "/Common/test-account_443",
"generation": 1,
"selfLink":
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.5:443",
"enabled": true,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"poolReference": {
"link":
},
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": true,
"vsIndex": 3,
"policiesReference": {
"link":
"isSubcollection": true
},
"profilesReference": {
"link":
"isSubcollection": true
}
}

This should help. You can iterate over the "items" key and get the required information
Ex:
parsed_json = {
"kind": "tm:ltm:virtual:virtualcollectionstate",
"items": [
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account",
"partition": "Common",
"fullPath": "/Common/test-account",
"generation": 1,
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.1:80",
"enabled": True,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": True,
"vsIndex": 2,
"rules": [
"/Common/redirect-http-to-https"
],
"rulesReference": [
{
"link": True
}
],
"policiesReference": {
"link": True,
"isSubcollection": True
},
"profilesReference": {
"link": True,
"isSubcollection": True
}
},
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account_443",
"partition": "Common",
"fullPath": "/Common/test-account_443",
"generation": 1,
"selfLink": True,
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.5:443",
"enabled": True,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"poolReference": {
"link": True
},
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": True,
"vsIndex": 3,
"policiesReference": {
"link": True,
"isSubcollection": True
},
"profilesReference": {
"link": True,
"isSubcollection": True
}
}]
}
res = []
for value in parsed_json["items"]:
print(value["name"])
print(value["destination"])
print(value["destination"].split("/")[-1].split(":")[0]) #if you need only the IP.
Output:
test-account
/Common/10.1.1.1:80
10.1.1.1
test-account_443
/Common/10.1.1.5:443
10.1.1.5