We Keep Coding

extract all Json key values

I'm not advanced with Python Json. I have these Json result:
{
"href": "https://api.spotify.com/v1/users/wizzler/playlists",
"items": [
{
"collaborative": false,
"external_urls": {
"spotify": "http://open.spotify.com/user/wizzler/playlists/53Y8wT46QIMz5H4WQ8O22c"
},
"href": "https://api.spotify.com/v1/users/wizzler/playlists/53Y8wT46QIMz5H4WQ8O22c",
"id": "53Y8wT46QIMz5H4WQ8O22c",
"images": [],
"name": "Wizzlers Big Playlist",
"owner": {
"external_urls": {
"spotify": "http://open.spotify.com/user/wizzler"
},
"href": "https://api.spotify.com/v1/users/wizzler",
"id": "wizzler",
"type": "user",
"uri": "spotify:user:wizzler"
},
"public": true,
"snapshot_id": "bNLWdmhh+HDsbHzhckXeDC0uyKyg4FjPI/KEsKjAE526usnz2LxwgyBoMShVL+z+",
"tracks": {
"href": "https://api.spotify.com/v1/users/wizzler/playlists/53Y8wT46QIMz5H4WQ8O22c/tracks",
"total": 30
},
"type": "playlist",
"uri": "spotify:user:wizzler:playlist:53Y8wT46QIMz5H4WQ8O22c"
},
{
"collaborative": false,
"external_urls": {
"spotify": "http://open.spotify.com/user/wizzlersmate/playlists/1AVZz0mBuGbCEoNRQdYQju"
},
"href": "https://api.spotify.com/v1/users/wizzlersmate/playlists/1AVZz0mBuGbCEoNRQdYQju",
"id": "1AVZz0mBuGbCEoNRQdYQju",
"images": [],
"name": "Another Playlist",
"owner": {
"external_urls": {
"spotify": "http://open.spotify.com/user/wizzlersmate"
},
"href": "https://api.spotify.com/v1/users/wizzlersmate",
"id": "wizzlersmate",
"type": "user",
"uri": "spotify:user:wizzlersmate"
},
"public": true,
"snapshot_id": "Y0qg/IT5T02DKpw4uQKc/9RUrqQJ07hbTKyEeDRPOo9LU0g0icBrIXwVkHfQZ/aD",
"tracks": {
"href": "https://api.spotify.com/v1/users/wizzlersmate/playlists/1AVZz0mBuGbCEoNRQdYQju/tracks",
"total": 58
},
"type": "playlist",
"uri": "spotify:user:wizzlersmate:playlist:1AVZz0mBuGbCEoNRQdYQju"
}
],
"limit": 9,
"next": null,
"offset": 0,
"previous": null,
"total": 9
}
Now I need to extract only the Playlist ids. How to do that?
Edit:
I get the Json Data from doing:
r = requests.get(BASE_URL + 'users/' + user_id + '/playlists', headers=headers)
r = r.json()
print(r) returning me the Json Data. When I try to data = json.load(r)
I get these error! AttributeError: 'dict' object has no attribute 'read'

First, load the JSON file using the built in json library.
import json
with open('path/to/json/file.json') as f:
data = json.load(f)
Then, use a list comprehension to get only the IDs.
playlist_ids = [item['id'] for item in data['items']]
Edit: Or, if you've got your JSON parsed already, just use the list comprehension. Don't do r = r.json(), that will reset the request object to the data. Set it to some other variable, data is OK - data = r.json()
playlist_ids = [item['id'] for item in data['items']]
Edit 2: If you only want it where the owner ID is "wizzler", then add a if clause to the list comprehension.
playlist_ids = [item['id'] for item in data['items'] if item['owner']['id'] == 'wizzler']

Issue in writing json to excel file in python

I am trying to get json output of via one of API request , which i wanted then load that into excel file
The problem is the response i get from api, if i dump it to json.dumps() method, its becoming not parsable. But if i try to parse it as text, then tried to format it json formatter its parsing
Though i wrote code to write to csv below, but i wanted it to excel file..
Here is my sample respone.text variable in my actual code looks like:
{
"value": [
{
"correlationId": "xxxxxxxxxx",
"eventName": {
"value": "EndRequest",
"localizedValue": "EndRequest"
},
"id": "/subscriptions/xxxxxxxxxx/resourcegroups/xxxxxxxxx/providers/Microsoft.Compute/virtualMachines/xxxxxx/extensions/enablevmaccess/events/xxxxxxxxxx/ticks/xxxxxxxx",
"level": "Informational",
"resourceGroupName": "xxxxxx",
"resourceProviderName": {
"value": "Microsoft.Compute",
"localizedValue": "Microsoft.Compute"
},
"operationName": {
"value": "Microsoft.Compute/virtualMachines/extensions/write",
"localizedValue": "Microsoft.Compute/virtualMachines/extensions/write"
},
"status": {
"value": "Succeeded",
"localizedValue": "Succeeded"
},
"eventTimestamp": "2020-08-06T12:47:02.0657952Z",
"submissionTimestamp": "2020-08-06T12:49:03.137537Z"
},
{
"correlationId": "xxxxxxxxxx",
"eventName": {
"value": "EndRequest",
"localizedValue": "EndRequest"
},
"id": "/subscriptions/xxxxxxxxxx/resourcegroups/xxxxxxxxx/providers/Microsoft.Compute/virtualMachines/xxxxxx/extensions/enablevmaccess/events/xxxxxxxxxx/ticks/xxxxxxxx",
"level": "Informational",
"resourceGroupName": "xxxxxx",
"resourceProviderName": {
"value": "Microsoft.Compute",
"localizedValue": "Microsoft.Compute"
},
"operationName": {
"value": "Microsoft.Compute/virtualMachines/extensions/write",
"localizedValue": "Microsoft.Compute/virtualMachines/extensions/write"
},
"status": {
"value": "Succeeded",
"localizedValue": "Succeeded"
},
"eventTimestamp": "2020-08-06T12:47:02.0657952Z",
"submissionTimestamp": "2020-08-06T12:49:03.137537Z"
},
]
}
Here the code I am trying:
d_date = datetime.datetime.now()
today = d_date.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
print(today)
N = 10
date_N_days_ago = datetime.datetime.now() - timedelta(days=N)
start_date = date_N_days_ago.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
print(start_date)
vm_list = compute_client.virtual_machines.list_all()
for vm_general in vm_list:
general_view = vm_general.id.split("/")
resource_group = general_view[4]
print(resource_group)
BASE_URL = f"https://management.azure.com/subscriptions/{subscription_id}/providers/microsoft.insights/eventtypes/management/values?api-version=2015-04-01&$filter=eventTimestamp ge {start_date} and eventTimestamp le {today} and resourceGroupName eq {resource_group}&$select=eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,correlationId,submissionTimestamp,level"
BASE_URL = BASE_URL
headers = {
"Authorization": 'Bearer ' + credential.token["access_token"]
}
response = requests.get(BASE_URL, headers=headers)
# if i convert below line to df_json = response.json() it says AttributeError: 'str' object has no attribute 'json'
df_json = response.text # this is a string but i am able to parse it properly in json forammter
print(df_json)
with open('c:\csv\logs_test.csv', 'w') as f:
for key in df_json.keys():
f.write("%s,%s\n" % (key, df_json[key]))
break
I am getting error like:
AttributeError: 'str' object has no attribute 'keys'
Expected result:
Actually I need to to write to xls (excel) format having columns as "correlationId,eventName,id,resourceGroupName,resourceProviderName,operationName,status,eventTimestamp,submissionTimestamp

You can actually use eval to convert the text to a dictionary and then use pandas to convert it to an excel file.
import pandas
response_dict = eval(response.text)
df = pd.DataFrame(response_dict['value'])
df['tag'] = "Managed by IT"
file_name = 'data.xls'
df.to_excel(file_name, index = False)

The easiest is to convert to pandas dataframe and then to xls file.
You will to have to install xlwt - pip install xlwt.
import pandas as pd
data = {
"value": [
{
"correlationId": "xxxxxxxxxx",
"eventName": {
"value": "EndRequest",
"localizedValue": "EndRequest"
},
"id": "/subscriptions/xxxxxxxxxx/resourcegroups/xxxxxxxxx/providers/Microsoft.Compute/virtualMachines/xxxxxx/extensions/enablevmaccess/events/xxxxxxxxxx/ticks/xxxxxxxx",
"level": "Informational",
"resourceGroupName": "xxxxxx",
"resourceProviderName": {
"value": "Microsoft.Compute",
"localizedValue": "Microsoft.Compute"
},
"operationName": {
"value": "Microsoft.Compute/virtualMachines/extensions/write",
"localizedValue": "Microsoft.Compute/virtualMachines/extensions/write"
},
"status": {
"value": "Succeeded",
"localizedValue": "Succeeded"
},
"eventTimestamp": "2020-08-06T12:47:02.0657952Z",
"submissionTimestamp": "2020-08-06T12:49:03.137537Z"
},
{
"correlationId": "xxxxxxxxxx",
"eventName": {
"value": "EndRequest",
"localizedValue": "EndRequest"
},
"id": "/subscriptions/xxxxxxxxxx/resourcegroups/xxxxxxxxx/providers/Microsoft.Compute/virtualMachines/xxxxxx/extensions/enablevmaccess/events/xxxxxxxxxx/ticks/xxxxxxxx",
"level": "Informational",
"resourceGroupName": "xxxxxx",
"resourceProviderName": {
"value": "Microsoft.Compute",
"localizedValue": "Microsoft.Compute"
},
"operationName": {
"value": "Microsoft.Compute/virtualMachines/extensions/write",
"localizedValue": "Microsoft.Compute/virtualMachines/extensions/write"
},
"status": {
"value": "Succeeded",
"localizedValue": "Succeeded"
},
"eventTimestamp": "2020-08-06T12:47:02.0657952Z",
"submissionTimestamp": "2020-08-06T12:49:03.137537Z"
}
]
}
df = pd.json_normalize(data['value'])
cols = ["correlationId","eventName.value","id","resourceGroupName","resourceProviderName.value","operationName.value","status.value","eventTimestamp","submissionTimestamp"]
df[cols].to_excel("data.xls", index=False)
Instead of json, use demjson. Install the library - pip install demjson because json parses correctly only if it's a proper json.
import demjson
data = demjson.decode(response.text)
# remaining code goes on

Accessing nested value in loop in json using python

I want to fetch the value of each api3 in this json object where each array has api3 value.
{
"count": 10,
"result": [
{
"type": "year",
"year": {
"month": {
"api1": {
"href": "https://Ap1.com"
},
"api2": {
"href": "FETCH-CONTENT"
},
"api3": {
"href": "https://Ap3.com"
},
"api4": {
"href": "https://Ap4.com"
}
},
"id": "sdvnkjsnvj",
"summary": "summeryc",
"type": "REST",
"apiId": "mlksmfmksdfs",
"idProvider": {
"id": "sfsmkfmskf",
"name": "Apikey"
},
"tags": []
}
},
{
"type": "year1",
"year": {
"month": {
"api1": {
"href": "https://Ap11.com"
},
"api2": {
"href": "FETCH-CONTENT-1"
},
"api3": {
"href": "https://Ap13.com"
},
"api4": {
"href": "https://Ap14.com"
}
},
"id": "sdvnkjsnvj",
"summary": "summeryc",
"type": "REST",
"apiId": "mlksmfmksdfs",
"idProvider": {
"id": "sfsmkfmskf",
"name": "Apikey"
},
"tags": []
}
},
I am able to get the whole json object and first value inside it.
with open('C:\python\examplee.json','r+') as fr:
data = json.load(fr)
print(data["result"])
Thank you in advance for helping me figuring this.

For each element in list of result key, get the value for the nested dictionary within item
print([item['year']['month']['api3'] for item in data['result']])
The output will be [{'href': 'https://Ap3.com'}, {'href': 'https://Ap13.com'}]
Or if you want to get the href value as well
print([item['year']['month']['api3']['href'] for item in data['result']])
The output will be
['https://Ap3.com', 'https://Ap13.com']
So your whole code will look like
data = {}
with open('C:\python\examplee.json','r+') as fr:
data = json.load(fr)
print([item['year']['month']['api3']['href'] for item in dct['result']])

Looks like your JSON schema is static so you can just use this:
print([x['year']['month']['api3']['href'] for x in data['result']])
will return you:
['https://Ap3.com', 'https://Ap13.com']

Not able to extract my friend images from Facebook API using python

I have got this response from the Facebook Graph API:
{
"taggable_friends": {
"data": [
{
"name": "Friend1 Name",
"picture": {
"data": {
"url": "https://fb-s-c-a.akamaihd.net/h-ak-fbx/v/t1.0-1/p200x200/completeUrl1"
}
},
"id": "response1d"
},
{
"name": "Friend2 name",
"picture": {
"data": {
"url": "https://fb-s-a-a.akamaihd.net/h-ak-fbx/v/t1.0-1/p200x200/completeURL2"
}
},
"id": "responseid2"
}
],
"paging": {
"cursors": {
"before": "xyz",
"after": "abc"
},
"next": "NextpageURl"
}
},
"id": "xxxxxxxxx"
}
I am willing to extract the URL part of the graph API response with field taggable_friends.
I have tried something like this:
for friends in data_json_liked_pages['taggable_friends']['data']:
friend_url = friends['picture']['data']['url']
print friend_url
I am getting the following error:
Exception Type: TypeError
Exception Value: list indices must be integers, not str
What can I do to improve this?

Iterating through JSON in Python using an OFFSET

I am trying to use the HubSpot CRM API to get "All Deals".
The API endpoint is: https://api.hubapi.com/deals/v1/deal/all?hapikey=demo
The JSON returned looks like this...
{
"deals": [
{
"portalId": 62515,
"dealId": 18039629,
"isDeleted": false,
"associations": {
"associatedVids": [],
"associatedCompanyIds": [],
"associatedDealIds": []
},
"properties": {
"dealname": {
"value": "Company",
"timestamp": 1457040864519,
"source": "API",
"sourceId": null
},
"amount": {
"value": "10",
"timestamp": 1457040864519,
"source": "API",
"sourceId": null
},
"closedate": {
"value": "",
"timestamp": 1457040864519,
"source": "API",
"sourceId": null
},
"hubspot_owner_id": {
"value": "11626092",
"timestamp": 1457046177648,
"source": "SALESFORCE",
"sourceId": null
},
"hs_lastmodifieddate": {
"value": "1457046177662",
"timestamp": 1457046177662,
"source": "CALCULATED",
"sourceId": null
},
"hubspot_owner_assigneddate": {
"value": "1457046177648",
"timestamp": 1457046177648,
"source": "SALESFORCE",
"sourceId": null
},
"num_associated_contacts": {
"value": "0",
"timestamp": 0,
"source": "CALCULATED",
"sourceId": null
},
"hs_createdate": {
"value": "1457040864535",
"timestamp": 1457040864535,
"source": null,
"sourceId": null
},
"createdate": {
"value": "1457040864535",
"timestamp": 1457040864535,
"source": null,
"sourceId": null
},
"hs_salesforceopportunityid": {
"value": "00628000007nRyuAAE",
"timestamp": 1457046177648,
"source": "SALESFORCE",
"sourceId": null
}
},
"imports": []
},
{
"portalId": 62515,
"dealId": 18040854,
"isDeleted": false,
"associations": {
"associatedVids": [],
"associatedCompanyIds": [],
"associatedDealIds": []
},
"properties": {
"dealname": {
"value": "5678",
"timestamp": 1457042290572,
"source": "API",
"sourceId": null
},
"amount": {
"value": "750000.0",
"timestamp": 1457042290572,
"source": "API",
"sourceId": null
},
"closedate": {
"value": "",
"timestamp": 1457042290572,
"source": "API",
"sourceId": null
},
"hs_lastmodifieddate": {
"value": "1457042290592",
"timestamp": 1457042290592,
"source": "CALCULATED",
"sourceId": null
},
"num_associated_contacts": {
"value": "0",
"timestamp": 0,
"source": "CALCULATED",
"sourceId": null
},
"hs_createdate": {
"value": "1457042290592",
"timestamp": 1457042290592,
"source": null,
"sourceId": null
},
"createdate": {
"value": "1457042290592",
"timestamp": 1457042290592,
"source": null,
"sourceId": null
}
},
"imports": []
}
],
"hasMore": true,
"offset": 1467187
}
And I understand that if hasMore==true, then you are supposed to grab the offset and include it in another API call something like this: https://api.hubapi.com/deals/v1/deal/all?hapikey=demo&offset=1467187
And then keep doing that until hasMore==false.
I am using the following code to extract the first chunk of JSON from the API:
import requests
url = "https://api.hubapi.com/deals/v1/deal/all"
querystring = {"hapikey":"demo"}
headers = {
'cache-control': "no-cache"
}
response = requests.request("GET", url, headers=headers, params=querystring)
print(response.text)
So... my question is that now I am getting my JSON, how do I:
1) Read one chunk of JSON
2) If hasMore==true then go do #1 again
3) ElseIf hasMore==false then combine ALL the JSON from ALL iterations of #1 above into one big JSON
4) Return the value from #3
Any help please?

Working solution
import json
import requests
url = "https://api.hubapi.com/deals/v1/deal/all"
querystring = {"hapikey":"demo"}
headers = {
'cache-control': "no-cache"
}
all_deals = []
response = requests.request("GET", url, headers=headers, params=querystring).json()
for deal in response['deals']:
all_deals.append(deal)
hasMore = response['hasMore']
offset = response['offset']
while hasMore:
querystring = {
"hapikey":"demo",
"offset":offset
}
response = requests.request("GET", url, headers=headers, params=querystring).json()
for deal in response['deals']:
all_deals.append(deal)
hasMore = response['hasMore']
offset = response['offset']
print(json.dumps(all_deals))