We Keep Coding

Python Taking API response and adding to JSON Key error

I have a script that takes an ID from a JSON file, adds it to a URL for an API request. The aim is to have a loop run through the 1000-ish ids and preduce one JSON file with all the information contained within.
The current code calls the first request and creates and populates the JSON file, but when run in a loop it throws a key error.
import json
import requests
fname = "NewdataTest.json"
def write_json(data, fname):
fname = "NewdataTest.json"
with open(fname, "w") as f:
json.dump(data, f, indent = 4)
with open (fname) as json_file:
data = json.load(json_file)
temp = data[0]
#print(newData)
y = newData
data.append(y)
# Read test.json to get tmdb IDs
tmdb_ids = []
with open('test.json', 'r') as json_fp:
imdb_info = json.load(json_fp)
tmdb_ids = [movie_info['tmdb_id'] for movies_chunk in imdb_info for movie_index, movie_info in movies_chunk.items()]
# Add IDs to API call URL
for tmdb_id in tmdb_ids:
print("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Send API Call
response_API = requests.get("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Check API Call Status
print(response_API.status_code)
write_json((response_API.json()), "NewdataTest.json")
The error is in this line "temp = data[0]" I have tried printing the keys for data, nothing. At this point, I have no idea where I am with this as I have hacked it about it barely resembles anything like a cohesive piece of code. My aim was to make a simple function to get the data from the JSON, one to produce the API call URLs, and one to write the results to the new JSON.
Example of API reponse JSON:
{
"adult": false,
"backdrop_path": "/e1cC9muSRtAHVtF5GJtKAfATYIT.jpg",
"belongs_to_collection": null,
"budget": 0,
"genres": [
{
"id": 10749,
"name": "Romance"
},
{
"id": 35,
"name": "Comedy"
}
],
"homepage": "",
"id": 1063242,
"imdb_id": "tt24640474",
"original_language": "fr",
"original_title": "Disconnect: The Wedding Planner",
"overview": "After falling victim to a scam, a desperate man races the clock as he attempts to plan a luxurious destination wedding for an important investor.",
"popularity": 34.201,
"poster_path": "/tGmCxGkVMOqig2TrbXAsE9dOVvX.jpg",
"production_companies": [],
"production_countries": [
{
"iso_3166_1": "KE",
"name": "Kenya"
},
{
"iso_3166_1": "NG",
"name": "Nigeria"
}
],
"release_date": "2023-01-13",
"revenue": 0,
"runtime": 107,
"spoken_languages": [
{
"english_name": "English",
"iso_639_1": "en",
"name": "English"
},
{
"english_name": "Afrikaans",
"iso_639_1": "af",
"name": "Afrikaans"
}
],
"status": "Released",
"tagline": "",
"title": "Disconnect: The Wedding Planner",
"video": false,
"vote_average": 5.8,
"vote_count": 3
}

You can store all results from the API calls into a list and then save this list in Json format into a file. For example:
#...
all_data = []
for tmdb_id in tmdb_ids:
print("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Send API Call
response_API = requests.get("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Check API Call Status
print(response_API.status_code)
if response_API.status_code == 200:
# store the Json data in a list:
all_data.append(response_API.json())
# write the list to file
with open('output.json', 'w') as f_out:
json.dump(all_data, f_out, indent=4)
This will produce output.json with all responses in Json format.

How can I convert a CSV file to a JSON file with a nested json object using Python?

I am stuck with a problem where I don't know how I can convert a "nested JSON object" inside a CSV file into a JSON object.
So I have a CSV file with the following value:
data.csv
1, 12385, {'message': 'test 1', 'EngineId': 3, 'PersonId': 1, 'GUID': '0ace2-02d8-4eb6-b2f0-63bb10829cd4s56'}, 6486D, TestSender1
2, 12347, {'message': 'test 2', 'EngineId': 3, 'PersonId': 2, 'GUID': 'c6d25672-cb17-45e8-87be-46a6cf14e76b'}, 8743F, TestSender2
I wrote a python script that converts this CSV file into a JSON file inside an array.
This I did with the following python script
csvToJson.py
import json
import csv
with open("data.csv","r") as f:
reader = csv.reader(f)
data = []
for row in reader:
data.append({"id": row[0],
"receiver": row[1],
"payload": row[2],
"operator": row[3],
"sender": row[4]})
with open("data.json", "w") as f:
json.dump(data, f, indent=4)
The problem I'm facing is that I'm not getting the right values inside "payload", which I would like to be a nested JSON object.
The result I get is the following:
data.json
[
{
"id": "1",
"receiver": " 12385",
"payload": " {'message': 'test 1'",
"operator": " 'EngineId': 3",
"sender": " 'PersonId': 1"
},
{
"id": "2",
"receiver": " 12347",
"payload": " {'message': 'test 2'",
"operator": " 'EngineId': 3",
"sender": " 'PersonId': 2"
}
]
So my question is, how can I create a nested JSON object for the "payload" while I'm doing the conversion from CSV to JSON?
I think the main problem is that it is seen as a string and not as an object.

Try the following. You can just do everything as previously, but merge back all elements that were in 3rd column and load it via ast.literal_eval.
import json
import csv
import ast
with open("data.csv","r") as f:
reader = csv.reader(f,skipinitialspace=True)
data = [{"id": ident,
"receiver": rcv,
"payload": ast.literal_eval(','.join(payload)),
"operator": op,
"sender": snd}
for ident,rcv,*payload,op,snd in reader]
with open("data.json", "w") as f:
json.dump(data, f, indent=4)

I cant append to my JSON array with Python

This doesn't seem to work with this line being the issue
data["Objects"]["Text"].append({"f":"var6", "g":"var7", "h":"var8"})
This is the entire function
def CreateTextJSON(TextObject):
print("s")
with open('base.json') as f:
data = json.load(f)
data["Objects"]["Text"].append({"f":"var6", "g":"var7", "h":"var8"})
with open('base.json', 'w') as f:
json.dump(data, f)
This is my JSON (base.json)
{
"Objects": [
{
"Type": "Menu",
"Path": "assets/images/mainmenu/mainmenu.png",
"Name": "MainMenu",
"X": 0,
"Y": 0,
"Width": 1920,
"Height": 1080,
"Buttons": [],
"Text": []
}]
}
Im trying to add to the [Text] array but I can't seem to.

Credit to #Grismar "Also, the problem would seem to be that "Objects" is not a dictionary, but a list with 1 element, which is the dictionary, so you need data["Objects"][0]["Text"].append( ..."

I have to send the content of a list to a string

I have to send the content of a list to a single string.
I've tried using a loop, but only could print the result:
for item in output_list:
for line in item['output'].split('\n'):
print(line)
that is the list
output_list =
{
"jsonrpc": "2.0",
"result": [
{},
{
"tablesLastChangeTime": 1483721367.4560423,
"tablesAgeOuts": 0,
"tablesInserts": 3,
"lldpNeighbors": [
{
"ttl": 120,
"neighborDevice": "HP830_LSW",
"neighborPort": "GigabitEthernet1/0/12",
"port": "Ethernet47"
},
{
"ttl": 120,
"neighborDevice": "HP_5500EI",
"neighborPort": "GigabitEthernet2/0/22",
"port": "Ethernet48"
},
{
"ttl": 120,
"neighborDevice": "HP_5500EI",
"neighborPort": "GigabitEthernet1/0/24",
"port": "Management1"
}
],
"tablesDeletes": 0,
"tablesDrops": 0
}
],
"id": "EapiExplorer-1"
}
I want to send the content of a list to a single string.

The json module contains a method called 'dumps' which can take in your dictionary and return a string.
import json
my_string = json.dumps(output_list)

There are a couple of methods here you could use. The first is to generically pretty-print:
import pprint
...
pprint.pprint(output_list)
The second is to output in json format, since your output_list looks like it belongs that way:
import json
...
print(json.dumps(output_list))

Reformat non-serializable JSON-ish data into a format suitable for value extraction in Python

With the following simple Python script:
import json
file = 'toy.json'
data = json.loads(file)
print(data['gas']) # example
My data generates the error ...is not JSON serializable.
With this, slightly more sophisticated, Python script:
import json
import sys
#load the data into an element
data = open('transactions000000000029.json', 'r')
#dumps the json object into an element
json_str = json.dumps(data)
#load the json to a string
resp = json.loads(json_str)
#extract an element in the response
print(resp['gas'])
The same.
What I'd like to do is extract all the values of a particular index, so ideally I'd like to render the input like so:
...
"hash": "0xf2b5b8fb173e371cbb427625b0339f6023f8b4ec3701b7a5c691fa9cef9daf63",
"gasUsed": "21000",
"hash": "0xf8f2a397b0f7bb1ff212b6bcc57e4a56ce3e27eb9f5839fef3e193c0252fab26"
"gasUsed": "21000"
...
The data looks like this:
{
"blockNumber": "1941794",
"blockHash": "0x41ee74e34cbf9ef4116febea958dbc260e2da3a6bf6f601bfaeb2cd9ab944a29",
"hash": "0xf2b5b8fb173e371cbb427625b0339f6023f8b4ec3701b7a5c691fa9cef9daf63",
"from": "0x3c0cbb196e3847d40cb4d77d7dd3b386222998d9",
"to": "0x2ba24c66cbff0bda0e3053ea07325479b3ed1393",
"gas": "121000",
"gasUsed": "21000",
"gasPrice": "20000000000",
"input": "",
"logs": [],
"nonce": "14",
"value": "0x24406420d09ce7440000",
"timestamp": "2016-07-24 20:28:11 UTC"
}
{
"blockNumber": "1941716",
"blockHash": "0x75e1602cad967a781f4a2ea9e19c97405fe1acaa8b9ad333fb7288d98f7b49e3",
"hash": "0xf8f2a397b0f7bb1ff212b6bcc57e4a56ce3e27eb9f5839fef3e193c0252fab26",
"from": "0xa0480c6f402b036e33e46f993d9c7b93913e7461",
"to": "0xb2ea1f1f997365d1036dd6f00c51b361e9a3f351",
"gas": "121000",
"gasUsed": "21000",
"gasPrice": "20000000000",
"input": "",
"logs": [],
"nonce": "1",
"value": "0xde0b6b3a7640000",
"timestamp": "2016-07-24 20:12:17 UTC"
}
What would be the best way to achieve that?
I've been thinking that perhaps the best way would be to reformat it as valid json?
Or maybe to just treat it like regex?

Your json file is not valid. This data should be a list of dictionaries. You should then separate each dictionary with a comma, Like this:
[
{
"blockNumber":"1941794",
"blockHash": "0x41ee74bf9ef411d9ab944a29",
"hash":"0xf2ef9daf63",
"from":"0x3c0cbb196e3847d40cb4d77d7dd3b386222998d9",
"to":"0x2ba24c66cbff0bda0e3053ea07325479b3ed1393",
"gas":"121000",
"gasUsed":"21000",
"gasPrice":"20000000000",
"input":"",
"logs":[
],
"nonce":"14",
"value":"0x24406420d09ce7440000",
"timestamp":"2016-07-24 20:28:11 UTC"
},
{
"blockNumber":"1941716",
"blockHash":"0x75e1602ca8d98f7b49e3",
"hash":"0xf8f2a397b0f7bb1ff212e193c0252fab26",
"from":"0xa0480c6f402b036e33e46f993d9c7b93913e7461",
"to":"0xb2ea1f1f997365d1036dd6f00c51b361e9a3f351",
"gas":"121000",
"gasUsed":"21000",
"gasPrice":"20000000000",
"input":"",
"logs":[
],
"nonce":"1",
"value":"0xde0b6b3a7640000",
"timestamp":"2016-07-24 20:12:17 UTC"
}
]
Then use this to open the file:
with open('toy.json') as data_file:
data = json.load(data_file)
You can then render the desired output like:
for item in data:
print item['hash']
print item['gasUsed']

If each block is valid JSON data you can parse them seperatly:
data = []
with open('transactions000000000029.json') as inpt:
lines = []
for line in inpt:
if line.startswith('{'): # block starts
lines = [line]
else:
lines.append(line)
if line.startswith('}'): # block ends
data.append(json.loads(''.join(lines)))
for block in data:
print("hash: {}".format(block['hash']))
print("gasUsed: {}".format(block['gasUsed']))