So for a school project I have to read in some data through an api. So far everything (kinda) works, but when I try to read in some data underneath a couple of links, I get keyerrors.
I would not mind posting the entire list, but it is mainly in dutch.
import json
from urllib.request import urlopen
with urlopen("http://api.buienradar.nl/data/public/2.0/jsonfeed ") as response:
source = response.read()
data = json.loads(source)
#pirnt(json.dumps(data, indent =2))
for item in data['actual']['stationmeasurements']:
del item['iconurl']
del item['graphUrl']
station = item['stationname']
#regio = item['regio]
stationid = item['winddirection']
print(station, stationid)
with open('uitlezen_buienradar.json', 'w') as f:
json.dump(data, f , indent=2)
This is a part of the list:
"$id": "1",
"buienradar": {
"$id": "2",
"copyright": "(C)opyright Buienradar / RTL. Alle rechten voorbehouden",
"terms": "Deze feed mag vrij worden gebruikt onder voorwaarde van bronvermelding buienradar.nl inclusief een hyperlink naar https://www.buienradar.nl. Aan de feed kunnen door gebruikers of andere personen geen rechten worden ontleend."
},
"actual": {
"$id": "3",
"actualradarurl": "https://api.buienradar.nl/image/1.0/RadarMapNL?w=500&h=512",
"sunrise": "2019-10-04T07:45:00",
"sunset": "2019-10-04T19:11:00",
"stationmeasurements": [
{
"$id": "4",
"stationid": 6391,
"stationname": "Meetstation Arcen",
"lat": 51.5,
"lon": 6.2,
"regio": "Venlo",
"timestamp": "2019-10-04T02:30:00",
"weatherdescription": "Zwaar bewolkt",
"iconurl": "https://www.buienradar.nl/resources/images/icons/weather/30x30/cc.png",
"graphUrl": "https://www.buienradar.nl/nederland/weerbericht/weergrafieken/cc",
"winddirection": "ZO",
"temperature": 10.0,
"groundtemperature": 9.9,
"feeltemperature": 9.6,
"windgusts": 2.4,
"windspeed": 1.5,
"windspeedBft": 1,
"humidity": 93.0,
"precipitation": 0.0,
"sunpower": 0.0,
"rainFallLast24Hour": 4.5,
"rainFallLastHour": 0.0,
"winddirectiondegrees": 143
After 'iconurl' and 'graphurl' it just spews out errors.
Use dict.get(key), then there will be no exception if the key does not exist. Incomplete data in your items ;)
import json
from urllib.request import urlopen
with urlopen("http://api.buienradar.nl/data/public/2.0/jsonfeed ") as response:
source = response.read()
data = json.loads(source)
#pirnt(json.dumps(data, indent =2))
for item in data['actual']['stationmeasurements']:
del item['iconurl']
del item['graphUrl']
station = item.get('stationname')
#regio = item['regio]
stationid = item.get('winddirection')
print(station, stationid)
with open('uitlezen_buienradar.json', 'w') as f:
json.dump(data, f , indent=2)
Related
I have a script that takes an ID from a JSON file, adds it to a URL for an API request. The aim is to have a loop run through the 1000-ish ids and preduce one JSON file with all the information contained within.
The current code calls the first request and creates and populates the JSON file, but when run in a loop it throws a key error.
import json
import requests
fname = "NewdataTest.json"
def write_json(data, fname):
fname = "NewdataTest.json"
with open(fname, "w") as f:
json.dump(data, f, indent = 4)
with open (fname) as json_file:
data = json.load(json_file)
temp = data[0]
#print(newData)
y = newData
data.append(y)
# Read test.json to get tmdb IDs
tmdb_ids = []
with open('test.json', 'r') as json_fp:
imdb_info = json.load(json_fp)
tmdb_ids = [movie_info['tmdb_id'] for movies_chunk in imdb_info for movie_index, movie_info in movies_chunk.items()]
# Add IDs to API call URL
for tmdb_id in tmdb_ids:
print("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Send API Call
response_API = requests.get("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Check API Call Status
print(response_API.status_code)
write_json((response_API.json()), "NewdataTest.json")
The error is in this line "temp = data[0]" I have tried printing the keys for data, nothing. At this point, I have no idea where I am with this as I have hacked it about it barely resembles anything like a cohesive piece of code. My aim was to make a simple function to get the data from the JSON, one to produce the API call URLs, and one to write the results to the new JSON.
Example of API reponse JSON:
{
"adult": false,
"backdrop_path": "/e1cC9muSRtAHVtF5GJtKAfATYIT.jpg",
"belongs_to_collection": null,
"budget": 0,
"genres": [
{
"id": 10749,
"name": "Romance"
},
{
"id": 35,
"name": "Comedy"
}
],
"homepage": "",
"id": 1063242,
"imdb_id": "tt24640474",
"original_language": "fr",
"original_title": "Disconnect: The Wedding Planner",
"overview": "After falling victim to a scam, a desperate man races the clock as he attempts to plan a luxurious destination wedding for an important investor.",
"popularity": 34.201,
"poster_path": "/tGmCxGkVMOqig2TrbXAsE9dOVvX.jpg",
"production_companies": [],
"production_countries": [
{
"iso_3166_1": "KE",
"name": "Kenya"
},
{
"iso_3166_1": "NG",
"name": "Nigeria"
}
],
"release_date": "2023-01-13",
"revenue": 0,
"runtime": 107,
"spoken_languages": [
{
"english_name": "English",
"iso_639_1": "en",
"name": "English"
},
{
"english_name": "Afrikaans",
"iso_639_1": "af",
"name": "Afrikaans"
}
],
"status": "Released",
"tagline": "",
"title": "Disconnect: The Wedding Planner",
"video": false,
"vote_average": 5.8,
"vote_count": 3
}
You can store all results from the API calls into a list and then save this list in Json format into a file. For example:
#...
all_data = []
for tmdb_id in tmdb_ids:
print("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Send API Call
response_API = requests.get("https://api.themoviedb.org/3/movie/" + str(tmdb_id) + "?api_key=****")
# Check API Call Status
print(response_API.status_code)
if response_API.status_code == 200:
# store the Json data in a list:
all_data.append(response_API.json())
# write the list to file
with open('output.json', 'w') as f_out:
json.dump(all_data, f_out, indent=4)
This will produce output.json with all responses in Json format.
edit
problem solved thanks to Oyono and chepner!
I'm trying to save a dictionary which includes long lists as a JSON file, without having each new item in the lists in a new line.
to save the dictionary, I'm using the command:
with open('file.json', 'w') as fp:
json.dump(coco, fp, indent=2)
The file is really big and I can not open it in Colab. so when I open it using VSC it's shown each item in the list in a separate line.
when I try to print only a small part of the dictionary in the Colab I'm getting everything in a single line.
Any ideas why could it happen or how to avoid it?
this is how it's look like in VSC:
"annotation": [
{
"segmentation": [
[
75.0,
74.5,
...(many more lines like this),
435.0,
435.5
]
],
"iscrowd": 0,
"category_id": 1,
"image_id": 43,
"id": 430,
"bbox": [
11.0,
280.0,
117.0,
156.0
],
"area": 9897,
}
],
]
}
And this is how I want it to look (and can't tell if there is actual difference between the files)
{
"segmentation": [ [ 316.0, 171.5, 320.5, 168.0, 332.5, 153.0, 332.5, 149.0, 330.0, 146.5, 305.0, 134.5, 292.0, 125.5, 280.0, 120.5, 275.0, 116.5, 270.0, 115.5, 261.5, 130.0, 258.0, 133.5, 251.5, 136.0, 255.0, 140.5, 282.0, 153.5, 285.0, 156.5, 289.0, 156.5, 296.0, 159.5, 310.0, 170.5, 316.0, 171.5 ] ],
"iscrowd": 0,
"image_id": 5,
"category_id": 1,
"id": 5,
"bbox": [ 251.5, 115.5, 81.0, 56.0 ],
"area": 2075.0
},
You have to remove indent=2 in your call to dump:
with open('file.json', 'w') as fp:
json.dump(coco, fp)
The indent keyword will save your dictionary object as a pretty printed json object, which means printing each list item in individual lines.
You can try this in a python console to see how indent influences your output json:
json.dumps({1: "a", "b": [1, 2]}, indent=2)
# will output: '{\n "1": "a",\n "b": [\n 1,\n 2\n ]\n}'
json.dumps({1: "a", "b": [1, 2]})
# will output: '{"1": "a", "b": [1, 2]}'
Aloha,
My python routine will retrieve json from site, then check the file and download another json given the first answer and eventually download a zip.
The first json file gives information about doc.
Here's an example :
[
{
"id": "d9789918772f935b2d686f523d066a7b",
"originalName": "130010259_AC2_R44_20200101",
"type": "SUP",
"status": "document.deleted",
"legalStatus": "APPROVED",
"name": "130010259_SUP_R44_AC2",
"grid": {
"name": "R44",
"title": "GRAND EST"
},
"bbox": [
3.4212881,
47.6171589,
8.1598899,
50.1338684
],
"documentSource": "UPLOAD",
"uploadDate": "2020-06-25T14:56:27+02:00",
"updateDate": "2021-01-19T14:33:35+01:00",
"fileIdentifier": "SUP-AC2-R44-130010259-20200101",
"legalControlStatus": 101
},
{
"id": "6a9013bdde6acfa632861aeb1a02942b",
"originalName": "130010259_AC2_R44_20210101",
"type": "SUP",
"status": "document.production",
"legalStatus": "APPROVED",
"name": "130010259_SUP_R44_AC2",
"grid": {
"name": "R44",
"title": "GRAND EST"
},
"bbox": [
3.4212881,
47.6171589,
8.1598899,
50.1338684
],
"documentSource": "UPLOAD",
"uploadDate": "2021-01-18T16:37:01+01:00",
"updateDate": "2021-01-19T14:33:29+01:00",
"fileIdentifier": "SUP-AC2-R44-130010259-20210101",
"legalControlStatus": 101
},
{
"id": "efd51feaf35b12248966cb82f603e403",
"originalName": "130010259_PM2_R44_20210101",
"type": "SUP",
"status": "document.production",
"legalStatus": "APPROVED",
"name": "130010259_SUP_R44_PM2",
"grid": {
"name": "R44",
"title": "GRAND EST"
},
"bbox": [
3.6535762,
47.665021,
7.9509455,
49.907347
],
"documentSource": "UPLOAD",
"uploadDate": "2021-01-28T09:52:31+01:00",
"updateDate": "2021-01-28T18:53:34+01:00",
"fileIdentifier": "SUP-PM2-R44-130010259-20210101",
"legalControlStatus": 101
},
{
"id": "2e1b6104fdc09c84077d54fd9e74a7a7",
"originalName": "444619258_I4_R44_20210211",
"type": "SUP",
"status": "document.pre_production",
"legalStatus": "APPROVED",
"name": "444619258_SUP_R44_I4",
"grid": {
"name": "R44",
"title": "GRAND EST"
},
"bbox": [
2.8698336,
47.3373246,
8.0881368,
50.3796449
],
"documentSource": "UPLOAD",
"uploadDate": "2021-04-19T10:20:20+02:00",
"updateDate": "2021-04-19T14:46:21+02:00",
"fileIdentifier": "SUP-I4-R44-444619258-20210211",
"legalControlStatus": 100
}
]
What I try to do is to retrieve "id" from this json file. (ex. "id": "2e1b6104fdc09c84077d54fd9e74a7a7",)
I've tried
import json
from jsonpath_rw import jsonpath, parse
import jsonpath_rw_ext as jp
with open('C:/temp/gpu/SUP/20210419/SUPGE.json') as f:
d = json.load(f)
data = json.dumps(d)
print("oriName: {}".format( jp.match1("$.id[*]",data) ) )
It doesn't work In fact, I'm not sure how jsonpath-rw is intended to work. Thankfully there was this blogpost But I'm still stuck.
Does anyone have a clue ?
With the id, I'll be able to download another json and in this json there'll be an archiveUrl to get the zipfile.
Thanks in advance.
import json
file = open('SUPGE.json')
with file as f:
d = json.load(f)
for i in d:
print(i.get('id'))
this will give you id only.
d9789918772f935b2d686f523d066a7b
6a9013bdde6acfa632861aeb1a02942b
efd51feaf35b12248966cb82f603e403
2e1b6104fdc09c84077d54fd9e74a7a7
Ok.
Here's what I've done.
import json
import urllib
# not sure it's the best way to load json from url, but it works fine
# and I could test most of code if needed.
def getResponse(url):
operUrl = urllib.request.urlopen(url)
if(operUrl.getcode()==200):
data = operUrl.read()
jsonData = json.loads(data)
else:
print("Erreur reçue", operUrl.getcode())
return jsonData
# Here I get the json from the url. *
# That part will be in the final script a parameter,
# because I got lot of territory to control
d = getResponse('https://www.geoportail-urbanisme.gouv.fr/api/document?documentFamily=SUP&grid=R44&legalStatus=APPROVED')
for i in d:
if i['status'] == 'document.production' :
print('id du doc en production :',i.get('id'))
# here we parse the id to fetch the whole document.
# Same server, same API but different url
_URL = 'https://www.geoportail-urbanisme.gouv.fr/api/document/' + i.get('id')+'/details'
d2 = getResponse(_URL)
print('archive',d2['archiveUrl'])
urllib.request.urlretrieve(d2['archiveUrl'], 'c:/temp/gpu/SUP/'+d2['metadata']+'.zip' )
# I used wget in the past and loved the progression bar.
# Maybe I'd switch to wget because of it.
# Works fine.
Thanks for your answer. I'm delighted to see that even with only the json library you could do amazing things. Just normal stuff. But amazing.
Feel free to comment if you think I've missed smthg.
I am trying to use csv file to read data and convert them into nested array using python.
my column values of csv are
"hallticket_Number ","student_name","gender","course_name","university_course_code ","university_college_code","caste","course_year","semester_yearly_exams","subject_name1","subject_code1","marks_or_grade_points_obtained1","maximum_marks_or_grade_points1","pass_mark1","no_of_credits1","pass_fail_absent1","subject_name2","subject_code2","marks_or_grade_points_obtained2","maximum_marks_or_grade_points2","no_of_credits2","pass_fail_absent2" ,"subject_name3","subject_code3", "marks_or_grade_points_obtained3","maximum_marks_or_grade_points3","no_of_credits3", "pass_fail_absent3" ,"subject_name4" ,"subject_code4" ,"marks_or_grade_points_obtained4","maximum_marks_or_grade_points4","no_of_credits4" , "pass_fail_absent4" ,"subject_code5", "marks_or_grade_points_obtained5" ,"maximum_marks_or_grade_points5","no_of_credits5","pass_fail_absent5","subject_name6","marks_or_grade_points_obtained6","maximum_marks_or_grade_points6", "no_of_credits6","pass_fail_absent","final_result_pass_fail","marks_or_sgpa_
The output i need in JSON is
{
"hallticket_": 22342,
"student_name": "abc",
"gender": "m",
"course_name":" fgd",
"course_code":52,
"college_code ":521,
"caste":"open",
"year":55,
"exam":"s1",
"subject": [ {
"subject_name1":"hh",
"subject_code1":52,
"marks_or_grade_points_obtained1":85,
"maximum_marks_or_grade_points1":50,
"pass_mark1":52,
"no_of_credits1":85,
"pass_fail_absent1":"pass"},]
"subject": [ {
"subject_name2":"hh",
"subject_code2":52,
"marks_or_grade_points_obtained2":85,
"maximum_marks_or_grade_points2":50,
"pass_mark2":52,
"no_of_credits2":85,
"pass_fail_absent2":"pass"},]
"subject": [ {
"subject_name3":"hh",
"subject_code3":52,
"marks_or_grade_points_obtained3":85,
"maximum_marks_or_grade_points3":50,
"pass_mark3":52,
"no_of_credits3":85,
"pass_fail_absent3":"pass"},]
"subject": [ {
"subject_name4":"hh",
"subject_code4":52,
"marks_or_grade_points_obtained4":85,
"maximum_marks_or_grade_points4":50,
"pass_mark4":52,
"no_of_credits4":85,
"pass_fail_absent4":"pass"},]
"subject": [ {
"subject_name5":"hh",
"subject_code5":52,
"marks_or_grade_points_obtained5":85,
"maximum_marks_or_grade_points5":50,
"pass_mark5":52,
"no_of_credits5":85,
"pass_fail_absent5":"pass"},]
"subject": [ {
"subject_name6":"hh",
"subject_code6":52,
"marks_or_grade_points_obtained6":85,
"maximum_marks_or_grade_points6":50,
"pass_mark6":52,
"no_of_credits6":85,
"pass_fail_absent6":"pass"},]
"final_result_pass_fail":"pass",
" marks_or_sgpa_obtained":"8.00",
"maximum_marks_sgpa":"10",
"total_credits":"135"
}
import csv
import json
# Open the CSV
f = open('data.csv', 'r')
reader = csv.DictReader(f)
# Parse the CSV into JSON
out = json.dumps([row for row in reader])
print(out)
Hopefully this will work as your expectations!
I have this json object and i am trying to make it into a python list but i am getting some characters along that i don't need
import json
data = {
"products": [
{
"product_cp": 100.0,
"product_sp": 120.0,
"product_name": "coke",
},
{
"product_cp": 100.5,
"product_sp": 120.0,
"product_name": "fanta",
},
{
"product_cp": 70.5,
"product_sp": 100.5,
"product_name": "pepsi",
}
]
}
data = json.dumps(data)
print(data)
print('\v')
data = json.loads(data)
data_list = list(data['products'])
when i do:
print(data_list)
i get:
[{u'product_cp': 100.0, u'product_sp': 120.0, u'product_name': u'coke'},{u'product_cp': 100.5, u'product_sp': 120.0, u'product_name': u'fanta'}, {u'product_cp': 70.5, u'product_sp': 100.5, u'product_name': u'pepsi'}]
please how i do i make it so that {,[,},] an 'u characters doesn't show up?
To print individual product details without the syntactical overhead({,[,],}) you can use a nested loop:
for product in data["products"]:
for info in product:
print("%s: %s" % (info, product[info]))
print()
The output is:
product_cp: 100.0
product_sp: 120.0
product_name: coke
product_cp: 100.5
product_sp: 120.0
product_name: fanta
product_cp: 70.5
product_sp: 100.5
product_name: pepsi
try this code under python 3.6, i am a python rookie as well.
jsonobject = json.dumps(data)
jsonobjectToString = json.loads(jsonobject)
for level1 in jsonobjectToString["products"]:
str = level1["product_cp"],level1["product_sp"],level1["product_name"]
resultresp.append(str)
print(resultresp)
===================
[(100.0, 120.0, 'coke'), (100.5, 120.0, 'fanta'), (70.5, 100.5, 'pepsi')]