How do I flatten nested list from JSON object? - python

I've exported fitbit sleep data and got a json file with nested variables + dict. I would like to convert the json file to a csv file that will display all "regular" variables, e.g. "dateOfSleep" but also the nested variables, e.g. "deep" & "wake" with all dictionary information.
I tried json_normalize; but I can only make it work for the first nested variables, e.g. "levels". Anybody has an idea?
Much appreciated.
[{
"logId" : 32072056107,
"dateOfSleep" : "2021-05-08",
"startTime" : "2021-05-07T23:22:00.000",
"endTime" : "2021-05-08T08:05:30.000",
"duration" : 31380000,
"minutesToFallAsleep" : 0,
"minutesAsleep" : 468,
"minutesAwake" : 55,
"minutesAfterWakeup" : 0,
"timeInBed" : 523,
"efficiency" : 93,
"type" : "stages",
"infoCode" : 0,
"levels" : {
"summary" : {
"deep" : {
"count" : 5,
"minutes" : 85,
"thirtyDayAvgMinutes" : 68
},
"wake" : {
"count" : 30,
"minutes" : 55,
"thirtyDayAvgMinutes" : 56
},
"light" : {
"count" : 30,
"minutes" : 267,
"thirtyDayAvgMinutes" : 235
},
"rem" : {
"count" : 10,
"minutes" : 116,
"thirtyDayAvgMinutes" : 94
}
},
.....

Use pd.json_normalize, all nested levels are flatten and join with dots (by default):
import pandas as pd
import json
with open('data.json') as fp:
data = json.load(fp)
df = pd.json_normalize(data)
Output:
>>> df
logId dateOfSleep startTime ... levels.summary.rem.count levels.summary.rem.minutes levels.summary.rem.thirtyDayAvgMinutes
0 32072056107 2021-05-08 2021-05-07T23:22:00.000 ... 10 116 94
[1 rows x 25 columns]
Content of data.json:
[{
"logId" : 32072056107,
"dateOfSleep" : "2021-05-08",
"startTime" : "2021-05-07T23:22:00.000",
"endTime" : "2021-05-08T08:05:30.000",
"duration" : 31380000,
"minutesToFallAsleep" : 0,
"minutesAsleep" : 468,
"minutesAwake" : 55,
"minutesAfterWakeup" : 0,
"timeInBed" : 523,
"efficiency" : 93,
"type" : "stages",
"infoCode" : 0,
"levels" : {
"summary" : {
"deep" : {
"count" : 5,
"minutes" : 85,
"thirtyDayAvgMinutes" : 68
},
"wake" : {
"count" : 30,
"minutes" : 55,
"thirtyDayAvgMinutes" : 56
},
"light" : {
"count" : 30,
"minutes" : 267,
"thirtyDayAvgMinutes" : 235
},
"rem" : {
"count" : 10,
"minutes" : 116,
"thirtyDayAvgMinutes" : 94
}
}
}
}]

Related

tuple indices must be integers or slices, not tuple

I'll try to make a JSON
for i in ktg_mhs:
chart_ktg ={
'value' : ktg_mhs[i][1],
'name' : ktg_mhs[i][0]
}
result.append(chart_ktg)
data = json.dumps(chart_ktg)
but, I get an error at this
'value' : ktg_mhs[i][1],
'name' : ktg_mhs[i][0]
How to solve it and why?
I have this input :
(('Coding', 51), ('Database', 27), ('UI/UX', 223), ('Manajemen', 34), ('Dasar Sistem Informasi', 7), ('Bisnis', 64), ('Kesehatan', 18), ('Lain-lain', 235))
And I want to make JSON like this :
{
value: 335,
name: 'Coding'
}, {
value: 310,
name: 'Database'
},
I try this :
result = []
chart_ktg =[
{
'value' : ktg_mhsi[1],
'name' : ktg_mhsi[0]
}
for ktg_mhsi in ktg_mhs
]
# result.append(chart_ktg)
data = json.dumps(chart_ktg)
And the output like this :
[{"value": 51, "name": "Coding"}, {"value": 27, "name": "Database"}, {"value": 223, "name": "UI/UX"}, {"value": 34, "name": "Manajemen"}, {"value": 7, "name": "Dasar Sistem Informasi"}, {"value": 64, "name": "Bisnis"}, {"value": 18, "name": "Kesehatan"}, {"value": 235, "name": "Lain-lain"}]
for i in ktg_mhs will not yield the indices, but the elements of the list, so you work with:
for ktg_mhsi in ktg_mhs:
chart_ktg ={
'value' : ktg_mhsi[1],
'name' : ktg_mhsi[0]
}
result.append(chart_ktg)
data = json.dumps(chart_ktg)
We can replace this with list comprehension with:
chart_ktg = [
{
'value' : value,
'name' : name
}
for name, value in ktg_mhs
]
data = json.dumps(chart_ktg)

Issues with turning json data into an variable in python

I have been working on an educational project a small part of it requires me to convert a single line of json data into an variable in python 3 which I recieve from domoticz (an external open source software) however due to my skill level with json I have expierenced some issues and I am not exactly sure what im doing wrong. I did get the 200 response everytime so I assume from what I understood that means the connection isnt the issue but rather the python code. (I censored the addressed but they are correct.)
The code im using:
import time
import re
import requests
from ctypes import c_int, c_char_p, byref, sizeof, c_uint16, c_int32, c_byte
from ctypes import c_void_p
from datetime import datetime
import os
import urllib.request
import json
import logging
import sys
from requests.exceptions import HTTPError
logger = logging.getLogger(__name__)
domoticzserver='ip'
switchid='3'
device='5'
tempbed=str(4)
def domoticzrequest (url):
request = urllib.request.Request(url)
response = urllib.request.urlopen(request)
return response.read()
import urllib.request, json
with urllib.request.urlopen("http://domoticzip/json.htm?type=devices&rid=4") as url:
data = json.loads(url.read().decode())
print(data)
The json I get back which i can see by typing clicking the url in python:
{
"ActTime" : 1606722346,
"AstrTwilightEnd" : "18:37",
"AstrTwilightStart" : "06:23",
"CivTwilightEnd" : "17:14",
"CivTwilightStart" : "07:47",
"DayLength" : "08:08",
"NautTwilightEnd" : "17:56",
"NautTwilightStart" : "07:04",
"ServerTime" : "2020-11-30 08:45:46",
"SunAtSouth" : "12:30",
"Sunrise" : "08:26",
"Sunset" : "16:34",
"app_version" : "2020.2",
"result" :
[
{
"AddjMulti" : 1.0,
"AddjMulti2" : 1.0,
"AddjValue" : 0.0,
"AddjValue2" : 0.0,
"BatteryLevel" : 255,
"CustomImage" : 0,
"Data" : "Normal",
"Description" : "",
"Favorite" : 0,
"HardwareID" : 1,
"HardwareName" : "Domoticz Internal",
"HardwareType" : "Domoticz Internal interface",
"HardwareTypeVal" : 67,
"HaveDimmer" : false,
"HaveGroupCmd" : false,
"HaveTimeout" : false,
"ID" : "148702",
"LastUpdate" : "2020-10-19 15:10:02",
"MaxDimLevel" : 0,
"Name" : "Domoticz Security Panel",
"Notifications" : "false",
"PlanID" : "0",
"PlanIDs" :
[
0
],
"Protected" : false,
"ShowNotifications" : true,
"SignalLevel" : "-",
"Status" : "Normal",
"StrParam1" : "",
"StrParam2" : "",
"SubType" : "Security Panel",
"SwitchType" : "Security",
"SwitchTypeVal" : 0,
"Timers" : "false",
"Type" : "Security",
"TypeImg" : "security",
"Unit" : 0,
"Used" : 0,
"XOffset" : "0",
"YOffset" : "0",
"idx" : "2"
},
{
"AddjMulti" : 1.0,
"AddjMulti2" : 1.0,
"AddjValue" : 0.0,
"AddjValue2" : 0.0,
"BatteryLevel" : 255,
"CustomImage" : 0,
"Data" : "-5.0 C",
"Description" : "",
"Favorite" : 1,
"HardwareID" : 2,
"HardwareName" : "Test sensor",
"HardwareType" : "Dummy (Does nothing, use for virtual switches only)",
"HardwareTypeVal" : 15,
"HaveTimeout" : true,
"ID" : "14053",
"LastUpdate" : "2020-11-09 09:03:34",
"Name" : "Temperatuur Kachel",
"Notifications" : "false",
"PlanID" : "0",
"PlanIDs" :
[
0
],
"Protected" : false,
"ShowNotifications" : true,
"SignalLevel" : "-",
"SubType" : "LaCrosse TX3",
"Temp" : -5.0,
"Timers" : "false",
"Type" : "Temp",
"TypeImg" : "temperature",
"Unit" : 1,
"Used" : 1,
"XOffset" : "0",
"YOffset" : "0",
"idx" : "3",
"trend" : 0
},
{
"AddjMulti" : 1.0,
"AddjMulti2" : 1.0,
"AddjValue" : 0.0,
"AddjValue2" : 0.0,
"BatteryLevel" : 255,
"CustomImage" : 0,
"Data" : "17.5",
"Description" : "",
"Favorite" : 1,
"HardwareID" : 3,
"HardwareName" : "Test switch",
"HardwareType" : "Dummy (Does nothing, use for virtual switches only)",
"HardwareTypeVal" : 15,
"HaveTimeout" : true,
"ID" : "0014054",
"LastUpdate" : "2020-11-06 11:51:09",
"Name" : "Temperatuur gewenst",
"Notifications" : "false",
"PlanID" : "0",
"PlanIDs" :
[
0
],
"Protected" : false,
"SetPoint" : "17.5",
"ShowNotifications" : true,
"SignalLevel" : "-",
"SubType" : "SetPoint",
"Timers" : "false",
"Type" : "Thermostat",
"TypeImg" : "override_mini",
"Unit" : 1,
"Used" : 1,
"XOffset" : "0",
"YOffset" : "0",
"idx" : "4"
}
],
"status" : "OK",
"title" : "Devices"
}
Basicly I want SetPoint(in the last tab of text also right above this) which is in this instance 17.5 as a variable in python and I will make the python code loop so it will grab that json url everytime to update the status of setpoint. But im having issues only grabbing the 17.5 to make it into a variable. I end up getting the entire json like this code is doing. or I will end up getting everything past and including the setpoint if I change some stuff. Does anyone know what im doing wrong and possibly where I should be looking for an solution? I am a bit unexpierenced with the json part of python and I have no clue where to get started as the code's I found and have tried seem to not work or give me errors.
Thank you very much for your time!
json.loads returns a python dictionary so maybe sth like this would do:
result = json['result']
set_point = 0.0
for res in result:
if 'SetPoint' in res:
set_point = res['SetPoint']
You are getting your data stored in data ={"key": argument} as a dictionary.
If you want to access a certain value you have to call for it. in Your case:
SetPoint = float(data["result"][-1]["SetPoint"])
To break it down:
data["result"] # gives you a list which elements are dictionaries.
the [-1] # calls for the last element in you list which contains the SetPoint
["SetPoint"] # then calls for the SetPoint Value which is a String "17.5"
float(...) converts the string to a float value

JSONDecoder erorr: Expecting ',' delimiter: line 10 column 29 (char 16011)

I am requesting URLs which contain JSON objects in python.
Some of these URLs contain several JSON objects (so several parts which are enclosed by these brackets: [])
When I tried to load these with json.loads() I got the error:
JSONDecodeError: Extra data: line 494 column 1 (char 50502)
Therefore I tried to split the JSON objects and write them into a list like this:
response = requests.get(url)
textinhalt = response.text
ref = textinhalt.rsplit('[')
tev = []
for line in ref:
daten = json.loads(line[line.find(r"{"):line.rfind("}")+1])
tev.append(daten)
But I get this error:
JSONDecodeError: Expecting ',' delimiter: line 10 column 29 (char 16011)
For example here is the part of the JSON that causes the Extra Data error(line 494):
474. "originalbild" : {
475. "alt" : "",
476. "height" : "3601",
477. "quelle" : "",
478. "src" : "/imgs/65/2/7/6/2/8/0/4/IMG_5630-3a13bb38ae440652.jpeg",
479. "untertitel" : "",
480. "width" : "5401"
481. },
482. "b" : "",
483. "redakteur_bid" : "",
484. "redakteur_email" : "",
485. "redakteur_inline" : "0",
486. "redakteur_kategorie" : "1",
487. "redakteur_kuerzel" : "",
488. "redakteur_nachname" : "",
489. "redakteur_redaktion" : "",
490. "redakteur_vorname" : "",
491. "ressort" : "",
492. "seitennavigation_liste" : [
493. {
494. "_baseurl" : "/Macoun-2019-Von-SwiftUI-bis-NFC-4547400.html",
495. "artikelseite" : 1,
496. "container_id" : 4547400,
497. "titel" : "Macoun 2019: Von SwiftUI bis NFC"
498. },
499. {
500. "artikelseite" : 2,
501. "titel" : "Motion Capturing in ARKit und RealityKit"
502. },
503. {
504. "artikelseite" : 3,
505. "titel" : "Bring deine Tests zum Rennen"
506. },
507. {
508. "artikelseite" : 4,
509. "titel" : "Tipps f\u00fcr Existenzgr\u00fcnder"
510. },
511. {
512. "artikelseite" : 5,
513. "titel" : "Cross Platform Entwicklung mit Kotlin"
514. }
515. ],
516. "seo_beschreibung" : "",
517. "seo_no_meta_description" : 0,
518. "seo_titel" : "",
519. "show_webdev_chooser" : null,
520. "socialbookmarks_keywords_ph_data" : "Apple, Entwickler, Programmieren, iOS, macOS",
521. "speakingurl_primitive" : "Macoun-2019-Von-SwiftUI-bis-NFC",
522. "teaser_anrisstext" : "",
523. "teaser_titel" : "",
524. "teaser_untertitel" : "",
525. "texte_anzahl_zeichen" : [
526. 12499
527. ],
What am I doing wrong?

Error while adding key/value to Python Dict in nested loop

I have a Json Structure as Follows:
{
"_id" : ObjectId("asdasda156121s"),
"Hp" : {
"bermud" : [
{
"abc" : {
"gfh" : 1,
"fgh" : 0.0,
"xyz" : [
{
"kjl" : "0",
"bnv" : 0,
}
],
"xvc" : "bv",
"hgth" : "INnn",
"sdf" : 0,
}
}
},
{
"abc" : {
"gfh" : 1,
"fgh" : 0.0,
"xyz" : [
{
"kjl" : "0",
"bnv" : 0,
}
],
"xvc" : "bv",
"hgth" : "INnn",
"sdf" : 0,
}
}
},
..
I am trying to parse this json and add a new value with key ['cat'] inside the object 'xyz',below is my py code.
data = []
for x in a:
for y in x['Hp'].values():
for z in y:
for k in z['abc']['xyz']:
for m in data:
det = m['response']
// Some processing with det whose output is stored in s
k['cat'] = s
print x
However when x is printed only the last value is being appended onto the whole dictionary, wheras there are different values for s. Its obvious that the 'cat' key is being overwritten everytime the loop rounds,but can't find a way to make it right.What mistake am I making?

MongoDB find in array of objects

I want to query Mongodb: find all users, that have 'artist'=Iowa in any array item of objects.
Here is Robomongo of my collection:
In Python I'm doing:
Vkuser._get_collection().find({
'member_of_group': 20548570,
'my_music': {
'items': {
'$elemMatch': {
'artist': 'Iowa'
}
}
}
})
but this returns nothing. Also tried this:
{'member_of_group': 20548570, 'my_music': {'$elemMatch': {'$.artist': 'Iowa'}}} and that didn't work.
Here is part of document with array:
"can_see_audio" : 1,
"my_music" : {
"items" : [
{
"name" : "Anastasia Plotnikova",
"photo" : "http://cs620227.vk.me/v620227451/9c47/w_okXehPbYc.jpg",
"id" : "864451",
"name_gen" : "Anastasia"
},
{
"title" : "Ain't Talkin' 'Bout Dub",
"url" : "http://cs4964.vk.me/u14671028/audios/c5b8a0735224.mp3?extra=jgV4ZQrFrsfxZCJf4gsRgnKWvdAfIqjE0M6eMtxGFpj2yp4vjs5DYgAGImPMp4mCUSUGJzoyGeh2Es6L-H51TPa3Q_Q",
"lyrics_id" : 24313846,
"artist" : "Apollo 440",
"genre_id" : 18,
"id" : 344280392,
"owner_id" : 864451,
"duration" : 279
},
{
"title" : "Animals",
"url" : "http://cs1316.vk.me/u4198685/audios/4b9e4536e1be.mp3?extra=TScqXzQ_qaEFKHG8trrwbFyNvjvJKEOLnwOWHJZl_cW5EA6K3a9vimaMpx-Yk5_k41vRPywzuThN_IHT8mbKlPcSigw",
"lyrics_id" : 166037,
"artist" : "Nickelback",
"id" : 344280351,
"owner_id" : 864451,
"duration" : 186
},
The following query should work. You can use the dot notation to query into sub documents and arrays.
Vkuser._get_collection().find({
'member_of_group': 20548570,
'my_music.items.artist':'Iowa'
})
The following query worked for me in the mongo shell
db.collection1.find({ "my_music.items.artist" : "Iowa" })

Categories