We Keep Coding

Python parse json object within object within object

I've been successful in reading in and parsing a json string using python including normalising and exploding arrays. However I have a second json format which I'm struggling with and making little head way.
I need to pull out the 'Entities' and 'Transitions' within the 'data' object.
My json structure is:
{
"model": {
"id": "639b2970ac4d16767484b2bd",
"name": "TestImport",
"description": "",
"type": "LineageModel",
"referenceModelType": null,
"owner": {
"id": "639b2904ac4d167674849e1e",
"name": "xyz",
"firstName": null,
"lastName": null
},
"members": {
"totalUsers": 1,
"totalGroups": 0,
"users": {
"Owner": [
{
"joinTime": "2022-12-15T14:04:32.076Z",
"modificationTime": "2022-12-15T14:04:32.076Z",
"email": "sxyz",
"username": null,
"hasPendingSiteInvite": false,
"isDisabled": false,
"id": "639b2904ac4d167674849e1e",
"name": "xyz",
"firstName": null,
"lastName": null
}
]
},
"groups": {}
},
"$type": "ModelInformation"
},
"data": {
"version": "simple",
"roots": [
"8a44e4d6-4062-4e1c-a46c-c7787cab4405",
"d1494635-9005-4337-8eab-227265b29332"
],
"entities": {
"29380f60-620e-4314-9969-4ad6fe5bbea6": {
"name": "Element",
"children": [],
"id": "29380f60-620e-4314-9969-4ad6fe5bbea6",
"properties": {}
},
"86361ab4-6002-4f3b-b6ca-7e35acd69f9b": {
"name": "Application",
"children": [
"29380f60-620e-4314-9969-4ad6fe5bbea6"
],
"id": "86361ab4-6002-4f3b-b6ca-7e35acd69f9b",
"properties": {}
},
"223d9749-feb2-425d-b512-17b5322cda96": {
"name": "S_Group",
"children": [
"86361ab4-6002-4f3b-b6ca-7e35acd69f9b"
],
"id": "223d9749-feb2-425d-b512-17b5322cda96",
"properties": {}
}
},
"transitions": {
"c4e2026a-9c57-4bb0-b2e2-f7068d9c6fe5": {
"source": "29380f60-620e-4314-9969-4ad6fe5bbea6",
"target": "040677a5-820f-4d17-ae50-1296c0e36273",
"id": "c4e2026a-9c57-4bb0-b2e2-f7068d9c6fe5",
"properties": {}
}
},
"metadata": {
"queries": "{\"version\":\"3\"}",
"views": "{\"views\":[{\"name\":\"MainView\",\"description\":\"\",\"applyOnLoad\":true,\"view\":{\"version\":20,\"zoom\":{\"scale\":1,\"layerWidth\":250,\"layerSpacing\":40},\"collapsed\":{\"expanded\":[\"961847ad-1693-47b5-aa29-88ee07700b5e\",\"0da88727-e272-44a4-adeb-5a0465970490\",\"223d9749-feb2-425d-b512-17b5322cda96\",\"86361ab4-6002-4f3b-b6ca-7e35acd69f9b\"],\"collapsed\":[]},\"trace\":{\"enabled\":true,\"lock\":null,\"highlightedTraceDepth\":1,\"isHighlightedTraceDepthAll\":true,\"isTraversable\":true},\"selection\":[],\"queries\":{\"styled\":[],\"filtered\":[],\"filterType\":{\"6eecd266-2c2e-46d1-a00a-ceec18a87cdb\":\"show\"},\"expandedModules\":[]},\"settings\":{\"hideEmptyContainers\":false,\"hideFilteredLayers\":false,\"expandFilteredEntities\":false,\"portHintsEnabled\":true,\"autoBundleTransitions\":1,\"autoStyleTransitions\":1,\"autoHideTransitions\":1,\"maxSpanningTransitionDepth\":10,\"rootEntityType\":\"Layer\"}},\"options\":{\"zoom\":true,\"collapsed\":true,\"trace\":true,\"selection\":true,\"queries\":true,\"settings\":true},\"id\":\"VIEW-tcPIi3jP\"}]}"
},
"queries": [
{
"id": "9eb5c7fe-8d4c-4d94-b1a7-fee08bb2f663",
"name": "TestID",
"description": "",
"displayRules": "[{\"id\":\"e119af2d-5061-43f3-901a-4c70f402bb5d\",\"type\":\"PROPERTY\",\"staticColour\":\"#E47833\",\"dynamicColouring\":false,\"cls\":\"dr-19\",\"property\":\"TestID\",\"prefix\":\"\",\"suffix\":\"\",\"center\":false,\"left\":false}]",
"querySource": "not isEmpty(TestID)",
"modulePath": "Uncategorised",
"importedFromModel": null
}
],
"propertyDefinitions": {
"propertyDefinitionsId": "2ab6a0c3-8d24-4235-9783-e241437bf860",
"modelId": "639b2970ac4d16767484b2bd",
"properties": {
"TestID": {
"type": "Number",
"defaultValue": null,
"options": [],
"optionInfos": {}
}
}
}
},
"importedModels": {},
"importedModelsForQueries": {},
"propertyDefinitionsForImports": {},
"templateCollections": {}
}
I have been using the following to convert a json to data frame:
fInput = 'filepath to json file'
with open(fInput, encoding='utf-16') as inputfile:
df = pd.read_json(inputfile)
fOutput = 'output file path'
df.to_csv(fOutput, encoding='utf-16, index = false)
I then normalise columns using which for other json formats works
pd.json_normalize(df['column'])
and I explode arrays in the columns using:
df2 = pd.DataFrame([d, tup.id) for df.intertuples() for d in tup.columnName])
What I can't work out is how to pull into a data frame the 'entities' object from the 'data' object. Once I can get to that then I should be able to parse the content.
I had got to:
df = df["data"]
df = df["entities"]
When I print that it looks promising, but if I try to output to csv it fails with "'dict' object has no attributes" so I'm going wrong somewhere. The traceback for the error is:
AttributeError
Input [48], in <cell line: 14>()
12 df = df["entities"]
13 print(df)
14 df.to_csv(fOutput, encoding='utf-16', index=false)
AttributeError: 'dict' object has no attribute 'to_csv'
Any pointers appreciated.

Couldn't you just do this?
I saved the json data you gave in a file named data.txt
import json
with open('data.txt','r') as file:
data = json.load(file)
entities = data['data']['entities']
transitions = data['data']['transitions']
print(f'Entities: {entities}\nTransitions: {transitions}')
Output
Entities: {'29380f60-620e-4314-9969-4ad6fe5bbea6': {'name': 'Element', 'children': [], 'id': '29380f60-620e-4314-9969-4ad6fe5bbea6', 'properties': {}}, '86361ab4-6002-4f3b-b6ca-7e35acd69f9b': {'name': 'Application', 'children': ['29380f60-620e-4314-9969-4ad6fe5bbea6'], 'id': '86361ab4-6002-4f3b-b6ca-7e35acd69f9b', 'properties': {}}, '223d9749-feb2-425d-b512-17b5322cda96': {'name': 'S_Group', 'children': ['86361ab4-6002-4f3b-b6ca-7e35acd69f9b'], 'id': '223d9749-feb2-425d-b512-17b5322cda96', 'properties': {}}}
Transitions: {'c4e2026a-9c57-4bb0-b2e2-f7068d9c6fe5': {'source': '29380f60-620e-4314-9969-4ad6fe5bbea6', 'target': '040677a5-820f-4d17-ae50-1296c0e36273', 'id': 'c4e2026a-9c57-4bb0-b2e2-f7068d9c6fe5', 'properties': {}}}

I am unable to create dictionary

I am unable to create a dictionary whose structure is like :
"aggs": {
"4": {
"date_histogram": {
"field": "#timestamp",
"interval": "1m",
"time_zone": "Asia/Kolkata",
"min_doc_count": 1,
},
"aggs": {
"5": {
"range": {
"field": "system.core.idle.pct",
"ranges": [{"from": 0, "to": 0.001}, {"from": 0.001, "to": 1}],
"keyed": true,
},
"aggs": {
"2": {"max": {"field": "system.core.system.pct"}},
"3": {"max": {"field": "system.core.idle.pct"}},
},
}
},
}
}
Mostly I am getting the issue in creating aggs:5 inside aggs:4 and then looping it. I need to do it for multiple aggs. the number of agg id can be upto 1000 also.
I am trying to derive it from the dictionary:
"aggs": [
{"id": "1", "enabled": true, "type": "count", "schema": "metric", "params": {}},
{
"id": "2",
"enabled": true,
"type": "max",
"schema": "metric",
"params": {"field": "system.core.system.pct", "customLabel": "system"},
},
{
"id": "3",
"enabled": true,
"type": "max",
"schema": "metric",
"params": {"field": "system.core.idle.pct", "customLabel": "idle"},
},
{
"id": "4",
"enabled": true,
"type": "date_histogram",
"schema": "bucket",
"params": {
"field": "#timestamp",
"interval": "m",
"customInterval": "2h",
"min_doc_count": 1,
"extended_bounds": {},
},
},
{
"id": "5",
"enabled": true,
"type": "range",
"schema": "bucket",
"params": {
"field": "system.core.idle.pct",
"ranges": [{"from": 0, "to": 0.001}, {"from": 0.001, "to": 1}],
},
},
]
Can anyone show the code how to execute it. Basically I am failing in creating nested dictionary.

It looks like you are using JSON data.
You can see this link https://realpython.com/python-json/.
You can use the inbuilt JSON lib to covert the whole thing in a dict
Basically something like this
json_string = """
{
"researcher": {
"name": "Ford Prefect",
"species": "Betelgeusian",
"relatives": [
{
"name": "Zaphod Beeblebrox",
"species": "Betelgeusian"
}
]
}
}
"""
data = json.loads(json_string)

add whole structure in {}, and replace true to True.
{"aggs": {
"4": {
"date_histogram": {
"field": "#timestamp",
"interval": "1m",
"time_zone": "Asia/Kolkata",
"min_doc_count": 1,
},
"aggs": {
"5": {
"range": {
"field": "system.core.idle.pct",
"ranges": [{"from": 0, "to": 0.001}, {"from": 0.001, "to": 1}],
"keyed": True,
},
"aggs": {
"2": {"max": {"field": "system.core.system.pct"}},
"3": {"max": {"field": "system.core.idle.pct"}},
},
}
},
}
}}
{'aggs': {'4': {'date_histogram': {'field': '#timestamp',
'interval': '1m',
'time_zone': 'Asia/Kolkata',
'min_doc_count': 1},
'aggs': {'5': {'range': {'field': 'system.core.idle.pct',
'ranges': [{'from': 0, 'to': 0.001}, {'from': 0.001, 'to': 1}],
'keyed': True},
'aggs': {'2': {'max': {'field': 'system.core.system.pct'}},
'3': {'max': {'field': 'system.core.idle.pct'}}}}}}}}

Remove all empty keys in a nested dictionary (via yield)

I have the following function to remove keys with an empty value or that start with an underscore. It works on a non-nested dictionary:
def _remove_missing_and_underscored_keys(d):
if not d: return d
for key in d.keys():
if not d.get(key):
del d[key]
elif key.startswith('_'):
del d[key]
return d
d = {"Name": "David",
"_Age": 50,
"Numbers": [1,2,3,4,5],
"Height": ""
}
>>> _remove_missing_and_underscored_keys(d)
{'Name': 'David', 'Numbers': [1, 2, 3, 4, 5]}
However, I would like to create the above so that it can remove nested items as well. I believe I need to use a yield statement for this, but I'm not having luck with implementing it properly. Here is an example of what I want it to do:
d = {
"PlatformID": "B00EU7XL9Q",
"Platform": "Amazon",
"Type": "Collection",
"Products": {
"UK": {
"URL": "http://www.amazon.co.uk/dp/B00EU7XL9Q",
"Rating": None,
"_IsAudited": True,
"Offers": {
"HDBUY": {
"Currency": "GBP",
"FutureReleaseStartDate": None,
"Cost": "14.99",
"IsFutureRelease": False
},
"SDBUY": {
"Currency": "GBP",
"FutureReleaseStartDate": None,
"Cost": "14.99",
"IsFutureRelease": False
}
}
}
}
}
>>> _remove_missing_and_underscored_keys(d)
{
"PlatformID": "B00EU7XL9Q",
"Platform": "Amazon",
"Type": "Collection",
"Products": {
"UK": {
"URL": "http://www.amazon.co.uk/dp/B00EU7XL9Q",
"Offers": {
"HDBUY": {
"Currency": "GBP",
"Cost": "14.99",
},
"SDBUY": {
"Currency": "GBP",
"Cost": "14.99",
}
}
}
}
}
In other words, it will do the above operation on all nested levels of the dict.

You can use recursion with a dictionary comprehension:
d = {'PlatformID': 'B00EU7XL9Q', 'Platform': 'Amazon', 'Type': 'Collection', 'Products': {'UK': {'URL': 'http://www.amazon.co.uk/dp/B00EU7XL9Q', 'Rating': None, '_IsAudited': True, 'Offers': {'HDBUY': {'Currency': 'GBP', 'FutureReleaseStartDate': None, 'Cost': '14.99', 'IsFutureRelease': False}, 'SDBUY': {'Currency': 'GBP', 'FutureReleaseStartDate': None, 'Cost': '14.99', 'IsFutureRelease': False}}}}}
def _del(_d):
return {a:_del(b) if isinstance(b, dict) else b for a, b in _d.items() if b and not a.startswith('_')}
import json
print(json.dumps(_del(d), indent=4))
Output:
{
"PlatformID": "B00EU7XL9Q",
"Platform": "Amazon",
"Type": "Collection",
"Products": {
"UK": {
"URL": "http://www.amazon.co.uk/dp/B00EU7XL9Q",
"Offers": {
"HDBUY": {
"Currency": "GBP",
"Cost": "14.99"
},
"SDBUY": {
"Currency": "GBP",
"Cost": "14.99"
}
}
}
}
}

def _remove_missing_and_underscored_keys(d):
if not d: return d
for key in d.keys():
if not d.get(key):
del d[key]
elif key.startswith('_'):
del d[key]
elif type(d[key]) == dict:
d[key] = _remove_missing_and_underscored_keys(d[key])
return d

I think you meant to say, you needed to use recursion to solve this. I don't think using a generator quite solves your problem.
Another caveat is that you shouldn't iterate through a variable you are changing. That's why I create the copy _d modify and return that and iterate through the original structure.
import pprint
def _remove_missing_and_underscored_keys(d):
if not d: return d
_d = d.copy()
for key in _d.keys():
if not _d.get(key):
del d[key]
elif key.startswith('_'):
del d[key]
elif isinstance(_d[key], dict):
_remove_missing_and_underscored_keys(_d[key])
return _d
_d = {
"PlatformID": "B00EU7XL9Q",
"Platform": "Amazon",
"Type": "Collection",
"Products": {
"UK": {
"URL": "http://www.amazon.co.uk/dp/B00EU7XL9Q",
"Rating": None,
"_IsAudited": True,
"Offers": {
"HDBUY": {
"Currency": "GBP",
"FutureReleaseStartDate": None,
"Cost": "14.99",
"IsFutureRelease": False
},
"SDBUY": {
"Currency": "GBP",
"FutureReleaseStartDate": None,
"Cost": "14.99",
"IsFutureRelease": False
}
}
}
}
}
foo = _remove_missing_and_underscored_keys(_d)
pprint.pprint(foo)
Output:
{'Platform': 'Amazon',
'PlatformID': 'B00EU7XL9Q',
'Products': {'UK': {'Offers': {'HDBUY': {'Cost': '14.99', 'Currency': 'GBP'},
'SDBUY': {'Cost': '14.99', 'Currency': 'GBP'}},
'URL': 'http://www.amazon.co.uk/dp/B00EU7XL9Q'}},
'Type': 'Collection'}

Just go recursive.
Add another check to see if a value in primary dict is a dict, and call the same function on it.
# your code goes here
def _remove_missing_and_underscored_keys(d):
if not d: return d
for key in d.keys():
if not d.get(key):
del d[key]
elif key.startswith('_'):
del d[key]
elif type(d[key]) is dict:
#print("key '{}' stores a dict '{}', need to cleanup recursively".format(key, d[key]))
d[key] = _remove_missing_and_underscored_keys(d[key])
# Keep below check if you want to treat empty dict as `empty` as well
if d[key] == None or d[key] == {}:
del d[key]
return d
d = {
"PlatformID": "B00EU7XL9Q",
"Platform": "Amazon",
"Type": "Collection",
"Products": {
"UK": {
"URL": "http://www.amazon.co.uk/dp/B00EU7XL9Q",
"Rating": None,
"_IsAudited": True,
"Offers": {
"HDBUY": {
"Currency": "GBP",
"FutureReleaseStartDate": None,
"Cost": "14.99",
"IsFutureRelease": False
},
"SDBUY": {
"Currency": "GBP",
"FutureReleaseStartDate": None,
"Cost": "14.99",
"IsFutureRelease": False
},
"x" : {
"y":None
}
}
}
}
}
e = _remove_missing_and_underscored_keys(d)
print(e)
See it in action: https://ideone.com/5xDDZl
Above code also handles empty dicts stored at any key or any dict that became empty after recursively cleaning it. You can remove that check if needed.

Json format for python

I'm rewriting a view based on what I know the final output should be in json but it's returning the dictionary as a string.
new output
{
"results":
["
{
'plot': u'',
'runtime': u'N/A',
'description': u'x',
'videos': [
{
'id': 823,
'name': u'x',
'youtube_id': u'FtcubOnXgZk'
}
],
'country': u'India',
'writer': u'Neetu Varma, Ranjeev Verma',
'name': u'Chalk N Duster',
'id': 940,
'director': u'Jayant Gilatar',
'hot': True,
'content': u'x',
'actors': u'Shabana Azmi, Arya Babbar, Gavie Chahal, Juhi Chawla',
'year': 2015,
'images': [
{'small': '/media/cache/62/fd/62fd5158d281c042e3cf1f919183e94e.jpg', 'medium': '/media/cache/5e/32/5e32ebb1a4d25bba0d0c70b4b448e948.jpg'}],
'trailer_youtube_id': u'FtcubOnXgZk',
'type': 'movie',
'slug': u'chalk-n-duster',
'categories': [{'parent_id': 2, 'id': 226, 'name': u'Drama'}],
'shows': {
'starts': '2016-01-16',
'booking_url': u'',
'venue': {
'address': u'',
'id': 854,
'name': u'Nyali Cinemax',
'area': {
'id': 52,
'parent': {
'id': 48,
'name': u'Mombasa'
},
'name': u'Nyali'
}
},
'starts_time': '18:30:00'
}
}", "{'plot': u'' ....
old output
"results": [
{
"actors": "x",
"categories": [
{
"id": 299,
"name": "Biography",
"parent_id": 2
},
],
"content": "x",
"country": "x",
"description": "x",
"director": "x",
"hot": true,
"id": 912,
"images": [
{
"medium": "/media/cache/d2/b3/d2b3a7885e7c39bfc5c2b297b66619c5.jpg",
"small": "/media/cache/e2/d0/e2d01b2c7c77d3590536666de4a7fd7d.jpg"
}
],
"name": "Bridge of Spies",
"plot": "x",
"runtime": "141 min",
"shows": [
{
"booking_url": "",
"starts": "2015-11-27",
"starts_time": "16:30:00",
"venue": {
"address": "The Junction Shopping Mall",
"area": {
"id": 68,
"name": "Ngong Road",
"parent": {
"id": 2,
"name": "Nairobi"
}
},
"id": 1631,
"name": "Century Cinemax Junction"
}
},
],
"slug": "bridge-of-spies",
"trailer_youtube_id": "",
"type": "movie",
"videos": [
{
"id": "795",
"name": "Bridge of Spies",
"youtube_id": "2-2x3r1m2I4"
}
],
"writer": "Matt Charman, Ethan Coen, Joel Coen",
"year": 2015
}, ...
]
Here's the view, I know the shows should also be a list, but in order to start testing I'll need the data to come in the right format. If it's involves too much rewriting I'm okay with links and explanation.
#memoize(timeout=60*60)
def movies_json():
today = datetime.date.today()
movies = Movie.objects.filter(shows__starts__gte=today)
results = []
number = len(movies)
for movie in movies:
print "Now Remaining: {0}".format(number)
number -= 1
medium = get_thumbnail(movie.picture(), '185x274', crop='center', quality=99).url
small = get_thumbnail(movie.picture(), '50x74', crop='center', quality=99).url
movie_details = {
'director':movie.director,
'plot':movie.plot,
'actors':movie.actors,
'content':movie.content,
'country':movie.country,
'description':movie.description,
'hot':movie.hot,
'id':movie.id,
'images':[{'medium':medium, 'small':small}],
'name':movie.name,
'plot':movie.plot,
'runtime':movie.runtime,
'slug':movie.slug,
'type':'movie',
'writer':movie.writer,
'year':movie.year,
}
youtube_details = movie.videos.filter(youtube_id__isnull=False)[0]
movie_details['trailer_youtube_id'] = youtube_details.youtube_id if youtube_details.youtube_id else ""
movie_details['videos'] = [
{
'id':youtube_details.id,
'name':movie.name,
'youtube_id':youtube_details.youtube_id,
}
]
shows = []
for show in movie.shows.all():
show_details = {
'booking_url':show.booking_url,
'starts':show.starts.isoformat(),
'starts_time':show.starts_time.isoformat(),
'venue': {
'address':show.venue.address,
'area': {
'id': show.venue.area.id,
'name': show.venue.area.name,
'parent': {
'id': show.venue.area.parent.id,
'name': show.venue.area.parent.name,
}
},
'id': show.venue.id,
'name': show.venue.name,
}
}
shows.append(show_details)
movie_details['shows'] = show_details
category_list = []
for category in movie.categories.all():
category_details = {
'id':category.id,
'name':category.name,
'parent_id':category.parent.id,
}
category_list.append(category_details)
movie_details['categories'] = category_list
results.append(movie_details)
return results
The data is returned by django rest framework 0.4.0

import json
json_obj = json.load(json_string)

Work with JSON file on Python

I have a Google Chrome Bookmark file, and it's in JSON format
{
"checksum": "b884cbfb1a6697fa9b9eea9cb2054183",
"roots": {
"bookmark_bar": {
"children": [ {
"date_added": "12989159740428363",
"id": "4",
"name": "test2",
"type": "url",
"url": "chrome://bookmarks/#1"
} ],
"date_added": "12989159700896551",
"date_modified": "12989159740428363",
"id": "1",
"name": "bookmark_bar",
"type": "folder"
},
"other": {
"children": [ {
"date_added": "12989159740428363",
"id": "4",
"name": "test",
"type": "url",
"url": "chrome://bookmarks/#1"
} ],
"date_added": "12989159700896557",
"date_modified": "0",
"id": "2",
"name": "aaa",
"type": "folder"
},
"synced": {
"children": [ ],
"date_added": "12989159700896558",
"date_modified": "0",
"id": "3",
"name": "bbb",
"type": "folder"
}
},
"version": 1
}
and in Python format:
{'checksum': 'b884cbfb1a6697fa9b9eea9cb2054183', 'version': 1, 'roots': {'synced': {'name': 'bbb', 'date_modified': '0', 'children': [], 'date_added': '12989159700896558', 'type': 'folder', 'id': '3'}, 'bookmark_bar': {'name': 'bookmark_bar', 'date_modified': '12989159740428363', 'children': [{'url': 'chrome://bookmarks/#1', 'date_added': '12989159740428363', 'type': 'url', 'id': '4', 'name': 'test2'}], 'date_added': '12989159700896551', 'type': 'folder', 'id': '1'}, 'other': {'name': 'aaa', 'date_modified': '0', 'children': [{'url': 'chrome://bookmarks/#1', 'date_added': '12989159740428363', 'type': 'url', 'id': '4', 'name': 'test'}], 'date_added': '12989159700896557', 'type': 'folder', 'id': '2'}}}
I'm writing a bookmark manager now.
I want to move the web pages by name.
For example: mv /bookmark_bar/test2 /other/test2
But every web pages are dictionaries, and they are in a list. So, I must use index to locate the web pages, I can't locate them by name.
Any ideas?

Is it what you need https://gist.github.com/3332055 ?

An example of how to iterate over the structure - exactly what you want to do with it then, is up to you:
for root, val in bm['roots'].iteritems():
print root, 'is named', val['name']
for child in val['children']:
print '\t', child['name'], 'is at', child['url']

# -*- coding: utf-8 -*-
import json
def hook(pairs):
o = {}
for k, v in pairs.iteritems():
o[str(k)] = v
return o
jsonString = """{"a":"a","b":"b","c":{"c1":"c1","c2":"c2"}}"""
r = json.loads(jsonString, object_hook=hook)
assert r['c']['c1'] == "c1"
del r['c']['c1']
assert not r['c'].has_key('c1')