How to merge strings from the yield generator of JSON into one JSON?
I have got Nested Dictionary by yield generator, and I aim to have one JSON file.
I have the output of these correct strings of nested dictionary.
{"domain.com": {"Chrome": "19362.344607264396"}}
{"domain.com": {"ChromeMobile": "7177.498437391487"}}
{"another.com": {"MobileSafari": "6237.433155080214"}}
{"another.com": {"Safari": "5895.409403430795"}}
and I want to merge into one JSON file
[
{
"domain.com": {
"Chrome": "19362.344607264396"
}
},
{
"domain.com": {
"ChromeMobile": "7177.498437391487"
}
},
{
"another.com": {
"MobileSafari": "6237.433155080214"
}
},
{
"another.com": {
"Safari": "5895.409403430795"
}
}
]
or ideally to have a JSON likes to this
{
"browsers": [
{
"domain.com": {
"Chrome": "19362.344607264396",
"ChromeMobile": "7177.498437391487",
},
"another.com": {
"MobileSafari": "6237.433155080214",
"Safari": "5895.409403430795"
}
}
]
}
My code
# Cloudflare zone bandwidth total
def browser_map_page_views(domain_zone):
cloudflare = prom.custom_query(
query="topk(5, sum by(family) (increase(browser_map_page_views_count{job='cloudflare', zone='"f'{domain_zone}'"'}[10d])))"
)
for domain_z in cloudflare:
user_agent = domain_z['metric']['family']
value = domain_z['value'][1]
yield {domain_zone: {user_agent: {'value': value}}}
# Get list of zones from Prometheus based on Host Tracker data
def domain_zones():
zones_domain = prom.custom_query(
query="host_tracker_uptime_percent{job='donodeexporter'}"
)
for domain_z in zones_domain:
yield domain_z['metric']['zone']
There is a final output.
# Get a list of domains and substitution each one into a request of Prometheus query.
for domain_list in domain_zones():
for dict in browser_map_page_views(domain_zone=domain_list):
dicts = dict
print(json.dumps(dicts))
Something like this
import json
DATA = [
{"domain.com": {"Chrome": "19362.344607264396"}},
{"domain.com": {"ChromeMobile": "7177.498437391487"}},
{"another.com": {"MobileSafari": "6237.433155080214"}},
{"another.com": {"Safari": "5895.409403430795"}}
]
def yield_data():
for d in DATA:
yield d
def merge1():
return list(yield_data())
def merge2():
output = {}
for d in DATA:
key = list(d.keys())[0]
output[key] = output.get(key, {})
output[key].update(d[key])
return {"browsers": [output]}
print(json.dumps(merge1(), indent=2))
print(json.dumps(merge2(), indent=2))
merge1's output looks like:
[
{
"domain.com": {
"Chrome": "19362.344607264396"
}
},
{
"domain.com": {
"ChromeMobile": "7177.498437391487"
}
},
{
"another.com": {
"MobileSafari": "6237.433155080214"
}
},
{
"another.com": {
"Safari": "5895.409403430795"
}
}
]
merge2's output looks like
{
"browsers": [
{
"domain.com": {
"Chrome": "19362.344607264396",
"ChromeMobile": "7177.498437391487"
},
"another.com": {
"MobileSafari": "6237.433155080214",
"Safari": "5895.409403430795"
}
}
]
}
Related
I have the following object in MongoDB:
{
"_id":"...",
"username":"XXX",
"keys":[
{
"key":"c443c2cc2754d3",
"ref":"Autos",
"lists":[
{
"list_name":"Toyota",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":false,
"bonnet":true,
"interior":{
"dashboard":true,
"electronics":false
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"parked"
},
{
"list_name":"Nissan",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":true,
"bonnet":false,
"interior":{
"dashboard":false,
"electronics":true
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"garage"
}
]
},
{
"key":"80d54bd834ff60",
"ref":"Trucks",
"lists":[
{
"list_name":"MAN",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":false,
"bonnet":false,
"interior":{
"dashboard":false,
"electronics":false
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"parked"
},
{
"list_name":"Toyota",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":true,
"bonnet":false,
"interior":{
"dashboard":true,
"electronics":true
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"leased"
}
]
}
]
}
and whenever I try to update the status of an item inside a list, it only updates the first object.
If I try to update the status of the 1st list, 2nd object, it will only update the first one:
self.users.update_one({"username": username,
"keys.key": key,
"keys.lists.list_name": listid
},
{"$set": {
"keys.0.lists.$.status": 'repaired'
}
}
)
What am I doing wrong and how can I update the list item I want? or even the 3rd nested object (dashboard, electronics)?
I have the list of path string like below. How to convert it into complete json object?
foldersList = [
'1/',
'1/2/',
'1/2/2.txt',
'1/2/5/',
'1/5.txt',
'2/',
'2/test.txt',
'test.json'
]
How to convert it into complete json object like below
{
"fileMenu":{
"list":[
{
"fileType":"d",
"name":"1",
"subFolders":[
{
"fileType":"-",
"name":"5.txt",
},
{
"fileType":"d",
"name":"2",
"subFolders":[
{
"fileType":"-",
"name":"2.txt",
},
{
"date":1594983597000,
"fileType":"d",
"name":"5",
"size":0,
"subFolders":[]
}]
}]
},
{
"fileType":"d",
"name":"2",
"subFolders":[{
"fileType":"-",
"name":"test.txt"
}]
},
{
"fileType":"-",
"name":"test.json"
}],
"status":"OK"
}
}
How to do this? I tried it with the some code snippets.
foldersList = [
'1/',
'1/2/',
'1/2/2.txt',
'1/2/5/',
'1/5.txt',
'2/',
'2/test.txt',
'test.json'
]
foldersJson = {}
nodeInfoList = []
nodeInfoDic = {}
for i, path in enumerate(foldersList):
nodeInfoDic = foldersJson
for j,node in enumerate(path.split('/')):
if node != '':
if nodeInfoDic.has_key(node) != True:
nodeInfoDic[node] = {}
nodeInfoDic = nodeInfoDic[node]
# print(foldersJson)
nodeInfoList.append(nodeInfoDic)
print(nodeInfoList)
# print(foldersJson)
I am trying to scrape some values through a json that looks like:
{
"attributes":{
"531":{
"id":"531",
"code":"taille",
"label":"taille",
"options":[
{
"id":"30",
"label":"40",
"is_in":"0"
},
{
"id":"31",
"label":"41",
"is_in":"1"
}
]
}
},
"template":"Helloworld"
}
My issue is that the number 531 is different in each json file that I am trying to scrape and what I am trying to grab through this json is the label and is_in value
What I have done so far is that I tried to do something like this but I am stuck and dont know how to do if the 531 is changing to something else
getOption = '{
"attributes":{
"531":{
"id":"531",
"code":"taille",
"label":"taille",
"options":[
{
"id":"30",
"label":"40",
"is_in":"0"
},
{
"id":"31",
"label":"41",
"is_in":"1"
}
]
}
},
"template":"Helloworld"
}'
for att, values in getOption.items():
print(values)
So how do I possible scrape the value label and is_in?
I'm not sure if you can have several 531 keys but you can loop through them.
getOption = {
"attributes":{
"531":{
"id":"531",
"code":"taille",
"label":"taille",
"options":[
{
"id":"30",
"label":"40",
"is_in":"0"
},
{
"id":"31",
"label":"41",
"is_in":"1"
}
]
}
},
"template":"Helloworld"
}
attributes = getOption['attributes']
for key in attributes.keys():
for item in attributes[key]['options']:
print(item['label'], item['is_in'])
I am attempting to parse a json response that looks like this:
{
"links": {
"next": "http://www.neowsapp.com/rest/v1/feed?start_date=2015-09-08&end_date=2015-09-09&detailed=false&api_key=xxx",
"prev": "http://www.neowsapp.com/rest/v1/feed?start_date=2015-09-06&end_date=2015-09-07&detailed=false&api_key=xxx",
"self": "http://www.neowsapp.com/rest/v1/feed?start_date=2015-09-07&end_date=2015-09-08&detailed=false&api_key=xxx"
},
"element_count": 22,
"near_earth_objects": {
"2015-09-08": [
{
"links": {
"self": "http://www.neowsapp.com/rest/v1/neo/3726710?api_key=xxx"
},
"id": "3726710",
"neo_reference_id": "3726710",
"name": "(2015 RC)",
"nasa_jpl_url": "http://ssd.jpl.nasa.gov/sbdb.cgi?sstr=3726710",
"absolute_magnitude_h": 24.3,
"estimated_diameter": {
"kilometers": {
"estimated_diameter_min": 0.0366906138,
"estimated_diameter_max": 0.0820427065
},
"meters": {
"estimated_diameter_min": 36.6906137531,
"estimated_diameter_max": 82.0427064882
},
"miles": {
"estimated_diameter_min": 0.0227984834,
"estimated_diameter_max": 0.0509789586
},
"feet": {
"estimated_diameter_min": 120.3760332259,
"estimated_diameter_max": 269.1689931548
}
},
"is_potentially_hazardous_asteroid": false,
"close_approach_data": [
{
"close_approach_date": "2015-09-08",
"close_approach_date_full": "2015-Sep-08 09:45",
"epoch_date_close_approach": 1441705500000,
"relative_velocity": {
"kilometers_per_second": "19.4850295284",
"kilometers_per_hour": "70146.106302123",
"miles_per_hour": "43586.0625520053"
},
"miss_distance": {
"astronomical": "0.0269230459",
"lunar": "10.4730648551",
"kilometers": "4027630.320552233",
"miles": "2502653.4316094954"
},
"orbiting_body": "Earth"
}
],
"is_sentry_object": false
},
}
I am trying to figure out how to parse through to get "miss_distance" dictionary values ? I am unable to wrap my head around it.
Here is what I have been able to do so far:
After I get a Response object from request.get()
response = request.get(url
I convert the response object to json object
data = response.json() #this returns dictionary object
I try to parse the first level of the dictionary:
for i in data:
if i == "near_earth_objects":
dataset1 = data["near_earth_objects"]["2015-09-08"]
#this returns the next object which is of type list
Please someone can explain me :
1. How to decipher this response in the first place.
2. How can I move forward in parsing the response object and get to miss_distance dictionary ?
Please any pointers/help is appreciated.
Thank you
Your data will will have multiple dictionaries for the each date, near earth object, and close approach:
near_earth_objects = data['near_earth_objects']
for date in near_earth_objects:
objects = near_earth_objects[date]
for object in objects:
close_approach_data = object['close_approach_data']
for close_approach in close_approach_data:
print(close_approach['miss_distance'])
The code below gives you a table of date, miss_distances for every object for every date
import json
raw_json = '''
{
"near_earth_objects": {
"2015-09-08": [
{
"close_approach_data": [
{
"miss_distance": {
"astronomical": "0.0269230459",
"lunar": "10.4730648551",
"kilometers": "4027630.320552233",
"miles": "2502653.4316094954"
},
"orbiting_body": "Earth"
}
]
}
]
}
}
'''
if __name__ == "__main__":
parsed = json.loads(raw_json)
# assuming this json includes more than one near_earch_object spread across dates
near_objects = []
for date, near_objs in parsed['near_earth_objects'].items():
for obj in near_objs:
for appr in obj['close_approach_data']:
o = {
'date': date,
'miss_distances': appr['miss_distance']
}
near_objects.append(o)
print(near_objects)
output:
[
{'date': '2015-09-08',
'miss_distances': {
'astronomical': '0.0269230459',
'lunar': '10.4730648551',
'kilometers': '4027630.320552233',
'miles': '2502653.4316094954'
}
}
]
data = response.json()
sortJson = json.dumps(data, sort_keys=True,
indent=2, separators=(',', ':'))
result = json.loads (data)
print ('"saleTotal":', result['trips']['tripOption']['pricing']['saleTotal'])
This is the code I have currently. I'm looking to parse through a nested JSON file, but each time I run this I get the following error:
TypeError: the JSON object must be str, not 'dict'
The JSON file when Pretty Printed turns out like this:
{
"kind":"qpxExpress#tripsSearch",
"trips":{
"data":{
"aircraft":[
{
"code":"321",
"kind":"qpxexpress#aircraftData",
"name":"Airbus A321"
}
],
"airport":[
{
"city":"ORL",
"code":"MCO",
"kind":"qpxexpress#airportData",
"name":"Orlando International"
},
{
"city":"CHI",
"code":"ORD",
"kind":"qpxexpress#airportData",
"name":"Chicago O'Hare"
}
],
"carrier":[
{
"code":"F9",
"kind":"qpxexpress#carrierData",
"name":"Frontier Airlines, Inc."
}
],
"city":[
{
"code":"CHI",
"kind":"qpxexpress#cityData",
"name":"Chicago"
},
{
"code":"ORL",
"kind":"qpxexpress#cityData",
"name":"Orlando"
}
],
"kind":"qpxexpress#data",
"tax":[
{
"id":"ZP",
"kind":"qpxexpress#taxData",
"name":"US Flight Segment Tax"
},
{
"id":"AY_001",
"kind":"qpxexpress#taxData",
"name":"US September 11th Security Fee"
},
{
"id":"US_001",
"kind":"qpxexpress#taxData",
"name":"US Transportation Tax"
},
{
"id":"XF",
"kind":"qpxexpress#taxData",
"name":"US Passenger Facility Charge"
}
]
},
"kind":"qpxexpress#tripOptions",
"requestId":"2z1TQ9iVMcSlUH8HW0O0eq",
"tripOption":[
{
"id":"WQZ8ICu2L8RLqt1MyMNFAQ001",
"kind":"qpxexpress#tripOption",
"pricing":[
{
"baseFareTotal":"USD37.11",
"fare":[
{
"basisCode":"Z00ZSS5",
"carrier":"F9",
"destination":"ORL",
"id":"AR5um4n2cToXHml3a125O0CU7toTISvPQER/01Xhbf2E",
"kind":"qpxexpress#fareInfo",
"origin":"CHI"
}
],
"fareCalculation":"ORD F9 MCO Q9.29 Q4.65 23.17Z00ZSS5 USD 37.11 END ZP ORD XT 2.79US 4.00ZP 5.60AY 4.50XF ORD4.50",
"kind":"qpxexpress#pricingInfo",
"latestTicketingTime":"2016-03-22T00:24-04:00",
"passengers":{
"adultCount":1,
"kind":"qpxexpress#passengerCounts"
},
"ptc":"ADT",
"saleFareTotal":"USD37.11",
"saleTaxTotal":"USD16.89",
"saleTotal":"USD54.00",
"segmentPricing":[
{
"fareId":"AR5um4n2cToXHml3a125O0CU7toTISvPQER/01Xhbf2E",
"kind":"qpxexpress#segmentPricing",
"segmentId":"GoIDkawPBE2TZk14"
}
],
"tax":[
{
"chargeType":"GOVERNMENT",
"code":"US",
"country":"US",
"id":"US_001",
"kind":"qpxexpress#taxInfo",
"salePrice":"USD2.79"
},
{
"chargeType":"GOVERNMENT",
"code":"AY",
"country":"US",
"id":"AY_001",
"kind":"qpxexpress#taxInfo",
"salePrice":"USD5.60"
},
{
"chargeType":"GOVERNMENT",
"code":"XF",
"country":"US",
"id":"XF",
"kind":"qpxexpress#taxInfo",
"salePrice":"USD4.50"
},
{
"chargeType":"GOVERNMENT",
"code":"ZP",
"country":"US",
"id":"ZP",
"kind":"qpxexpress#taxInfo",
"salePrice":"USD4.00"
}
]
}
],
"saleTotal":"USD54.00",
"slice":[
{
"duration":167,
"kind":"qpxexpress#sliceInfo",
"segment":[
{
"bookingCode":"Z",
"bookingCodeCount":9,
"cabin":"COACH",
"duration":167,
"flight":{
"carrier":"F9",
"number":"1294"
},
"id":"GoIDkawPBE2TZk14",
"kind":"qpxexpress#segmentInfo",
"leg":[
{
"aircraft":"321",
"arrivalTime":"2016-05-11T09:42-04:00",
"departureTime":"2016-05-11T05:55-05:00",
"destination":"MCO",
"duration":167,
"id":"LQKIza3yQIpaLyDq",
"kind":"qpxexpress#legInfo",
"meal":"Food and Beverages for Purchase",
"mileage":1006,
"origin":"ORD",
"originTerminal":"3",
"secure":true
}
],
"marriedSegmentGroup":"0"
}
]
}
]
}
]
}
}
Ultimately I'm trying to find the saleTotal and have the corresponding value printed alongside it:
saleTotal: 54.00
tripOption and pricing are lists of objects, if you take that into account it will work:
print ('"saleTotal":', result['trips']['tripOption'][0]['pricing'][0]['saleTotal'])
Output:
"saleTotal": USD54.00