I have the following object in MongoDB:
{
"_id":"...",
"username":"XXX",
"keys":[
{
"key":"c443c2cc2754d3",
"ref":"Autos",
"lists":[
{
"list_name":"Toyota",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":false,
"bonnet":true,
"interior":{
"dashboard":true,
"electronics":false
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"parked"
},
{
"list_name":"Nissan",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":true,
"bonnet":false,
"interior":{
"dashboard":false,
"electronics":true
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"garage"
}
]
},
{
"key":"80d54bd834ff60",
"ref":"Trucks",
"lists":[
{
"list_name":"MAN",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":false,
"bonnet":false,
"interior":{
"dashboard":false,
"electronics":false
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"parked"
},
{
"list_name":"Toyota",
"key":"c443c2cc2754d3",
"broken_parts":{
"headlights":true,
"bonnet":false,
"interior":{
"dashboard":true,
"electronics":true
}
},
"timestamp":"2023-01-26T13:00:21.803Z",
"status":"leased"
}
]
}
]
}
and whenever I try to update the status of an item inside a list, it only updates the first object.
If I try to update the status of the 1st list, 2nd object, it will only update the first one:
self.users.update_one({"username": username,
"keys.key": key,
"keys.lists.list_name": listid
},
{"$set": {
"keys.0.lists.$.status": 'repaired'
}
}
)
What am I doing wrong and how can I update the list item I want? or even the 3rd nested object (dashboard, electronics)?
How to merge strings from the yield generator of JSON into one JSON?
I have got Nested Dictionary by yield generator, and I aim to have one JSON file.
I have the output of these correct strings of nested dictionary.
{"domain.com": {"Chrome": "19362.344607264396"}}
{"domain.com": {"ChromeMobile": "7177.498437391487"}}
{"another.com": {"MobileSafari": "6237.433155080214"}}
{"another.com": {"Safari": "5895.409403430795"}}
and I want to merge into one JSON file
[
{
"domain.com": {
"Chrome": "19362.344607264396"
}
},
{
"domain.com": {
"ChromeMobile": "7177.498437391487"
}
},
{
"another.com": {
"MobileSafari": "6237.433155080214"
}
},
{
"another.com": {
"Safari": "5895.409403430795"
}
}
]
or ideally to have a JSON likes to this
{
"browsers": [
{
"domain.com": {
"Chrome": "19362.344607264396",
"ChromeMobile": "7177.498437391487",
},
"another.com": {
"MobileSafari": "6237.433155080214",
"Safari": "5895.409403430795"
}
}
]
}
My code
# Cloudflare zone bandwidth total
def browser_map_page_views(domain_zone):
cloudflare = prom.custom_query(
query="topk(5, sum by(family) (increase(browser_map_page_views_count{job='cloudflare', zone='"f'{domain_zone}'"'}[10d])))"
)
for domain_z in cloudflare:
user_agent = domain_z['metric']['family']
value = domain_z['value'][1]
yield {domain_zone: {user_agent: {'value': value}}}
# Get list of zones from Prometheus based on Host Tracker data
def domain_zones():
zones_domain = prom.custom_query(
query="host_tracker_uptime_percent{job='donodeexporter'}"
)
for domain_z in zones_domain:
yield domain_z['metric']['zone']
There is a final output.
# Get a list of domains and substitution each one into a request of Prometheus query.
for domain_list in domain_zones():
for dict in browser_map_page_views(domain_zone=domain_list):
dicts = dict
print(json.dumps(dicts))
Something like this
import json
DATA = [
{"domain.com": {"Chrome": "19362.344607264396"}},
{"domain.com": {"ChromeMobile": "7177.498437391487"}},
{"another.com": {"MobileSafari": "6237.433155080214"}},
{"another.com": {"Safari": "5895.409403430795"}}
]
def yield_data():
for d in DATA:
yield d
def merge1():
return list(yield_data())
def merge2():
output = {}
for d in DATA:
key = list(d.keys())[0]
output[key] = output.get(key, {})
output[key].update(d[key])
return {"browsers": [output]}
print(json.dumps(merge1(), indent=2))
print(json.dumps(merge2(), indent=2))
merge1's output looks like:
[
{
"domain.com": {
"Chrome": "19362.344607264396"
}
},
{
"domain.com": {
"ChromeMobile": "7177.498437391487"
}
},
{
"another.com": {
"MobileSafari": "6237.433155080214"
}
},
{
"another.com": {
"Safari": "5895.409403430795"
}
}
]
merge2's output looks like
{
"browsers": [
{
"domain.com": {
"Chrome": "19362.344607264396",
"ChromeMobile": "7177.498437391487"
},
"another.com": {
"MobileSafari": "6237.433155080214",
"Safari": "5895.409403430795"
}
}
]
}
I am trying to scrape some values through a json that looks like:
{
"attributes":{
"531":{
"id":"531",
"code":"taille",
"label":"taille",
"options":[
{
"id":"30",
"label":"40",
"is_in":"0"
},
{
"id":"31",
"label":"41",
"is_in":"1"
}
]
}
},
"template":"Helloworld"
}
My issue is that the number 531 is different in each json file that I am trying to scrape and what I am trying to grab through this json is the label and is_in value
What I have done so far is that I tried to do something like this but I am stuck and dont know how to do if the 531 is changing to something else
getOption = '{
"attributes":{
"531":{
"id":"531",
"code":"taille",
"label":"taille",
"options":[
{
"id":"30",
"label":"40",
"is_in":"0"
},
{
"id":"31",
"label":"41",
"is_in":"1"
}
]
}
},
"template":"Helloworld"
}'
for att, values in getOption.items():
print(values)
So how do I possible scrape the value label and is_in?
I'm not sure if you can have several 531 keys but you can loop through them.
getOption = {
"attributes":{
"531":{
"id":"531",
"code":"taille",
"label":"taille",
"options":[
{
"id":"30",
"label":"40",
"is_in":"0"
},
{
"id":"31",
"label":"41",
"is_in":"1"
}
]
}
},
"template":"Helloworld"
}
attributes = getOption['attributes']
for key in attributes.keys():
for item in attributes[key]['options']:
print(item['label'], item['is_in'])
I am attempting to parse a json response that looks like this:
{
"links": {
"next": "http://www.neowsapp.com/rest/v1/feed?start_date=2015-09-08&end_date=2015-09-09&detailed=false&api_key=xxx",
"prev": "http://www.neowsapp.com/rest/v1/feed?start_date=2015-09-06&end_date=2015-09-07&detailed=false&api_key=xxx",
"self": "http://www.neowsapp.com/rest/v1/feed?start_date=2015-09-07&end_date=2015-09-08&detailed=false&api_key=xxx"
},
"element_count": 22,
"near_earth_objects": {
"2015-09-08": [
{
"links": {
"self": "http://www.neowsapp.com/rest/v1/neo/3726710?api_key=xxx"
},
"id": "3726710",
"neo_reference_id": "3726710",
"name": "(2015 RC)",
"nasa_jpl_url": "http://ssd.jpl.nasa.gov/sbdb.cgi?sstr=3726710",
"absolute_magnitude_h": 24.3,
"estimated_diameter": {
"kilometers": {
"estimated_diameter_min": 0.0366906138,
"estimated_diameter_max": 0.0820427065
},
"meters": {
"estimated_diameter_min": 36.6906137531,
"estimated_diameter_max": 82.0427064882
},
"miles": {
"estimated_diameter_min": 0.0227984834,
"estimated_diameter_max": 0.0509789586
},
"feet": {
"estimated_diameter_min": 120.3760332259,
"estimated_diameter_max": 269.1689931548
}
},
"is_potentially_hazardous_asteroid": false,
"close_approach_data": [
{
"close_approach_date": "2015-09-08",
"close_approach_date_full": "2015-Sep-08 09:45",
"epoch_date_close_approach": 1441705500000,
"relative_velocity": {
"kilometers_per_second": "19.4850295284",
"kilometers_per_hour": "70146.106302123",
"miles_per_hour": "43586.0625520053"
},
"miss_distance": {
"astronomical": "0.0269230459",
"lunar": "10.4730648551",
"kilometers": "4027630.320552233",
"miles": "2502653.4316094954"
},
"orbiting_body": "Earth"
}
],
"is_sentry_object": false
},
}
I am trying to figure out how to parse through to get "miss_distance" dictionary values ? I am unable to wrap my head around it.
Here is what I have been able to do so far:
After I get a Response object from request.get()
response = request.get(url
I convert the response object to json object
data = response.json() #this returns dictionary object
I try to parse the first level of the dictionary:
for i in data:
if i == "near_earth_objects":
dataset1 = data["near_earth_objects"]["2015-09-08"]
#this returns the next object which is of type list
Please someone can explain me :
1. How to decipher this response in the first place.
2. How can I move forward in parsing the response object and get to miss_distance dictionary ?
Please any pointers/help is appreciated.
Thank you
Your data will will have multiple dictionaries for the each date, near earth object, and close approach:
near_earth_objects = data['near_earth_objects']
for date in near_earth_objects:
objects = near_earth_objects[date]
for object in objects:
close_approach_data = object['close_approach_data']
for close_approach in close_approach_data:
print(close_approach['miss_distance'])
The code below gives you a table of date, miss_distances for every object for every date
import json
raw_json = '''
{
"near_earth_objects": {
"2015-09-08": [
{
"close_approach_data": [
{
"miss_distance": {
"astronomical": "0.0269230459",
"lunar": "10.4730648551",
"kilometers": "4027630.320552233",
"miles": "2502653.4316094954"
},
"orbiting_body": "Earth"
}
]
}
]
}
}
'''
if __name__ == "__main__":
parsed = json.loads(raw_json)
# assuming this json includes more than one near_earch_object spread across dates
near_objects = []
for date, near_objs in parsed['near_earth_objects'].items():
for obj in near_objs:
for appr in obj['close_approach_data']:
o = {
'date': date,
'miss_distances': appr['miss_distance']
}
near_objects.append(o)
print(near_objects)
output:
[
{'date': '2015-09-08',
'miss_distances': {
'astronomical': '0.0269230459',
'lunar': '10.4730648551',
'kilometers': '4027630.320552233',
'miles': '2502653.4316094954'
}
}
]
I have the following JSON file - test.json (names, keys and addresses changed for security reasons)
[
{
"accountMode":"Live",
"acquirer":"TEST",
"acquirerConstraints":{
"cardTypes":[
"MASTERCARD",
"MAESTRO",
"VISA"
],
"cvcRegexp":"^[0-9]{3}$",
"cvcRequired":true,
"maxAmount":500000,
"minAmount":50
},
"acquirerDetails":{
"TEST":"Studio",
"ERROR_LIST":[
],
"MERCHANT_CODE":"218331",
"VALID":true,
"_mId":"T712484",
"_status":"INPROCESS",
"email":"test7#gmail.com",
"name":"Studio",
"valid":true
},
"acquirerValidations":null,
"allowedCurrencies":[
"EUR",
"USD",
"GBP"
],
"apiKeyPairs":[
{
"accountMode":"Live",
"label":"Virtual Terminal",
"publishableKey":"niunibiubniunijknkjknj",
"source":"VIRTUAL_TERMINAL"
},
{
"accountMode":"Live",
"label":"Default",
"publishableKey":"iiuhiuhiu",
"source":"ECOMMERCE"
}
],
"appLogoUrl":null,
"applicationId":"541d75e0-7db8b343a31f",
"authorizationCode":"",
"closedDate":null,
"closureReason":null,
"declineAvsAddressFailure":false,
"declineAvsZipFailure":false,
"declineCvcFailure":false,
"defaultCurrency":"EUR",
"descriptor":null,
"email":"test1#gmail.com",
"id":"ddddeff",
"invitationCode":null,
"locale":"en_IE",
"merchantApplication":{
"accountNumber":null,
"acquirer":"TEST",
"annualAmount":null,
"annualVolume":null,
"applicationType":"APPROVAL",
"bankName":"UNKNOWN",
"brand":null,
"businessAddress":"54 My St, 1",
"businessAddress2":null,
"businessCity":"Abbey",
"businessCountry":"IRL",
"businessPhone":null,
"businessState":"DUBLIN",
"businessZip":null,
"data":null,
"email":"test#gmail.com",
"escalationPhone":null,
"fax":null,
"legalName":"UAB \"Studio\"",
"maxTransactionAmount":null,
"mccCode":"5712",
"merchantPromotionCode":null,
"mobile":null,
"monthlyAmount":null,
"monthlyVolume":null,
"ownerFirstName":"tlana",
"ownerLastName":"nava",
"phone":"37647",
"GuideAccepted":null,
"privacyAccepted":true,
"privacyVersion":"1a",
"referenceId":"9104d65i08d071",
"routingNumber":null,
"singleTransactionAmount":null,
"statementName":"UAB \"Studio\"",
"taxId":null,
"termsAccepted":true,
"termsVersion":"1a",
"url":"http://www.design.lt"
},
"merchantId":"12484",
"merchantPromotionCode":null,
"mposEnabled":true,
"name":"Studio",
"netonfiguration":null,
"onboardedDate":1505513232485,
"onboardingMethod":null,
"onboardingStatus":"INPROCESS",
"partner":null,
"saqCompliant":false,
"saqExpires":null,
"settings":[
{
"key":"MERCHANT_DETAILS",
"value":"{\"zip\":\"Wicklow\",\"phone\":\"342647\",\"email\":\"suppoor#outlook.com\",\"address\":\"Bck 6\",\"state\":\"Ireland\",\"addressLine2\":\"Unit 8, Bl Par\",\"city\":\"Wicklow\"}"
},
{
"key":"VAT_NUMBER",
"value":"/evzaqen/"
}
],
"timezone":"Europe/Dublin",
"tinStatus":null
},
{
"accountMode":"Live",
"acquirer":"TEST",
"acquirerConstraints":{
"cardTypes":[
"MASTERCARD",
"MAESTRO",
"VISA"
],
"cvcRegexp":"^[0-9]{3}$",
"cvcRequired":true,
"maxAmount":500000,
"minAmount":50
},
"acquirerDetails":{
"TEST":"test",
"ERROR_LIST":[
],
"MERCHANT_CODE":"594920",
"MID_ASSIGNED":true,
"VALID":true,
"_mId":"103558",
"_status":"APPROVED",
"acquiringMid":"1036598",
"descriptor":"test 8885551212",
"email":"test#gmail.com",
"gatewayMid":"SIMP337",
"id":"SIMP337",
"level4Mid":"76576576",
"name":"test",
"status":"APPROVED",
"transactionCurrency":"USD;EUR;GBP",
"valid":true,
"paymentGatewayKey":"ytfytfytfyt"
},
"acquirerValidations":null,
"allowedCurrencies":[
"EUR",
"USD",
"GBP"
],
"apiKeyPairs":[
],
"appLogoUrl":null,
"applicationId":"949bdde5-07-d8d58f4c3d01",
"authorizationCode":"",
"closedDate":null,
"closureReason":null,
"declineAvsAddressFailure":false,
"declineAvsZipFailure":false,
"declineCvcFailure":false,
"defaultCurrency":"EUR",
"descriptor":"test85551212",
"email":"test#gmail.com",
"id":"9f3a7d7",
"invitationCode":null,
"locale":"en_US",
"merchantApplication":{
"accountNumber":null,
"acquirer":"TEST",
"annualAmount":null,
"annualVolume":null,
"applicationType":"APPROVAL",
"bankName":"UNKNOWN",
"brand":null,
"businessAddress":"123 test",
"businessAddress2":null,
"businessCity":"Atlanta",
"businessCountry":"IRL",
"businessPhone":null,
"businessState":"CARLOW",
"businessZip":null,
"data":null,
"email":"test#gmail.com",
"escalationPhone":null,
"fax":null,
"legalName":"stest",
"maxTransactionAmount":null,
"mccCode":"521",
"merchantPromotionCode":null,
"mobile":null,
"monthlyAmount":null,
"monthlyVolume":null,
"ownerFirstName":"moto",
"ownerLastName":"test",
"phone":"3141212",
"GuideAccepted":null,
"privacyAccepted":true,
"privacyVersion":"1a",
"referenceId":"2920",
"routingNumber":null,
"singleTransactionAmount":null,
"statementName":"test",
"taxId":null,
"termsAccepted":true,
"termsVersion":"1a",
"url":null
},
"merchantId":"1036558",
"merchantPromotionCode":null,
"mposEnabled":true,
"name":"test",
"netonfiguration":null,
"onboardedDate":1456846054925,
"onboardingMethod":null,
"onboardingStatus":"CLOSED",
"partner":null,
"saqCompliant":false,
"saqExpires":null,
"settings":[
],
"timezone":"Europe/Dublin",
"tinStatus":"InCompliance"
}
]
I want to process this file and take some of the information and populate a CSV file with it. To do this I am using the following:
import unicodecsv
import json
json_data = open("test.json")
data = json.load(json_data)
f = unicodecsv.writer(open("results.csv","wb+"))
for entry in data:
if "merchantApplication" in entry:
ma = entry["merchantApplication"]
if "email" in ma:
f.writerow([ma["ownerFirstName"],ma["ownerLastName"],ma["email"],ma["legalName"],ma["businessAddress"],ma["businessAddress2"],ma["businessCity"],ma["businessCountry"],ma["businessState"],ma["businessZip"],ma["phone"],ma["mobile"]])
json_data.close()
This Works fine but does not print the headers above the columns. How do I add in the headers? I am using Python 2.7.10
How do I add in the headers?
Well quite simply by calling f.writerow((<your>,<headers>,<here>)) before your for loop.