Python Fattening Complex JSON - python

I have a LinkedIn dataset of follower statistics in the following JSON (Removed many key pair values for easy understanding). In this, each key has a different number of inner key pair values.
Can somebody help convert this to a CSV output using python?
{
"paging": { "start": 0, "count": 10, "links": [] },
"elements": [
{
"followerCountsByAssociationType": [
{
"followerCounts": {
"organicFollowerCount": 2775,
"paidFollowerCount": 0
}
},
{
"followerCounts": {
"organicFollowerCount": 13,
"paidFollowerCount": 0
},
"associationType": "EMPLOYEE"
}
],
"followerCountsByRegion": [
{
"region": "urn:li:region:7312",
"followerCounts": {
"organicFollowerCount": 2,
"paidFollowerCount": 0
}
},
{
"region": "urn:li:region:6981",
"followerCounts": {
"organicFollowerCount": 2,
"paidFollowerCount": 0
}
},
{
"region": "urn:li:region:620",
"followerCounts": {
"organicFollowerCount": 2,
"paidFollowerCount": 0
}
}
],
"followerCountsBySeniority": [
{
"followerCounts": {
"organicFollowerCount": 12,
"paidFollowerCount": 0
},
"seniority": "urn:li:seniority:8"
},
{
"followerCounts": {
"organicFollowerCount": 5,
"paidFollowerCount": 0
},
"seniority": "urn:li:seniority:9"
},
{
"followerCounts": {
"organicFollowerCount": 2,
"paidFollowerCount": 0
},
"seniority": "urn:li:seniority:1"
}
],
"followerCountsByIndustry": [
{
"followerCounts": {
"organicFollowerCount": 1,
"paidFollowerCount": 0
},
"industry": "urn:li:industry:51"
},
{
"followerCounts": {
"organicFollowerCount": 1,
"paidFollowerCount": 0
},
"industry": "urn:li:industry:74"
},
{
"followerCounts": {
"organicFollowerCount": 1,
"paidFollowerCount": 0
},
"industry": "urn:li:industry:77"
},
{
"followerCounts": {
"organicFollowerCount": 1,
"paidFollowerCount": 0
},
"industry": "urn:li:industry:78"
},
],
"followerCountsByFunction": [
{
"followerCounts": {
"organicFollowerCount": 3,
"paidFollowerCount": 0
},
"function": "urn:li:function:14"
},
{
"followerCounts": {
"organicFollowerCount": 3,
"paidFollowerCount": 0
},
"function": "urn:li:function:21"
},
{
"followerCounts": {
"organicFollowerCount": 2,
"paidFollowerCount": 0
},
"function": "urn:li:function:11"
},
{
"followerCounts": {
"organicFollowerCount": 2,
"paidFollowerCount": 0
},
"function": "urn:li:function:17"
},
{
"followerCounts": {
"organicFollowerCount": 2,
"paidFollowerCount": 0
},
"function": "urn:li:function:1"
},
],
"followerCountsByStaffCountRange": [
{
"followerCounts": {
"organicFollowerCount": 267,
"paidFollowerCount": 0
},
"staffCountRange": "SIZE_1001_TO_5000"
},
{
"followerCounts": {
"organicFollowerCount": 185,
"paidFollowerCount": 0
},
"staffCountRange": "SIZE_201_TO_500"
},
{
"followerCounts": {
"organicFollowerCount": 131,
"paidFollowerCount": 0
},
"staffCountRange": "SIZE_501_TO_1000"
},
{
"followerCounts": {
"organicFollowerCount": 81,
"paidFollowerCount": 0
},
"staffCountRange": "SIZE_5001_TO_10000"
},
{
"followerCounts": {
"organicFollowerCount": 74,
"paidFollowerCount": 0
},
"staffCountRange": "SIZE_2_TO_10"
},
{
"followerCounts": {
"organicFollowerCount": 10,
"paidFollowerCount": 0
},
"staffCountRange": "SIZE_1"
}
],
"followerCountsByCountry": [
{
"followerCounts": {
"organicFollowerCount": 1,
"paidFollowerCount": 0
},
"country": "urn:li:country:es"
},
{
"followerCounts": {
"organicFollowerCount": 1,
"paidFollowerCount": 0
},
"country": "urn:li:country:ph"
},
{
"followerCounts": {
"organicFollowerCount": 1,
"paidFollowerCount": 0
},
"country": "urn:li:country:ng"
}
],
"organizationalEntity": "urn:li:organization:28849398"
}
]
}
I tried using the json_normalize(data['Elements']) but that gives the following Output
I am sure that there must be some parameter in json_normalize() that can simplify the inner nesting.
The desired output is as follows-
FollwerCounByAssociationorganicfollowecount
FollwerCounByAssociationpaidfollowecount
AssociationType
Region
RegionOrganicFollowercount
RegionpaidFollowercount
2775
0
Employee
urn:li:region:7312
2
0
urn:li:region:6981
2
0
.......And so on
Now I have only made a small part of the output but largely for as many entries in the last the columns will go on (while for the others it will be null)
Would appreciate any help possible! Thanks!

Related

Convert a MultiIndex pandas DataFrame to a nested JSON

I have the following Dataframe with MultiIndex rows in pandas.
time available_slots status
month day
1 1 10:00:00 1 AVAILABLE
1 12:00:00 1 AVAILABLE
1 14:00:00 1 AVAILABLE
1 16:00:00 1 AVAILABLE
1 18:00:00 1 AVAILABLE
2 10:00:00 1 AVAILABLE
... ... ... ...
2 28 12:00:00 1 AVAILABLE
28 14:00:00 1 AVAILABLE
28 16:00:00 1 AVAILABLE
28 18:00:00 1 AVAILABLE
28 20:00:00 1 AVAILABLE
And I need to transform it to a hierarchical nested JSON as this:
[
{
"month": 1,
"days": [
{
"day": 1,
"slots": [
{
"time": "10:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "12:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
...
]
},
{
"day": 2,
"slots": [
...
]
}
]
},
{
"month": 2,
"days":[
{
"day": 1,
"slots": [
...
]
}
]
},
...
]
Unfortunately, it is not as easy as doing df.to_json(orient="index").
Does anyone know if there is a method in pandas to perform this kind of transformations? or in what way I could iterate over the DataFrame to build the final object?

Here's one way. Basically repeated groupby + apply(to_dict) + reset_index until we get the desired shape:
out = (df.groupby(level=[0,1])
.apply(lambda x: x.to_dict('records'))
.reset_index()
.rename(columns={0:'slots'})
.groupby('month')
.apply(lambda x: x[['day','slots']].to_dict('records'))
.reset_index()
.rename(columns={0:'days'})
.to_json(orient='records', indent=True)
)
Output:
[
{
"month":1,
"days":[
{
"day":1,
"slots":[
{
"time":"10:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"12:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"14:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"16:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"18:00:00",
"available_slots":1,
"status":"AVAILABLE"
}
]
},
{
"day":2,
"slots":[
{
"time":"10:00:00",
"available_slots":1,
"status":"AVAILABLE"
}
]
}
]
},
{
"month":2,
"days":[
{
"day":28,
"slots":[
{
"time":"12:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"14:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"16:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"18:00:00",
"available_slots":1,
"status":"AVAILABLE"
},
{
"time":"20:00:00",
"available_slots":1,
"status":"AVAILABLE"
}
]
}
]
}
]

You can use a double loop for each level of your index:
data = []
for month, df1 in df.groupby(level=0):
data.append({'month': month, 'days': []})
for day, df2 in df1.groupby(level=1):
data[-1]['days'].append({'day': day, 'slots': df2.to_dict('records')})
Output:
import json
print(json.dumps(data, indent=2))
[
{
"month": 1,
"days": [
{
"day": 1,
"slots": [
{
"time": "10:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "12:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "14:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "16:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "18:00:00",
"available_slots": 1,
"status": "AVAILABLE"
}
]
},
{
"day": 2,
"slots": [
{
"time": "10:00:00",
"available_slots": 1,
"status": "AVAILABLE"
}
]
}
]
},
{
"month": 2,
"days": [
{
"day": 28,
"slots": [
{
"time": "12:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "14:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "18:00:00",
"available_slots": 1,
"status": "AVAILABLE"
},
{
"time": "20:00:00",
"available_slots": 1,
"status": "AVAILABLE"
}
]
}
]
}
]

Python - Get Nested Data from Multiple Levels

Wasn't sure how to title this question but I am working with the Quickbooks Online API and when querying a report like BalanceSheet or GeneralLedger the API returns data rows in multiple nested levels which is quite frustrating to parse through.
Example of the BalanceSheet return included below. I am only interested in the data from "Row" objects but as you can see that can be returned in 1, 2, 3 or more different levels of data. I am thinking of going through each level to check for Rows and then get each Row but that seems overly complex as I would need multiple for loops for each level.
I'm wondering if there is a better way to get each "Row" in that data without regard to which level it is on? Any ideas would be appreciated!
Here's an example of a return from their sandbox data:
{
"Header": {
"Time": "2021-04-28T14:12:17-07:00",
"ReportName": "BalanceSheet",
"DateMacro": "this calendar year-to-date",
"ReportBasis": "Accrual",
"StartPeriod": "2021-01-01",
"EndPeriod": "2021-04-28",
"SummarizeColumnsBy": "Month",
"Currency": "USD",
"Option": [
{
"Name": "AccountingStandard",
"Value": "GAAP"
},
{
"Name": "NoReportData",
"Value": "false"
}
]
},
"Columns": {
"Column": [
{
"ColTitle": "",
"ColType": "Account",
"MetaData": [
{
"Name": "ColKey",
"Value": "account"
}
]
},
{
"ColTitle": "Jan 2021",
"ColType": "Money",
"MetaData": [
{
"Name": "StartDate",
"Value": "2021-01-01"
},
{
"Name": "EndDate",
"Value": "2021-01-31"
},
{
"Name": "ColKey",
"Value": "Jan 2021"
}
]
},
{
"ColTitle": "Feb 2021",
"ColType": "Money",
"MetaData": [
{
"Name": "StartDate",
"Value": "2021-02-01"
},
{
"Name": "EndDate",
"Value": "2021-02-28"
},
{
"Name": "ColKey",
"Value": "Feb 2021"
}
]
},
{
"ColTitle": "Mar 2021",
"ColType": "Money",
"MetaData": [
{
"Name": "StartDate",
"Value": "2021-03-01"
},
{
"Name": "EndDate",
"Value": "2021-03-31"
},
{
"Name": "ColKey",
"Value": "Mar 2021"
}
]
},
{
"ColTitle": "Apr 1-28, 2021",
"ColType": "Money",
"MetaData": [
{
"Name": "StartDate",
"Value": "2021-04-01"
},
{
"Name": "EndDate",
"Value": "2021-04-28"
},
{
"Name": "ColKey",
"Value": "Apr 1-28, 2021"
}
]
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "ASSETS"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Current Assets"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Bank Accounts"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Checking",
"id": "35"
},
{
"value": "1201.00"
},
{
"value": "1201.00"
},
{
"value": "1201.00"
},
{
"value": "1201.00"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Savings",
"id": "36"
},
{
"value": "800.00"
},
{
"value": "800.00"
},
{
"value": "800.00"
},
{
"value": "800.00"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Bank Accounts"
},
{
"value": "2001.00"
},
{
"value": "2001.00"
},
{
"value": "2001.00"
},
{
"value": "2001.00"
}
]
},
"type": "Section",
"group": "BankAccounts"
},
{
"Header": {
"ColData": [
{
"value": "Accounts Receivable"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Accounts Receivable (A/R)",
"id": "84"
},
{
"value": "5281.52"
},
{
"value": "5281.52"
},
{
"value": "5281.52"
},
{
"value": "5281.52"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Accounts Receivable"
},
{
"value": "5281.52"
},
{
"value": "5281.52"
},
{
"value": "5281.52"
},
{
"value": "5281.52"
}
]
},
"type": "Section",
"group": "AR"
},
{
"Header": {
"ColData": [
{
"value": "Other Current Assets"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Inventory Asset",
"id": "81"
},
{
"value": "596.25"
},
{
"value": "596.25"
},
{
"value": "596.25"
},
{
"value": "596.25"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Undeposited Funds",
"id": "4"
},
{
"value": "2062.52"
},
{
"value": "2062.52"
},
{
"value": "2062.52"
},
{
"value": "2062.52"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Other Current Assets"
},
{
"value": "2658.77"
},
{
"value": "2658.77"
},
{
"value": "2658.77"
},
{
"value": "2658.77"
}
]
},
"type": "Section",
"group": "OtherCurrentAssets"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Current Assets"
},
{
"value": "9941.29"
},
{
"value": "9941.29"
},
{
"value": "9941.29"
},
{
"value": "9941.29"
}
]
},
"type": "Section",
"group": "CurrentAssets"
},
{
"Header": {
"ColData": [
{
"value": "Fixed Assets"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Truck",
"id": "37"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Original Cost",
"id": "38"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Truck"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
}
]
},
"type": "Section"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Fixed Assets"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
},
{
"value": "13495.00"
}
]
},
"type": "Section",
"group": "FixedAssets"
}
]
},
"Summary": {
"ColData": [
{
"value": "TOTAL ASSETS"
},
{
"value": "23436.29"
},
{
"value": "23436.29"
},
{
"value": "23436.29"
},
{
"value": "23436.29"
}
]
},
"type": "Section",
"group": "TotalAssets"
},
{
"Header": {
"ColData": [
{
"value": "LIABILITIES AND EQUITY"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Liabilities"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Current Liabilities"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"Header": {
"ColData": [
{
"value": "Accounts Payable"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Accounts Payable (A/P)",
"id": "33"
},
{
"value": "1602.67"
},
{
"value": "1602.67"
},
{
"value": "1602.67"
},
{
"value": "1602.67"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Accounts Payable"
},
{
"value": "1602.67"
},
{
"value": "1602.67"
},
{
"value": "1602.67"
},
{
"value": "1602.67"
}
]
},
"type": "Section",
"group": "AP"
},
{
"Header": {
"ColData": [
{
"value": "Credit Cards"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Mastercard",
"id": "41"
},
{
"value": "157.72"
},
{
"value": "157.72"
},
{
"value": "157.72"
},
{
"value": "157.72"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Credit Cards"
},
{
"value": "157.72"
},
{
"value": "157.72"
},
{
"value": "157.72"
},
{
"value": "157.72"
}
]
},
"type": "Section",
"group": "CreditCards"
},
{
"Header": {
"ColData": [
{
"value": "Other Current Liabilities"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Arizona Dept. of Revenue Payable",
"id": "89"
},
{
"value": "0.00"
},
{
"value": "0.00"
},
{
"value": "0.00"
},
{
"value": "0.00"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Board of Equalization Payable",
"id": "90"
},
{
"value": "370.94"
},
{
"value": "370.94"
},
{
"value": "370.94"
},
{
"value": "370.94"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Loan Payable",
"id": "43"
},
{
"value": "4000.00"
},
{
"value": "4000.00"
},
{
"value": "4000.00"
},
{
"value": "4000.00"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Other Current Liabilities"
},
{
"value": "4370.94"
},
{
"value": "4370.94"
},
{
"value": "4370.94"
},
{
"value": "4370.94"
}
]
},
"type": "Section",
"group": "OtherCurrentLiabilities"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Current Liabilities"
},
{
"value": "6131.33"
},
{
"value": "6131.33"
},
{
"value": "6131.33"
},
{
"value": "6131.33"
}
]
},
"type": "Section",
"group": "CurrentLiabilities"
},
{
"Header": {
"ColData": [
{
"value": "Long-Term Liabilities"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Notes Payable",
"id": "44"
},
{
"value": "25000.00"
},
{
"value": "25000.00"
},
{
"value": "25000.00"
},
{
"value": "25000.00"
}
],
"type": "Data"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Long-Term Liabilities"
},
{
"value": "25000.00"
},
{
"value": "25000.00"
},
{
"value": "25000.00"
},
{
"value": "25000.00"
}
]
},
"type": "Section",
"group": "LongTermLiabilities"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Liabilities"
},
{
"value": "31131.33"
},
{
"value": "31131.33"
},
{
"value": "31131.33"
},
{
"value": "31131.33"
}
]
},
"type": "Section",
"group": "Liabilities"
},
{
"Header": {
"ColData": [
{
"value": "Equity"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
]
},
"Rows": {
"Row": [
{
"ColData": [
{
"value": "Opening Balance Equity",
"id": "34"
},
{
"value": "-9337.50"
},
{
"value": "-9337.50"
},
{
"value": "-9337.50"
},
{
"value": "-9337.50"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Retained Earnings",
"id": "2"
},
{
"value": "1642.46"
},
{
"value": "1642.46"
},
{
"value": "1642.46"
},
{
"value": "1642.46"
}
],
"type": "Data"
},
{
"ColData": [
{
"value": "Net Income"
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
},
{
"value": ""
}
],
"type": "Data",
"group": "NetIncome"
}
]
},
"Summary": {
"ColData": [
{
"value": "Total Equity"
},
{
"value": "-7695.04"
},
{
"value": "-7695.04"
},
{
"value": "-7695.04"
},
{
"value": "-7695.04"
}
]
},
"type": "Section",
"group": "Equity"
}
]
},
"Summary": {
"ColData": [
{
"value": "TOTAL LIABILITIES AND EQUITY"
},
{
"value": "23436.29"
},
{
"value": "23436.29"
},
{
"value": "23436.29"
},
{
"value": "23436.29"
}
]
},
"type": "Section",
"group": "TotalLiabilitiesAndEquity"
}
]
}
}

Parsing an array of dictionnaries in a django for loop

In a Django application, I want to use a dictionary as elements of a result.html page:
<tbody>
{% for element in products%}
<tr>
<td>{{ element['q0']['Results'][0]['Name'] }}</td>
</tr>
{% endfor %}
</tbody>
But it returns Could not parse the remainder: '['q0']['Results'][0]['Name']' from 'element['q0']['Results'][0]['Name']':
return render(request, 'todo/result.html', {'products': top_products})
File "C:\Python36\lib\site-packages\django\shortcuts.py", line 19, in render
content = loader.render_to_string(template_name, context, request, using=using)
File "C:\Python36\lib\site-packages\django\template\loader.py", line 61, in render_to_string
template = get_template(template_name, using=using)
File "C:\Python36\lib\site-packages\django\template\loader.py", line 15, in get_template
return engine.get_template(template_name)
File "C:\Python36\lib\site-packages\django\template\backends\django.py", line 34, in get_template
return Template(self.engine.get_template(template_name), self)
File "C:\Python36\lib\site-packages\django\template\engine.py", line 143, in get_template
template, origin = self.find_template(template_name)
File "C:\Python36\lib\site-packages\django\template\engine.py", line 125, in find_template
template = loader.get_template(name, skip=skip)
File "C:\Python36\lib\site-packages\django\template\loaders\base.py", line 30, in get_template
contents, origin, origin.template_name, self.engine,
File "C:\Python36\lib\site-packages\django\template\base.py", line 155, in __init__
self.nodelist = self.compile_nodelist()
File "C:\Python36\lib\site-packages\django\template\base.py", line 193, in compile_nodelist
return parser.parse()
File "C:\Python36\lib\site-packages\django\template\base.py", line 478, in parse
raise self.error(token, e)
File "C:\Python36\lib\site-packages\django\template\base.py", line 476, in parse
compiled_result = compile_func(self, token)
File "C:\Python36\lib\site-packages\django\template\defaulttags.py", line 814, in do_for
nodelist_loop = parser.parse(('empty', 'endfor',))
File "C:\Python36\lib\site-packages\django\template\base.py", line 449, in parse
raise self.error(token, e)
File "C:\Python36\lib\site-packages\django\template\base.py", line 447, in parse
filter_expression = self.compile_filter(token.contents)
File "C:\Python36\lib\site-packages\django\template\base.py", line 563, in compile_filter
return FilterExpression(token, self)
File "C:\Python36\lib\site-packages\django\template\base.py", line 663, in __init__
"from '%s'" % (token[upto:], token))
django.template.exceptions.TemplateSyntaxError: Could not parse the remainder: '['q0']['Results'][0]['Name']' from 'element['q0']['Results'][0]['Name']'
It was sent by views.py:
def getmatch(request):
# cosas cosas cosas para obtener top_products
print(top_products[0])
return render(request, 'todo/result.html', {'products': top_products})
Here is an example of a product top_products[0]:
{
"q1": {
"Id": "q1",
"Limit": 20,
"Offset": 0,
"TotalResults": 0,
"Locale": "fr_FR",
"Results": [],
"Includes": {},
"HasE rrors": false,
"Errors": []
},
"q0": {
"Id": "q0",
"Limit": 10,
"Offset": 0,
"TotalResults": 1,
"Locale": "fr_FR",
"Results": [
{
"EANs": [
"8011003827336"
],
"Description": "L’aur a divine d’une femme habillée d’une essence éblouissante et sensuelle…\nEros pour Femme est le mythe signé Versace, qui déclenche la passion débordante d’Eros au pre mier regard.\n\nMais qui séduit qui ?\nEros pour Femme est une invitation à s’abandonner au désir, en osmose avec les forces de la nature apaisée.\n\nAudacieuse, cré ative et sensuelle, comme seule peut l’être la maison Versace, cette Eau de Toilette révèle une aura radieuse et une séduction irrésistible.",
"ImageUrl": "https://w ww.sephora.fr/dw/image/v2/BCVW_PRD/on/demandware.static/-/Sites-masterCatalog_Sephora/default/dw99b648b2/images/hi-res/SKU/SKU_5/359845_swatch.jpg?sw=250&sh=250&sm=f it",
"Name": "Eros pour Femme - Eau de Toilette",
"Id": "P2615007",
"CategoryId": "parfum_719097",
"BrandExternalId": "versace_c45bfd",
"Brand": {
"Id": "versace_c45b fd",
"Name": "VERSACE"
},
"Active": true,
"ProductPageUrl": "https://www.sephora.fr/p/eros-pour-femme---eau-de-toilette-359845.html",
"Disabled": false,
"ISBNs": [],
"FamilyIds": [],
"UPCs": [],
"StoryIds": [],
"ModelNumbers": [],
"Attributes": {},
"QuestionIds": [],
"AttributesOrder": [],
"ReviewIds": [],
"ManufacturerPartNumber s": [],
"QAStatistics": {
"QuestionHelpfulVoteCount": 0,
"FirstAnswerTime": "None",
"LastQuestionAnswerTime": "None",
"FirstQuestionTime": "None",
"FeaturedAnswerCount": 0,
"LastAnswerTime": "None",
"TagDistribution": {},
"ContextDataDistribution": {},
"TotalAnswerCount": 0,
"FeaturedQuestionCount": 0,
"LastQuestionTime": "None",
"Question NotHelpfulVoteCount": 0,
"BestAnswerCount": 0,
"TagDistributionOrder": [],
"AnswerHelpfulVoteCount": 0,
"HelpfulVoteCount": 0,
"AnswerNotHelpfulVoteCount": 0,
"Total QuestionCount": 0,
"ContextDataDistributionOrder": []
},
"TotalQuestionCount": 0,
"TotalAnswerCount": 0,
"ReviewStatistics": {
"ContextDataDistributionOrder": [
"Gender ",
"Age",
"Eyes",
"Skin",
"loyalty"
],
"ContextDataDistribution": {
"Gender": {
"Id": "Gender",
"Values": [
{
"Count": 7,
"Value": "Female"
}
]
},
"Age": {
"Id": "Age",
"Valu es": [
{
"Count": 1,
"Value": "13to17"
},
{
"Count": 2,
"Value": "18to24"
},
{
"Count": 1,
"Value": "25to34"
},
{
"Count": 1,
"Value": "35to44"
},
{
"Count": 1,
"Value": "45to 54"
},
{
"Count": 1,
"Value": "plus54"
}
]
},
"Eyes": {
"Id": "Eyes",
"Values": [
{
"Count": 2,
"Value": "Marrons"
},
{
"Count": 3,
"Value": "Bleus"
},
{
"Count": 1,
"Value": "N oirs"
}
]
},
"Skin": {
"Id": "Skin",
"Values": [
{
"Count": 1,
"Value": "Normale"
},
{
"Count": 2,
"Value": "Seche"
},
{
"Count": 2,
"Value": "Mixte"
},
{
"Count": 1,
"Value": " Deshydratee"
}
]
},
"loyalty": {
"Id": "loyalty",
"Values": [
{
"Count": 2,
"Value": "Yes--Im-a-VIB"
},
{
"Count": 2,
"Value": "Yes--Im-a-VIB-Rouge"
},
{
"Count": 2,
"Value": "No"
}
]
}
},
"AverageOverallRating": 4.428571428571429,
"NotHelpfulVoteCount": 1,
"FeaturedReviewCount": 0,
"NotRecommendedCount": 1,
"HelpfulVoteCount": 19,
"RatingDis tribution": [
{
"RatingValue": 5,
"Count": 5
},
{
"RatingValue": 2,
"Count": 1
},
{
"RatingValue": 4,
"Count": 1
}
],
"RecommendedCount": 5,
"RatingsOnlyReviewCount": 0,
"To talReviewCount": 7,
"FirstSubmissionTime": "2017-05-28T22:46:00.000+00:00",
"LastSubmissionTime": "2020-03-21T19:01:26.000+00:00",
"SecondaryRatingsAveragesOrder": [],
"SecondaryRatingsAverages": {},
"OverallRatingRange": 5,
"TagDistributionOrder": [],
"TagDistribution": {}
},
"TotalReviewCount": 7,
"FilteredQAStatistics": {
"Ques tionHelpfulVoteCount": 0,
"FirstAnswerTime": "None",
"LastQuestionAnswerTime": "None",
"FirstQuestionTime": "None",
"FeaturedAnswerCount": 0,
"LastAnswerTime": "None",
"TagD istribution": {},
"ContextDataDistribution": {},
"TotalAnswerCount": 0,
"FeaturedQuestionCount": 0,
"LastQuestionTime": "None",
"QuestionNotHelpfulVoteCount": 0,
"Best AnswerCount": 0,
"TagDistributionOrder": [],
"AnswerHelpfulVoteCount": 0,
"HelpfulVoteCount": 0,
"AnswerNotHelpfulVoteCount": 0,
"TotalQuestionCount": 0,
"ContextDat aDistributionOrder": []
},
"FilteredReviewStatistics": {
"ContextDataDistributionOrder": [
"Gender",
"Age",
"Eyes",
"Skin",
"loyalty"
],
"ContextDataDistribution": {
"Gen der": {
"Id": "Gender",
"Values": [
{
"Count": 7,
"Value": "Female"
}
]
},
"Age": {
"Id": "Age",
"Values": [
{
"Count": 1,
"Value": "13to17"
},
{
"Count": 2,
"Value": "18to24"
},
{
"Count": 1,
"Value": "25to34"
},
{
"Count": 1,
"Value": "35to44"
},
{
"Count": 1,
"Value": "45to54"
},
{
"Count": 1,
"Value": "plus54"
}
]
},
"Eyes": {
"Id": "Eyes",
"Value s": [
{
"Count": 2,
"Value": "Marrons"
},
{
"Count": 3,
"Value": "Bleus"
},
{
"Count": 1,
"Value": "Noirs"
}
]
},
"Skin": {
"Id": "Skin",
"Values": [
{
"Count": 1,
"Value": "Nor male"
},
{
"Count": 2,
"Value": "Seche"
},
{
"Count": 2,
"Value": "Mixte"
},
{
"Count": 1,
"Value": "Deshydratee"
}
]
},
"loyalty": {
"Id": "loyalty",
"Values": [
{
"Count": 2,
"Value": "Yes--Im-a-VIB"
},
{
"Count": 2,
"Value": "Yes--Im-a-VIB-Rouge"
},
{
"Count": 2,
"Value": "No"
}
]
}
},
"AverageOverallRating": 4.428571428571429,
"NotHelpfulVoteCo unt": 1,
"FeaturedReviewCount": 0,
"NotRecommendedCount": 1,
"HelpfulVoteCount": 19,
"RatingDistribution": [
{
"RatingValue": 5,
"Count": 5
},
{
"RatingValue": 2,
"Count ": 1
},
{
"RatingValue": 4,
"Count": 1
}
],
"RecommendedCount": 5,
"RatingsOnlyReviewCount": 0,
"TotalReviewCount": 7,
"FirstSubmissionTime": "2017-05-28T22:46:00.000+00 :00",
"LastSubmissionTime": "2020-03-21T19:01:26.000+00:00",
"SecondaryRatingsAveragesOrder": [],
"SecondaryRatingsAverages": {},
"OverallRatingRange": 5,
"TagDistri butionOrder": [],
"TagDistribution": {}
}
}
],
"Includes": {},
"HasErrors": false,
"Errors": []
},
"d": {
"attributs": {
"Doux": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Délicat": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Elegant": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Mature": {
"claimed_benefit": 0,
" perceived_benefit": 0
},
"Sexy": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Féminin": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Frais": {
"claimed_ benefit": 0,
"perceived_benefit": 0.14285714285714285
},
"Classe": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Mou": {
"claimed_benefit": 0,
"perceived_benefit": 0.14285714285714285
},
"Décontracté": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Comme les autres": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Jeu ne femme": {
"claimed_benefit": 1,
"perceived_benefit": 0.14285714285714285
},
"charmant": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Gai": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Propre": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Eté": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Rafraîchissant ": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Chaud": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Masculin": {
"claimed_benefit": 0,
"perceived_benefit ": 0
},
"Fiable": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Mystérieux": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Furtif": {
"claimed_benefit": 0,
"perceived_benefit": 0.14285714285714285
},
"Fort": {
"claimed_benefit": 0,
"perceived_benefit": 0.14285714285714285
},
"Hivernal": {
"claimed_benefit": 0,
"perceived_ benefit": 0
},
"Herbacé": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Plantes": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Big brands": {
"claimed_be nefit": 0,
"perceived_benefit": 0
},
"Luxueux": {
"claimed_benefit": 0,
"perceived_benefit": 0
},
"Connu": {
"claimed_benefit": 0,
"perceived_benefit": 0.2857142857142857
},
"A la mode": {
"claimed_benefit": 0,
"perceived_benefit": 0
}
}
},
"total": 0
}

Instead of using square bracket notations, Django Template Language uses dots. So the result should be: {{ element.q0.Results.0.Name }}

Sort query by length of field in Elstic Search

I have a field in each of my documents like so
'some_field': 3, 5, 10
But each document could have a very length of numbers,
'some_field': 3, 5, 10 # Doc 1
'some_field': 5 # Doc 2
'some_field': 3, 5, 10, 20, 9 # Doc 3
Is there a way to query and sort by the length so that my results would be arranged as so:
'some_field': 3, 5, 10, 20, 9 # Doc 3
'some_field': 3, 5, 10 # Doc 1
'some_field': 5 # Doc 2
My current query, sorting by _id at the moment
es_object.search(index='index', size=500, body={
"sort": [
{"_id": "desc"}
],
"query": {
"bool": {
"must": [
{
"match_all": {}
},
{
"exists": {
"field": "some_field"
}
}
],
"filter": [],
"should": [],
"must_not": []
}
}})

You can do script sort
{
"sort": {
"_script": {
"script": "doc['some_field'].value.length()",
"type": "number",
"order": "asc"
}
},
"query": {
"bool": {
"must": [{
"match_all": {}
},
{
"exists": {
"field": "some_field"
}
}
],
"filter": [],
"should": [],
"must_not": []
}
}
}

How to merge two aggregation queries into one

How can I merge these 2 queries into one:
Query 1
db.Response.aggregate([
{
"$and": [
{ "job_details.owner_id" : 428 },
{ "job_details.owner_type" : 'searches' }
]
},
{
"$group": {
"_id": "$candidate_city_name_string",
"count": { "$sum": 1 }
}
}
])
Query 2
db.Response.aggregate([
{
"$and": [
{ "job_details.owner_id" : 428 },
{ "job_details.owner_type" : 'searches' }
]
},
{
"$group": {
"_id": "$skill",
"count": { "$sum": 1 }
}
}
])
The result of this query like this
output 1:
{
"result": [
{ _id: 'Bangalore', count: 8 },
{ _id: 'cochi', count: 9 }
]
"ok":1
}
output 2:
{
"result": [
{ _id: 'java', count: 7 },
{ _id: 'python', count: 10 }
],
"ok":1
}
How can I get these 2 results in one query?
I need an output like this:
Expected output:
{
"result": [
"candidate_city_name_string": [
{ _id: 'Bangalore', count: 8 },
{ _id: 'cochi', count: 9 }
],
"skill": [
{ _id: 'java', count: 7 },
{ _id: 'python', count: 10 }
]
],
"ok":1
}
Is it possible? Somewhere I saw something about $facet but I didn't understand that.

db.Response.aggregate([
{"$match":{"$and":[{"job_details.owner_id" : 482},{"job_details.owner_type" : 'searches'}]}},
{$facet: {
"candidate_sublocation_name_string": [
{"$group": {"_id":"$candidate_sublocation_name_string","count": {"$sum": 1 }}}
],
"skill": [
{"$group": {"_id":"$skill","count": {"$sum": 1 }}}
]
}}])

Categories