I'm having trouble iterating through a JSON inside the django template.
My JOSN looks like,
{
"main_cats": [
{
"pk": 1,
"model": "BeeForms.maincategory",
"fields": {
"is_visible": true,
"name": "Test Main",
"order_by_asc": true,
"base_category": 1,
"help_text": "",
"is_seller_field": false,
"position": 10,
"show_seller": true,
"order_by_type": true
}
}
],
"sub_cats": [
{
"pk": 15,
"model": "BeeForms.subcategory",
"fields": {
"regex": "[0-9a-zA-Z]",
"library_id": "TextBox",
"input_length": 255,
"name": "TextBox",
"order_by_asc": false,
"field_type": "text",
"main_category": 1,
"is_active": true,
"is_visible": true,
"help_text": "",
"position": 10,
"is_required": true,
"field_display_type": "input",
"order_by_type": true,
"has_enumerations": false
}
},
{
"pk": 16,
"model": "BeeForms.subcategory",
"fields": {
"regex": "[A-Za-z0-9]",
"library_id": "SelectBox",
"input_length": 255,
"name": "SelectBox",
"order_by_asc": false,
"field_type": "",
"main_category": 1,
"is_active": true,
"is_visible": true,
"help_text": "",
"position": 10,
"is_required": false,
"field_display_type": "select",
"order_by_type": true,
"has_enumerations": true
}
},
{
"pk": 18,
"model": "BeeForms.subcategory",
"fields": {
"regex": "[0-9a-zA-Z]",
"library_id": "TextArea",
"input_length": 255,
"name": "TextArea",
"order_by_asc": false,
"field_type": "varchar",
"main_category": 1,
"is_active": true,
"is_visible": true,
"help_text": "",
"position": 10,
"is_required": false,
"field_display_type": "textarea",
"order_by_type": true,
"has_enumerations": false
}
},
{
"pk": 22,
"model": "BeeForms.subcategory",
"fields": {
"regex": "[A-Za-z0-9]",
"library_id": "Checkbox",
"input_length": 255,
"name": "Checkbox",
"order_by_asc": false,
"field_type": "",
"main_category": 1,
"is_active": true,
"is_visible": true,
"help_text": "",
"position": 10,
"is_required": false,
"field_display_type": "checkbox",
"order_by_type": true,
"has_enumerations": true
}
}
],
"base_cats": [
{
"pk": 1,
"model": "BeeForms.basecategory",
"fields": {
"form_type": 1,
"position": 10,
"order_by_asc": true,
"name": "Sales Information",
"order_by_type": true
}
},
{
"pk": 2,
"model": "BeeForms.basecategory",
"fields": {
"form_type": 1,
"position": 20,
"order_by_asc": true,
"name": "Listing information",
"order_by_type": true
}
},
{
"pk": 3,
"model": "BeeForms.basecategory",
"fields": {
"form_type": 1,
"position": 30,
"order_by_asc": true,
"name": "Land & Tax",
"order_by_type": true
}
},
{
"pk": 4,
"model": "BeeForms.basecategory",
"fields": {
"form_type": 1,
"position": 40,
"order_by_asc": true,
"name": "Exterior Construction",
"order_by_type": true
}
},
{
"pk": 5,
"model": "BeeForms.basecategory",
"fields": {
"form_type": 1,
"position": 50,
"order_by_asc": true,
"name": "Interior/Utilities",
"order_by_type": true
}
},
{
"pk": 6,
"model": "BeeForms.basecategory",
"fields": {
"form_type": 1,
"position": 60,
"order_by_asc": true,
"name": "HOA/Community",
"order_by_type": true
}
},
{
"pk": 7,
"model": "BeeForms.basecategory",
"fields": {
"form_type": 1,
"position": 70,
"order_by_asc": true,
"name": "Remarks",
"order_by_type": true
}
}
],
"enum_content": [
{
"pk": 58,
"model": "BeeForms.enumerationcontent",
"fields": {
"library_id": "SelectYesA1",
"name": "SelectYesA1",
"order_by_asc": false,
"is_visible": true,
"is_active": true,
"enum_type_category": 53,
"position": 10,
"order_by_type": true
}
},
{
"pk": 59,
"model": "BeeForms.enumerationcontent",
"fields": {
"library_id": "SelectNoA1",
"name": "SelectNoA1",
"order_by_asc": false,
"is_visible": true,
"is_active": true,
"enum_type_category": 53,
"position": 10,
"order_by_type": true
}
},
{
"pk": 60,
"model": "BeeForms.enumerationcontent",
"fields": {
"library_id": "SelectCheckBoxYes",
"name": "SelectCheckBoxYes",
"order_by_asc": false,
"is_visible": true,
"is_active": true,
"enum_type_category": 54,
"position": 10,
"order_by_type": true
}
},
{
"pk": 61,
"model": "BeeForms.enumerationcontent",
"fields": {
"library_id": "SelectCheckBoxNo",
"name": "SelectCheckBoxNo",
"order_by_asc": false,
"is_visible": true,
"is_active": true,
"enum_type_category": 54,
"position": 10,
"order_by_type": true
}
}
],
"enums": [
{
"pk": 20,
"model": "BeeForms.enumerations",
"fields": {
"library_id": "SelectYes",
"name": "SelectYes",
"order_by_asc": false,
"is_visible": true,
"has_type": true,
"is_active": true,
"position": 10,
"order_by_type": true,
"sub_category": 16
}
},
{
"pk": 21,
"model": "BeeForms.enumerations",
"fields": {
"library_id": "SelectNo",
"name": "SelectNo",
"order_by_asc": false,
"is_visible": true,
"has_type": true,
"is_active": true,
"position": 10,
"order_by_type": true,
"sub_category": 16
}
},
{
"pk": 30,
"model": "BeeForms.enumerations",
"fields": {
"library_id": "Yes",
"name": "Yes",
"order_by_asc": false,
"is_visible": true,
"has_type": true,
"is_active": true,
"position": 10,
"order_by_type": true,
"sub_category": 22
}
},
{
"pk": 31,
"model": "BeeForms.enumerations",
"fields": {
"library_id": "No",
"name": "No",
"order_by_asc": false,
"is_visible": true,
"has_type": false,
"is_active": true,
"position": 10,
"order_by_type": true,
"sub_category": 22
}
}
],
"enum_types": [
{
"pk": 53,
"model": "BeeForms.enumerationtype",
"fields": {
"regex": "[A-Za-z0-9]",
"is_visible": true,
"name": "SelectYesA",
"library_id": "SelectYesA",
"field_type": "",
"is_active": true,
"has_content": true,
"enum_category": 20,
"field_display_type": "select",
"input_length": 255
}
},
{
"pk": 54,
"model": "BeeForms.enumerationtype",
"fields": {
"regex": "[A-Za-z0-9]",
"is_visible": true,
"name": "",
"library_id": "",
"field_type": "",
"is_active": true,
"has_content": true,
"enum_category": 21,
"field_display_type": "checkbox",
"input_length": 255
}
}
]
}
The code I use to create the json file is as follows.
def serialize(self, instance_name, form, destination):
""" Seralizes the information passed in."""
# we need to make sure that the directory is real.
if not os.path.exists(destination):
os.makedirs(destination)
# Lets grab all the information needed and serailize it all at the same time.
base_cats = serializers.serialize('json', BaseCategory.objects.all().filter(form_type_id=form.id).order_by('id'))
main_cats = serializers.serialize('json', MainCategory.objects.all().order_by('id'))
sub_cats = serializers.serialize('json', SubCategory.objects.all().order_by('id'))
enums = serializers.serialize('json', Enumerations.objects.all().order_by('id'))
enum_types = serializers.serialize('json', EnumerationType.objects.all().order_by('id'))
enum_content = serializers.serialize('json', EnumerationContent.objects.all().order_by('id'))
# now that they are all serailized make it into a list of simplejson
base_list = simplejson.loads(base_cats)
main_list = simplejson.loads(main_cats)
sub_list = simplejson.loads(sub_cats)
enums_list = simplejson.loads(enums)
enum_type_list = simplejson.loads(enum_types)
enum_content_list = simplejson.loads(enum_content)
# now we combine them all together...
self.json_data = simplejson.dumps({'base_cats': base_list, 'main_cats': main_list, 'sub_cats': sub_list,
'enums': enums_list, 'enum_types': enum_type_list,
'enum_content': enum_content_list}, indent=4)
print(self.json_data)
# now that we combined them into json_data. Lets write it out to a file.
with open(destination + instance_name + '.json', 'w') as json_write:
json_write.write(self.json_data)
I load the JSON into a Python Object with the following.
with open(instance_location + '.json', 'r') as json_read:
self.json_data = json_read.read()
# now lets take the data and load into json array
data = simplejson.loads(self.json_data)
# now lets grab each list.
base_cats = data['base_cats']
main_cats = data['main_cats']
sub_cats = data['sub_cats']
enums = data['enums']
enum_types = data['enum_types']
enum_content = data['enum_content']
# lets return it all!
return base_cats, main_cats, sub_cats, enums, enum_types, enum_content
In straight Python I'm able to loop through (for example base_cats) and get the information I need with the following for loop
for base in base_dict:
print base['pk']
print base['fields']['name']
When I try that for loop in the django template I get the error of:
Could not parse the remainder: '['pk']' from 'base['pk']'
With how the JSON is structered how would I loop through it in the django template as the following is not working.
{% for base in base_cats %}
{{ base['pk'] }}
{% endfor %}
Access the fields with the dot operator in the template, not with brackets:
{% for base in base_cats %}
{{ base.pk }}
{% endfor %}
For reference:
https://docs.djangoproject.com/en/1.10/ref/templates/language/#template-variables
Related
import scrapy
from ..items import CentriItem
from scrapy_playwright.page import PageMethod
class NikespiderSpider(scrapy.Spider):
name = 'nikespider'
def start_requests(self):
url = "https://www.nike.com/ch/en/w/mens-nik1?q=shirts"
yield scrapy.Request(url, meta=dict(
playwright=True,
playwright_include_page=True,
playwright_page_methods=[
PageMethod("wait_for_selector", ".product-card__link-overlay"),
PageMethod("evaluate", "window.scrollBy(0, document.body.scrollHeight)"),
PageMethod("wait_for_selector",".css-c2ovjx:nth-child(25) .product-card__link-overlay"),
]
))
async def parse(self, response):
page = response.meta["playwright_page"]
await page.close()
i = 0
for link in response.css(".product-card__link-overlay::attr(href)"):
i = i + 1
print(i)
async def parse_items(self, response,link):
pass
This is my code for scraping Nike. The url which I am scraping has infinite scrolling. It loads 24 items at first and after you scroll down it loads 24 more. I am trying to add this functionality using scrapy-playwright but it is not working. Anyone able to help me?
It looks like it would probably be easier to extract the data from their api.
import scrapy
class NikespiderSpider(scrapy.Spider):
name = 'nikespider'
def start_requests(self):
for i in range(1, 20):
v = i * 12
url = f"https://api.nike.com/cic/browse/v2?queryid=products&anonymousId=699ECBF06AACCD31DCB69E0C71964748&country=ch&endpoint=%2Fproduct_feed%2Frollup_threads%2Fv2%3Ffilter%3Dmarketplace(CH)%26filter%3Dlanguage(en-GB)%26filter%3DemployeePrice(true)%26filter%3DattributeIds(0f64ecc7-d624-4e91-b171-b83a03dd8550)%26searchTerms%3Dshirts%26anchor%3D{v}%26consumerChannelId%3Dd9a5bc42-4b9c-4976-858a-f159cf99c647%26count%3D24&language=en-GB&localizedRangeStr=%7BlowestPrice%7D%E2%80%94%7BhighestPrice%7D"
yield scrapy.Request(url=url)
def parse(self, response):
yield response.json()
output for the first page looks like this:
{
"data": {
"products": {
"errors": null,
"products": [
{
"cardType": "default",
"cloudProductId": "a44f2253-c745-5143-bd1e-f2c841da1550",
"colorDescription": "Obsidian/Pepper Red/Gorge Green/Gold Dart",
"colorways": [
{
"colorDescription": "Obsidian/Pepper Red/Gorge Green/Gold Dart",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/a77364ff-d0e8-4eaa-83eb-2a206896b28d/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/e1374e74-f614-40dd-aecd-ac4c0bd1a5bb/portugal-french-terry-football-hoodie-rF3Vcw.png"
},
"pdpUrl": "{countryLang}/t/portugal-french-terry-football-hoodie-rF3Vcw/DH4826-451",
"price": {
"currency": "CHF",
"currentPrice": 109.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 109.95,
"minimumAdvertisedPrice": null
},
"cloudProductId": "a44f2253-c745-5143-bd1e-f2c841da1550",
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNew": true,
"label": "JUST_IN",
"pid": "14061770",
"prebuildId": null,
"productInstanceId": null
}
],
"customizable": false,
"hasExtendedSizing": false,
"id": "96633291-36b7-4a6d-a1ab-ef0e264da2c2",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/a77364ff-d0e8-4eaa-83eb-2a206896b28d/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/e1374e74-f614-40dd-aecd-ac4c0bd1a5bb/portugal-french-terry-football-hoodie-rF3Vcw.png"
},
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isGiftCard": false,
"isJersey": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNBA": false,
"isNFL": false,
"isSustainable": false,
"label": "JUST_IN",
"nbyColorway": null,
"pid": "14061770",
"prebuildId": null,
"price": {
"currency": "CHF",
"currentPrice": 109.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 109.95,
"minimumAdvertisedPrice": null
},
"productInstanceId": null,
"productType": "APPAREL",
"properties": null,
"salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
"subtitle": "Men's Nike French Terry Football Hoodie",
"title": "Portugal",
"url": "{countryLang}/t/portugal-french-terry-football-hoodie-rF3Vcw/DH4826-451"
},
{
"cardType": "default",
"cloudProductId": "a05bd5c6-290e-5d8c-b583-9c8fd49fee3b",
"colorDescription": "Coastal Blue/Green Spark/Dynamic Yellow",
"colorways": [
{
"colorDescription": "Coastal Blue/Green Spark/Dynamic Yellow",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/af0e8a3f-7f4b-417c-b7b1-6acd09607ee2/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/d6150c37-9086-4b1a-b1f0-66e59f3258cb/brazil-travel-fleece-football-hoodie-H4PKQk.png"
},
"pdpUrl": "{countryLang}/t/brazil-travel-fleece-football-hoodie-H4PKQk/DH4822-490",
"price": {
"currency": "CHF",
"currentPrice": 109.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 109.95,
"minimumAdvertisedPrice": null
},
"cloudProductId": "a05bd5c6-290e-5d8c-b583-9c8fd49fee3b",
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNew": true,
"label": "JUST_IN",
"pid": "14061554",
"prebuildId": null,
"productInstanceId": null
}
],
"customizable": false,
"hasExtendedSizing": false,
"id": "9ba8eaff-d94d-445a-b569-8bae4c1d741b",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/af0e8a3f-7f4b-417c-b7b1-6acd09607ee2/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/d6150c37-9086-4b1a-b1f0-66e59f3258cb/brazil-travel-fleece-football-hoodie-H4PKQk.png"
},
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isGiftCard": false,
"isJersey": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNBA": false,
"isNFL": false,
"isSustainable": false,
"label": "JUST_IN",
"nbyColorway": null,
"pid": "14061554",
"prebuildId": null,
"price": {
"currency": "CHF",
"currentPrice": 109.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 109.95,
"minimumAdvertisedPrice": null
},
"productInstanceId": null,
"productType": "APPAREL",
"properties": null,
"salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
"subtitle": "Men's Fleece Football Hoodie",
"title": "Brazil Travel",
"url": "{countryLang}/t/brazil-travel-fleece-football-hoodie-H4PKQk/DH4822-490"
},
{
"cardType": "default",
"cloudProductId": "40d88287-0129-52c7-ad5e-646ba8333389",
"colorDescription": "Game Royal/Blue Fury/Blue Fury",
"colorways": [
{
"colorDescription": "Game Royal/Blue Fury/Blue Fury",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/b66e9663-2fd1-4398-9270-d4e4d46ed83e/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/873eaab7-cfec-4b70-b6af-19880a563637/england-strike-dri-fit-short-sleeve-football-top-1m2GSc.png"
},
"pdpUrl": "{countryLang}/t/england-strike-dri-fit-short-sleeve-football-top-1m2GSc/DH6443-480",
"price": {
"currency": "CHF",
"currentPrice": 59.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 59.95,
"minimumAdvertisedPrice": null
},
"cloudProductId": "40d88287-0129-52c7-ad5e-646ba8333389",
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNew": true,
"label": "JUST_IN",
"pid": "14062596",
"prebuildId": null,
"productInstanceId": null
}
],
"customizable": false,
"hasExtendedSizing": false,
"id": "f73148f5-4b63-4b1f-84cc-b03bcc408ca1",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/b66e9663-2fd1-4398-9270-d4e4d46ed83e/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/873eaab7-cfec-4b70-b6af-19880a563637/england-strike-dri-fit-short-sleeve-football-top-1m2GSc.png"
},
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isGiftCard": false,
"isJersey": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNBA": false,
"isNFL": false,
"isSustainable": true,
"label": "JUST_IN",
"nbyColorway": null,
"pid": "14062596",
"prebuildId": null,
"price": {
"currency": "CHF",
"currentPrice": 59.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 59.95,
"minimumAdvertisedPrice": null
},
"productInstanceId": null,
"productType": "APPAREL",
"properties": null,
"salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
"subtitle": "Men's Nike Dri-FIT Short-Sleeve Football Top",
"title": "England Strike",
"url": "{countryLang}/t/england-strike-dri-fit-short-sleeve-football-top-1m2GSc/DH6443-480"
},
{
"cardType": "default",
"cloudProductId": "e04c4bb3-4382-5df4-9f03-7c1d393f9d05",
"colorDescription": "Obsidian/Green Glow/Green Glow",
"colorways": [
{
"colorDescription": "Obsidian/Green Glow/Green Glow",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/4cbd4672-d5ac-453b-b023-2d5160b33341/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/7f04a8ed-649c-4705-b110-0c605b852f7b/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs.png"
},
"pdpUrl": "{countryLang}/t/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs/DN0676-451",
"price": {
"currency": "CHF",
"currentPrice": 119.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 119.95,
"minimumAdvertisedPrice": null
},
"cloudProductId": "e04c4bb3-4382-5df4-9f03-7c1d393f9d05",
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNew": true,
"label": "JUST_IN",
"pid": "14063595",
"prebuildId": null,
"productInstanceId": null
}
],
"customizable": false,
"hasExtendedSizing": false,
"id": "0027a0c6-0e97-445c-b6ed-a92a6bfdf18b",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/4cbd4672-d5ac-453b-b023-2d5160b33341/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/7f04a8ed-649c-4705-b110-0c605b852f7b/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs.png"
},
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isGiftCard": false,
"isJersey": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNBA": false,
"isNFL": false,
"isSustainable": true,
"label": "JUST_IN",
"nbyColorway": null,
"pid": "14063595",
"prebuildId": null,
"price": {
"currency": "CHF",
"currentPrice": 119.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 119.95,
"minimumAdvertisedPrice": null
},
"productInstanceId": null,
"productType": "APPAREL",
"properties": null,
"salesChannel": ["NikeApp", "Nike.com", "Nike Store Experiences"],
"subtitle": "Men's Nike Dri-FIT Football Shirt",
"title": "Australia 2022/23 Stadium Away",
"url": "{countryLang}/t/australia-2022-23-stadium-away-dri-fit-football-shirt-XKZCLs/DN0676-451"
},
{
"cardType": "default",
"cloudProductId": "758da5f5-ea79-5bf7-a401-8ccb945a5f16",
"colorDescription": "Off-Noir/White",
"colorways": [
{
"colorDescription": "Off-Noir/White",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/1efdbff3-1c9e-4984-9bf2-8da11d8f5161/image.jpg",
"squarishURL": "https://static.nike.com/a/images/t_default/5ca1f9a9-f1fa-42fe-a552-571ffbcc8ba6/nikecourt-dri-fit-rafa-challenger-short-sleeve-tennis-top-wNcD3N.png"
},
"pdpUrl": "{countryLang}/t/nikecourt-dri-fit-rafa-challenger-short-sleeve-tennis-top-wNcD3N/DD8547-045",
"price": {
"currency": "CHF",
"currentPrice": 84.95,
"discounted": false,
"employeePrice": 0,
"fullPrice": 84.95,
"minimumAdvertisedPrice": null
},
"cloudProductId": "758da5f5-ea79-5bf7-a401-8ccb945a5f16",
"inStock": true,
"isComingSoon": false,
"isBestSeller": false,
"isExcluded": false,
"isLaunch": false,
"isMemberExclusive": false,
"isNew": true,
"label": "JUST_IN",
"pid": "14168043",
"prebuildId": null,
"productInstanceId": null
},
{
"colorDescription": "Pure Platinum/Pink Gaze/Black",
"images": {
"portraitURL": "https://static.nike.com/a/images/c_limit,w_400,f_auto/t_product_v1/dbf2edc2-da8b-4e7e-976f-5d16f207b449/image.jpg",
...
I have a JSON response that I want to convert to a Pandas dataframe for further analysis.
{
"isCurrentUserFollowing": False,
"id": 1,
"contactType": "Household",
"isPrivate": False,
"name": "Mrs Mattie Mahoney",
"informalName": "Mattie",
"description": None,
"website": None,
"maritalStatus": None,
"anniversaryMonth": None,
"anniversaryDay": None,
"anniversaryYear": None,
"mergedIntoContactId": None,
"address": {
"id": 1,
"label": "Primary",
"address1": "xyz",
"address2": None,
"city": "abc",
"state": "aa",
"postal": "11111",
"country": "United States",
"isPrimary": True,
"canBePrimary": False,
"startMonth": 1,
"startDay": 1,
"endMonth": 12,
"endDay": 31
},
"giftAskAmount": "$250",
"giftAskType": "Recurring Gift",
"lifeToDateGiving": "$225",
"yearToDateGiving": "$225",
"lastGiftAmount": "$225",
"lastGiftDate": "7/2/2021",
"contactIndividuals": [
{
"id": 1,
"contactId": 1,
"prefix": "Mrs",
"firstName": "Joan",
"middleName": None,
"lastName": "Arc",
"suffix": None,
"gender": "Female",
"isPrimary": True,
"canBePrimary": False,
"isSecondary": False,
"canBeSecondary": False,
"birthMonth": 1,
"birthDay": 1,
"birthYear": 1900,
"birthDate": "April 1",
"approximateAge": 100,
"isDeceased": False,
"passion": None,
"avatarUrl": "www.abc.com",
"contactMethods": [
{
"id": 1,
"type": "Home Email",
"value": "abc#hotmail.com",
"isOptedIn": False,
"isPrimary": True,
"canBePrimary": False
},
{
"id": 2,
"type": "Mobile Phone",
"value": "(111) 1111111",
"isOptedIn": False,
"isPrimary": True,
"canBePrimary": False
}
],
"createDateTimeUtc": "2021-04-15T17:41:31.2166667",
"modifiedDateTimeUtc": "2021-04-15T17:41:31.2166667",
"customFields": [],
"customCollections": []
},
{
"id": 2,
"contactId": 1,
"prefix": None,
"firstName": "John",
"middleName": None,
"lastName": "Smith",
"suffix": None,
"gender": "Female",
"isPrimary": False,
"canBePrimary": True,
"isSecondary": True,
"canBeSecondary": False,
"birthMonth": 1,
"birthDay": 11,
"birthYear": 1990,
"birthDate": "July 1",
"approximateAge": 100,
"isDeceased": False,
"passion": None,
"avatarUrl": "www.google.com",
"contactMethods": [
{
"id": 3,
"type": "Home Email",
"value": "mno#gmail.com",
"isOptedIn": False,
"isPrimary": True,
"canBePrimary": False
}
],
"createDateTimeUtc": "2021-04-15T17:41:31.2166667",
"modifiedDateTimeUtc": "2021-04-15T17:41:31.2166667",
"customFields": [],
"customCollections": []
}
],
"contactGiftsUrl": "/api/Gift/ByContact/1",
"contactPassthroughGiftsUrl": "/api/Gift/Passthrough/ByContact/1",
"contactPlannedGiftsUrl": "/api/PlannedGift/ByContact/1",
"contactRecurringGiftsUrl": "/api/RecurringGift/ByContact/1",
"contactImportantNotesUrl": "/api/ContactNote/Important/ByContact/1",
"contactNotesUrl": "/api/ContactNote/ByContact/1",
"contactTagsUrl": "/api/ContactTag/ByContact/1",
"contactRelationshipsUrl": "/api/Relationship/ByContact/1",
"primaryAvatarUrl": "www.apple.com",
"contactReferences": [],
"originSegmentId": None,
"originSegment": None,
"createDateTimeUtc": "2021-04-15T17:41:31.217",
"modifiedDateTimeUtc": "2021-04-15T17:41:31.217",
"customFields": [],
"customCollections": []
}
I am struggling with the code I need to write to get the relevant fields out
I am able to get some fields but I cannot get the gender and emails to be retrieved. Can someone assist ?
df = pd.DataFrame(
columns=[
"contactcustomerid",
"contactType",
"contactName",
"contactaddress",
"contactcity",
"contactstate",
"contactzip",
"contactcountry",
"giftAskAmount",
"giftAskType",
"lifetodateGiving",
"yeartoDateGiving",
"lastGiftAmount",
"lastGiftDate",
"contactGender",
"contactEmail",
]
)
if json_data:
row = {
"contactcustomerid": json_data.get("id"),
"contactType": json_data.get("contactType"),
"contactName": json_data.get("name"),
"contactaddress": json_data.get("address").get("address1"),
"contactcity": json_data.get("address").get("city"),
"contactstate": json_data.get("address").get("state"),
"contactzip": json_data.get("address").get("postal"),
"contactcountry": json_data.get("address").get("country"),
"giftAskAmount": json_data.get("giftAskAmount"),
"giftAskType": json_data.get("giftAskType"),
"lifetodateGiving": json_data.get("lifeToDateGiving"),
"yeartoDateGiving": json_data.get("yearToDateGiving"),
"lastGiftAmount": json_data.get("lastGiftAmount"),
"lastGiftDate": json_data.get("lastGiftDate"),
}
df = df.append(row, ignore_index="True")
df = df.fillna(0)
I have been trying to work with a JSON object where I have been trying to get values from two different keys. What I want to do is to check if in object 1 contains in object 2 and has the value over 0 then I want to print it out.
get_json = json.dumps({
"attributes": {
"203": {
"id": "203",
"code": "sizefootwear_conf",
"label": "EU",
"options": [{
"id": "6320",
"label": "38",
"products": ["69813"]
},
{
"id": "6351",
"label": "38,5",
"products": ["69817"]
},
{
"id": "6335",
"label": "39",
"products": ["69818"]
},
{
"id": "6354",
"label": "40",
"products": ["69819"]
},
{
"id": "6338",
"label": "40,5",
"products": ["69820"]
},
{
"id": "6357",
"label": "41",
"products": ["69821"]
},
{
"id": "6326",
"label": "42",
"products": ["69822"]
},
{
"id": "6362",
"label": "42,5",
"products": ["69823"]
},
{
"id": "6341",
"label": "43",
"products": ["69824"]
},
{
"id": "6365",
"label": "44",
"products": ["69814"]
},
{
"id": "6344",
"label": "44,5",
"products": ["69815"]
},
{
"id": "6370",
"label": "45,5",
"products": ["69816"]
}
],
"position": "0"
},
"205": {
"id": "205",
"code": "sizefootwearus_conf",
"label": "US",
"options": [{
"id": "6319",
"label": "5,5",
"products": ["69813"]
},
{
"id": "6372",
"label": "6,0",
"products": ["69817"]
},
{
"id": "6334",
"label": "6,5",
"products": ["69818"]
},
{
"id": "6350",
"label": "7,0",
"products": ["69819"]
},
{
"id": "6337",
"label": "7,5",
"products": ["69820"]
},
{
"id": "6353",
"label": "8,0",
"products": ["69821"]
},
{
"id": "6325",
"label": "8,5",
"products": ["69822"]
},
{
"id": "6356",
"label": "9,0",
"products": ["69823"]
},
{
"id": "6340",
"label": "9,5",
"products": ["69824"]
},
{
"id": "6364",
"label": "10,0",
"products": ["69814"]
},
{
"id": "6343",
"label": "10,5",
"products": ["69815"]
},
{
"id": "6328",
"label": "11,5",
"products": ["69816"]
}
],
"position": "1"
},
"204": {
"id": "204",
"code": "sizefootwearuk_conf",
"label": "UK",
"options": [{
"id": "6318",
"label": "5,0",
"products": ["69813"]
},
{
"id": "6352",
"label": "5,5",
"products": ["69817"]
},
{
"id": "6743",
"label": "6,0-EU39",
"products": ["69818"]
},
{
"id": "6744",
"label": "6,0-EU40",
"products": ["69819"]
},
{
"id": "6355",
"label": "6,5",
"products": ["69820"]
},
{
"id": "6336",
"label": "7,0",
"products": ["69821"]
},
{
"id": "6361",
"label": "7,5",
"products": ["69822"]
},
{
"id": "6324",
"label": "8,0",
"products": ["69823"]
},
{
"id": "6363",
"label": "8,5",
"products": ["69824"]
},
{
"id": "6339",
"label": "9,0",
"products": ["69814"]
},
{
"id": "6366",
"label": "9,5",
"products": ["69815"]
},
{
"id": "6369",
"label": "10,5",
"products": ["69816"]
}
],
"position": "2"
}
},
"productStockAlert": {
"entity": "69825",
"map": {
"203": {
"label": "52,5",
"": "",
"6610": "6610",
"6498": "6498",
"6582": "6582",
"6516": "6516",
"6501": "6501",
"6518": "6518",
"6504": "6504",
"6395": "6395",
"6404": "6404",
"6533": "6533",
"6407": "6407",
"6530": "6530",
"6410": "6410",
"6413": "6413",
"6416": "6416",
"6534": "6534",
"6419": "6419",
"6422": "6422",
"6425": "6425",
"6398": "6398",
"6401": "6401",
"6531": "6531",
"6431": "6431",
"6443": "6443",
"6446": "6446",
"6495": "6495",
"6449": "6449",
"6452": "6452",
"6455": "6455",
"6458": "6458",
"6461": "6461",
"6807": "6807",
"6464": "6464",
"6434": "6434",
"6437": "6437",
"6558": "6558",
"6440": "6440",
"6480": "6480",
"6481": "6481",
"6382": "6382",
"6465": "6465",
"6631": "6631",
"6332": "6332",
"6466": "6466",
"6348": "6348",
"6634": "6634",
"6320": "6320",
"6351": "6351",
"6384": "6384",
"6659": "6659",
"6335": "6335",
"6388": "6388",
"6508": "6508",
"6354": "6354",
"6338": "6338",
"6389": "6389",
"6664": "6664",
"6357": "6357",
"6390": "6390",
"6506": "6506",
"6637": "6637",
"6326": "6326",
"6362": "6362",
"6391": "6391",
"6640": "6640",
"6341": "6341",
"6392": "6392",
"6560": "6560",
"6365": "6365",
"6344": "6344",
"6385": "6385",
"6838": "6838",
"6368": "6368",
"6386": "6386",
"6370": "6370",
"6643": "6643",
"6628": "6628",
"6329": "6329",
"6529": "6529",
"6387": "6387",
"6843": "6843",
"6347": "6347",
"6470": "6470",
"6360": "6360",
"6646": "6646",
"6472": "6472",
"6323": "6323",
"6564": "6564",
"6593": "6593",
"6474": "6474",
"6376": "6376",
"6565": "6565",
"6561": "6561",
"6567": "6567",
"6604": "6604",
"6607": "6607"
},
"205": {
"label": "18,0",
"": "",
"6513": "6513",
"6497": "6497",
"6583": "6583",
"6500": "6500",
"6821": "6821",
"6503": "6503",
"6532": "6532",
"6394": "6394",
"6403": "6403",
"6406": "6406",
"6409": "6409",
"6412": "6412",
"6415": "6415",
"6418": "6418",
"6421": "6421",
"6424": "6424",
"6397": "6397",
"6400": "6400",
"6430": "6430",
"6442": "6442",
"6445": "6445",
"6448": "6448",
"6451": "6451",
"6454": "6454",
"6457": "6457",
"6460": "6460",
"6463": "6463",
"6433": "6433",
"6436": "6436",
"6439": "6439",
"6555": "6555",
"6468": "6468",
"6507": "6507",
"6632": "6632",
"6331": "6331",
"6319": "6319",
"6635": "6635",
"6372": "6372",
"6334": "6334",
"6661": "6661",
"6350": "6350",
"6337": "6337",
"6663": "6663",
"6353": "6353",
"6619": "6619",
"6325": "6325",
"6621": "6621",
"6638": "6638",
"6356": "6356",
"6340": "6340",
"6623": "6623",
"6641": "6641",
"6364": "6364",
"6343": "6343",
"6625": "6625",
"6840": "6840",
"6367": "6367",
"6328": "6328",
"6644": "6644",
"6371": "6371",
"6346": "6346",
"6842": "6842",
"6359": "6359",
"6322": "6322",
"6647": "6647",
"6373": "6373",
"6566": "6566",
"6375": "6375",
"6562": "6562",
"6605": "6605",
"6608": "6608"
},
"204": {
"label": "17,0",
"": "",
"6611": "6611",
"6514": "6514",
"6496": "6496",
"6515": "6515",
"6499": "6499",
"6517": "6517",
"6502": "6502",
"6393": "6393",
"6505": "6505",
"6402": "6402",
"6405": "6405",
"6408": "6408",
"6411": "6411",
"6414": "6414",
"6417": "6417",
"6420": "6420",
"6423": "6423",
"6396": "6396",
"6399": "6399",
"6429": "6429",
"6745": "6745",
"6441": "6441",
"6444": "6444",
"6447": "6447",
"6450": "6450",
"6453": "6453",
"6456": "6456",
"6459": "6459",
"6462": "6462",
"6432": "6432",
"6435": "6435",
"6438": "6438",
"6467": "6467",
"6381": "6381",
"6633": "6633",
"6330": "6330",
"6349": "6349",
"6636": "6636",
"6318": "6318",
"6352": "6352",
"6660": "6660",
"6333": "6333",
"6743": "6743",
"6744": "6744",
"6355": "6355",
"6662": "6662",
"6336": "6336",
"6620": "6620",
"6361": "6361",
"6622": "6622",
"6639": "6639",
"6324": "6324",
"6363": "6363",
"6624": "6624",
"6642": "6642",
"6339": "6339",
"6366": "6366",
"6626": "6626",
"6839": "6839",
"6342": "6342",
"6627": "6627",
"6369": "6369",
"6645": "6645",
"6327": "6327",
"6358": "6358",
"6841": "6841",
"6345": "6345",
"6471": "6471",
"6648": "6648",
"6321": "6321",
"6473": "6473",
"6374": "6374",
"6563": "6563",
"6606": "6606",
"6609": "6609"
}
},
"child": {
"6320_6319_6318_": {
"entity": "69813",
"stock_number": 0,
"stock_status": false,
"productId": "69813",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6365_6364_6339_": {
"entity": "69814",
"stock_number": 5,
"stock_status": true,
"productId": "69814",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6344_6343_6366_": {
"entity": "69815",
"stock_number": 3,
"stock_status": true,
"productId": "69815",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6370_6328_6369_": {
"entity": "69816",
"stock_number": 1,
"stock_status": true,
"productId": "69816",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6351_6372_6352_": {
"entity": "69817",
"stock_number": 0,
"stock_status": false,
"productId": "69817",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6335_6334_6743_": {
"entity": "69818",
"stock_number": 0,
"stock_status": false,
"productId": "69818",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6354_6350_6744_": {
"entity": "69819",
"stock_number": 0,
"stock_status": false,
"productId": "69819",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6338_6337_6355_": {
"entity": "69820",
"stock_number": 0,
"stock_status": false,
"productId": "69820",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6357_6353_6336_": {
"entity": "69821",
"stock_number": 3,
"stock_status": true,
"productId": "69821",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6326_6325_6361_": {
"entity": "69822",
"stock_number": 4,
"stock_status": true,
"productId": "69822",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6362_6356_6324_": {
"entity": "69823",
"stock_number": 6,
"stock_status": true,
"productId": "69823",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
},
"6341_6340_6363_": {
"entity": "69824",
"stock_number": 6,
"stock_status": true,
"productId": "69824",
"parent_url": "https://www.bstn.com/eu_en/p/jordan-jordan-why-not-zer0-4-dd4889-006-0250549"
}
}
}
}
)
So what I did is that I created two dicts within a list:
first_loop = []
second_loop = []
total_stock = 0
for idx, sizes in json_value["attributes"].items():
for getId in sizes["options"]:
first_loop.append({getId["label"]: getId["products"][0]})
break
for idx, test in json_value["productStockAlert"]["child"].items():
total_stock += test["stock_number"]
second_loop.append({test["productId"]: test["stock_number"]})
print("first_loop", first_loop)
print("second_loop", second_loop)
print("total_stock", total_stock)
which returns:
first_loop [{'38': '69813'}, {'38,5': '69817'}, {'39': '69818'}, {'40': '69819'}, {'40,5': '69820'}, {'41': '69821'}, {'42': '69822'}, {'42,5': '69823'}, {'43': '69824'}, {'44': '69814'}, {'44,5': '69815'}, {'45,5': '69816'}]
second_loop [{'69813': 0}, {'69814': 5}, {'69815': 3}, {'69816': 1}, {'69817': 0}, {'69818': 0}, {'69819': 0}, {'69820': 0}, {'69821': 3}, {'69822': 4}, {'69823': 6}, {'69824': 6}]
total_stock 28
My issue is how can I compare from first_loop where I check the ID (etc 69816) is in second_loop and has the value above 0, if its above 0 then I want to add it to a append it to a new list etc: 45,5 (1) (Which is the size number from first_loop and the number (value) from second_loop.
Output would end up:
>>> ["41 (3)", "42 (4)", "42,5 (6)", "43 (6)", "44 (5)", "44,5 (3)", "45,5 (1)"]
Basically, you just need to create id-label mapping, id-count mapping and merge them:
id_label_mapping = {o["products"][0]: o["label"] for o in next(iter(json_value["attributes"].values()))["options"]}
id_count_mapping = {o["productId"]: o["stock_number"] for o in json_value["productStockAlert"]["child"].values()}
result = [f"{l} ({id_count_mapping[k]})" for k, l in id_label_mapping.items() if id_count_mapping.get(k)]
In your code you've done 2 major mistakes which makes implementation of last step (merging) much harder.
You're creating list of dicts instead of single dict with different keys;
In first_loop you're using label as key, but in second_loop you're using productId.
If we will fix this 2 gaps, your code will work:
first_loop = {}
second_loop = {}
total_stock = 0
for idx, sizes in json_value["attributes"].items():
for getId in sizes["options"]:
first_loop[getId["products"][0]] = getId["label"]
break
for idx, test in json_value["productStockAlert"]["child"].items():
total_stock += test["stock_number"]
second_loop[test["productId"]] = test["stock_number"]
result = []
for product_id, label in first_loop.items():
count = second_loop.get(product_id)
if count: # filters both None (key doesn't exit) and 0
result.append(f"{label} ({count})")
print("result", result)
print("total_stock", total_stock)
Not sure if it's the most efficient way, but you could:
make dicts not lists, does it need to be a list?
swap the key-value of the first_loop
intersect the sets
get the values from original, print only if > 0
[Code not tested]
first_loop = {}
second_loop = {}
total_stock = 0
for idx, sizes in json_value["attributes"].items():
for getId in sizes["options"]:
first_loop[getId["products"][0]] = getId["label"]
break
for idx, test in json_value["productStockAlert"]["child"].items():
total_stock += test["stock_number"]
second_loop[test["productId"]] = test["stock_number"]
matching = set(first_loop.keys()).intersection(second_loop.keys())
for prod_id in matching:
stock = second_loop.get(prod_id)
if stock > 0:
print(f"{first_loop.get(prod_id) ({stock})")
Lastly, you have a break statement, that will make it run only one time... In which case you do not need a for loop...
It's quite too case specific, but hope it helps...
I'm new to python and I've been trying to parse the output of from an F5 API request and I'm struggling a bit. I would like to extract all the "name" and "destination" values and save those to a file. This should be a name and the corresponding IP address. From the output below, it would be:
test-account 10.1.1.1
test-account_443 10.1.1.5
I can extract the individual elements, but I'm struggling to figure out how to iterate over the output file so I can get all of the key values I want. I've cut the output short to make it easier to read.
parsed_json = json.loads(data)
destination = parsed_json['items'][0]['destination']
name = parsed_json['items'][0]['name']
This only extracts one key value from the whole output file. Many thanks in advance.
{
"kind": "tm:ltm:virtual:virtualcollectionstate",
"items": [
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account",
"partition": "Common",
"fullPath": "/Common/test-account",
"generation": 1,
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.1:80",
"enabled": true,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": true,
"vsIndex": 2,
"rules": [
"/Common/redirect-http-to-https"
],
"rulesReference": [
{
"link":
}
],
"policiesReference": {
"link":
"isSubcollection": true
},
"profilesReference": {
"link":
"isSubcollection": true
}
},
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account_443",
"partition": "Common",
"fullPath": "/Common/test-account_443",
"generation": 1,
"selfLink":
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.5:443",
"enabled": true,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"poolReference": {
"link":
},
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": true,
"vsIndex": 3,
"policiesReference": {
"link":
"isSubcollection": true
},
"profilesReference": {
"link":
"isSubcollection": true
}
}
This should help. You can iterate over the "items" key and get the required information
Ex:
parsed_json = {
"kind": "tm:ltm:virtual:virtualcollectionstate",
"items": [
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account",
"partition": "Common",
"fullPath": "/Common/test-account",
"generation": 1,
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.1:80",
"enabled": True,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": True,
"vsIndex": 2,
"rules": [
"/Common/redirect-http-to-https"
],
"rulesReference": [
{
"link": True
}
],
"policiesReference": {
"link": True,
"isSubcollection": True
},
"profilesReference": {
"link": True,
"isSubcollection": True
}
},
{
"kind": "tm:ltm:virtual:virtualstate",
"name": "test-account_443",
"partition": "Common",
"fullPath": "/Common/test-account_443",
"generation": 1,
"selfLink": True,
"addressStatus": "yes",
"autoLasthop": "default",
"cmpEnabled": "yes",
"connectionLimit": 0,
"destination": "/Common/10.1.1.5:443",
"enabled": True,
"gtmScore": 0,
"ipProtocol": "tcp",
"mask": "255.255.255.255",
"mirror": "disabled",
"mobileAppTunnel": "disabled",
"nat64": "disabled",
"pool": "/Common/test-account",
"poolReference": {
"link": True
},
"rateLimit": "disabled",
"rateLimitDstMask": 0,
"rateLimitMode": "object",
"rateLimitSrcMask": 0,
"serviceDownImmediateAction": "none",
"source": "0.0.0.0/0",
"sourceAddressTranslation": {
"type": "automap"
},
"sourcePort": "preserve",
"synCookieStatus": "not-activated",
"translateAddress": "enabled",
"translatePort": "enabled",
"vlansDisabled": True,
"vsIndex": 3,
"policiesReference": {
"link": True,
"isSubcollection": True
},
"profilesReference": {
"link": True,
"isSubcollection": True
}
}]
}
res = []
for value in parsed_json["items"]:
print(value["name"])
print(value["destination"])
print(value["destination"].split("/")[-1].split(":")[0]) #if you need only the IP.
Output:
test-account
/Common/10.1.1.1:80
10.1.1.1
test-account_443
/Common/10.1.1.5:443
10.1.1.5
I m trying to access a post on a Facebook Page which has the content in Hindi. So the raw response that I get from Facebook API is as below.
In this response the message is actually in Hindi, how do I detect the encoding of the message and print it in Hindi?
{
"id": "182929845081087_579535732087161",
"from": {
"id": "182929845081087",
"category": "Non-profit organization",
"name": "Brahma Kumaris"
},
"message": "\u092e\u0941\u0930\u0932\u0940 \u0938\u093e\u0930:- \u092e\u0940\u0920\u0947 \u092c\u091a\u094d\u091a\u0947-\u0924\u0941\u092e\u094d\u0939\u0947\u0902 \u0905\u0928\u094d\u0924 \u0924\u0915 \u092f\u0939 \u092e\u0940\u0920\u0940 \u0928\u0949\u0932\u0947\u091c \u0938\u0941\u0928\u0924\u0947 \u0930\u0939\u0928\u093e \u0939\u0948 \u091c\u092c \u0924\u0915 \u091c\u0940\u0928\u093e \u0939\u0948-\u092a\u0922\u093c\u0928\u093e \u0914\u0930 \u092f\u094b\u0917 \u0938\u0940\u0916\u0928\u093e \u0939\u0948 \n \u092a\u094d\u0930\u0936\u094d\u0928:- \u092c\u093e\u092a \u0915\u0947 \u0938\u093e\u0925-\u0938\u093e\u0925 \u0924\u0941\u092e \u092c\u091a\u094d\u091a\u0947 \u0915\u093f\u0938 \u0938\u0947\u0935\u093e \u0915\u0947 \u0928\u093f\u092e\u093f\u0924\u094d\u0924 \u092c\u0928\u0947 \u0939\u0941\u090f \u0939\u094b? \n \u0909\u0924\u094d\u0924\u0930:- \u091c\u0948\u0938\u0947 \u092c\u093e\u092a \u0938\u093e\u0930\u0947 \u0935\u093f\u0936\u094d\u0935 \u0915\u094b \u0932\u093f\u092c\u0930\u0947\u091f \u0915\u0930\u0924\u0947 \u0939\u0948\u0902, \u0938\u092c \u092a\u0930 \u092c\u094d\u0932\u093f\u0938 \u0915\u0930\u0924\u0947 \u0939\u0948\u0902, \u092a\u0940\u0938 \u092e\u0947\u0915\u0930 \u092c\u0928 \u092a\u0940\u0938 \u0938\u094d\u0925\u093e\u092a\u0928 \u0915\u0930\u0924\u0947 \u0939\u0948\u0902 \u0910\u0938\u0947 \u0924\u0941\u092e \u092c\u091a\u094d\u091a\u0947 \u092d\u0940 \u092c\u093e\u092a \u0915\u0947 \u0938\u093e\u0925 \u0907\u0938 \u0938\u0947\u0935\u093e \u0915\u0947 \u0928\u093f\u092e\u093f\u0924\u094d\u0924 \u0939\u094b\u0964 \u0924\u0941\u092e \u0939\u094b \u0938\u0948\u0932\u0935\u0947\u0936\u0928 \u0906\u0930\u094d\u092e\u0940\u0964 \u0924\u0941\u092e\u094d\u0939\u0947\u0902 \u092d\u093e\u0930\u0924 \u0915\u0947 \u0921\u0942\u092c\u0947 \u0939\u0941\u090f \u092c\u0947\u095c\u0947 \u0915\u094b \u0938\u0948\u0932\u0935\u0947\u091c \u0915\u0930\u0928\u093e \u0939\u0948\u0964 21 \u091c\u0928\u094d\u092e\u094b\u0902 \u0915\u0947 \u0932\u093f\u090f \u0938\u092c\u0915\u094b \u0938\u092e\u094d\u092a\u0924\u094d\u0924\u093f\u0935\u093e\u0928 \u092c\u0928\u093e\u0928\u093e \u0939\u0948\u0964 \u0910\u0938\u0940 \u0938\u0947\u0935\u093e \u0924\u0941\u092e \u092c\u091a\u094d\u091a\u094b\u0902 \u0915\u0947 \u0938\u093f\u0935\u093e\u090f \u0914\u0930 \u0915\u094b\u0908 \u0915\u0930 \u0928\u0939\u0940\u0902 \u0938\u0915\u0924\u093e\u0964 \n \u0927\u093e\u0930\u0923\u093e \u0915\u0947 \u0932\u093f\u090f \u092e\u0941\u0916\u094d\u092f \u0938\u093e\u0930:- \n 1) \u0935\u093f\u0915\u0930\u094d\u092e\u093e\u091c\u0940\u0924 \u092c\u0928\u0928\u0947 \u0915\u0947 \u0932\u093f\u090f \u091a\u0932\u0924\u0947 \u092b\u093f\u0930\u0924\u0947 \u092c\u093e\u092a \u0915\u094b \u092f\u093e\u0926 \u0915\u0930\u0928\u0947 \u0915\u093e \u0905\u092d\u094d\u092f\u093e\u0938 \u0915\u0930\u0928\u093e \u0939\u0948\u0964 \u092f\u093e\u0926 \u0915\u093e \u091a\u093e\u0930\u094d\u091f \u091c\u0930\u0942\u0930 \u0930\u0916\u0928\u093e \u0939\u0948\u0964 \n 2) \u0905\u092a\u0928\u0940 \u0939\u0930 \u091a\u0932\u0928 \u0938\u0947 \u092e\u093e\u0924-\u092a\u093f\u0924\u093e \u0914\u0930 \u091f\u0940\u091a\u0930 \u0915\u093e \u0936\u094b \u0915\u0930\u0928\u093e \u0939\u0948\u0964 \u0935\u093f\u0928\u093e\u0936 \u0915\u093e\u0932 \u092e\u0947\u0902 \u092a\u094d\u0930\u0940\u0924 \u092c\u0941\u0926\u094d\u0927\u093f \u092c\u0928\u0915\u0930 \u0930\u0939\u0928\u093e \u0939\u0948\u0964 \u0930\u0942\u0939\u093e\u0928\u0940 \u0938\u0947\u0935\u093e \u0915\u0930\u0928\u0940 \u0939\u0948\u0964 \n \u0935\u0930\u0926\u093e\u0928:- \u0935\u093e\u092f\u0926\u094b\u0902 \u0915\u0940 \u0938\u094d\u092e\u0943\u0924\u093f \u0926\u094d\u0935\u093e\u0930\u093e \u095e\u093e\u092f\u0926\u093e \u0909\u0920\u093e\u0928\u0947 \u0935\u093e\u0932\u0947 \u0938\u0926\u093e \u092c\u093e\u092a \u0915\u0940 \u092c\u094d\u0932\u0948\u0938\u093f\u0902\u0917 \u0915\u0947 \u092a\u093e\u0924\u094d\u0930 \u092d\u0935 \n \u091c\u094b \u092d\u0940 \u0935\u093e\u092f\u0926\u0947 \u092e\u0928 \u0938\u0947, \u092c\u094b\u0932 \u0938\u0947 \u0905\u0925\u0935\u093e \u0932\u093f\u0916\u0915\u0930 \u0915\u0930\u0924\u0947 \u0939\u094b, \u0909\u0928\u094d\u0939\u0947\u0902 \u0938\u094d\u092e\u0943\u0924\u093f \u092e\u0947\u0902 \u0930\u0916\u094b \u0924\u094b \u0935\u093e\u092f\u0926\u0947 \u0915\u093e \u092a\u0942\u0930\u093e \u092b\u093e\u092f\u0926\u093e \u0909\u0920\u093e \u0938\u0915\u0924\u0947 \u0939\u094b\u0964 \u091a\u0947\u0915 \u0915\u0930\u094b \u0915\u093f \u0915\u093f\u0924\u0928\u0947 \u092c\u093e\u0930 \u0935\u093e\u092f\u0926\u093e \u0915\u093f\u092f\u093e \u0939\u0948 \u0914\u0930 \u0915\u093f\u0924\u0928\u093e \u0928\u093f\u092d\u093e\u092f\u093e \u0939\u0948! \u0935\u093e\u092f\u0926\u093e \u0914\u0930 \u095e\u093e\u092f\u0926\u093e - \u0907\u0928 \u0926\u094b\u0928\u094b\u0902 \u0915\u093e \u092c\u0948\u0932\u0947\u0928\u094d\u0938 \u0930\u0939\u0947 \u0924\u094b \u0935\u0930\u0926\u093e\u0924\u093e \u092c\u093e\u092a \u0926\u094d\u0935\u093e\u0930\u093e \u092c\u094d\u0932\u0948\u0938\u093f\u0902\u0917 \u092e\u093f\u0932\u0924\u0940 \u0930\u0939\u0947\u0917\u0940\u0964 \u091c\u0948\u0938\u0947 \u0938\u0902\u0915\u0932\u094d\u092a \u0936\u094d\u0930\u0947\u0937\u094d\u0920 \u0915\u0930\u0924\u0947 \u0939\u094b \u0910\u0938\u0947 \u0915\u0930\u094d\u092e \u092d\u0940 \u0936\u094d\u0930\u0947\u0937\u094d\u0920 \u0939\u094b\u0902 \u0924\u094b \u0938\u092b\u0932\u0924\u093e \u092e\u0942\u0930\u094d\u0924 \u092c\u0928 \u091c\u093e\u092f\u0947\u0902\u0917\u0947\u0964 \n \u0938\u094d\u0932\u094b\u0917\u0928:- \u0938\u094d\u0935\u092f\u0902 \u0915\u094b \u0910\u0938\u093e \u0926\u093f\u0935\u094d\u092f \u0906\u0907\u0928\u093e \u092c\u0928\u093e\u0913 \u091c\u093f\u0938\u092e\u0947\u0902 \u092c\u093e\u092a \u0939\u0940 \u0926\u093f\u0916\u093e\u0908 \u0926\u0947 \u0924\u092c \u0915\u0939\u0947\u0902\u0917\u0947 \u0938\u091a\u094d\u091a\u0940 \u0938\u0947\u0935\u093e\u0964",
"actions": [
{
"name": "Comment",
"link": "http://www.facebook.com/182929845081087/posts/579535732087161"
},
{
"name": "Like",
"link": "http://www.facebook.com/182929845081087/posts/579535732087161"
}
],
"privacy": {
"description": "Public",
"value": "EVERYONE",
"friends": "",
"networks": "",
"allow": "",
"deny": ""
},
"type": "status",
"status_type": "mobile_status_update",
"application": {
"name": "UpdateYou",
"id": "351985104836764"
},
"created_time": "2013-05-30T03:00:08+0000",
"updated_time": "2013-05-30T15:08:42+0000",
"shares": {
"count": 2
},
"likes": {
"data": [
{
"name": "Bhumika Mahant",
"id": "100002238635044"
},
{
"name": "Kumar DrVinay",
"id": "100002736938311"
},
{
"name": "Namrata Trehan Pathria",
"id": "100000281688593"
},
{
"name": "Devesh Sharma",
"id": "100001192346711"
}
],
"count": 37
},
"comments": {
"data": [
{
"id": "579535732087161_6364194",
"from": {
"name": "Namrata Trehan Pathria",
"id": "100000281688593"
},
"message": "Om shanti meet he baba",
"can_remove": true,
"created_time": "2013-05-30T15:08:42+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6363607",
"from": {
"name": "Cetan Patil",
"id": "100003155153074"
},
"message": "om shanti",
"can_remove": true,
"created_time": "2013-05-30T11:06:27+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6363549",
"from": {
"name": "Maya Ramchandani",
"id": "100003705148351"
},
"message": "Omshanti",
"can_remove": true,
"created_time": "2013-05-30T10:38:39+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6363525",
"from": {
"name": "Subhash Bambal",
"id": "100002808519452"
},
"message": "Om Shanti",
"can_remove": true,
"created_time": "2013-05-30T10:29:05+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6363354",
"from": {
"name": "Poonam Dhanuka",
"id": "100004088191006"
},
"message": "om shanti baba",
"can_remove": true,
"created_time": "2013-05-30T09:12:35+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6363232",
"from": {
"name": "Hemprakash Pant",
"id": "100004354350224"
},
"message": "Om Shanti Baba",
"can_remove": true,
"created_time": "2013-05-30T07:45:38+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362963",
"from": {
"name": "Barun Sharma",
"id": "100005696734282"
},
"message": "om shanti....My baba beloved baba sweet baba.....",
"can_remove": true,
"created_time": "2013-05-30T05:33:26+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362770",
"from": {
"name": "Arya Singh",
"id": "100001924554892"
},
"message": "om shanti...................",
"can_remove": true,
"created_time": "2013-05-30T04:41:45+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362737",
"from": {
"name": "Khushi Dhurve",
"id": "100001700564503"
},
"message": "Om shanti...gm...mere pyare baapdada...awm...love lots...\u003C3:-):-*:-*",
"can_remove": true,
"created_time": "2013-05-30T04:23:53+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362675",
"from": {
"name": "NNibedita Behera",
"id": "100002645048155"
},
"message": "Om shanti baba",
"can_remove": true,
"created_time": "2013-05-30T03:59:49+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362654",
"from": {
"name": "Sonali Supe",
"id": "100002108817901"
},
"message": "OM SHANTI MERE PYARE BABA......................",
"can_remove": true,
"created_time": "2013-05-30T03:52:05+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362647",
"from": {
"name": "Tejsingh Gurjar",
"id": "100004301563182"
},
"message": "om shanti",
"can_remove": true,
"created_time": "2013-05-30T03:47:46+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362620",
"from": {
"name": "Swati Sonar",
"id": "100002927228747"
},
"message": "om shanti",
"can_remove": true,
"created_time": "2013-05-30T03:34:13+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362605",
"from": {
"name": "Megha Gulati",
"id": "100004777265970"
},
"message": "gd mrng baba.om shanti",
"can_remove": true,
"created_time": "2013-05-30T03:28:23+0000",
"like_count": 0,
"user_likes": false
},
{
"id": "579535732087161_6362579",
"from": {
"name": "Jay Rathod",
"id": "100005154643627"
},
"message": "OM SHANTI",
"can_remove": true,
"created_time": "2013-05-30T03:14:18+0000",
"like_count": 0,
"user_likes": false
}
],
"paging": {
"cursors": {
"after": "MQ==",
"before": "MTU="
}
}
}
},
Correctly detecting encoding is it impossible. There are libraries that try to guess and work very well but you can't completely trust them. Normally in web environments encodings come in response headers (Content-Encoding), have you checked there?
Then, when you know what encoding is (by guessing or by looking at charset encoding header) you then will have to parse the JSON dictionary and in the proper fields where the strings are encoding (message, for example) you may do message.decode('').
That will return unicode decode string and you may work with that.
Now, it seems to me that you're already getting the response decoded in unicode. The reason why I think that is that in message you get \u092e character which is DEVANAGARI LETTER MA.
So probably you have already decode in Unicode the string and you may processing as you want in python. For instance, do something like: message.encode('utf-8') and there you'll have it encoded in utf-8.
Hope this bring some light!