We Keep Coding

Explode json without pandas

I have a JSON object:
{
"data": {
"geography": [
{
"id": "1",
"state": "USA",
"properties": [
{
"code": "CMD-01",
"value": "34"
},
{
"code": "CMD-02",
"value": "24"
}
]
},
{
"id": "2",
"state": "Canada",
"properties": [
{
"code": "CMD-04",
"value": "50"
},
{
"code": "CMD-05",
"value": "60"
}
]
}
]
}
}
I want to get the result as a new JSON, but without using pandas (and all those explode, flatten and normalize functions...). Is there any option to get this structure without using pandas or having an Out of memory issue?
The output should be:
{ "id": "1",
"state": "USA",
"code": "CMD-01",
"value": "34"
},
{ "id": "1",
"state": "USA",
"code": "CMD-02",
"value": "24",
},
{ "id": "2",
"state": "Canada",
"code": "CMD-04",
"value": "50"
},
{ "id": "2",
"state": "Canada",
"code": "CMD-05",
"value": "60"
},

You can simply loop over the list associated with "geography" and build new dictionaries that you will add to a newly created list:
dict_in = {
"data": {
"geography": [
{
"id": "1",
"state": "USA",
"properties": [
{
"code": "CMD-01",
"value": "34"
},
{
"code": "CMD-02",
"value": "24"
}
]
},
{
"id": "2",
"state": "Canada",
"properties": [
{
"code": "CMD-04",
"value": "50"
},
{
"code": "CMD-05",
"value": "60"
}
]
}
]
}
}
import json
rec_out = []
for obj in dict_in["data"]["geography"]:
for prop in obj["properties"]:
dict_out = {
"id": obj["id"],
"state": obj["state"]
}
dict_out.update(prop)
rec_out.append(dict_out)
print(json.dumps(rec_out, indent=4))
Output:
[
{
"id": "1",
"state": "USA",
"code": "CMD-01",
"value": "34"
},
{
"id": "1",
"state": "USA",
"code": "CMD-02",
"value": "24"
},
{
"id": "2",
"state": "Canada",
"code": "CMD-04",
"value": "50"
},
{
"id": "2",
"state": "Canada",
"code": "CMD-05",
"value": "60"
}
]

Compare keys value on two list of dict

I'm trying to compare the contents of a dictionary from 2 lists of dict, I have data like this
data1 = [
{
"name": "MoreDependentsInd",
"description": "Form 1040-SP Indicator",
"data_type": "CheckboxType",
"xpath": "",
},
{
"name": "DependentFirstNm",
"description": "Form 1040-SR Indicator",
"data_type": "PersonNameControlType",
"xpath": "",
},
]
data2 = [
{
"id": 29,
"kind": "IN",
"container": 1,
"content": "null",
"uid": "IRS 1040-DependentDetail",
"title": "DependentDetail",
"display_name": "null",
"description": "null",
"extra_info": {
"kind": "null",
"include": {
"fields": [
{
"kind": "PersonFirstNameType",
"name": "DependentFirstNm",
"annotation": {
"documentation": {
"description": "Dependent First Name",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "PersonLastNameType",
"name": "DependentLastNm",
"annotation": {
"documentation": {
"description": "Dependent Last Name",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "PersonNameControlType",
"name": "DependentNameControlTxt",
"annotation": {
"documentation": {
"description": "Dependent Name Control",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "null",
"name": "IdentityProtectionPIN",
"annotation": {
"documentation": {
"description": "Dependent/Qualifying Child Identity Protection PIN",
"line_number": "(2)",
}
},
"restriction": {"rule": "null"},
},
{
"kind": "null",
"name": "DependentRelationshipCd",
"annotation": {
"documentation": {
"description": "Dependent Relationship Code",
"line_number": "(3)",
}
},
"restriction": {
"rule": {
"kind": "TextType",
"enumeration": [
"SON",
"DAUGHTER",
"STEPCHILD",
"FOSTER CHILD",
"BROTHER",
"SISTER",
"STEPBROTHER",
"STEPSISTER",
"HALF BROTHER",
"HALF SISTER",
"GRANDCHILD",
"NIECE",
"NEPHEW",
"PARENT",
"GRANDPARENT",
"AUNT",
"UNCLE",
"OTHER",
"NONE",
],
}
},
},
{
"kind": "PersonFirstNameType",
"name": "DependentFirstNm",
"annotation": {
"documentation": {
"description": "Dependent First Name",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "PersonLastNameType",
"name": "DependentLastNm",
"annotation": {
"documentation": {
"description": "Dependent Last Name",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "PersonNameControlType",
"name": "DependentNameControlTxt",
"annotation": {
"documentation": {
"description": "Dependent Name Control",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "null",
"name": "IdentityProtectionPIN",
"annotation": {
"documentation": {
"description": "Dependent/Qualifying Child Identity Protection PIN",
"line_number": "(2)",
}
},
"restriction": {"rule": "null"},
},
{
"kind": "null",
"name": "DependentRelationshipCd",
"annotation": {
"documentation": {
"description": "Dependent Relationship Code",
"line_number": "(3)",
}
},
"restriction": {
"rule": {
"kind": "TextType",
"enumeration": [
"SON",
"DAUGHTER",
"STEPCHILD",
"FOSTER CHILD",
"BROTHER",
"SISTER",
"STEPBROTHER",
"STEPSISTER",
"HALF BROTHER",
"HALF SISTER",
"GRANDCHILD",
"NIECE",
"NEPHEW",
"PARENT",
"GRANDPARENT",
"AUNT",
"UNCLE",
"OTHER",
"NONE",
],
}
},
},
{
"kind": "PersonFirstNameType",
"name": "DependentFirstNm",
"annotation": {
"documentation": {
"description": "Dependent First Name",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "PersonLastNameType",
"name": "DependentLastNm",
"annotation": {
"documentation": {
"description": "Dependent Last Name",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "PersonNameControlType",
"name": "DependentNameControlTxt",
"annotation": {
"documentation": {
"description": "Dependent Name Control",
"line_number": "(1)",
}
},
"restriction": "null",
},
{
"kind": "null",
"name": "IdentityProtectionPIN",
"annotation": {
"documentation": {
"description": "Dependent/Qualifying Child Identity Protection PIN",
"line_number": "(2)",
}
},
"restriction": {"rule": "null"},
},
{
"kind": "null",
"name": "DependentRelationshipCd",
"annotation": {
"documentation": {
"description": "Dependent Relationship Code",
"line_number": "(3)",
}
},
"restriction": {
"rule": {
"kind": "TextType",
"enumeration": [
"SON",
"DAUGHTER",
"STEPCHILD",
"FOSTER CHILD",
"BROTHER",
"SISTER",
"STEPBROTHER",
"STEPSISTER",
"HALF BROTHER",
"HALF SISTER",
"GRANDCHILD",
"NIECE",
"NEPHEW",
"PARENT",
"GRANDPARENT",
"AUNT",
"UNCLE",
"OTHER",
"NONE",
],
}
},
},
],
"reference": "null",
},
"annotation": "null",
"max_occurs": "100",
"min_occurs": "0",
"restriction": "null",
},
"instruction": "null",
"created_at": "2022-04-26T09:53:36.426118Z",
"updated_at": "2022-04-26T09:53:36.426203Z",
},
{
"id": 30,
"kind": "IN",
"container": 1,
"content": "null",
"uid": "IRS 1040-MoreDependentsInd",
"title": "MoreDependentsInd",
"display_name": "null",
"description": "null",
"extra_info": {
"kind": "CheckboxType",
"include": "null",
"annotation": {
"documentation": {
"description": "More Dependents Indicator",
"line_number": "null",
}
},
"max_occurs": "null",
"min_occurs": "0",
"restriction": "null",
},
"instruction": "null",
"created_at": "2022-04-26T09:53:36.427552Z",
"updated_at": "2022-04-26T09:53:36.427647Z",
},
]
for i in data1:
for j in data2:
if i["name"] == j["title"]:
print("success")
the result is a success, but in my code, there is a deficiency, if it processes a lot of data then the performance will feel slow, how to fix it? and what if the value such as the data type is in a key that we don't know is located?
for example, I search value data_type from data1 I compare it with CheckboxType
"extra_info": {
"kind": "CheckboxType",
"include": "null",
"annotation": {
"documentation": {
"description": "More Dependents Indicator",
"line_number": "null",
}
},
from data2 on index 1 data2[1] but the value CheckboxType is also exist in another key on data2 in another index
Thanks!

You can do the test in O(n) instead of O(n^2) by building sets of name and title (which is O(n)) and then taking their intersection (which is also O(n) now that they're sets):
>>> {i["name"] for i in data1} & {i["title"] for i in data2}
{'MoreDependentsInd'}
Non-empty sets are truthy so you can use this in a conditional if you don't care about what the intersection is:
>>> if {i["name"] for i in data1} & {i["title"] for i in data2}:
... print("success")
...
success
If you want to look at all string values without having to know that name and title are the intersecting keys, use another generator expression to put all string values into the two sets:
>>> {v for i in data1 for v in i.values() if isinstance(v, str)} & {v for i in data2 for v in i.values() if isinstance(v, str)}
{'MoreDependentsInd'}

Why am I receiving an error when attempting to parse JSON object within for loop?

Everything with my script runs fine until I try to run it through a for loop. Specifically, when I attempt to index a specific array within the object. Before I get to the The script is intended to grab the delivery date for each tracking number in my list.
This is my script:
import requests
import json
TrackList = ['1Z3X756E0310496105','1ZX0373R0303581450','1ZX0373R0103574417']
url = 'https://onlinetools.ups.com/rest/Track'
para1 = '...beginning of JSON request string...'
para2 = '...end of JSON request string...'
for TrackNum in TrackList:
parameters = para1+TrackNum+para2
resp = requests.post(url = url, data = parameters, verify=False)
data = json.loads(resp.text)
DelDate = data['TrackResponse']['Shipment']['Package'][0]['Activity'][0]['Date']
print(DelDate)
JSON API Response (if needed):
{
"TrackResponse": {
"Response": {
"ResponseStatus": {
"Code": "1",
"Description": "Success"
},
"TransactionReference": {
"CustomerContext": "Analytics Inquiry"
}
},
"Shipment": {
"InquiryNumber": {
"Code": "01",
"Description": "ShipmentIdentificationNumber",
"Value": "1ZX0373R0103574417"
},
"Package": {
"Activity": [
{
"ActivityLocation": {
"Address": {
"City": "OKLAHOMA CITY",
"CountryCode": "US",
"PostalCode": "73128",
"StateProvinceCode": "OK"
},
"Code": "M3",
"Description": "Front Desk",
"SignedForByName": "CUMMINGS"
},
"Date": "20190520",
"Status": {
"Code": "9E",
"Description": "Delivered",
"Type": "D"
},
"Time": "091513"
},
{
"ActivityLocation": {
"Address": {
"City": "Oklahoma City",
"CountryCode": "US",
"StateProvinceCode": "OK"
},
"Description": "Front Desk"
},
"Date": "20190520",
"Status": {
"Code": "OT",
"Description": "Out For Delivery Today",
"Type": "I"
},
"Time": "085943"
},
{
"ActivityLocation": {
"Address": {
"City": "Oklahoma City",
"CountryCode": "US",
"StateProvinceCode": "OK"
},
"Description": "Front Desk"
},
"Date": "20190520",
"Status": {
"Code": "DS",
"Description": "Destination Scan",
"Type": "I"
},
"Time": "011819"
},
{
"ActivityLocation": {
"Address": {
"City": "Oklahoma City",
"CountryCode": "US",
"StateProvinceCode": "OK"
},
"Description": "Front Desk"
},
"Date": "20190519",
"Status": {
"Code": "AR",
"Description": "Arrival Scan",
"Type": "I"
},
"Time": "235100"
},
{
"ActivityLocation": {
"Address": {
"City": "DFW Airport",
"CountryCode": "US",
"StateProvinceCode": "TX"
},
"Description": "Front Desk"
},
"Date": "20190519",
"Status": {
"Code": "DP",
"Description": "Departure Scan",
"Type": "I"
},
"Time": "195500"
},
{
"ActivityLocation": {
"Address": {
"City": "DFW Airport",
"CountryCode": "US",
"StateProvinceCode": "TX"
},
"Description": "Front Desk"
},
"Date": "20190517",
"Status": {
"Code": "OR",
"Description": "Origin Scan",
"Type": "I"
},
"Time": "192938"
},
{
"ActivityLocation": {
"Address": {
"CountryCode": "US"
},
"Description": "Front Desk"
},
"Date": "20190517",
"Status": {
"Code": "MP",
"Description": "Order Processed: Ready for UPS",
"Type": "M"
},
"Time": "184621"
}
],
"PackageWeight": {
"UnitOfMeasurement": {
"Code": "LBS"
},
"Weight": "2.00"
},
"ReferenceNumber": [
{
"Code": "01",
"Value": "8472745558"
},
{
"Code": "01",
"Value": "5637807:1007379402:BN81-17077A:1"
},
{
"Code": "01",
"Value": "5637807"
}
],
"TrackingNumber": "1ZX0373R0103574417"
},
"PickupDate": "20190517",
"Service": {
"Code": "001",
"Description": "UPS Next Day Air"
},
"ShipmentAddress": [
{
"Address": {
"AddressLine": "S 600 ROYAL LN",
"City": "COPPELL",
"CountryCode": "US",
"PostalCode": "750193827",
"StateProvinceCode": "TX"
},
"Type": {
"Code": "01",
"Description": "Shipper Address"
}
},
{
"Address": {
"City": "OKLAHOMA CITY",
"CountryCode": "US",
"PostalCode": "73128",
"StateProvinceCode": "OK"
},
"Type": {
"Code": "02",
"Description": "ShipTo Address"
}
}
],
"ShipmentWeight": {
"UnitOfMeasurement": {
"Code": "LBS"
},
"Weight": "2.00"
},
"ShipperNumber": "X0373R"
}
}
}
Below is the error I receive:
Traceback (most recent call last):
File "/Users/***/Library/Preferences/PyCharmCE2019.1/scratches/UPS_API.py", line 15, in <module>
DelDate = data['TrackResponse']['Shipment']['Package'][0]['Activity'][0]['Date']
KeyError: 0

You're trying to index "Package" at index 0, but it's an object not an array. So you should be accessing ['Package']['Activity']

just take away the [0] because there is no [1] or [2]

Build relation between elasticsearch aggregators - Nested groupings

I need to create nested groupings between fields.
Let us consider the example given below,
Documents:
{
"keyword": "abc",
"country": "IN",
"state": "TN",
"city": "Chennai"
},
{
"keyword": "abc",
"country": "IN",
"state": "TN",
"city": "Trichy"
},
{
"keyword": "abc",
"country": "IN",
"state": "KL",
"city": "TVM"
},
{
"keyword": "abc",
"country": "US",
"state": "Cal",
"city": "California"
}
Required output(Something like this):
{
"country": "IN",
"TN": [
"Chennai",
"Trichy"
],
"KL": [
"TVM"
]
},
{
"country": "US",
"Cal": [
"California"
]
}
Query used:
{
"from": 0,
"size": 1,
"aggs": {
"country": {
"terms": {
"field": "country.keyword",
"size": 50000
}
},
"state": {
"terms": {
"field": "state.keyword",
"size": 50000
}
},
"city": {
"terms": {
"field": "city.keyword",
"size": 50000
}
}
},
"query": {
"query_string": {
"query": "(keyword:abc) "
}
}
}
For this query I got separate bucket as output for city , state and country.
But what I need is city should be grouped under state and state should be grouped under country.
Thanks in advance.

Following query with aggregation should work for you
{
"query": {
"query_string": {
"query": "(keyword:abc)"
}
},
"size": 0,
"aggs": {
"country_agg": {
"terms": {
"field": "country.keyword",
"size": 10
},
"aggs": {
"state_agg": {
"terms": {
"field": "state.keyword",
"size": 10
},
"aggs": {
"city_agg": {
"terms": {
"field": "city.keyword",
"size": 10
}
}
}
}
}
}
}
}

"object mapping [prices] can't be changed from nested to non-nested" on Bulk Python

I'm trying to insert a doc in ElasticSearch but every time i try to insert in python, its return me an error. But if i try to insert from Kibana or cUrl, its succeed.
I already tried the elasticserach-dsl but i've got the same error.
(Sorry for my bad english, i'm from brazil :D)
Error i've got:
elasticsearch.helpers.BulkIndexError: ((...)'status': 400, 'error': {'type':
'illegal_argument_exception', 'reason': "object mapping [prices] can't be changed from nested to non-nested"}}}])
My code:
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
doc = [{
"_index": "products",
"_type": "test_products",
"_source": {
[...]
"prices": {
"latest": {
"value": 89,
"when": 1502795602848
},
"old": [
{
"value": 0,
"when": 1502795602848
}
]
},
"sizes": [
{
"name": "P",
"available": True
},
{
"name": "M",
"available": True
}
],
"created": "2017-08-15T08:13:22.848284"
}
}]
bulk(self.es, doc, index="products")
My ES mapping:
{
"test_products": {
"mappings": {
"products": {
"properties": {
"approved": {
"type": "boolean"
},
"available": {
"type": "boolean"
},
"brand": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"buyClicks": {
"type": "integer"
},
"category": {
"type": "keyword"
},
"code": {
"type": "keyword"
},
"color": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"created": {
"type": "date"
},
"description": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"gender": {
"type": "keyword"
},
"images": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"likes": {
"type": "integer"
},
"link": {
"type": "keyword"
},
"name": {
"type": "text",
"term_vector": "yes",
"analyzer": "nGram_analyzer",
"search_analyzer": "whitespace_analyzer"
},
"prices": {
"type": "nested",
"properties": {
"latest": {
"type": "nested",
"properties": {
"value": {
"type": "long"
},
"when": {
"type": "date",
"format": "dd-MM-yyyy||epoch_millis"
}
}
},
"old": {
"type": "nested",
"properties": {
"value": {
"type": "long"
},
"when": {
"type": "date",
"format": "dd-MM-yyyy||epoch_millis"
}
}
}
}
},
"redirectClicks": {
"type": "integer"
},
"sizes": {
"type": "nested",
"properties": {
"available": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
"quantity": {
"type": "integer"
}
}
},
"slug": {
"type": "keyword"
},
"store": {
"type": "keyword"
},
"subCategories": {
"type": "nested",
"properties": {
"name": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"tags": {
"type": "text",
"fields": {
"raw": {
"type": "text",
"term_vector": "yes",
"analyzer": "nGram_analyzer",
"search_analyzer": "whitespace_analyzer"
}
}
},
"thumbnails": {
"type": "keyword"
}
}
}
}
}
}