We Keep Coding

glom assign based on data

In the following code, I am trying to mask personal information based on data. I have two scenarioes. In scenario 1, I want to update when type = 'FirstName', update or assign valueString value to "Masked". In scenario 2, I want to update when type matches the pattern "first****Name", update or assign valueString value to "Masked". I was wondering if anyone have suggestions for writing glom assign statements to solve the above cases.
Example Json String
{
"id": "985babac-9999-8888-8887",
"entity": [
{
"what": {
"reference": "4lincoln-123-11eb-bc1a-732f"
},
"detail": [
{
"type": "uuid",
"valueString": "4obama-f199-77eb-bc1a-555555704d2f"
},
{
"type": "firstName",
"valueString": "John"
},
{
"type": "userName",
"valueString": "Johns"
},
{
"type": "middleInitial",
"valueString": "S"
},
{
"type": "lastName",
"valueString": "Trump"
},
{
"type": "first-4fa999-f1999-Name",
"valueString": "John"
},
{
"type": "birth-4fa999-f1999-Date",
"valueString": "2010-01-01"
}
]
}
]
}
Updated output should look like the following
{
"id": "985babac-9999-8888-8887",
"entity": [
{
"what": {
"reference": "4lincoln-123-11eb-bc1a-732f"
},
"detail": [
{
"type": "uuid",
"valueString": "4obama-f199-77eb-bc1a-555555704d2f"
},
{
"type": "firstName",
"valueString": "Masked"
},
{
"type": "userName",
"valueString": "Johns"
},
{
"type": "middleInitial",
"valueString": "S"
},
{
"type": "lastName",
"valueString": "Trump"
},
{
"type": "first-4fa999-f1999-Name",
"valueString": "Masked"
},
{
"type": "birth-4fa999-f1999-Date",
"valueString": "2010-01-01"
}
]
}
]
}

I came up with the following solution. I was wondering if this can be done in one glom call instead of calling multiple times?
import json
import logging
import sys
import time
import re
from glom import glom, assign, Coalesce, SKIP, Spec, Path, Call, T, Iter, Inspect
LOGGING_FORMAT = '%(asctime)s - [%(filename)s:%(name)s:%(lineno)d] - %(levelname)s - %(message)s'
LOGLEVEL = logging.INFO
logging.basicConfig(level=LOGLEVEL,format=LOGGING_FORMAT)
logger = logging.getLogger(__name__)
start_time = time.time()
target = {
"id": "985babac-9999-8888-8887",
"entity": [
{
"what": {
"reference": "4lincoln-123-11eb-bc1a-732f"
},
"detail": [
{
"type": "uuid",
"valueString": "4obama-f199-77eb-bc1a-555555704d2f"
},
{
"type": "firstName",
"valueString": "John"
},
{
"type": "userName",
"valueString": "Johns"
},
{
"type": "middleInitial",
"valueString": "S"
},
{
"type": "lastName",
"valueString": "Trump"
},
{
"type": "first-4fa999-f1999-Name",
"valueString": "John"
},
{
"type": "birth-4fa999-f1999-Date",
"valueString": "2010-01-01"
}
]
}
]
}
# def myupdate(x):
# for count, item in enumerate(x):
# myspec = 'entity.0.detail.{}.valueString'.format(count)
# if item == 'firstName':
# _ = assign(target,myspec,'Masked')
piiRegex = re.compile(r'^first.*Name$|^last.*Name$|^middle.*Initial$|^birth.*Date$')
def myupdate(x):
for count, item in enumerate(x):
myspec = 'entity.0.detail.{}.valueString'.format(count)
mo = piiRegex.search(item)
if mo:
_ = assign(target,myspec,'Masked')
spec = {'result': ('entity.0.detail', ['type'], myupdate)}
xyz = glom(target, spec)
print(xyz)
print(target)
logger.info("Program completed in --- %s seconds ---" % (time.time() - start_time))
===============
Result:
{'result': None}
{'id': '985babac-9999-8888-8887', 'entity': [{'what': {'reference': '4lincoln-123-11eb-bc1a-732f'}, 'detail': [{'type': 'uuid', 'valueString': '4obama-f199-77eb-bc1a-555555704d2f'}, {'type': 'firstName', 'valueString': 'Masked'}, {'type': 'userName', 'valueString': 'Johns'}, {'type': 'middleInitial', 'valueString': 'Masked'}, {'type': 'lastName', 'valueString': 'Masked'}, {'type': 'first-4fa999-f1999-Name', 'valueString': 'Masked'}, {'type': 'birth-4fa999-f1999-Date', 'valueString': 'Masked'}]}]}

How to read fields without numeric index in JSON

I have a json file where I need to read it in a structured way to insert in a database each value in its respective column, but in the tag "customFields" the fields change index, example: "Tribe / Customer" can be index 0 (row['customFields'][0]) in a json block, and in the other one be index 3 (row['customFields'][3]), so I tried to read the data using the name of the row field ['customFields'] ['Tribe / Customer'], but I got the error below:
TypeError: list indices must be integers or slices, not str
Script:
def getCustomField(ModelData):
for row in ModelData["data"]["squads"][0]["cards"]:
print(row['identifier'],
row['customFields']['Tribe / Customer'],
row['customFields']['Stopped with'],
row['customFields']['Sub-Activity'],
row['customFields']['Activity'],
row['customFields']['Complexity'],
row['customFields']['Effort'])
if __name__ == "__main__":
f = open('test.json')
json_file = json.load(f)
getCustomField(json_file)
JSON:
{
"data": {
"squads": [
{
"name": "TESTE",
"cards": [
{
"identifier": "0102",
"title": "TESTE",
"description": " TESTE ",
"status": "on_track",
"priority": null,
"assignees": [
{
"fullname": "TESTE",
"email": "TESTE"
}
],
"createdAt": "2020-04-16T15:00:31-03:00",
"secondaryLabel": null,
"primaryLabels": [
"TESTE",
"TESTE"
],
"swimlane": "TESTE",
"workstate": "Active",
"customFields": [
{
"name": "Tribe / Customer",
"value": "TESTE 1"
},
{
"name": "Checkpoint",
"value": "GNN"
},
{
"name": "Stopped with",
"value": null
},
{
"name": "Sub-Activity",
"value": "DEPLOY"
},
{
"name": "Activity",
"value": "TOOL"
},
{
"name": "Complexity",
"value": "HIGH"
},
{
"name": "Effort",
"value": "20"
}
]
},
{
"identifier": "0103",
"title": "TESTE",
"description": " TESTE ",
"status": "on_track",
"priority": null,
"assignees": [
{
"fullname": "TESTE",
"email": "TESTE"
}
],
"createdAt": "2020-04-16T15:00:31-03:00",
"secondaryLabel": null,
"primaryLabels": [
"TESTE",
"TESTE"
],
"swimlane": "TESTE",
"workstate": "Active",
"customFields": [
{
"name": "Tribe / Customer",
"value": "TESTE 1"
},
{
"name": "Stopped with",
"value": null
},
{
"name": "Checkpoint",
"value": "GNN"
},
{
"name": "Sub-Activity",
"value": "DEPLOY"
},
{
"name": "Activity",
"value": "TOOL"
},
{
"name": "Complexity",
"value": "HIGH"
},
{
"name": "Effort",
"value": "20"
}
]
}
]
}
]
}
}

You'll have to parse the list of custom fields into something you can access by name. Since you're accessing multiple entries from the same list, a dictionary is the most appropriate choice.
for row in ModelData["data"]["squads"][0]["cards"]:
custom_fields_dict = {field['name']: field['value'] for field in row['customFields']}
print(row['identifier'],
custom_fields_dict['Tribe / Customer'],
...
)
If you only wanted a single field you could traverse the list looking for a match, but it would be less efficient to do that repeatedly.
I'm skipping over dealing with missing fields - you'd probably want to use get('Tribe / Customer', some_reasonable_default) if there's any possibility of the field not being present in the json list.

How to access certain values inside a string in Python

I need to extract only a particular element value inside the string.
Below is the code which I used to get the AdsInsight data using Facebook AdsInsight API.
class LibFacebook:
def __init__(self, app_id, app_secret, access_token, ad_account_id):
FacebookAdsApi.init(app_id, app_secret, access_token)
self.account = AdAccount(ad_account_id)
#get ads insight
insights = self.account.get_insights(fields=[
AdsInsights.Field.campaign_id,
AdsInsights.Field.actions,
], params={
'level': AdsInsights.Level.campaign,
})
print(insights)
Output
<AdsInsights> {
"campaign_id": "23843294609751234",
"actions": [
{
"action_type": "post_reaction",
"value": "1"
},
{
"action_type": "landing_page_view",
"value": "78"
},
{
"action_type": "link_click",
"value": "163"
}
]
Question : Along with campaign_id value(23843294609751234) , I need the value of only landing_page_view i.e 78 (and not other action items)and put it in a df. How do I access them ?
Further Information: AdsInsights.Field.actions is of type string.
type(AdsInsights.Field.actions)
str

hope this will work,
lets take your data is a list of AdsInsights objects
obj = [{
"campaign_id": "23843294609751234",
"actions" : [
{
"action_type": "post_reaction",
"value": "1"
},
{
"action_type": "landing_page_view",
"value": "78"
},
{
"action_type": "link_click",
"value": "163"
}
]
},
{
"campaign_id": "112233",
"actions" : [
{
"action_type": "post_reaction",
"value": "1"
},
{
"action_type": "landing_page_view",
"value": "100"
},
{
"action_type": "link_click",
"value": "163"
}
]
}]
you can get result like this
result_arr = []
for i in obj:
datadict = {}
datadict["campaign_id"] = i.get("campaign_id")
for action in i.get("actions"):
if action.get("action_type") == "landing_page_view":
datadict["value"]= action.get("value")
result_arr.append(datadict)
result_arr would be
[{'campaign_id': '23843294609751234', 'value': '78'},
{'campaign_id': '112233', 'value': '100'}]
next convert list of dictionaries to a dataframe
df=pd.DataFrame(result_arr)

How can I extract data from json

I want to extract code from JSON format.
import json
json_data = '''
{
"Body": {
"stkCallback": {
"MerchantRequestID": "22531-976234-1",
"CheckoutRequestID": "ws_CO_DMZ_250600506_23022019144745852",
"ResultCode": 0,
"ResultDesc": "The service request is processed successfully.",
"CallbackMetadata": {
"Item": [
{
"Name": "Amount",
"Value": 1.0
},
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
},
{
"Name": "Balance"
},
{
"Name": "TransactionDate",
"Value": 20190223144807
},
{
"Name": "PhoneNumber",
"Value": 254725696042
}
]
}
}
}
}
'''
json_da = data['Body']
list_data = data['Body']['MpesaReceiptNumber']
print (json_da)
print (list_data)
I want to print this: NBN52K8A1J

The problem is that you have a list of dicts you need to search first:
>>> for obj in data['Body']['stkCallback']['CallbackMetadata']['Item']:
... print(obj)
...
{'Name': 'Amount', 'Value': 1.0}
{'Name': 'MpesaReceiptNumber', 'Value': 'NBN52K8A1J'}
{'Name': 'Balance'}
{'Name': 'TransactionDate', 'Value': 20190223144807}
{'Name': 'PhoneNumber', 'Value': 254725696042}
One possibility is
>>> [x['Value'] for x in data['Body']['stkCallback']['CallbackMetadata']['Item'] if x['Name'] == 'MpesaReceiptNumber'][0]
'NBN52K8A1J'

Just use the library json. Then you can print its inner elements
import json
json_data = '{"Body":{"stkCallback":{"MerchantRequestID":"22531-976234-1","CheckoutRequestID":"ws_CO_DMZ_250600506_23022019144745852","ResultCode":0,"ResultDesc":"The service request is processed successfully.","CallbackMetadata":{"Item":[{"Name":"Amount","Value":1.00},{"Name":"MpesaReceiptNumber","Value":"NBN52K8A1J"},{"Name":"Balance"},{"Name":"TransactionDate","Value":20190223144807},{"Name":"PhoneNumber","Value":254725696042}]}}}}'
a = json.loads(json_data)
print(a["Body"]["stkCallback"]["CallbackMetadata"]["Item"][1]["Value"])

You were almost there just have to get the the key value pair itself from the dicts and check if it is the name you wanted:
data = json.loads(json_data)
list_data = data['Body']["stkCallback"]['CallbackMetadata']['Item']
var: str
for x in list_data:
if x['Name'] == 'MpesaReceiptNumber':
var = x['Value']
break
print(var)
You can use this in the future easily by replacing the if check with the name of something else so you can grab the value depending on a variable.

I find using pprint to get the shape of the data structure is helpful when you're learning how to navigate it all out.
import json
import pprint
json_data = '{"Body":{"stkCallback":{"MerchantRequestID":"22531-976234-1","CheckoutRequestID":"ws_CO_DMZ_250600506_23022019144745852","ResultCode":0,"ResultDesc":"The service request is processed successfully.","CallbackMetadata":{"Item":[{"Name":"Amount","Value":1.00},{"Name":"MpesaReceiptNumber","Value":"NBN52K8A1J"},{"Name":"Balance"},{"Name":"TransactionDate","Value":20190223144807},{"Name":"PhoneNumber","Value":254725696042}]}}}}'
data = json.loads(json_data)
pprint.pprint(data)
Results in:
{'Body': {'stkCallback': {'CallbackMetadata': {'Item': [{'Name': 'Amount', 'Value': 1.0},
{'Name': 'MpesaReceiptNumber', 'Value': 'NBN52K8A1J'},
{'Name': 'Balance'},
{'Name': 'TransactionDate', 'Value': 20190223144807},
{'Name': 'PhoneNumber', 'Value': 254725696042}]},
'CheckoutRequestID': 'ws_CO_DMZ_250600506_23022019144745852',
'MerchantRequestID': '22531-976234-1',
'ResultCode': 0,
'ResultDesc': 'The service request is processed successfully.'}}}
So you should be able to see that data["Body"]["stkCallback"]["CallbackMetadata"]["Item"] gets to to the depth you need for your data.
>>> pprint.pprint(data["Body"]["stkCallback"]["CallbackMetadata"]["Item"])
[{'Name': 'Amount', 'Value': 1.0},
{'Name': 'MpesaReceiptNumber', 'Value': 'NBN52K8A1J'},
{'Name': 'Balance'},
{'Name': 'TransactionDate', 'Value': 20190223144807},
{'Name': 'PhoneNumber', 'Value': 254725696042}]
So next you need to iterate through that list and find a match (if one exists) for the MpesaReceiptNumber key.
receipt_no = None
for item in data["Body"]["stkCallback"]["CallbackMetadata"]["Item"]:
if item.get('Name') == 'MpesaReceiptNumber':
receipt_no = item.get('Value')
print(f"The receipt # is: {receipt_no}")

If you parse the json you will notice that the path to the data is not simply ['Body']['MpesaReceiptNumber']. In fact you have a list of dicts inside ['Item'] that needs to be searched.
Parsed data tree
One suggestion is to run the following code to find the data you are looking for:
import json
json_data = '{"Body":{"stkCallback":{"MerchantRequestID":"22531-976234-1","CheckoutRequestID":"ws_CO_DMZ_250600506_23022019144745852","ResultCode":0,"ResultDesc":"The service request is processed successfully.","CallbackMetadata":{"Item":[{"Name":"Amount","Value":1.00},{"Name":"MpesaReceiptNumber","Value":"NBN52K8A1J"},{"Name":"Balance"},{"Name":"TransactionDate","Value":20190223144807},{"Name":"PhoneNumber","Value":254725696042}]}}}}'
data = (json.loads(json_data))
list_data = data['Body']['stkCallback']['CallbackMetadata']['Item']
# Returns:
# [{'Name': 'Amount', 'Value': 1.0}, {'Name': 'MpesaReceiptNumber', 'Value':'NBN52K8A1J'}, {'Name': 'Balance'}, {'Name': 'TransactionDate', 'Value': 20190223144807}, {'Name': 'PhoneNumber', 'Value': 254725696042}]
# Now find Name: 'MpesaReceiptNumber' inside the dict list
find_it = next(item for item in list_data if item["Name"] == "MpesaReceiptNumber")
find_it = find_it['Value']
print (find_it)
Result
NBN52K8A1J

Use jq.
First off, it can "pretty print" any JSON data.
Put the value of json_data into a file test.json, and then show the formatted output of the JSON data with:
$ jq <test.json
{
"Body": {
"stkCallback": {
"MerchantRequestID": "22531-976234-1",
"CheckoutRequestID": "ws_CO_DMZ_250600506_23022019144745852",
"ResultCode": 0,
"ResultDesc": "The service request is processed successfully.",
"CallbackMetadata": {
"Item": [
{
"Name": "Amount",
"Value": 1
},
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
},
{
"Name": "Balance"
},
{
"Name": "TransactionDate",
"Value": 20190223144807
},
{
"Name": "PhoneNumber",
"Value": 254725696042
}
]
}
}
}
}
Next, to extract values, a selector path needs to be given on the jq command line:
jq '.Body.stkCallback.CallbackMetadata.Item|.[]|select(.Name == "MpesaReceiptNumber")|.Value' test.json
"NBN52K8A1J"
Now to make this sequence easier to understand, let's break it down component by component.
To extract and return only the .Body:
$ jq '.Body' <test.json
{
"stkCallback": {
"MerchantRequestID": "22531-976234-1",
"CheckoutRequestID": "ws_CO_DMZ_250600506_23022019144745852",
"ResultCode": 0,
"ResultDesc": "The service request is processed successfully.",
"CallbackMetadata": {
"Item": [
{
"Name": "Amount",
"Value": 1
},
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
},
{
"Name": "Balance"
},
{
"Name": "TransactionDate",
"Value": 20190223144807
},
{
"Name": "PhoneNumber",
"Value": 254725696042
}
]
}
}
}
Now let's fetch the stkCallback component:
$ jq '.Body.stkCallback' <test.json
{
"MerchantRequestID": "22531-976234-1",
"CheckoutRequestID": "ws_CO_DMZ_250600506_23022019144745852",
"ResultCode": 0,
"ResultDesc": "The service request is processed successfully.",
"CallbackMetadata": {
"Item": [
{
"Name": "Amount",
"Value": 1
},
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
},
{
"Name": "Balance"
},
{
"Name": "TransactionDate",
"Value": 20190223144807
},
{
"Name": "PhoneNumber",
"Value": 254725696042
}
]
}
}
Ok, now the callbackMetadata:
$ jq '.Body.stkCallback.CallbackMetadata' <test.json
{
"Item": [
{
"Name": "Amount",
"Value": 1
},
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
},
{
"Name": "Balance"
},
{
"Name": "TransactionDate",
"Value": 20190223144807
},
{
"Name": "PhoneNumber",
"Value": 254725696042
}
]
}
Next, the Item part:
$ jq '.Body.stkCallback.CallbackMetadata.Item' <test.json
[
{
"Name": "Amount",
"Value": 1
},
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
},
{
"Name": "Balance"
},
{
"Name": "TransactionDate",
"Value": 20190223144807
},
{
"Name": "PhoneNumber",
"Value": 254725696042
}
]
Notice that the result is an array of list items? Let's filter the data out of the array:
$ jq '.Body.stkCallback.CallbackMetadata.Item|.[]' <test.json
{
"Name": "Amount",
"Value": 1
}
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
}
{
"Name": "Balance"
}
{
"Name": "TransactionDate",
"Value": 20190223144807
}
{
"Name": "PhoneNumber",
"Value": 254725696042
}
Now the result is just the list of tuples, each with a "Name" and "Value". So, let's select just the one we (you) wanted:
$ jq '.Body.stkCallback.CallbackMetadata.Item|.[]|select(.Name == "MpesaReceiptNumber")' <test.json
{
"Name": "MpesaReceiptNumber",
"Value": "NBN52K8A1J"
}
Cool. We've got the tuple we wanted. Let's extract just the value now:
$ jq '.Body.stkCallback.CallbackMetadata.Item|.[]|select(.Name == "MpesaReceiptNumber")|.Value' <test.json
"NBN52K8A1J"
And, there you go.

Simplest way to get the value associated with a specific CallbackMetadata item name:
json_string = '''
{
"Body": {
"stkCallback": {
...
}
'''
json_data = json.loads(json_string)
for item in json_data["Body"]["stkCallback"]["CallbackMetadata"]["Item"]:
if item["Name"] == "MpesaReceiptNumber":
print(item["Value"]) # -> NBN52K8A1J

Create dynamic json object in python

I have a dictionary which is contain multiple keys and values and the values also contain the key, value pair. I am not getting how to create dynamic json using this dictionary in python. Here's the dictionary:
image_dict = {"IMAGE_1":{"img0":"IMAGE_2","img1":"IMAGE_3","img2":"IMAGE_4"},"IMAGE_2":{"img0":"IMAGE_1", "img1" : "IMAGE_3"},"IMAGE_3":{"img0":"IMAGE_1", "img1":"IMAGE_2"},"IMAGE_4":{"img0":"IMAGE_1"}}
My expected result like this :
{
"data": [
{
"image": {
"imageId": {
"id": "IMAGE_1"
},
"link": {
"target": {
"id": "IMAGE_2"
},
"target": {
"id": "IMAGE_3"
},
"target": {
"id": "IMAGE_4"
}
}
},
"updateData": "link"
},
{
"image": {
"imageId": {
"id": "IMAGE_2"
},
"link": {
"target": {
"id": "IMAGE_1"
},
"target": {
"id": "IMAGE_3"
}
}
},
"updateData": "link"
},
{
"image": {
"imageId": {
"id": "IMAGE_3"
},
"link": {
"target": {
"id": "IMAGE_1"
},
"target": {
"id": "IMAGE_2"
}
}
},
"updateData": "link"
} ,
{
"image": {
"imageId": {
"id": "IMAGE_4"
},
"link": {
"target": {
"id": "IMAGE_1"
}
}
},
"updateData": "link"
}
]
}
I tried to solve it but I didn't get expected result.
result = {"data":[]}
for k,v in sorted(image_dict.items()):
for a in sorted(v.values()):
result["data"].append({"image":{"imageId":{"id": k},
"link":{"target":{"id": a}}},"updateData": "link"})
print(json.dumps(result, indent=4))

In Python dictionaries you can't have 2 values with the same key. So you can't have multiple targets all called "target". So you can index them. Also I don't know what this question has to do with dynamic objects but here's the code I got working:
import re
dict_res = {}
ind = 0
for image in image_dict:
lin_ind = 0
sub_dict = {'image' + str(ind): {'imageId': {image}, 'link': {}}}
for sub in image_dict[image].values():
sub_dict['image' + str(ind)]['link'].update({'target' + str(lin_ind): {'id': sub}})
lin_ind += 1
dict_res.update(sub_dict)
ind += 1
dict_res = re.sub('target\d', 'target', re.sub('image\d', 'image', str(dict_res)))
print dict_res