We Keep Coding

Comparing dictionary of list of dictionary/nested dictionary

There are two dict main and input, I want to validate the "input" such that all the keys in the list of dictionary and nested dictionary (if present/all keys are optional) matches that of the main if not the wrong/different key should be returned as the output.
main = "app":[{
"name": str,
"info": [
{
"role": str,
"scope": {"groups": list}
}
]
},{
"name": str,
"info": [
{"role": str}
]
}]
input_data = "app":[{
'name': 'nms',
'info': [
{
'role': 'user',
'scope': {'groups': ['xyz']
}
}]
},{
'name': 'abc',
'info': [
{'rol': 'user'}
]
}]
when compared input with main the wrong/different key should be given as output, in this case
['rol']

The schema module does exactly this.
You can catch SchemaUnexpectedTypeError to see which data doesn't match your pattern.
Also, make sure you don't use the word input as a variable name, as it's the name of a built-in function.

keys = []
def print_dict(d):
if type(d) == dict:
for val in d.keys():
df = d[val]
try:
if type(df) == list:
for i in range(0,len(df)):
if type(df[i]) == dict:
print_dict(df[i])
except AttributeError:
pass
keys.append(val)
else:
try:
x = d[0]
if type(x) == dict:
print_dict(d[0])
except:
pass
return keys
keys_input = print_dict(input)
keys = []
keys_main = print_dict(main)
print(keys_input)
print(keys_main)
for i in keys_input[:]:
if i in keys_main:
keys_input.remove(i)
print(keys_input)
This has worked for me. you can check above code snippet and if any changes provide more information so any chances if required.

Dictionary and lists compare theire content nested by default.
input_data == main should result in the right output if you format your dicts correctly. Try adding curly brackets "{"/"}" arround your dicts. It should probably look like something like this:
main = {"app": [{
"name": str,
"info": [
{
"role": str,
"scope": {"groups": list}
}
]
},{
"name": str,
"info": [
{"role": str}
]
}]}
input_data = {"app":[{
'name': 'nms',
'info': [
{
'role': 'user',
'scope': {'groups': ['xyz']
}
}]
},{
'name': 'abc',
'info': [
{'rol': 'user'}
]
}]}
input_data2 = {"app": [{
'name': 'nms',
'info': [
{
'role': 'user',
'scope': {'groups': ['xyz']
}
}]
}, {
'name': 'abc',
'info': [
{'rol': 'user'}
]
}]}
Comparision results should look like this:
input_data2 == input_data # True
main == input_data # False

In Python, trying to filter out item from dict using pop

I'm pulling data from an API call, the JSON output has some specific keys that I want to get rid of, for the most part I've used pop('id', None) but I have a circumstance where the item has a nested list. i.e.
[
{
"enabled":true,
"is_local":true,
"name":"no_squash",
"policy_type":"nfs-export",
"rules":[
{
"access":"no-squash",
"anongid":"None",
"anonuid":"None",
"atime":true,
"client":"*",
"fileid_32bit":false,
"id":"6680ab71-1823-48fc-bc84-920059d218c5",
"index":1,
"name":"no_squash.1",
"permission":"rw",
"policy":{
"id":"e399c67e-595a-8b21-18dc-678164e360bd",
"name":"no_squash",
"resource_type":"nfs-export-policies"
},
"policy_version":"ffffffff-d747-c55b-0000-0000411034c5",
"secure":false,
"security":[
"krb5",
"krb5i",
"krb5p",
"sys"
]
}
],
"version":"ffffffff-d747-c55b-0000-0000411034c5"
}
]
More so the rules section, how do I target ['rules']['id'] specifically and remove it?
check = requests.get(standard_url+api_version+"/nfs-export-policies", headers=auth_headers, verify=False)
centry = check.json()['items']
for item in centry:
item.pop('id', None)
item.pop('location', None)
The above has already removed the id key from the initial entry but not the subelement.
I've worked out that I can achieve the desired results with nested for loops, but is there a better way?
Raw JSON as received from the API get call:
{'continuation_token': None, 'total_item_count': 1, 'items': [{'name': 'no_squash', 'id': 'e399c67e-595a-8b21-18dc-678164e360bd', 'enabled': True, 'is_local': True, 'location': {'name': 'fbstaines03', 'id': 'a7c7d4ad-b515-4f6b-a396-562cdad2063d', 'resource_type': 'arrays'}, 'version': 'ffffffff-d747-c55b-0000-0000411034c5', 'rules': [{'name': 'no_squash.1', 'id': '6680ab71-1823-48fc-bc84-920059d218c5', 'policy': {'name': 'no_squash', 'id': 'e399c67e-595a-8b21-18dc-678164e360bd', 'resource_type': 'nfs-export-policies'}, 'access': 'no-squash', 'anongid': None, 'anonuid': None, 'atime': True, 'client': '*', 'fileid_32bit': False, 'index': 1, 'permission': 'rw', 'secure': False, 'security': ['krb5', 'krb5i', 'krb5p', 'sys'], 'policy_version': 'ffffffff-d747-c55b-0000-0000411034c5'}], 'policy_type': 'nfs-export'}]}
Any help gratefully received.

IIUC it looks as though any 'id' key in any dictionary needs to be removed. If so, this is best dealt with recursively:
import json
J = """[
{
"enabled":true,
"is_local":true,
"name":"no_squash",
"policy_type":"nfs-export",
"rules":[
{
"access":"no-squash",
"anongid":"None",
"anonuid":"None",
"atime":true,
"client":"*",
"fileid_32bit":false,
"id":"6680ab71-1823-48fc-bc84-920059d218c5",
"index":1,
"name":"no_squash.1",
"permission":"rw",
"policy":{
"id":"e399c67e-595a-8b21-18dc-678164e360bd",
"name":"no_squash",
"resource_type":"nfs-export-policies"
},
"policy_version":"ffffffff-d747-c55b-0000-0000411034c5",
"secure":false,
"security":[
"krb5",
"krb5i",
"krb5p",
"sys"
]
}
],
"version":"ffffffff-d747-c55b-0000-0000411034c5"
}
]"""
def rmid(e):
if isinstance(e, dict):
e.pop('id', None)
for v in e.values():
rmid(v)
elif isinstance(e, list):
for v in e:
rmid(v)
data = json.loads(J)
for d in data:
rmid(d)
print(json.dumps(data, indent=2))
Output:
[
{
"enabled": true,
"is_local": true,
"name": "no_squash",
"policy_type": "nfs-export",
"rules": [
{
"access": "no-squash",
"anongid": "None",
"anonuid": "None",
"atime": true,
"client": "*",
"fileid_32bit": false,
"index": 1,
"name": "no_squash.1",
"permission": "rw",
"policy": {
"name": "no_squash",
"resource_type": "nfs-export-policies"
},
"policy_version": "ffffffff-d747-c55b-0000-0000411034c5",
"secure": false,
"security": [
"krb5",
"krb5i",
"krb5p",
"sys"
]
}
],
"version": "ffffffff-d747-c55b-0000-0000411034c5"
}
]

How to get this list of dict in DynamoDB using boto3?

This is an example of items I want to add in DynamoDB:
Item 1:
user :{'Id': '123456789', 'Name': 'user2', 'Keys': [{'KeyId': '987654321', 'keyStatus': 'Active'}]}
Item 2:
user :{'Id': '59940303', 'Name': 'user2', 'Keys': [{'KeyId': '987654321', 'keyStatus': 'Active'},{'KeyId': '6382920434', 'keyStatus': 'Active'}]}
As you can see the Keys section contains a list of dicts. Item1 contains one dict, item2 contains two dicts.
How can I add this in DynamoDB?
This works (without the list of dicts):
table = dynamodb_resource.Table("name")
table.update_item(
TableName=table_name,
Key={"Id": user["Id"]},
UpdateExpression="set Name=:n",
ExpressionAttributeValues={
":n": user["Name"]
},
ReturnValues="UPDATED_NEW"
But How should I update this for a list of dicts?

Updates work fine for me as follows:
table.update_item(
Key={"id": user["Id"]},
UpdateExpression="set #n=:n, #k=:k",
ExpressionAttributeValues={":n": user["Name"], ":k": user["Keys"]},
ExpressionAttributeNames={"#n": "Name", "#k": "Keys"},
ReturnValues="UPDATED_NEW",
)

Here's a working example:
from boto3.dynamodb.conditions import Key
users = [
{"Id": "A123456789", "Name": "user1", "Keys": [{"KeyId": "A987654321", "keyStatus": "Active"}],},
{ "Id": "B123456789", "Name": "user2", "Keys": [{"KeyId": "Z987654321", "keyStatus": "Active"},{"KeyId": "C6382920434", "keyStatus": "Active"},],},
]
# create the two records
for user in users:
tbl.update_item(
Key={
"Id": user["Id"],
},
UpdateExpression="set #name=:n, #keys=:k",
ExpressionAttributeValues={
":n": user["Name"],
":k": user["Keys"],
},
ExpressionAttributeNames={"#name": "Name", "#keys": "Keys"}, # fields are reserved keywords! we must use substitution
ReturnValues="UPDATED_NEW",
)
# query a record
res = tbl.query(KeyConditionExpression=Key('Id').eq('B123456789'))
print(res["Items"][0])
# {'Id': 'B123456789', 'Keys': [{'keyStatus': 'Active', 'KeyId': 'Z987654321'}, {'keyStatus': 'Active', 'KeyId': 'C6382920434'}],}
# update a nested attribute - set the first key status to inactive
tbl.update_item(
Key={"Id": "B123456789"},
UpdateExpression="set #keys[0].#status=:s",
ExpressionAttributeValues={
":s": "Inactive",
},
ExpressionAttributeNames={"#keys": "Keys", "#status": "keyStatus"},
ReturnValues="UPDATED_NEW",
)
# query the updated record again
res = tbl.query(KeyConditionExpression=Key('Id').eq('B123456789'))
print(res["Items"][0])
# {'Id': 'B123456789', 'Keys': [{'KeyId': 'Z987654321', 'keyStatus': 'Inactive'}, {'keyStatus': 'Active', 'KeyId': 'C6382920434'}], 'Name': 'user2'}

Merge dictionaries with same key from two lists of dicts in python

I have two dictionaries, as below. Both dictionaries have a list of dictionaries as the value associated with their properties key; each dictionary within these lists has an id key. I wish to merge my two dictionaries into one such that the properties list in the resulting dictionary only has one dictionary for each id.
{
"name":"harry",
"properties":[
{
"id":"N3",
"status":"OPEN",
"type":"energetic"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
}
]
}
and the other list:
{
"name":"harry",
"properties":[
{
"id":"N3",
"type":"energetic",
"language": "english"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
The output I am trying to achieve is:
"name":"harry",
"properties":[
{
"id":"N3",
"status":"OPEN",
"type":"energetic",
"language": "english"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
As id: N3 is common in both the lists, those 2 dicts should be merged with all the fields. So far I have tried using itertools and
ds = [d1, d2]
d = {}
for k in d1.keys():
d[k] = tuple(d[k] for d in ds)
Could someone please help in figuring this out?

Here is one of the approach:
a = {
"name":"harry",
"properties":[
{
"id":"N3",
"status":"OPEN",
"type":"energetic"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
}
]
}
b = {
"name":"harry",
"properties":[
{
"id":"N3",
"type":"energetic",
"language": "english"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
# Create dic maintaining the index of each id in resp dict
a_ids = {item['id']: index for index,item in enumerate(a['properties'])} #{'N3': 0, 'N5': 1}
b_ids = {item['id']: index for index,item in enumerate(b['properties'])} #{'N3': 0, 'N6': 1}
# Loop through one of the dict created
for id in a_ids.keys():
# If same ID exists in another dict, update it with the key value
if id in b_ids:
b['properties'][b_ids[id]].update(a['properties'][a_ids[id]])
# If it does not exist, then just append the new dict
else:
b['properties'].append(a['properties'][a_ids[id]])
print (b)
Output:
{'name': 'harry', 'properties': [{'id': 'N3', 'type': 'energetic', 'language': 'english', 'status': 'OPEN'}, {'id': 'N6', 'status': 'OPEN', 'type': 'cool'}, {'id': 'N5', 'status': 'OPEN', 'type': 'hot'}]}

It might help to treat the two objects as elements each in their own lists. Maybe you have other objects with different name values, such as might come out of a JSON-formatted REST request.
Then you could do a left outer join on both name and id keys:
#!/usr/bin/env python
a = [
{
"name": "harry",
"properties": [
{
"id":"N3",
"status":"OPEN",
"type":"energetic"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
}
]
}
]
b = [
{
"name": "harry",
"properties": [
{
"id":"N3",
"type":"energetic",
"language": "english"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
]
a_names = set()
a_prop_ids_by_name = {}
a_by_name = {}
for ao in a:
an = ao['name']
a_names.add(an)
if an not in a_prop_ids_by_name:
a_prop_ids_by_name[an] = set()
for ap in ao['properties']:
api = ap['id']
a_prop_ids_by_name[an].add(api)
a_by_name[an] = ao
res = []
for bo in b:
bn = bo['name']
if bn not in a_names:
res.append(bo)
else:
ao = a_by_name[bn]
bp = bo['properties']
for bpo in bp:
if bpo['id'] not in a_prop_ids_by_name[bn]:
ao['properties'].append(bpo)
res.append(ao)
print(res)
The idea above is to process list a for names and ids. The names and ids-by-name are instances of a Python set. So members are always unique.
Once you have these sets, you can do the left outer join on the contents of list b.
Either there's an object in b that doesn't exist in a (i.e. shares a common name), in which case you add that object to the result as-is. But if there is an object in b that does exist in a (which shares a common name), then you iterate over that object's id values and look for ids not already in the a ids-by-name set. You add missing properties to a, and then add that processed object to the result.
Output:
[{'name': 'harry', 'properties': [{'id': 'N3', 'status': 'OPEN', 'type': 'energetic'}, {'id': 'N5', 'status': 'OPEN', 'type': 'hot'}, {'id': 'N6', 'status': 'OPEN', 'type': 'cool'}]}]
This doesn't do any error checking on input. This relies on name values being unique per object. So if you have duplicate keys in objects in both lists, you may get garbage (incorrect or unexpected output).

How to deepmerge dicts/list for a json in python

I'm trying to deepmerge lists to get a specific json.
what I want to achieve is this format (the order of the elements is irelevant):
{
"report": {
"context": [{
"report_id": [
"Report ID 30"
],
"status": [
"Status 7"
],
"fallzahl": [
"Fallzahl 52"
],
"izahl": [
"IZahl 20"
]
}
],
"körpergewicht": [{
"any_event_en": [{
"gewicht": [{
"|magnitude": 185.44,
"|unit": "kg"
}
],
"kommentar": [
"Kommentar 94"
],
"bekleidung": [{
"|code": "at0011"
}
]
}
]
}
]
}
}
I try to deepmerge dicts and lists to achieve this specific format. My baseline are some dicts:
{'körpergewicht': [{'any_event_en': [{'gewicht': [{'|magnitude': '100', '|unit': 'kg'}]}]}]}
{'körpergewicht': [{'any_event_en': [{'bekleidung': [{'|code': 'at0013'}]}]}]}
{'körpergewicht': [{'any_event_en': [{'kommentar': ['none']}]}]}
{'context': [{'status': ['fatty']}]}
{'context': [{'fallzahl': ['123']}]}
{'context': [{'report_id': ['123']}]}
{'context': [{'izahl': ['123']}]}
what I tried to do is following I have a dict called tmp_dict in that I hold a baseline dict as I loop through. The so called collect_dict is the dict in that I try to merge my baseline dicts. element holds my current baseline dict.
if (index == (len(element)-1)): #when the baseline dict is traversed completely
if tmp_dict:
first_key_of_tmp_dict=list(tmp_dict.keys())[0]
if not (first_key_of_tmp_dict in collect_dict):
collect_dict.update(tmp_dict)
else:
merge(tmp_dict,collect_dict)
else:
collect_dict.update(tmp_dict)
and I also wrote a merge method:
def merge(tmp_dict,collect_dict):
first_common_key_of_dicts=list(tmp_dict.keys())[0]
second_depth_key_of_tmp_dict=list(tmp_dict[first_common_key_of_dicts][0].keys())[0]
second_depth_tmp_dict=tmp_dict[first_common_key_of_dicts][0]
second_depth_key_of_coll_dict=collect_dict[first_common_key_of_dicts][0]
if not second_depth_key_of_tmp_dict in second_depth_key_of_coll_dict:
collect_dict.update(second_depth_tmp_dict)
else:
merge(second_depth_tmp_dict,second_depth_key_of_coll_dict)
what I'm coming up with goes in the right direction but is far from beeing my desired output:
{"report": {
"k\u00f6rpergewicht": [{
"any_event_en": [{
"kommentar": ["none"]
}
],
"bekleidung": [{
"|code": "at0013"
}
],
"gewicht": [{
"|magnitude": "100",
"|unit": "kg"
}
]
}
],
"context": [{
"fallzahl": ["234"]
}
],
"report_id": ["234"],
"status": ["s"],
"izahl": ["234"]
}
}
With another set of inputs:
{'atemfrequenz': {'context': [{'status': [{'|code': 'at0012'}]}]}},
{'atemfrequenz': {'context': [{'kategorie': ['Kategorie']}]}},
{'atemfrequenz': {'atemfrequenz': [{'messwert': [{'|magnitude': '123', '|unit': 'min'}]}]}}
I would like to achieve the following output:
"atemfrequenz": {
"context": [
{
"status": [
{
"|code": "at0012"
}
],
"kategorie": [
"Kategorie"
]
}
],
"atemfrequenz": [
{
"messwert": [
{
"|magnitude": 123,
"|unit": "/min"
}
]
}
]
}

This code should get the correct answer. I removed the special character (ö) to prevent errors.
dd = [
{'korpergewicht': [{'any_event_en': [{'gewicht': [{'|magnitude': '100', '|unit': 'kg'}]}]}] },
{'korpergewicht': [{'any_event_en': [{'bekleidung': [{'|code': 'at0013'}]}]}]},
{'korpergewicht': [{'any_event_en': [{'kommentar': ['none']}]}]},
{'context': [{'status': ['fatty']}]},
{'context': [{'fallzahl': ['123']}]},
{'context': [{'report_id': ['123']}]},
{'context': [{'izahl': ['123']}]}
]
def merge(d):
if (type(d) != type([])): return d
if (type(list(d[0].values())[0])) == type(""): return d
keys = list(set(list(k.keys())[0] for k in d))
lst = [{k:[]} for k in keys]
for e in lst:
for k in d:
if (list(e.keys())[0] == list(k.keys())[0]):
e[list(e.keys())[0]] += k[list(k.keys())[0]]
for e in lst:
if (type(e[list(e.keys())[0]][0]) == type({})):
e[list(e.keys())[0]] = merge(e[list(e.keys())[0]])
for i in lst[1:]: lst[0].update(i)
lst2 = [] # return list of single dictionary
lst2.append(lst[0])
return lst2
dx = merge(dd)
dx = {'report': dx[0]} # no list at lowest level
print(dx)
Output (formatted)
{'report': {
'korpergewicht': [{
'any_event_en': [{
'kommentar': ['none'],
'bekleidung': [{'|code': 'at0013'}],
'gewicht': [{'|magnitude': '100', '|unit': 'kg'}]}]}],
'context': [{
'report_id': ['123'],
'izahl': ['123'],
'fallzahl': ['123'],
'status': ['fatty']}]}}
Concerning the second data set provided, the data needs to structured to match the previous data set.
This data set works correctly:
dd = [
{'atemfrequenz': [{'context': [{'status': [{'|code': 'at0012'}]}]}]},
{'atemfrequenz': [{'context': [{'kategorie': ['Kategorie']}]}]},
{'atemfrequenz': [{'atemfrequenz': [{'messwert': [{'|magnitude': '123', '|unit': 'min'}]}]}]}
]
Output (formatted)
{'report': {
'atemfrequenz': [{
'atemfrequenz': [{
'messwert': [{'|magnitude': '123', '|unit': 'min'}]}],
'context': [{
'kategorie': ['Kategorie'],
'status': [{'|code': 'at0012'}]}]}]}}