We Keep Coding

Merge dictionaries with same key from two lists of dicts in python

I have two dictionaries, as below. Both dictionaries have a list of dictionaries as the value associated with their properties key; each dictionary within these lists has an id key. I wish to merge my two dictionaries into one such that the properties list in the resulting dictionary only has one dictionary for each id.
{
"name":"harry",
"properties":[
{
"id":"N3",
"status":"OPEN",
"type":"energetic"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
}
]
}
and the other list:
{
"name":"harry",
"properties":[
{
"id":"N3",
"type":"energetic",
"language": "english"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
The output I am trying to achieve is:
"name":"harry",
"properties":[
{
"id":"N3",
"status":"OPEN",
"type":"energetic",
"language": "english"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
As id: N3 is common in both the lists, those 2 dicts should be merged with all the fields. So far I have tried using itertools and
ds = [d1, d2]
d = {}
for k in d1.keys():
d[k] = tuple(d[k] for d in ds)
Could someone please help in figuring this out?

Here is one of the approach:
a = {
"name":"harry",
"properties":[
{
"id":"N3",
"status":"OPEN",
"type":"energetic"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
}
]
}
b = {
"name":"harry",
"properties":[
{
"id":"N3",
"type":"energetic",
"language": "english"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
# Create dic maintaining the index of each id in resp dict
a_ids = {item['id']: index for index,item in enumerate(a['properties'])} #{'N3': 0, 'N5': 1}
b_ids = {item['id']: index for index,item in enumerate(b['properties'])} #{'N3': 0, 'N6': 1}
# Loop through one of the dict created
for id in a_ids.keys():
# If same ID exists in another dict, update it with the key value
if id in b_ids:
b['properties'][b_ids[id]].update(a['properties'][a_ids[id]])
# If it does not exist, then just append the new dict
else:
b['properties'].append(a['properties'][a_ids[id]])
print (b)
Output:
{'name': 'harry', 'properties': [{'id': 'N3', 'type': 'energetic', 'language': 'english', 'status': 'OPEN'}, {'id': 'N6', 'status': 'OPEN', 'type': 'cool'}, {'id': 'N5', 'status': 'OPEN', 'type': 'hot'}]}

It might help to treat the two objects as elements each in their own lists. Maybe you have other objects with different name values, such as might come out of a JSON-formatted REST request.
Then you could do a left outer join on both name and id keys:
#!/usr/bin/env python
a = [
{
"name": "harry",
"properties": [
{
"id":"N3",
"status":"OPEN",
"type":"energetic"
},
{
"id":"N5",
"status":"OPEN",
"type":"hot"
}
]
}
]
b = [
{
"name": "harry",
"properties": [
{
"id":"N3",
"type":"energetic",
"language": "english"
},
{
"id":"N6",
"status":"OPEN",
"type":"cool"
}
]
}
]
a_names = set()
a_prop_ids_by_name = {}
a_by_name = {}
for ao in a:
an = ao['name']
a_names.add(an)
if an not in a_prop_ids_by_name:
a_prop_ids_by_name[an] = set()
for ap in ao['properties']:
api = ap['id']
a_prop_ids_by_name[an].add(api)
a_by_name[an] = ao
res = []
for bo in b:
bn = bo['name']
if bn not in a_names:
res.append(bo)
else:
ao = a_by_name[bn]
bp = bo['properties']
for bpo in bp:
if bpo['id'] not in a_prop_ids_by_name[bn]:
ao['properties'].append(bpo)
res.append(ao)
print(res)
The idea above is to process list a for names and ids. The names and ids-by-name are instances of a Python set. So members are always unique.
Once you have these sets, you can do the left outer join on the contents of list b.
Either there's an object in b that doesn't exist in a (i.e. shares a common name), in which case you add that object to the result as-is. But if there is an object in b that does exist in a (which shares a common name), then you iterate over that object's id values and look for ids not already in the a ids-by-name set. You add missing properties to a, and then add that processed object to the result.
Output:
[{'name': 'harry', 'properties': [{'id': 'N3', 'status': 'OPEN', 'type': 'energetic'}, {'id': 'N5', 'status': 'OPEN', 'type': 'hot'}, {'id': 'N6', 'status': 'OPEN', 'type': 'cool'}]}]
This doesn't do any error checking on input. This relies on name values being unique per object. So if you have duplicate keys in objects in both lists, you may get garbage (incorrect or unexpected output).

How to move sub-dictionary to another dictionary?

I am writing a function that takes 2 strings as inputs and would move a section of the dictionary to another.
def move(item_to_move, destination):
# do something....
My initial dictionary looks like this.
directories = {
'beers': {
'ipa': {
'stone': {}
}
},
'wines': {
'red': {
'cabernet': {}
}
},
'other' : {}
}
I would like to move either a subsection or section of the dictionary to another section. The sections are represented by each key of the path delimited by a '/'. For example, the inputs for my function would be:
item_to_move='beers/ipa'
destination='other'
move(directories, item_to_move,destination)
The output would be:
{
'wines': {
'red': {
'cabernet': {}
},
},
'other' :{
'beers': {
'ipa': {
'stone': {}
} }
},
}
NOTE: I am assuming all input paths for items_to_move are valid.

Find the origin's parent dictionary and the target's dictionary, then update the the target's dictionary with the origin's key and value (removing it from the origin's parent):
def move(tree,originPath,targetPath):
originKey = None
for originName in originPath.split("/"):
originParent = originParent[originKey] if originKey else tree
originKey = originName
targetDict = tree
for targetName in targetPath.split("/"):
targetDict = targetDict[targetName]
targetDict.update({originKey:originParent.pop(originKey)})
output:
directories = {
'beers': {
'ipa': {
'stone': {}
}
},
'wines': {
'red': {
'cabernet': {}
}
},
'other' : {}
}
move(directories,'beers/ipa','other')
print(directories)
{ 'beers': {},
'wines': { 'red': {'cabernet': {}} },
'other': { 'ipa': {'stone': {}} }
}

find all dictionary matching key=value pairs in dynamic dictionary

If I have a python dictionary like the following:
conf = {
'memory': {
'alarm': {
'active': 'yes',
'pagefile_error': {
'active':'no'
}
}
},
'disk': {
'alarm':{
'active':'yes',
'fixed':{
'#dev':{
'active':'yes',
'something':'else'
}
}
}
},
'cpu': {
'alarm': {
'active':'no',
'highcpu': {
'active':'yes'
}
}
}
}
how can I filter only the paths that end in 'active':'yes' and not show any other info.
In addition, for parent items that show up as active: no, I would want to disregard whatever comes after those.
conf = {
'memory': {
'alarm': {
'active': 'yes'
}
},
'disk' : {
'alarm':{
'active':'yes',
'fixed': {
'#dev': {
'active':'yes'
}
}
}
}
}
I don't have any working code for this yet as I'm not sure where to start. all I have at the moment is the starting dictionary.

Using recursion :
def keep_active_only(my_dict):
result_dict = {}
for key, value in my_dict.items():
# If there is embedded dict
if isinstance(value, dict):
# Compute the embedded dict using recursion
result_subdict = keep_active_only(value)
# Keeping result only if not empty
if result_subdict:
result_dict[key] = result_subdict
# Keep active key if value is yes
elif key == "active" and value == "yes":
result_dict[key] = value
# Returns empty dict if active is no
elif key == "active" and value == "no":
return {}
return result_dict
Output :
>>> keep_active_only(conf)
{
'disk': {
'alarm': {
'active': 'yes',
'fixed': {
'#dev': {
'active': 'yes'
}
}
}
},
'memory': {
'alarm': {
'active': 'yes'
}
}
}

You can use recursion:
def active(d):
_r, _flag = [], False
for a, b in d.items():
if a == 'active' and not _flag:
_r.append(b == 'yes')
_flag = True
if not _flag and isinstance(b, dict):
_r.append(active(b))
return all(_r)
def build(d, flag = False):
return {a:b if not isinstance(b, dict) else build(b, 'active' in b)
for a, b in d.items() if ((not isinstance(b, dict) and not flag) or a == 'active') or (isinstance(b, dict) and active(b))}
import json
print(json.dumps(build(conf), indent=4))
Output:
{
"memory": {
"alarm": {
"active": "yes"
}
},
"disk": {
"alarm": {
"active": "yes",
"fixed": {
"#dev": {
"active": "yes"
}
}
}
}
}

Not sure if I understand correctly, but here is a function that discards all data from the dict that does not take you to a particular key and value:
def filter_dict(d, key, value):
new_dict = {}
for d_key, d_value in d.items():
if d_key == key and d_value == value:
new_dict[d_key] = d_value
elif isinstance(d_value, dict):
child = filter_dict(d_value, key, value)
if child:
new_dict[d_key] = child
return new_dict
Here is how you would use it in your example:
from pprint import pprint
conf = {
'memory': {
'alarm': {
'active': 'yes',
'pagefile_error': {
'active':'no'
}
}
},
'disk': {
'alarm': {
'active': 'yes',
'fixed': {
'#dev': {
'active': 'yes',
'something': 'else'
}
}
}
}
}
pprint(filter_dict(conf, 'active', 'yes'))
# {'disk': {'alarm': {'active': 'yes', 'fixed': {'#dev': {'active': 'yes'}}}},
# 'memory': {'alarm': {'active': 'yes'}}}

Comparing Nested Python dict with list and dict

I've seen similar questions but none that exactly match what I'm doing and I believe other developers might face same issue if they are working with MongoDB.
I'm looking to compare two nested dict objects with dict and arrays and return a dict with additions and deletion (like you would git diff two files)
Here is what I have so far:
def dict_diff(alpha, beta, recurse_adds=False, recurse_dels=False):
"""
:return: differences between two python dict with adds and dels
example:
(This is the expected output)
{
'adds':
{
'specific_hours': [{'ends_at': '2015-12-25'}],
}
'dels':
{
'specific_hours': [{'ends_at': '2015-12-24'}],
'subscription_products': {'review_management': {'thiswillbedeleted': 'deleteme'}}
}
}
"""
if type(alpha) is dict and type(beta) is dict:
a_keys = alpha.keys()
b_keys = beta.keys()
dels = {}
adds = {}
for key in a_keys:
if type(alpha[key]) is list:
if alpha[key] != beta[key]:
adds[key] = dict_diff(alpha[key], beta[key], recurse_adds=True)
dels[key] = dict_diff(alpha[key], beta[key], recurse_dels=True)
elif type(alpha[key]) is dict:
if alpha[key] != beta[key]:
adds[key] = dict_diff(alpha[key], beta[key], recurse_adds=True)
dels[key] = dict_diff(alpha[key], beta[key], recurse_dels=True)
elif key not in b_keys:
dels[key] = alpha[key]
elif alpha[key] != beta[key]:
adds[key] = beta[key]
dels[key] = alpha[key]
for key in b_keys:
if key not in a_keys:
adds[key] = beta[key]
elif type(alpha) is list and type(beta) is list:
index = 0
adds=[]
dels=[]
for elem in alpha:
if alpha[index] != beta[index]:
dels.append(alpha[index])
adds.append(beta[index])
# print('update', adds, dels)
index+=1
else:
raise Exception("dict_diff function can only get dict objects")
if recurse_adds:
if bool(adds):
return adds
return {}
if recurse_dels:
if bool(dels):
return dels
return {}
return {'adds': adds, 'dels': dels}
The result I'm getting now is:
{'adds': {'specific_hours': [{'ends_at': '2015-12-24',
'open_hours': ['07:30-11:30', '12:30-21:30'],
'starts_at': '2015-12-22'},
{'ends_at': '2015-01-03',
'open_hours': ['07:30-11:30'],
'starts_at': '2015-01-0'}],
'subscription_products': {'review_management': {}}},
'dels': {'specific_hours': [{'ends_at': '2015-12-24',
'open_hours': ['07:30-11:30', '12:30-21:30'],
'starts_at': '2015-12-2'},
{'ends_at': '2015-01-03',
'open_hours': ['07:30-11:30'],
'starts_at': '2015-01-0'}],
'subscription_products': {'review_management': {'thiswillbedeleted': 'deleteme'}}}}
And this is the two objects I'm trying to compare:
alpha = {
'specific_hours': [
{
"starts_at": "2015-12-2",
"ends_at": "2015-12-24",
"open_hours": [
"07:30-11:30",
"12:30-21:30"
]
},
{
"starts_at": "2015-01-0",
"ends_at": "2015-01-03",
"open_hours": [
"07:30-11:30"
]
}
],
'subscription_products': {'presence_management':
{'expiration_date': 1953291600,
'payment_type': {
'free': 'iamfree',
'test': "test",
},
},
'review_management':
{'expiration_date': 1511799660,
'payment_type': {
'free': 'iamfree',
'test': "test",
},
'thiswillbedeleted': "deleteme",
}
},
}
beta = {
'specific_hours': [
{
"starts_at": "2015-12-22",
"ends_at": "2015-12-24",
"open_hours": [
"07:30-11:30",
"12:30-21:30"
]
},
{
"starts_at": "2015-01-0",
"ends_at": "2015-01-03",
"open_hours": [
"07:30-11:30"
]
}
],
'subscription_products': {'presence_management':
{'expiration_date': 1953291600,
'payment_type': {
'free': 'iamfree',
'test': "test",
},
},
'review_management':
{'expiration_date': 1511799660,
'payment_type': {
'free': 'iamfree',
'test': "test",
},
}
},
}

replace information in Json string based on a condition

I have a very large json file with several nested keys. From whaat I've read so far, if you do:
x = json.loads(data)
Python will interpret it as a dictionary (correct me if I'm wrong). The fourth level of nesting in the json file contains several elements named by an ID number and all of them contain an element called children, something like this:
{"level1":
{"level2":
{"level3":
{"ID1":
{"children": [1,2,3,4,5]}
}
{"ID2":
{"children": []}
}
{"ID3":
{"children": [6,7,8,9,10]}
}
}
}
}
What I need to do is to replace all items in all the "children" elements with nothing, meaning "children": [] if the ID number is in a list called new_ids and then convert it back to json. I've been reading on the subject for a few hours now but I haven't found anything similar to this to try to help myself.
I'm running Python 3.3.3. Any ideas are greatly appreciated!!
Thanks!!
EDIT
List:
new_ids=["ID1","ID3"]
Expected result:
{"level1":
{"level2":
{"level3":
{"ID1":
{"children": []}
}
{"ID2":
{"children": []}
}
{"ID3":
{"children": []}
}
}
}
}

First of all, your JSON is invalid. I assume you want this:
{"level1":
{"level2":
{"level3":
{
"ID1":{"children": [1,2,3,4,5]},
"ID2":{"children": []},
"ID3":{"children": [6,7,8,9,10]}
}
}
}
}
Now, load your data as a dictionary:
>>> with open('file', 'r') as f:
... x = json.load(f)
...
>>> x
{u'level1': {u'level2': {u'level3': {u'ID2': {u'children': []}, u'ID3': {u'children': [6, 7, 8, 9, 10]}, u'ID1': {u'children': [1, 2, 3, 4, 5]}}}}}
Now you can loop over the keys in x['level1']['level2']['level3'] and check whether they are in your new_ids.
>>> new_ids=["ID1","ID3"]
>>> for key in x['level1']['level2']['level3']:
... if key in new_ids:
... x['level1']['level2']['level3'][key]['children'] = []
...
>>> x
{u'level1': {u'level2': {u'level3': {u'ID2': {u'children': []}, u'ID3': {u'children': []}, u'ID1': {u'children': []}}}}}
You can now write x back to a file like this:
with open('myfile', 'w') as f:
f.write(json.dumps(x))
If your new_ids list is large, consider making it a set.

If you have simple dictionary like this
data_dict = {
"level1": {
"level2":{
"level3":{
"ID1":{"children": [1,2,3,4,5]},
"ID2":{"children": [] },
"ID3":{"children": [6,7,8,9,10]},
}
}
}
}
than you need only this:
data_dict = {
"level1": {
"level2":{
"level3":{
"ID1":{"children": [1,2,3,4,5]},
"ID2":{"children": [] },
"ID3":{"children": [6,7,8,9,10]},
}
}
}
}
new_ids=["ID1","ID3"]
for idx in new_ids:
if idx in data_dict['level1']["level2"]["level3"]:
data_dict['level1']["level2"]["level3"][idx]['children'] = []
print data_dict
'''
{
'level1': {
'level2': {
'level3': {
'ID2': {'children': []},
'ID3': {'children': []},
'ID1': {'children': []}
}
}
}
}
'''
but if you have more complicated dictionary
data_dict = {
"level1a": {
"level2a":{
"level3a":{
"ID2":{"children": [] },
"ID3":{"children": [6,7,8,9,10]},
}
}
},
"level1b": {
"level2b":{
"level3b":{
"ID1":{"children": [1,2,3,4,5]},
}
}
}
}
new_ids =["ID1","ID3"]
for level1 in data_dict.values():
for level2 in level1.values():
for level3 in level2.values():
for idx in new_ids:
if idx in level3:
level3[idx]['children'] = []
print data_dict
'''
{
'level1a': {
'level2a': {
'level3a': {
'ID2': {'children': []},
'ID3': {'children': []}
}
}
},
'level1b': {
'level2b': {
'level3b': {
'ID1': {'children': []}
}
}
}
}
'''