We Keep Coding

Modify nested dict key name by its parent

I have a following kind of structure to be handled:
payload = {
"name":"Event1",
"events":[
{
"name":"A",
"data":[
{
"name":"subscriptionId",
"data_id":0,
"data":0
},
{
"name":"updateCounter",
"data_id":1,
"data":0
},
{
"name":"noOfMessages",
"data_id":2,
"data":0
},
{
"name":"counter",
"data_id":3,
"data":0
},
{
"name":"resourceElements",
"data_id":4,
"data":0
},
{
"name":"type",
"data_id":5,
"data":0
},
{
"name":"subscription",
"data_id":6,
"data":0
},
{
"name":"element",
"data_id":7,
"data":[
{
"name":"type",
"data_id":0,
"data":0
},
{
"name":"plugLockState",
"data_id":1,
"data":{
"value":""
}
},
{
"name":"lockState",
"data_id":2,
"data":{
"value":""
}
},
{
"name":"flapState",
"data_id":6,
"data":{
"value":""
}
},
{
"name":"plugState",
"data_id":3,
"data":0
},
{
"name":"plugConnectionState",
"data_id":4,
"data":0
},
{
"name":"infrastructureState",
"data_id":5,
"data":0
}
]
}
]
}
]
}
I want to replace any key name within the nested structure by the parent, so the ideal result should look like this:
{
"name":"Event1",
"events":[
{
"name":"Event1.A",
"data":[
{
"name":"Event1.A.subscriptionId",
"data_id":0,
"data":0
},
{
"name":"Event1.A.updateCounter",
"data_id":1,
"data":0
},
{
"name":"Event1.A.noOfMessages",
"data_id":2,
"data":0
},
{
"name":"Event1.A.counter",
"data_id":3,
"data":0
},
{
"name":"Event1.A.resourceElements",
"data_id":4,
"data":0
},
{
"name":"Event1.A.type",
"data_id":5,
"data":0
},
{
"name":"Event1.A.subscription",
"data_id":6,
"data":0
},
{
"name":"Event1.A.element",
"data_id":7,
"data":[
{
"name":"Event1.A.element.type",
"data_id":0,
"data":0
},
{
"name":"Event1.A.element.plugLockState",
"data_id":1,
"data":{
"value":""
}
},
{
"name":"Event1.A.element.lockState",
"data_id":2,
"data":{
"value":""
}
},
{
"name":"Event1.A.element.flapState",
"data_id":6,
"data":{
"value":""
}
},
{
"name":"Event1.A.element.plugState",
"data_id":3,
"data":0
},
{
"name":"Event1.A.element.plugConnectionState",
"data_id":4,
"data":0
},
{
"name":"Event1.A.element.infrastructureState",
"data_id":5,
"data":0
}
]
}
]
}
]
}
so far I have written this recursive method:
def iterate_recursively(dictionary: dict, names=None):
if names is None:
names = []
for k, v in dictionary.items():
if isinstance(v, dict):
iterate_recursively(v)
elif isinstance(v, list):
for d in v:
if isinstance(d, dict):
names.append(d["name"])
iterate_recursively(d)
but I simply don't get it. How can the keys, based on my requirement, be changed while iterating recursively?

Here's a variant that returns a new dictionary (and thus leaving the original one unchanged).
code00.py:
#!/usr/bin/env python
import sys
from pprint import pprint as pp
payload = {
"name": "Event1",
"events": [
{
"name": "A",
"data": [
{
"name": "subscriptionId",
"data_id": 0,
"data": 0
},
{
"name": "updateCounter",
"data_id": 1,
"data": 0
},
{
"name": "noOfMessages",
"data_id": 2,
"data": 0
},
{
"name": "counter",
"data_id": 3,
"data": 0
},
{
"name": "resourceElements",
"data_id": 4,
"data": 0
},
{
"name": "type",
"data_id": 5,
"data": 0
},
{
"name": "subscription",
"data_id": 6,
"data": 0
},
{
"name": "element",
"data_id": 7,
"data": [
{
"name": "type",
"data_id": 0,
"data": 0
},
{
"name": "plugLockState",
"data_id": 1,
"data": {
"value": ""
}
},
{
"name": "lockState",
"data_id": 2,
"data": {
"value": ""
}
},
{
"name": "flapState",
"data_id": 6,
"data": {
"value": ""
}
},
{
"name": "plugState",
"data_id": 3,
"data": 0
},
{
"name": "plugConnectionState",
"data_id": 4,
"data": 0
},
{
"name": "infrastructureState",
"data_id": 5,
"data": 0
}
]
}
]
}
]
}
def concat_names(data, names=()):
if isinstance(data, dict):
name = data.get("name")
new_names = names + (name,) if name is not None else names
return {k: concat_names(v, names=new_names) if k != "name" else ".".join(new_names) for k, v in data.items()}
elif isinstance(data, (list, tuple)):
return [concat_names(e, names=names) for e in data]
else:
return data
def main(*argv):
pp(concat_names(payload), indent=2, sort_dicts=False)
if __name__ == "__main__":
print("Python {:s} {:03d}bit on {:s}\n".format(" ".join(elem.strip() for elem in sys.version.split("\n")),
64 if sys.maxsize > 0x100000000 else 32, sys.platform))
rc = main(*sys.argv[1:])
print("\nDone.")
sys.exit(rc)
Output:
[cfati#CFATI-5510-0:e:\Work\Dev\StackOverflow\q073621243]> "e:\Work\Dev\VEnvs\py_pc064_03.09_test0\Scripts\python.exe" ./code00.py
Python 3.9.9 (tags/v3.9.9:ccb0e6a, Nov 15 2021, 18:08:50) [MSC v.1929 64 bit (AMD64)] 064bit on win32
{ 'name': 'Event1',
'events': [ { 'name': 'Event1.A',
'data': [ { 'name': 'Event1.A.subscriptionId',
'data_id': 0,
'data': 0},
{ 'name': 'Event1.A.updateCounter',
'data_id': 1,
'data': 0},
{ 'name': 'Event1.A.noOfMessages',
'data_id': 2,
'data': 0},
{'name': 'Event1.A.counter', 'data_id': 3, 'data': 0},
{ 'name': 'Event1.A.resourceElements',
'data_id': 4,
'data': 0},
{'name': 'Event1.A.type', 'data_id': 5, 'data': 0},
{ 'name': 'Event1.A.subscription',
'data_id': 6,
'data': 0},
{ 'name': 'Event1.A.element',
'data_id': 7,
'data': [ { 'name': 'Event1.A.element.type',
'data_id': 0,
'data': 0},
{ 'name': 'Event1.A.element.plugLockState',
'data_id': 1,
'data': {'value': ''}},
{ 'name': 'Event1.A.element.lockState',
'data_id': 2,
'data': {'value': ''}},
{ 'name': 'Event1.A.element.flapState',
'data_id': 6,
'data': {'value': ''}},
{ 'name': 'Event1.A.element.plugState',
'data_id': 3,
'data': 0},
{ 'name': 'Event1.A.element.plugConnectionState',
'data_id': 4,
'data': 0},
{ 'name': 'Event1.A.element.infrastructureState',
'data_id': 5,
'data': 0}]}]}]}
Done.

You can do something like this:
def iterate_recursively(dictionary: dict, prefix_name=None):
if 'name' in dictionary:
if prefix_name is None:
prefix_name = dictionary['name']
else:
prefix_name += '.' + dictionary['name']
dictionary['name'] = prefix_name
for k, v in dictionary.items():
if isinstance(v, dict):
iterate_recursively(v, prefix_name)
elif isinstance(v, list):
for d in v:
iterate_recursively(d, prefix_name)

How can i insert this JSON file into a MongoDB collection?

I'm trying to insert this object into a mongo DB collection. I've tried a lot of ways and haven't gotten any results. I was wondering if anyone here could help me. The main problem is when passing the array of items into the key items.
The JSON File is similar to this:
{
'code': 'iuhuilknlkn',
'description': 'nllksnd',
'currency': 'Mxn',
'items': [
{
'item': {
'_id': {
'$oid': '60065d253ef6d468ced3603f'
},
'code': '2',
'description': '22',
'currency': 'Mxn',
'MU': 'Hr',
'sellingPrice': 1,
'buyingPrice': 3,
'supplier': None,
'cid': '5fbd81b32b325e5ca15fe5c9',
'itemAmount': 1,
'itemProductPrice': 1,
'itemAmountPrice': 1
}
},
{
'item': {
'_id': {
'$oid': '6011c18883a280ae0e5b8185'
},
'code': 'prb-001',
'description': 'prueba 1 artículo',
'currency': 'Mxn',
'MU': 'Ser',
'sellingPrice': 100.59,
'buyingPrice': 12,
'supplier': None,
'cid': '5fbd81b32b325e5ca15fe5c9',
'itemAmount': 1,
'itemProductPrice': 100.59,
'itemAmountPrice': 100.59
}
}
],
'price': 101.59
}
and the code I'm using right now is the following:
productsM.insert({
'code':newP['code'],
'description':newP['description'],
'currency' :newP['currency'],
'items' : [([val for dic in newP['items'] for val in
dic.values()])],
'price' : newP['price'],
'cid': csHe })
The error I get is the next one:
key '$oid' must not start with '$'

Your error is quite clear. Simply removing the $ from the $oid keys in your dictionary/json will resolve the issue. I don't think you can have a $ in key names since they are reserved for operators such as $in or $regex.
I removed the $ from the $oid keys and it worked like a charm. All I did was the following:
data = {
"code": "iuhuilknlkn",
"description": "nllksnd",
"currency": "Mxn",
"items": [
{
"item": {
"_id": {
"oid": "60065d253ef6d468ced3603f"
},
"code": "2",
"description": "22",
"currency": "Mxn",
"MU": "Hr",
"sellingPrice": 1,
"buyingPrice": 3,
"supplier": None,
"cid": "5fbd81b32b325e5ca15fe5c9",
"itemAmount": 1,
"itemProductPrice": 1,
"itemAmountPrice": 1
}
},
{
"item": {
"_id": {
"oid": "6011c18883a280ae0e5b8185"
},
"code": "prb-001",
"description": "prueba 1 artículo",
"currency": "Mxn",
"MU": "Ser",
"sellingPrice": 100.59,
"buyingPrice": 12,
"supplier": None,
"cid": "5fbd81b32b325e5ca15fe5c9",
"itemAmount": 1,
"itemProductPrice": 100.59,
"itemAmountPrice": 100.59
}
}
],
"price": 101.59
}
db.insert(data)

Remove duplicate values from list of nested dictionaries

I have list of dictionaries with nested structure. I need to remove all duplicate values. I'm newbie in Python and can't solve this task. Anyone can help me?
My list looks like:
[
{
"task_id":123,
"results":[
{
"url":"site.com",
"date":"04.18.2019"
},
{
"url":"another_site.com",
"date":"04.18.2019"
},
{
"url":"site1.com",
"date":"04.18.2019"
}
]
},
{
"task_id":456,
"results":[
{
"url":"site3.com",
"date":"04.18.2019"
},
{
"url":"site.com",
"date":"04.18.2019"
}
]
},
{
"task_id":789,
"results":[
{
"url":"site7.com",
"date":"04.18.2019"
},
{
"url":"site9.com",
"date":"04.18.2019"
},
{
"url":"site.com",
"date":"04.18.2019"
}
]
}
]
I need to set site.com only once. If any value of url is duplicated - exclude it from dict.
As result:
task 123 with 3 dicts in results
task 456 with 1 dict in results (exclude site.com)
task 789 with 2 dict in results (exclude site.com)
Desired output should looks like:
[
{
"task_id":123,
"results":[
{
"url":"site.com",
"date":"04.18.2019"
},
{
"url":"another_site.com",
"date":"04.18.2019"
},
{
"url":"site1.com",
"date":"04.18.2019"
}
]
},
{
"task_id":456,
"results":[
{
"url":"site3.com",
"date":"04.18.2019"
}
]
},
{
"task_id":789,
"results":[
{
"url":"site7.com",
"date":"04.18.2019"
},
{
"url":"site9.com",
"date":"04.18.2019"
}
]
}
]

let results to be your array.
u = set()
final = []
for dict in results:
for res in dict["results"]:
if res["url"] not in u:
u.add(res["url"])
final.append(res)
print(final)

You can use a list comprehension:
d = [{'task_id': 123, 'results': [{'url': 'site.com', 'date': '04.18.2019'}, {'url': 'another_site.com', 'date': '04.18.2019'}, {'url': 'site1.com', 'date': '04.18.2019'}]}, {'task_id': 456, 'results': [{'url': 'site3.com', 'date': '04.18.2019'}, {'url': 'site.com', 'date': '04.18.2019'}]}, {'task_id': 789, 'results': [{'url': 'site7.com', 'date': '04.18.2019'}, {'url': 'site9.com', 'date': '04.18.2019'}, {'url': 'site.com', 'date': '04.18.2019'}]}]
new_d = [{**a, 'results':[c for c in a['results'] if all(c not in b['results'] for b in d[:i])]} for i, a in enumerate(d)]
Output:
[
{
"task_id": 123,
"results": [
{
"url": "site.com",
"date": "04.18.2019"
},
{
"url": "another_site.com",
"date": "04.18.2019"
},
{
"url": "site1.com",
"date": "04.18.2019"
}
]
},
{
"task_id": 456,
"results": [
{
"url": "site3.com",
"date": "04.18.2019"
}
]
},
{
"task_id": 789,
"results": [
{
"url": "site7.com",
"date": "04.18.2019"
},
{
"url": "site9.com",
"date": "04.18.2019"
}
]
}
]

people = {
1: {'name': 'John',},
2: {'name': 'Marie'},
3: {'name': 'Ann',},
4: {'name': 'John'},
}
print(people)
unique = {}
for key, value in people.items():
if value not in unique.values():
unique[key] = value
print(unique)
try these

Expanding a dictionary using dot notation

I have the following json document:
{
"id": "5c26321bd8f4113d43b91141",
"idMemberCreator": "5b203bc7e47d817a8138bc37",
"data": {
"list": {
"name": "Sorji for QA",
"id": "5b0a2543b89acdbdb85f7b42"
},
"board": {
"shortLink": "iyCzZ5jx",
"name": "FlicksIO",
"id": "5b0a251f68a9e74b8ec3b3ac"
},
"card": {
"shortLink": "vOt2vO7v",
"idShort": 92,
"name": "New column in main for Storefront provider correlation.",
"id": "5b9c0023533f7c26424ea4ed",
"closed": true
},
"old": {
"closed": false
}
},
"type": "updateCard",
"date": "2018-12-28T14:24:27.455Z",
"limits": {},
"memberCreator": {
"id": "5b203bc7e47d817a8138bc37",
"avatarHash": "73bfa48c76c3c92615fe89ff79a6c5ae",
"avatarUrl": "https://trello-avatars.s3.amazonaws.com/73bfa48f79a6c5ae",
"fullName": "Marie Bond",
"idMemberReferrer": null,
"initials": "MB",
"username": "mb"
}
}
I would like to expand this out to be a single level with dot notation. That is, it should look like:
{
"id": "5c26321bd8f4113d43b91141",
"idMemberCreator": "5b203bc7e47d817a8138bc37",
"data.list.name": "Sorji for QA",
"data.list.id": "5b0a2543b89acdbdb85f7b42"
"data.board.shortLink": "iyCzZ5jx",
"data.board.name": "FlicksIO",
"data.board.id": "5b0a251f68a9e74b8ec3b3ac"
"data.card.shortLink": "vOt2vO7v",
"data.card.idShort": 92,
"data.card.name": "New column in main for Storefront provider correlation.",
"data.card.id": "5b9c0023533f7c26424ea4ed",
"data.card.closed": true
"data.old.closed": false
"type": "updateCard",
"date": "2018-12-28T14:24:27.455Z",
"limits": {},
"memberCreator.id": "5b203bc7e47d817a8138bc37",
"memberCreator.avatarHash": "73bfa48c76c3c92615fe89ff79a6c5ae",
"memberCreator.avatarUrl": "https://trello-avatars.s3.amazonaws.com/73bfa48f79a6c5ae",
"memberCreator.fullName": "Marie Bond",
"memberCreator.idMemberReferrer": null,
"memberCreator.initials": "MB",
"memberCreator.username": "mb"
}
Would it be possible to do this with a generator object? I've been working a lot on recursion today, and have been trying to move from while loops to using generator objects and yields, etc.

You can keep a parameter in the signature of the recursive function to store the paths:
data = {'id': '5c26321bd8f4113d43b91141', 'idMemberCreator': '5b203bc7e47d817a8138bc37', 'data': {'list': {'name': 'Sorji for QA', 'id': '5b0a2543b89acdbdb85f7b42'}, 'board': {'shortLink': 'iyCzZ5jx', 'name': 'FlicksIO', 'id': '5b0a251f68a9e74b8ec3b3ac'}, 'card': {'shortLink': 'vOt2vO7v', 'idShort': 92, 'name': 'New column in main for Storefront provider correlation.', 'id': '5b9c0023533f7c26424ea4ed', 'closed': True}, 'old': {'closed': False}}, 'type': 'updateCard', 'date': '2018-12-28T14:24:27.455Z', 'limits': {}, 'memberCreator': {'id': '5b203bc7e47d817a8138bc37', 'avatarHash': '73bfa48c76c3c92615fe89ff79a6c5ae', 'avatarUrl': 'https://trello-avatars.s3.amazonaws.com/73bfa48f79a6c5ae', 'fullName': 'Marie Bond', 'idMemberReferrer': None, 'initials': 'MB', 'username': 'mb'}}
def dot_paths(d, _paths = []):
for a, b in d.items():
if not b or not isinstance(b, dict):
yield ['.'.join(_paths+[a]), b]
else:
yield from dot_paths(b, _paths+[a])
import json
print(json.dumps(dict(dot_paths(data)), indent=4))
Output:
{
"id": "5c26321bd8f4113d43b91141",
"idMemberCreator": "5b203bc7e47d817a8138bc37",
"data.list.name": "Sorji for QA",
"data.list.id": "5b0a2543b89acdbdb85f7b42",
"data.board.shortLink": "iyCzZ5jx",
"data.board.name": "FlicksIO",
"data.board.id": "5b0a251f68a9e74b8ec3b3ac",
"data.card.shortLink": "vOt2vO7v",
"data.card.idShort": 92,
"data.card.name": "New column in main for Storefront provider correlation.",
"data.card.id": "5b9c0023533f7c26424ea4ed",
"data.card.closed": true,
"data.old.closed": false,
"type": "updateCard",
"date": "2018-12-28T14:24:27.455Z",
"limits": {},
"memberCreator.id": "5b203bc7e47d817a8138bc37",
"memberCreator.avatarHash": "73bfa48c76c3c92615fe89ff79a6c5ae",
"memberCreator.avatarUrl": "https://trello-avatars.s3.amazonaws.com/73bfa48f79a6c5ae",
"memberCreator.fullName": "Marie Bond",
"memberCreator.idMemberReferrer": null,
"memberCreator.initials": "MB",
"memberCreator.username": "mb"
}

Json format for python

I'm rewriting a view based on what I know the final output should be in json but it's returning the dictionary as a string.
new output
{
"results":
["
{
'plot': u'',
'runtime': u'N/A',
'description': u'x',
'videos': [
{
'id': 823,
'name': u'x',
'youtube_id': u'FtcubOnXgZk'
}
],
'country': u'India',
'writer': u'Neetu Varma, Ranjeev Verma',
'name': u'Chalk N Duster',
'id': 940,
'director': u'Jayant Gilatar',
'hot': True,
'content': u'x',
'actors': u'Shabana Azmi, Arya Babbar, Gavie Chahal, Juhi Chawla',
'year': 2015,
'images': [
{'small': '/media/cache/62/fd/62fd5158d281c042e3cf1f919183e94e.jpg', 'medium': '/media/cache/5e/32/5e32ebb1a4d25bba0d0c70b4b448e948.jpg'}],
'trailer_youtube_id': u'FtcubOnXgZk',
'type': 'movie',
'slug': u'chalk-n-duster',
'categories': [{'parent_id': 2, 'id': 226, 'name': u'Drama'}],
'shows': {
'starts': '2016-01-16',
'booking_url': u'',
'venue': {
'address': u'',
'id': 854,
'name': u'Nyali Cinemax',
'area': {
'id': 52,
'parent': {
'id': 48,
'name': u'Mombasa'
},
'name': u'Nyali'
}
},
'starts_time': '18:30:00'
}
}", "{'plot': u'' ....
old output
"results": [
{
"actors": "x",
"categories": [
{
"id": 299,
"name": "Biography",
"parent_id": 2
},
],
"content": "x",
"country": "x",
"description": "x",
"director": "x",
"hot": true,
"id": 912,
"images": [
{
"medium": "/media/cache/d2/b3/d2b3a7885e7c39bfc5c2b297b66619c5.jpg",
"small": "/media/cache/e2/d0/e2d01b2c7c77d3590536666de4a7fd7d.jpg"
}
],
"name": "Bridge of Spies",
"plot": "x",
"runtime": "141 min",
"shows": [
{
"booking_url": "",
"starts": "2015-11-27",
"starts_time": "16:30:00",
"venue": {
"address": "The Junction Shopping Mall",
"area": {
"id": 68,
"name": "Ngong Road",
"parent": {
"id": 2,
"name": "Nairobi"
}
},
"id": 1631,
"name": "Century Cinemax Junction"
}
},
],
"slug": "bridge-of-spies",
"trailer_youtube_id": "",
"type": "movie",
"videos": [
{
"id": "795",
"name": "Bridge of Spies",
"youtube_id": "2-2x3r1m2I4"
}
],
"writer": "Matt Charman, Ethan Coen, Joel Coen",
"year": 2015
}, ...
]
Here's the view, I know the shows should also be a list, but in order to start testing I'll need the data to come in the right format. If it's involves too much rewriting I'm okay with links and explanation.
#memoize(timeout=60*60)
def movies_json():
today = datetime.date.today()
movies = Movie.objects.filter(shows__starts__gte=today)
results = []
number = len(movies)
for movie in movies:
print "Now Remaining: {0}".format(number)
number -= 1
medium = get_thumbnail(movie.picture(), '185x274', crop='center', quality=99).url
small = get_thumbnail(movie.picture(), '50x74', crop='center', quality=99).url
movie_details = {
'director':movie.director,
'plot':movie.plot,
'actors':movie.actors,
'content':movie.content,
'country':movie.country,
'description':movie.description,
'hot':movie.hot,
'id':movie.id,
'images':[{'medium':medium, 'small':small}],
'name':movie.name,
'plot':movie.plot,
'runtime':movie.runtime,
'slug':movie.slug,
'type':'movie',
'writer':movie.writer,
'year':movie.year,
}
youtube_details = movie.videos.filter(youtube_id__isnull=False)[0]
movie_details['trailer_youtube_id'] = youtube_details.youtube_id if youtube_details.youtube_id else ""
movie_details['videos'] = [
{
'id':youtube_details.id,
'name':movie.name,
'youtube_id':youtube_details.youtube_id,
}
]
shows = []
for show in movie.shows.all():
show_details = {
'booking_url':show.booking_url,
'starts':show.starts.isoformat(),
'starts_time':show.starts_time.isoformat(),
'venue': {
'address':show.venue.address,
'area': {
'id': show.venue.area.id,
'name': show.venue.area.name,
'parent': {
'id': show.venue.area.parent.id,
'name': show.venue.area.parent.name,
}
},
'id': show.venue.id,
'name': show.venue.name,
}
}
shows.append(show_details)
movie_details['shows'] = show_details
category_list = []
for category in movie.categories.all():
category_details = {
'id':category.id,
'name':category.name,
'parent_id':category.parent.id,
}
category_list.append(category_details)
movie_details['categories'] = category_list
results.append(movie_details)
return results
The data is returned by django rest framework 0.4.0

import json
json_obj = json.load(json_string)