how to flatten complex nested json using spark dataframe - python

I have a code which flattens json data properly but its little slow as data is complex(input is taken as dict), but I want a similar code which works on spark dataframe.
#Flattening code
from functools import (partial,
singledispatch)
from itertools import chain
from typing import (Dict,
List,
TypeVar)
Serializable = TypeVar('Serializable', None, int, bool, float, str,
dict, list, tuple)
Array = List[Serializable]
Object = Dict[str, Serializable]
def flatten(object_: Object,
*,
path_separator: str = '.') -> Array[Object]:
"""
Flattens given JSON object into list of objects with non-nested values.
>>> flatten({'a': 1})
[{'a': 1}]
>>> flatten({'a': [1, 2]})
[{'a': 1}, {'a': 2}]
>>> flatten({'a': {'b': None}})
[{'a.b': None}]
>>> flatten({'a': [1, 2], 'b': []})
[{'a': 1}, {'a': 2}]
"""
keys = set(object_)
result = [dict(object_)]
while keys:
key = keys.pop()
new_result = []
for index, record in enumerate(result):
try:
value = record[key]
except KeyError:
new_result.append(record)
else:
if isinstance(value, dict):
del record[key]
new_value = flatten_nested_objects(
value,
prefix=key + path_separator,
path_separator=path_separator
)
keys.update(new_value.keys())
new_result.append({**new_value, **record})
elif isinstance(value, list):
del record[key]
new_records = [
flatten_nested_objects(sub_value,
prefix=key + path_separator,
path_separator=path_separator)
for sub_value in value
]
keys.update(chain.from_iterable(map(dict.keys,
new_records)))
if new_records:
new_result.extend({**new_record, **record}
for new_record in new_records)
else:
new_result.append(record)
else:
new_result.append(record)
result = new_result
return result
#singledispatch
def flatten_nested_objects(object_: Serializable,
*,
prefix: str = '',
path_separator: str) -> Object:
return {prefix[:-len(path_separator)]: object_}
#flatten_nested_objects.register(dict)
def _(object_: Object,
*,
prefix: str = '',
path_separator: str) -> Object:
result = dict(object_)
for key in list(result):
result.update(flatten_nested_objects(result.pop(key),
prefix=(prefix + key
+ path_separator),
path_separator=path_separator))
return result
#flatten_nested_objects.register(list)
def _(object_: Array,
*,
prefix: str = '',
path_separator: str) -> Object:
return {prefix[:-len(path_separator)]: list(map(partial(
flatten_nested_objects,
path_separator=path_separator),
object_))}
how to perform similar operation with spark dataframe to handle any kind of structure and flatten to leaf level.
similar code which works on spark dataframe and flatten properly and handles any complex types

Related

how to transfer this flat dict to a nested dict?

There is a flat dict like this:
{"a.b": "foo", "a.c": "bar", "d.e.f":"baz"}
And how to use Python to transfer the dict to this:
{
"a":
{
"b": "foo",
"c": "bar"
},
"d":
{
"e":
{
"f": "baz"
}
}
}
You can split the your keys on . into the last value and everything before with:
*parents, key = k.split('.')
You can dig down into a nested dictionary (creating them if necessary) with something like:
end = reduce(lambda x, y: x.setdefault(y, {}) , parents, new_d)
here end will be the dictionary at the end of the chain. So at that point you can just assign the value. Something like:
from functools import reduce
d = {"a.b": "foo", "a.c": "bar", "d.e.f":"baz", "d.e.g":"M", 'l':"single_key_test" }
new_d = {}
for k in d:
*parents, key = k.split('.')
end = reduce(lambda x, y: x.setdefault(y, {}) , parents, new_d)
end[key] = d[k]
print(new_d)
result:
{'a': {'b': 'foo', 'c': 'bar'}, 'd': {'e': {'f': 'baz', 'g': 'M'}}, 'l': 'single_key_test'}
You would first have to split your keys by '.' to get the paths.
Then you can create a function that creates a nested dictionary from one path:
def make_nested_dict(iterable, final):
"""Make nested dict from iterable"""
if iterable:
head, *tail = iterable
return {head: make_nested_dict(tail, final)}
else:
return final
Which works as follows:
d = {"a.b": "foo", "a.c": "bar", "d.e.f":"baz"}
for key in d:
paths = key.split('.')
nested_path = make_nested_dict(paths, d[key])
print(nested_path)
And gives the following paths:
{'a': {'b': 'foo'}}
{'a': {'c': 'bar'}}
{'d': {'e': {'f': 'baz'}}}
Then you can create a function that merges two nested dictionaries together recursively:
def merge_nested_dicts(d1, d2):
"""Merge two nested dictionaries together"""
for key in d2:
if key in d1:
if isinstance(d2[key], dict) and isinstance(d1[key], dict):
merge_nested_dicts(d1[key], d2[key])
else:
d1[key] = d2[key]
return d1
Which you can merge by updating a resultant nested dict:
nested_dict = {}
for key in d:
paths = key.split('.')
nested_path = make_nested_dict(paths, d[key])
nested_dict = merge_nested_dicts(nested_dict, nested_path)
print(nested_dict)
And now this gives the following:
{'a': {'b': 'foo', 'c': 'bar'}, 'd': {'e': {'f': 'baz'}}}
Full code with some comments:
def make_nested_dict(iterable, final):
"""Make nested dictionary path with a final attribute"""
# If iterable, keep recursing
if iterable:
# Unpack key and rest of dict
head, *tail = iterable
# Return new dictionary, recursing on tail value
return {head: make_nested_dict(tail, final)}
# Otherwise assign final attribute
else:
return final
def merge_nested_dicts(d1, d2):
"""Merge two nested dictionaries together"""
for key in d2:
# If we have matching keys
if key in d1:
# Only merge if both values are dictionaries
if isinstance(d2[key], dict) and isinstance(d1[key], dict):
merge_nested_dicts(d1[key], d2[key])
# Otherwise assign normally
else:
d1[key] = d2[key]
return d1
if __name__ == "__main__":
d = {"a.b": "foo", "a.c": "bar", "d.e.f":"baz", "d.e.g":"M", 'l':"single_key_test" }
nested_dict = {}
for key in d:
# Create path
paths = key.split('.')
# Create nested path
nested_path = make_nested_dict(paths, d[key])
# Update result dict by merging with new dict
nested_dict = merge_nested_dicts(nested_dict, nested_path)
print(nested_dict)

Iterating over a nested dictionary

Purpose this code that works to iterate over a nested dictionary but I'm looking for an output that gives a tuple or list of [keys] and then [values]. Here's the code:
from collections import Mapping, Set, Sequence
string_types = (str, unicode) if str is bytes else (str, bytes)
iteritems = lambda mapping: getattr(mapping, 'iteritems', mapping.items)()
def recurse(obj, path=(), memo=None):
if memo is None:
memo = set()
iterator = None
if isinstance(obj, Mapping):
iterator = iteritems
elif isinstance(obj, (Sequence, Set)) and not isinstance(obj, string_types):
iterator = enumerate
if iterator:
if id(obj) not in memo:
memo.add(id(obj))
for path_component, value in iterator(obj):
for result in recurse(value, path + (path_component,), memo):
yield result
memo.remove(id(obj))
else:
yield path, obj
class addNestedDict(dict):
def __missing__(self, key):
value = self[key] = type(self)()
return value
loansDict=addNestedDict()
loansDict[15]['A']['B']=[1,2,3,4]
for k,v in recurse(loansDict):
print(k,v)
The output I'm looking for is one line (15 ,'A','B') [1,2,3,4] so that I can be able to reference k[0],k[1] and v[0] etc...
This seems to work:
results = AddNestedDict()
for k,v in recurse(loansDict):
results.setdefault(k[:-1], []).append(v)
result_key, result_value = results.items()[0]
print('{} {}'.format(result_key, result_value)) # -> (15, 'A', 'B') [1, 2, 3, 4]
I renamed your class AddNestedDict so it follows PEP 8 guidelines.

Convert tree to list of lists

I have a tree structure, something like:
Company -> Department -> Roles ->
I have a triple for loop-structure like this:
for company in self.Companies():
cName = company.Name()
for dept in company.Departments():
dName = department.Name()
for role in dept.Roles():
rName = role.Name()
roleID = role.ID()
The .Name() function returns a dept name like Android-Sales. Companies can zero or more departments.
So far the above is all I have. I am trying to develop this so I can get a list of lists:
Ideally this is what I would like. If something repeats, the next item in list should leave it blank. Or it might not have a field, in that case leave it blank too.
[
['Google', 'Android-Sales', 'Marketer', 'A123'],
['','Google-Play','Developer', 'A435'],
['','','Tester','A125'],
['','','','A126'],
['My Small Company','','Super Role','A123']
]
Or this would work too...
[
['Google', 'Android-Sales', 'Marketer', 'A123'],
['Google','Google-Play','Developer', 'A435'],
['Google','Google-Play','Tester','A125'],
['Google','Google-Play','Tester','A126'],
['My Small Company','','Super Role','A123'] (Here "My Small Company" has no > departments.
]
Each inner list should be of length 4.
Try something like this:
tree = {"Google":{"Android":"yes", "Nexus":"no"}}
list_of_lists = []
def listbuilder(sub_tree, current_list):
for key in sub_tree:
if isinstance(sub_tree[key], dict):
listbuilder(sub_tree[key], current_list + [key])
else:
list_of_lists.append(current_list + [key] + [sub_tree[key]])
listbuilder(tree,[])
print str(list_of_lists)
Output:
[['Google', 'Nexus', 'no'], ['Google', 'Android', 'yes']]
A while ago I needed to insert JSON files in a structured way into a database. Based on the solution described by #Gillespie, I was able to convert the file hierarchy into lists of lists.
The following script expects to read a JSON file whose list of items is initially identified by the parent key "value". Here is a simple example of the data.json file.
JSON File
{
"value":
[
{
"A": 0,
"B": 1,
"C": 2
},
{
"C": {
"E": 3,
"F": 4
},
"D": [
{
"G": 5
},
{
"H": 6
}
]
}
]
}
The following script reads a JSON file and turns it into tuple lists to be inserted into a database. Remember to remove special characters before creating tables and columns in the database.
Python Script
import json
def dict_to_list(sub_tree: dict, current_list: [str], items_list: [[str]]) -> [[str]]:
try: # Tree branches.
for key in sub_tree:
if isinstance(sub_tree[key], list):
for sub_item in sub_tree[key]:
dict_to_list(sub_tree=sub_item, current_list=current_list + [key], items_list=items_list)
elif isinstance(sub_tree[key], dict):
dict_to_list(sub_tree=sub_tree[key], current_list=current_list + [key], items_list=items_list)
else:
items_list.append(current_list + [key] + [sub_tree[key]])
except: # Tree leaf.
items_list.append(current_list + [str(sub_tree)])
return items_list
def json_data_to_samples_list(json_data: dict, data_key: str = 'value', column_sep: str = "_") -> [[(str, str)]]:
samples_list = []
for parent in json_data[data_key]:
column_value_tuples = []
for child in dict_to_list(sub_tree=parent, current_list=[], items_list=[]):
column = column_sep.join(child[:-1])
value = child[-1]
column_value_tuples.append((column, value))
samples_list.append(column_value_tuples)
return samples_list
def main() -> None:
json_file_path = "data.json"
with open(json_file_path, mode="r", encoding="utf-8") as file:
json_data = json.load(file)
samples_list = json_data_to_samples_list(json_data=json_data)
print(f"\nExtracted {len(samples_list)} items from the JSON file:")
for idx, parent in enumerate(samples_list):
print(f"\n>>>>> Child #{idx}:")
for child in parent:
print(f"\t\t{child}")
print()
if __name__ == '__main__':
main()
Expected Output
Extracted 2 items from the JSON file:
>>>>> Child #0:
('A', 0)
('B', 1)
('C', 2)
>>>>> Child #1:
('C_E', 3)
('C_F', 4)
('D_G', 5)
('D_H', 6)
In addition to my previous answer, you might also want to convert a JSON file to tuples that can be inserted into a database. In that case, you can use the following approach.
Before running the following script, you must import the Dict class defined in this post (just copy and paste it). This class is fundamental to our approach to serializing objects in JSON.
Python Script
Definition of the JSON class.
from __future__ import annotations
from collections import OrderedDict
import logging
logging.basicConfig(level="DEBUG")
LOGGER = logging.getLogger(__name__)
class JSON:
def __init__(self, dict_list: [dict], key_sep: str = "_", dump_objects: bool = False):
"""
Instantiates a JSON processing object.
Parameters
----------
dict_list: [dict]
List of dictionaries.
key_sep: str
Nested keys separator.
dump_objects: bool
Whether to dump objects.
References
----------
[1] 'JSON' class: https://stackoverflow.com/a/70791993/16109419
[2] 'Dict' class: https://stackoverflow.com/a/70908985/16109419
"""
self.key_sep = key_sep
self.dump_objects = dump_objects
# Serializing dictionaries before processing them:
self.dict_list = [self.serialize(data=d, dump_objects=dump_objects) for d in dict_list]
#staticmethod
def serialize(data: dict, dump_objects: bool = False) -> [dict]:
"""
Serializes the objects contained in the dictionary.
Parameters
----------
data: dict
Dictionary.
dump_objects: bool
Whether to dump objects.
Returns
-------
data: dict
Dictionary.
Notes
-----
This method is required to handle data types not supported by the JSON standard.
For instance, only native data types are supported in Python (e.g., str, int).
Custom objects values are dumped into the dictionaries structure.
"""
serialized_d = Dict(data=data)
for keys, value in serialized_d.items():
parsed, parsed_value = False, None
if hasattr(value, 'isoformat'): # Date/Datetime object.
parsed = True
parsed_value = value.isoformat() if dump_objects else value
elif hasattr(value, '__dict__'): # Custom object.
parsed = True
value_vars = vars(value)
value_vars_str = str(value_vars)
value_str = str(value)
if value_vars_str == value_str: # Dict-based object.
parsed_value = JSON.serialize(data=value_vars, dump_objects=dump_objects)
else: # Not dict-based object.
if dump_objects:
parsed_value = JSON.serialize(data=value_vars, dump_objects=dump_objects)
else:
parsed_value = value_str
if parsed:
serialized_d.set(keys=keys, value=parsed_value)
data = serialized_d.data
return data
def dict_to_list(
self,
sub_tree: dict,
current_list: [str],
items_list: [[object]]
) -> [[object]]:
"""
Convert dictionary to items list.
Parameters
----------
sub_tree
current_list
items_list
Returns
-------
items_list: [[object]]
List of items list.
"""
try: # Tree branches.
for key in sub_tree:
if isinstance(sub_tree[key], (list, tuple)):
for sub_item in sub_tree[key]:
self.dict_to_list(
sub_tree=sub_item,
current_list=current_list + [key],
items_list=items_list
)
elif isinstance(sub_tree[key], dict):
self.dict_to_list(
sub_tree=sub_tree[key],
current_list=current_list + [key],
items_list=items_list
)
else:
items_list.append(current_list + [key] + [sub_tree[key]])
except: # Tree leaf.
items_list.append(current_list + [sub_tree])
return items_list
def extract_entries(self) -> [[(str, object)]]:
"""
Extracts entries from a dictionary.
Returns
-------
entries: [[(str, object)]]
List of key-value items list.
"""
entries = []
for parent in self.dict_list:
key_value_tuples = []
for child in self.dict_to_list(sub_tree=parent, current_list=[], items_list=[]):
key_parts = child[:-1]
key = self.key_sep.join(key_parts)
value = child[-1]
key_value_tuples.append((key, value))
entries.append(key_value_tuples)
return entries
#staticmethod
def get_nth_element(
items: [(str, object)],
element: str,
nth: int = 1
) -> ((str, object), bool):
"""
Get nth element (occurrence) from items list.
Parameters
----------
items: [(str, object)]
Items list.
element: str
Item key.
nth: int
Nth element position.
Returns
-------
nth_element, index_out_of_bounds: ((str, object), bool)
Nth element, and whether it was not found.
"""
assert nth >= 1, f"'nth' ({nth}) must be >= 1."
occurrences = [i for i in items if i[0] == element]
n_occurrences = len(occurrences)
if n_occurrences:
index_out_of_bounds = True if nth > n_occurrences else False
nth_element = occurrences[min(nth, n_occurrences) - 1]
else:
nth_element = None
index_out_of_bounds = True
return nth_element, index_out_of_bounds
def to_tuples(self) -> ([str], [tuple]):
"""
Convert JSON semi-structured data into structured tuples data.
Returns
-------
keys, values: ([str], [tuple])
List of keys and values.
Examples
--------
data = {
"values":
[
{
"A": 0,
"B": 1,
"C": 2
},
{
"C": {
"E": 3,
"F": 4
},
"D": [
{
"G": 5
},
{
"H": 6
}
]
}
]
}
self.dict_list = data['values']
... return (
["A", "B", "C", "C_E", "C_F", "D_G", "D_H"],
[
(0, 1, 2, None, None, None, None),
(None, None, None, 3, 4, 5, 6)
]
)
"""
LOGGER.info(f"Extracting values tuples from JSON file...")
entries = self.extract_entries()
keys = list(
OrderedDict.fromkeys(
[
key_value_tuple[0]
for samples in entries
for key_value_tuple in samples
]
)
)
n_entries = len(entries)
n_keys = len(keys)
values = []
for tuples, index in zip(entries, range(1, n_entries + 1)):
LOGGER.debug(f"Processing values from entry {index}/{n_entries} ({((index / n_entries) * 100):.2f}%)...")
for i in range(1, len(tuples) + 1):
index_out_of_bounds_count = 0
row = []
for c in keys:
key_value_tuple, index_out_of_bounds = self.get_nth_element(items=tuples, element=c, nth=i)
row.append(key_value_tuple[1]) if key_value_tuple else row.append(None)
if index_out_of_bounds:
index_out_of_bounds_count += 1
if index_out_of_bounds_count == n_keys:
break
if row.count(None) != n_keys:
values.append(row)
LOGGER.info(f"Successfully extracted values tuples from JSON file!")
return keys, values
Definition of the main method to show examples.
from datetime import date, datetime
class B:
def __init__(self) -> B:
self.data = {'a': {'b': [{'c': 0}, {'d': 1}]}}
class A:
def __init__(self) -> A:
self.name = "my_name"
self.attr = "my_attr"
self.b = B()
def main(dump_objects: bool) -> None:
data = {'values': [{'A': A(), 'C': 3, 'D': date.today(), 'E': datetime.now(), 'F': {'G': {'H': 8}}}]}
json_data = JSON(dict_list=data['values'], dump_objects=dump_objects)
print("JSON to tuples test:\n")
for i, d in enumerate(json_data.dict_list, start=1):
print(f"d{i} = {d}\n")
keys, values = json_data.to_tuples()
print(f"\nKeys:")
for i, key in enumerate(keys, start=1):
print(f"\t{i}. {key}")
print(f"\nValues:")
for i, row in enumerate(values, start=1):
print(f"\t{i}:")
for j, value in enumerate(row, start=1):
print(f"\t\t{j}. {value} ({type(value)})")
Example 1: not dumping objects
Calling the main method.
main(dump_objects=False)
Output:
JSON to tuples test:
d1 = {'A': '<__main__.A object at 0x000002002F436290>', 'C': 3, 'D': datetime.date(2022, 2, 3), 'E': datetime.datetime(2022, 2, 3, 15, 54, 4, 874847), 'F': {'G': {'H': 8}}}
INFO:__main__:Extracting values tuples from JSON file...
DEBUG:__main__:Processing values from entry 1/1 (100.00%)...
INFO:__main__:Successfully extracted values tuples from JSON file!
Keys:
1. A
2. C
3. D
4. E
5. F_G_H
Values:
1:
1. <__main__.A object at 0x000002002F436290> (<class 'str'>)
2. 3 (<class 'int'>)
3. 2022-02-03 (<class 'datetime.date'>)
4. 2022-02-03 15:54:04.874847 (<class 'datetime.datetime'>)
5. 8 (<class 'int'>)
Example 2: dumping objects
Calling the main method.
main(dump_objects=True)
Output:
JSON to tuples test:
d1 = {'A': {'name': 'my_name', 'attr': 'my_attr', 'b': {'data': {'a': {'b': [{'c': 0}, {'d': 1}]}}}}, 'C': 3, 'D': '2022-02-03', 'E': '2022-02-03T15:55:29.014941', 'F': {'G': {'H': 8}}}
INFO:__main__:Extracting values tuples from JSON file...
DEBUG:__main__:Processing values from entry 1/1 (100.00%)...
INFO:__main__:Successfully extracted values tuples from JSON file!
Keys:
1. A_name
2. A_attr
3. A_b_data_a_b_c
4. A_b_data_a_b_d
5. C
6. D
7. E
8. F_G_H
Values:
1:
1. my_name (<class 'str'>)
2. my_attr (<class 'str'>)
3. 0 (<class 'int'>)
4. 1 (<class 'int'>)
5. 3 (<class 'int'>)
6. 2022-02-03 (<class 'str'>)
7. 2022-02-03T15:55:29.014941 (<class 'str'>)
8. 8 (<class 'int'>)

urlencode a multidimensional dictionary in python

How can I get a URL-encoded version of a multidimensional dictionary in Python? Unfortunately, urllib.urlencode() only works in a single dimension. I would need a version capable of recursively encoding the dictionary.
For example, if I have the following dictionary:
{'a': 'b', 'c': {'d': 'e'}}
I want to obtain the following string:
a=b&c[d]=e
OK people. I implemented it myself:
import urllib
def recursive_urlencode(d):
"""URL-encode a multidimensional dictionary.
>>> data = {'a': 'b&c', 'd': {'e': {'f&g': 'h*i'}}, 'j': 'k'}
>>> recursive_urlencode(data)
u'a=b%26c&j=k&d[e][f%26g]=h%2Ai'
"""
def recursion(d, base=[]):
pairs = []
for key, value in d.items():
new_base = base + [key]
if hasattr(value, 'values'):
pairs += recursion(value, new_base)
else:
new_pair = None
if len(new_base) > 1:
first = urllib.quote(new_base.pop(0))
rest = map(lambda x: urllib.quote(x), new_base)
new_pair = "%s[%s]=%s" % (first, ']['.join(rest), urllib.quote(unicode(value)))
else:
new_pair = "%s=%s" % (urllib.quote(unicode(key)), urllib.quote(unicode(value)))
pairs.append(new_pair)
return pairs
return '&'.join(recursion(d))
if __name__ == "__main__":
import doctest
doctest.testmod()
Still, I'd be interested to know if there's a better way to do this. I can't believe Python's standard library doesn't implement this.
Something like this?
a = {'a': 'b', 'c': {'d': 'e'}}
url = urllib.urlencode([('%s[%s]'%(k,v.keys()[0]), v.values()[0] ) if type(v)==dict else (k,v) for k,v in a.iteritems()])
url = 'a=b&c%5Bd%5D=e'
Based on the code of #malaney, I think that the code below emulates the PHP function http_build_query() quite well.
#!/usr/bin/env python3
import urllib.parse
def http_build_query(data):
parents = list()
pairs = dict()
def renderKey(parents):
depth, outStr = 0, ''
for x in parents:
s = "[%s]" if depth > 0 or isinstance(x, int) else "%s"
outStr += s % str(x)
depth += 1
return outStr
def r_urlencode(data):
if isinstance(data, list) or isinstance(data, tuple):
for i in range(len(data)):
parents.append(i)
r_urlencode(data[i])
parents.pop()
elif isinstance(data, dict):
for key, value in data.items():
parents.append(key)
r_urlencode(value)
parents.pop()
else:
pairs[renderKey(parents)] = str(data)
return pairs
return urllib.parse.urlencode(r_urlencode(data))
if __name__ == '__main__':
payload = {
'action': 'add',
'controller': 'invoice',
'code': 'debtor',
'InvoiceLines': [
{'PriceExcl': 150, 'Description': 'Setupfee'},
{'PriceExcl':49.99, 'Description':'Subscription'}
],
'date': '2016-08-01',
'key': 'Yikes&ampersand'
}
print(http_build_query(payload))
payload2 = [
'item1',
'item2'
]
print(http_build_query(payload2))
I think the code below may be what you want
import urllib.parse
def url_encoder(params):
g_encode_params = {}
def _encode_params(params, p_key=None):
encode_params = {}
if isinstance(params, dict):
for key in params:
encode_key = '{}[{}]'.format(p_key,key)
encode_params[encode_key] = params[key]
elif isinstance(params, (list, tuple)):
for offset,value in enumerate(params):
encode_key = '{}[{}]'.format(p_key, offset)
encode_params[encode_key] = value
else:
g_encode_params[p_key] = params
for key in encode_params:
value = encode_params[key]
_encode_params(value, key)
if isinstance(params, dict):
for key in params:
_encode_params(params[key], key)
return urllib.parse.urlencode(g_encode_params)
if __name__ == '__main__':
params = {'name': 'interface_name', 'interfaces': [{'interface': 'inter1'}, {'interface': 'inter2'}]}
print(url_encoder(params))
the output is
interfaces%5B1%5D%5Binterface%5D=inter2&name=interface_name&interfaces%5B0%5D%5Binterface%5D=inter1
which is look like
interfaces[1][interface]=inter2&name=interface_name&interfaces[0][interface]=inter1
PS: you may want use OrderDict to replace dict above
The above solution only works for arrays with depth < 2. The code below will properly urlencode a multidimensional array of any depth.
#!/usr/bin/env python
import sys
import urllib
def recursive_urlencode(data):
def r_urlencode(data, parent=None, pairs=None):
if pairs is None:
pairs = {}
if parent is None:
parents = []
else:
parents = parent
for key, value in data.items():
if hasattr(value, 'values'):
parents.append(key)
r_urlencode(value, parents, pairs)
parents.pop()
else:
pairs[renderKey(parents + [key])] = renderVal(value)
return pairs
return urllib.urlencode(r_urlencode(data))
def renderKey(parents):
depth, outStr = 0, ''
for x in parents:
str = "[%s]" if depth > 0 else "%s"
outStr += str % renderVal(x)
depth += 1
return outStr
def renderVal(val):
return urllib.quote(unicode(val))
def main():
print recursive_urlencode(payload)
if __name__ == '__main__':
sys.exit(main())
The function get_encoded_url_params() takes a dict as argument and returns url encoded form of the dict.
def get_encoded_url_params(d):
"""URL-encode a nested dictionary.
:param d = dict
:returns url encoded string with dict key-value pairs as query parameters
e.g.
if d = { "addr":{ "country": "US", "line": ["a","b"] },
"routing_number": "011100915", "token": "asdf"
}
:returns 'addr[country]=US&addr[line][0]=a&addr[line][1]=b&routing_number=011100915&token=asdf'
or 'addr%5Bcountry%5D=US&addr%5Bline%5D%5B0%5D=a&addr%5Bline%5D%5B1%5D=b&routing_number=011100915&token=asdf'
(which is url encoded form of the former using quote_plus())
"""
def get_pairs(value, base):
if isinstance(value, dict):
return get_dict_pairs(value, base)
elif isinstance(value, list):
return get_list_pairs(value, base)
else:
return [base + '=' + str(value)]
# use quote_plus() to get url encoded string
# return [quote_plus(base) + '=' + quote_plus(str(value))]
def get_list_pairs(li, base):
pairs = []
for idx, value in enumerate(li):
new_base = base + '[' + str(idx) + ']'
pairs += get_pairs(value, new_base)
return pairs
def get_dict_pairs(d, base=''):
pairs = []
for key, value in d.items():
new_base = key if base == '' else base + '[' + key + ']'
pairs += get_pairs(value, new_base)
return pairs
return '&'.join(get_dict_pairs(d))
what about json.dumps and json.loads?
d = {'a': 'b', 'c': {'d': 'e'}}
s = json.dumps(d) # s: '{"a": "b", "c": {"d": "e"}}'
json.loads(s) # -> d
what about this simplified version:
def _clean(value):
return urllib.quote(unicode(value))
'&'.join([ v for val in [[ "%s[%s]=%s"%(k,ik, _(iv))
for ik, iv in v.items()] if type(v)==dict else ["%s=%s"%(k,_(v))]
for k,v in data.items() ]
for v in val ])
I agree is not readable, maybe flattening the list can be better done with itertools.chain instead of another list comprehension.
This only goes 1 level deeper, yours can go N levels deeper if you would add some logic to manage N numbers of "[%s]" depending on the level, but I guess is not that necesary
If you want to convert python dict/list/nested to PHP Array like urlencoded string.
In python, most of the data type you want to convert to urlencoded maybe: dict list tuple nested of them, Like
a = [1, 2]
print(recursive_urlencode(a))
# 0=1&1=2
a2 = (1, '2')
print(recursive_urlencode(a2))
# 0=1&1=2
b = {'a': 11, 'b': 'foo'}
print(recursive_urlencode(b))
# a=11&b=foo
c = {'a': 11, 'b': [1, 2]}
print(recursive_urlencode(c))
# a=11&b[0]=1&b[1]=2
d = [1, {'a': 11, 'b': 22}]
print(recursive_urlencode(d))
# 0=1&1[a]=11&1[b]=22
e = {'a': 11, 'b': [1, {'c': 123}, [3, 'foo']]}
print(recursive_urlencode(e))
# a=11&b[0]=1&b[1][c]=123&b[2][0]=3&b[2][1]=foo
https://github.com/Viky-zhang/to_php_post_arr
P.S. some code from: https://stackoverflow.com/a/4014164/2752670

How to convert a nested Python dict to object?

I'm searching for an elegant way to get data using attribute access on a dict with some nested dicts and lists (i.e. javascript-style object syntax).
For example:
>>> d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
Should be accessible in this way:
>>> x = dict2obj(d)
>>> x.a
1
>>> x.b.c
2
>>> x.d[1].foo
bar
I think, this is not possible without recursion, but what would be a nice way to get an object style for dicts?
Update: In Python 2.6 and onwards, consider whether the namedtuple data structure suits your needs:
>>> from collections import namedtuple
>>> MyStruct = namedtuple('MyStruct', 'a b d')
>>> s = MyStruct(a=1, b={'c': 2}, d=['hi'])
>>> s
MyStruct(a=1, b={'c': 2}, d=['hi'])
>>> s.a
1
>>> s.b
{'c': 2}
>>> s.c
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
AttributeError: 'MyStruct' object has no attribute 'c'
>>> s.d
['hi']
The alternative (original answer contents) is:
class Struct:
def __init__(self, **entries):
self.__dict__.update(entries)
Then, you can use:
>>> args = {'a': 1, 'b': 2}
>>> s = Struct(**args)
>>> s
<__main__.Struct instance at 0x01D6A738>
>>> s.a
1
>>> s.b
2
Surprisingly no one has mentioned Bunch. This library is exclusively meant to provide attribute style access to dict objects and does exactly what the OP wants. A demonstration:
>>> from bunch import bunchify
>>> d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
>>> x = bunchify(d)
>>> x.a
1
>>> x.b.c
2
>>> x.d[1].foo
'bar'
A Python 3 library is available at https://github.com/Infinidat/munch - Credit goes to codyzu
>>> from munch import DefaultMunch
>>> d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
>>> obj = DefaultMunch.fromDict(d)
>>> obj.b.c
2
>>> obj.a
1
>>> obj.d[1].foo
'bar'
class obj(object):
def __init__(self, d):
for k, v in d.items():
if isinstance(k, (list, tuple)):
setattr(self, k, [obj(x) if isinstance(x, dict) else x for x in v])
else:
setattr(self, k, obj(v) if isinstance(v, dict) else v)
>>> d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
>>> x = obj(d)
>>> x.b.c
2
>>> x.d[1].foo
'bar'
x = type('new_dict', (object,), d)
then add recursion to this and you're done.
edit this is how I'd implement it:
>>> d
{'a': 1, 'b': {'c': 2}, 'd': ['hi', {'foo': 'bar'}]}
>>> def obj_dic(d):
top = type('new', (object,), d)
seqs = tuple, list, set, frozenset
for i, j in d.items():
if isinstance(j, dict):
setattr(top, i, obj_dic(j))
elif isinstance(j, seqs):
setattr(top, i,
type(j)(obj_dic(sj) if isinstance(sj, dict) else sj for sj in j))
else:
setattr(top, i, j)
return top
>>> x = obj_dic(d)
>>> x.a
1
>>> x.b.c
2
>>> x.d[1].foo
'bar'
# Applies to Python-3 Standard Library
class Struct(object):
def __init__(self, data):
for name, value in data.items():
setattr(self, name, self._wrap(value))
def _wrap(self, value):
if isinstance(value, (tuple, list, set, frozenset)):
return type(value)([self._wrap(v) for v in value])
else:
return Struct(value) if isinstance(value, dict) else value
# Applies to Python-2 Standard Library
class Struct(object):
def __init__(self, data):
for name, value in data.iteritems():
setattr(self, name, self._wrap(value))
def _wrap(self, value):
if isinstance(value, (tuple, list, set, frozenset)):
return type(value)([self._wrap(v) for v in value])
else:
return Struct(value) if isinstance(value, dict) else value
Can be used with any sequence/dict/value structure of any depth.
There's a
collection helper called namedtuple, that can do this for you:
from collections import namedtuple
d_named = namedtuple('Struct', d.keys())(*d.values())
In [7]: d_named
Out[7]: Struct(a=1, b={'c': 2}, d=['hi', {'foo': 'bar'}])
In [8]: d_named.a
Out[8]: 1
If your dict is coming from json.loads(), you can turn it into an object instead (rather than a dict) in one line:
import json
from collections import namedtuple
json.loads(data, object_hook=lambda d: namedtuple('X', d.keys())(*d.values()))
See also How to convert JSON data into a Python object.
You can leverage the json module of the standard library with a custom object hook:
import json
class obj(object):
def __init__(self, dict_):
self.__dict__.update(dict_)
def dict2obj(d):
return json.loads(json.dumps(d), object_hook=obj)
Example usage:
>>> d = {'a': 1, 'b': {'c': 2}, 'd': ['hi', {'foo': 'bar'}]}
>>> o = dict2obj(d)
>>> o.a
1
>>> o.b.c
2
>>> o.d[0]
u'hi'
>>> o.d[1].foo
u'bar'
And it is not strictly read-only as it is with namedtuple, i.e. you can change values – not structure:
>>> o.b.c = 3
>>> o.b.c
3
Taking what I feel are the best aspects of the previous examples, here's what I came up with:
class Struct:
"""The recursive class for building and representing objects with."""
def __init__(self, obj):
for k, v in obj.items():
if isinstance(v, dict):
setattr(self, k, Struct(v))
else:
setattr(self, k, v)
def __getitem__(self, val):
return self.__dict__[val]
def __repr__(self):
return '{%s}' % str(', '.join('%s : %s' % (k, repr(v)) for (k, v) in self.__dict__.items()))
I ended up trying BOTH the AttrDict and the Bunch libraries and found them to be way too slow for my uses. After a friend and I looked into it, we found that the main method for writing these libraries results in the library aggressively recursing through a nested object and making copies of the dictionary object throughout. With this in mind, we made two key changes. 1) We made attributes lazy-loaded 2) instead of creating copies of a dictionary object, we create copies of a light-weight proxy object. This is the final implementation. The performance increase of using this code is incredible. When using AttrDict or Bunch, these two libraries alone consumed 1/2 and 1/3 respectively of my request time(what!?). This code reduced that time to almost nothing(somewhere in the range of 0.5ms). This of course depends on your needs, but if you are using this functionality quite a bit in your code, definitely go with something simple like this.
class DictProxy(object):
def __init__(self, obj):
self.obj = obj
def __getitem__(self, key):
return wrap(self.obj[key])
def __getattr__(self, key):
try:
return wrap(getattr(self.obj, key))
except AttributeError:
try:
return self[key]
except KeyError:
raise AttributeError(key)
# you probably also want to proxy important list properties along like
# items(), iteritems() and __len__
class ListProxy(object):
def __init__(self, obj):
self.obj = obj
def __getitem__(self, key):
return wrap(self.obj[key])
# you probably also want to proxy important list properties along like
# __iter__ and __len__
def wrap(value):
if isinstance(value, dict):
return DictProxy(value)
if isinstance(value, (tuple, list)):
return ListProxy(value)
return value
See the original implementation here by https://stackoverflow.com/users/704327/michael-merickel.
The other thing to note, is that this implementation is pretty simple and doesn't implement all of the methods you might need. You'll need to write those as required on the DictProxy or ListProxy objects.
If you want to access dict keys as an object (or as a dict for difficult keys), do it recursively, and also be able to update the original dict, you could do:
class Dictate(object):
"""Object view of a dict, updating the passed in dict when values are set
or deleted. "Dictate" the contents of a dict...: """
def __init__(self, d):
# since __setattr__ is overridden, self.__dict = d doesn't work
object.__setattr__(self, '_Dictate__dict', d)
# Dictionary-like access / updates
def __getitem__(self, name):
value = self.__dict[name]
if isinstance(value, dict): # recursively view sub-dicts as objects
value = Dictate(value)
return value
def __setitem__(self, name, value):
self.__dict[name] = value
def __delitem__(self, name):
del self.__dict[name]
# Object-like access / updates
def __getattr__(self, name):
return self[name]
def __setattr__(self, name, value):
self[name] = value
def __delattr__(self, name):
del self[name]
def __repr__(self):
return "%s(%r)" % (type(self).__name__, self.__dict)
def __str__(self):
return str(self.__dict)
Example usage:
d = {'a': 'b', 1: 2}
dd = Dictate(d)
assert dd.a == 'b' # Access like an object
assert dd[1] == 2 # Access like a dict
# Updates affect d
dd.c = 'd'
assert d['c'] == 'd'
del dd.a
del dd[1]
# Inner dicts are mapped
dd.e = {}
dd.e.f = 'g'
assert dd['e'].f == 'g'
assert d == {'c': 'd', 'e': {'f': 'g'}}
>>> def dict2obj(d):
if isinstance(d, list):
d = [dict2obj(x) for x in d]
if not isinstance(d, dict):
return d
class C(object):
pass
o = C()
for k in d:
o.__dict__[k] = dict2obj(d[k])
return o
>>> d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
>>> x = dict2obj(d)
>>> x.a
1
>>> x.b.c
2
>>> x.d[1].foo
'bar'
In 2021, use pydantic BaseModel - will convert nested dicts and nested json objects to python objects and vice versa:
https://pydantic-docs.helpmanual.io/usage/models/
>>> class Foo(BaseModel):
... count: int
... size: float = None
...
>>>
>>> class Bar(BaseModel):
... apple = 'x'
... banana = 'y'
...
>>>
>>> class Spam(BaseModel):
... foo: Foo
... bars: List[Bar]
...
>>>
>>> m = Spam(foo={'count': 4}, bars=[{'apple': 'x1'}, {'apple': 'x2'}])
Object to dict
>>> print(m.dict())
{'foo': {'count': 4, 'size': None}, 'bars': [{'apple': 'x1', 'banana': 'y'}, {'apple': 'x2', 'banana': 'y'}]}
Object to JSON
>>> print(m.json())
{"foo": {"count": 4, "size": null}, "bars": [{"apple": "x1", "banana": "y"}, {"apple": "x2", "banana": "y"}]}
Dict to object
>>> spam = Spam.parse_obj({'foo': {'count': 4, 'size': None}, 'bars': [{'apple': 'x1', 'banana': 'y'}, {'apple': 'x2', 'banana': 'y2'}]})
>>> spam
Spam(foo=Foo(count=4, size=None), bars=[Bar(apple='x1', banana='y'), Bar(apple='x2', banana='y2')])
JSON to object
>>> spam = Spam.parse_raw('{"foo": {"count": 4, "size": null}, "bars": [{"apple": "x1", "banana": "y"}, {"apple": "x2", "banana": "y"}]}')
>>> spam
Spam(foo=Foo(count=4, size=None), bars=[Bar(apple='x1', banana='y'), Bar(apple='x2', banana='y')])
x.__dict__.update(d) should do fine.
Typically you want to mirror dict hierarchy into your object but not list or tuples which are typically at lowest level. So this is how I did this:
class defDictToObject(object):
def __init__(self, myDict):
for key, value in myDict.items():
if type(value) == dict:
setattr(self, key, defDictToObject(value))
else:
setattr(self, key, value)
So we do:
myDict = { 'a': 1,
'b': {
'b1': {'x': 1,
'y': 2} },
'c': ['hi', 'bar']
}
and get:
x.b.b1.x 1
x.c ['hi', 'bar']
This should get your started:
class dict2obj(object):
def __init__(self, d):
self.__dict__['d'] = d
def __getattr__(self, key):
value = self.__dict__['d'][key]
if type(value) == type({}):
return dict2obj(value)
return value
d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
x = dict2obj(d)
print x.a
print x.b.c
print x.d[1].foo
It doesn't work for lists, yet. You'll have to wrap the lists in a UserList and overload __getitem__ to wrap dicts.
I know there's already a lot of answers here already and I'm late to the party but this method will recursively and 'in place' convert a dictionary to an object-like structure... Works in 3.x.x
def dictToObject(d):
for k,v in d.items():
if isinstance(v, dict):
d[k] = dictToObject(v)
return namedtuple('object', d.keys())(*d.values())
# Dictionary created from JSON file
d = {
'primaryKey': 'id',
'metadata':
{
'rows': 0,
'lastID': 0
},
'columns':
{
'col2': {
'dataType': 'string',
'name': 'addressLine1'
},
'col1': {
'datatype': 'string',
'name': 'postcode'
},
'col3': {
'dataType': 'string',
'name': 'addressLine2'
},
'col0': {
'datatype': 'integer',
'name': 'id'
},
'col4': {
'dataType': 'string',
'name': 'contactNumber'
}
},
'secondaryKeys': {}
}
d1 = dictToObject(d)
d1.columns.col1 # == object(datatype='string', name='postcode')
d1.metadata.rows # == 0
from mock import Mock
d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
my_data = Mock(**d)
# We got
# my_data.a == 1
This also works well too
class DObj(object):
pass
dobj = Dobj()
dobj.__dict__ = {'a': 'aaa', 'b': 'bbb'}
print dobj.a
>>> aaa
print dobj.b
>>> bbb
The simplest way would be using collections.namedtuple.
I find the following 4-liner the most beautiful, which supports nested dictionaries:
def dict_to_namedtuple(typename, data):
return namedtuple(typename, data.keys())(
*(dict_to_namedtuple(typename + '_' + k, v) if isinstance(v, dict) else v for k, v in data.items())
)
The output will look good as well:
>>> nt = dict_to_namedtuple('config', {
... 'path': '/app',
... 'debug': {'level': 'error', 'stream': 'stdout'}
... })
>>> print(nt)
config(path='/app', debug=config_debug(level='error', stream='stdout'))
>>> print(nt.debug.level)
'error'
Let me explain a solution I almost used some time ago. But first, the reason I did not is illustrated by the fact that the following code:
d = {'from': 1}
x = dict2obj(d)
print x.from
gives this error:
File "test.py", line 20
print x.from == 1
^
SyntaxError: invalid syntax
Because "from" is a Python keyword there are certain dictionary keys you cannot allow.
Now my solution allows access to the dictionary items by using their names directly. But it also allows you to use "dictionary semantics". Here is the code with example usage:
class dict2obj(dict):
def __init__(self, dict_):
super(dict2obj, self).__init__(dict_)
for key in self:
item = self[key]
if isinstance(item, list):
for idx, it in enumerate(item):
if isinstance(it, dict):
item[idx] = dict2obj(it)
elif isinstance(item, dict):
self[key] = dict2obj(item)
def __getattr__(self, key):
return self[key]
d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
x = dict2obj(d)
assert x.a == x['a'] == 1
assert x.b.c == x['b']['c'] == 2
assert x.d[1].foo == x['d'][1]['foo'] == "bar"
Old Q&A, but I get something more to talk. Seems no one talk about recursive dict. This is my code:
#!/usr/bin/env python
class Object( dict ):
def __init__( self, data = None ):
super( Object, self ).__init__()
if data:
self.__update( data, {} )
def __update( self, data, did ):
dataid = id(data)
did[ dataid ] = self
for k in data:
dkid = id(data[k])
if did.has_key(dkid):
self[k] = did[dkid]
elif isinstance( data[k], Object ):
self[k] = data[k]
elif isinstance( data[k], dict ):
obj = Object()
obj.__update( data[k], did )
self[k] = obj
obj = None
else:
self[k] = data[k]
def __getattr__( self, key ):
return self.get( key, None )
def __setattr__( self, key, value ):
if isinstance(value,dict):
self[key] = Object( value )
else:
self[key] = value
def update( self, *args ):
for obj in args:
for k in obj:
if isinstance(obj[k],dict):
self[k] = Object( obj[k] )
else:
self[k] = obj[k]
return self
def merge( self, *args ):
for obj in args:
for k in obj:
if self.has_key(k):
if isinstance(self[k],list) and isinstance(obj[k],list):
self[k] += obj[k]
elif isinstance(self[k],list):
self[k].append( obj[k] )
elif isinstance(obj[k],list):
self[k] = [self[k]] + obj[k]
elif isinstance(self[k],Object) and isinstance(obj[k],Object):
self[k].merge( obj[k] )
elif isinstance(self[k],Object) and isinstance(obj[k],dict):
self[k].merge( obj[k] )
else:
self[k] = [ self[k], obj[k] ]
else:
if isinstance(obj[k],dict):
self[k] = Object( obj[k] )
else:
self[k] = obj[k]
return self
def test01():
class UObject( Object ):
pass
obj = Object({1:2})
d = {}
d.update({
"a": 1,
"b": {
"c": 2,
"d": [ 3, 4, 5 ],
"e": [ [6,7], (8,9) ],
"self": d,
},
1: 10,
"1": 11,
"obj": obj,
})
x = UObject(d)
assert x.a == x["a"] == 1
assert x.b.c == x["b"]["c"] == 2
assert x.b.d[0] == 3
assert x.b.d[1] == 4
assert x.b.e[0][0] == 6
assert x.b.e[1][0] == 8
assert x[1] == 10
assert x["1"] == 11
assert x[1] != x["1"]
assert id(x) == id(x.b.self.b.self) == id(x.b.self)
assert x.b.self.a == x.b.self.b.self.a == 1
x.x = 12
assert x.x == x["x"] == 12
x.y = {"a":13,"b":[14,15]}
assert x.y.a == 13
assert x.y.b[0] == 14
def test02():
x = Object({
"a": {
"b": 1,
"c": [ 2, 3 ]
},
1: 6,
2: [ 8, 9 ],
3: 11,
})
y = Object({
"a": {
"b": 4,
"c": [ 5 ]
},
1: 7,
2: 10,
3: [ 12 , 13 ],
})
z = {
3: 14,
2: 15,
"a": {
"b": 16,
"c": 17,
}
}
x.merge( y, z )
assert 2 in x.a.c
assert 3 in x.a.c
assert 5 in x.a.c
assert 1 in x.a.b
assert 4 in x.a.b
assert 8 in x[2]
assert 9 in x[2]
assert 10 in x[2]
assert 11 in x[3]
assert 12 in x[3]
assert 13 in x[3]
assert 14 in x[3]
assert 15 in x[2]
assert 16 in x.a.b
assert 17 in x.a.c
if __name__ == '__main__':
test01()
test02()
Wanted to upload my version of this little paradigm.
class Struct(dict):
def __init__(self,data):
for key, value in data.items():
if isinstance(value, dict):
setattr(self, key, Struct(value))
else:
setattr(self, key, type(value).__init__(value))
dict.__init__(self,data)
It preserves the attributes for the type that's imported into the class. My only concern would be overwriting methods from within the dictionary your parsing. But otherwise seems solid!
Here is another way to implement SilentGhost's original suggestion:
def dict2obj(d):
if isinstance(d, dict):
n = {}
for item in d:
if isinstance(d[item], dict):
n[item] = dict2obj(d[item])
elif isinstance(d[item], (list, tuple)):
n[item] = [dict2obj(elem) for elem in d[item]]
else:
n[item] = d[item]
return type('obj_from_dict', (object,), n)
else:
return d
I stumbled upon the case I needed to recursively convert a list of dicts to list of objects, so based on Roberto's snippet here what did the work for me:
def dict2obj(d):
if isinstance(d, dict):
n = {}
for item in d:
if isinstance(d[item], dict):
n[item] = dict2obj(d[item])
elif isinstance(d[item], (list, tuple)):
n[item] = [dict2obj(elem) for elem in d[item]]
else:
n[item] = d[item]
return type('obj_from_dict', (object,), n)
elif isinstance(d, (list, tuple,)):
l = []
for item in d:
l.append(dict2obj(item))
return l
else:
return d
Note that any tuple will be converted to its list equivalent, for obvious reasons.
Hope this helps someone as much as all your answers did for me, guys.
What about just assigning your dict to the __dict__ of an empty object?
class Object:
"""If your dict is "flat", this is a simple way to create an object from a dict
>>> obj = Object()
>>> obj.__dict__ = d
>>> d.a
1
"""
pass
Of course this fails on your nested dict example unless you walk the dict recursively:
# For a nested dict, you need to recursively update __dict__
def dict2obj(d):
"""Convert a dict to an object
>>> d = {'a': 1, 'b': {'c': 2}, 'd': ["hi", {'foo': "bar"}]}
>>> obj = dict2obj(d)
>>> obj.b.c
2
>>> obj.d
["hi", {'foo': "bar"}]
"""
try:
d = dict(d)
except (TypeError, ValueError):
return d
obj = Object()
for k, v in d.iteritems():
obj.__dict__[k] = dict2obj(v)
return obj
And your example list element was probably meant to be a Mapping, a list of (key, value) pairs like this:
>>> d = {'a': 1, 'b': {'c': 2}, 'd': [("hi", {'foo': "bar"})]}
>>> obj = dict2obj(d)
>>> obj.d.hi.foo
"bar"
Here's another implementation:
class DictObj(object):
def __init__(self, d):
self.__dict__ = d
def dict_to_obj(d):
if isinstance(d, (list, tuple)): return map(dict_to_obj, d)
elif not isinstance(d, dict): return d
return DictObj(dict((k, dict_to_obj(v)) for (k,v) in d.iteritems()))
[Edit] Missed bit about also handling dicts within lists, not just other dicts. Added fix.
class Struct(dict):
def __getattr__(self, name):
try:
return self[name]
except KeyError:
raise AttributeError(name)
def __setattr__(self, name, value):
self[name] = value
def copy(self):
return Struct(dict.copy(self))
Usage:
points = Struct(x=1, y=2)
# Changing
points['x'] = 2
points.y = 1
# Accessing
points['x'], points.x, points.get('x') # 2 2 2
points['y'], points.y, points.get('y') # 1 1 1
# Accessing inexistent keys/attrs
points['z'] # KeyError: z
points.z # AttributeError: z
# Copying
points_copy = points.copy()
points.x = 2
points_copy.x # 1
How about this:
from functools import partial
d2o=partial(type, "d2o", ())
This can then be used like this:
>>> o=d2o({"a" : 5, "b" : 3})
>>> print o.a
5
>>> print o.b
3
I think a dict consists of number, string and dict is enough most time.
So I ignore the situation that tuples, lists and other types not appearing in the final dimension of a dict.
Considering inheritance, combined with recursion, it solves the print problem conveniently and also provides two ways to query a data,one way to edit a data.
See the example below, a dict that describes some information about students:
group=["class1","class2","class3","class4",]
rank=["rank1","rank2","rank3","rank4","rank5",]
data=["name","sex","height","weight","score"]
#build a dict based on the lists above
student_dic=dict([(g,dict([(r,dict([(d,'') for d in data])) for r in rank ]))for g in group])
#this is the solution
class dic2class(dict):
def __init__(self, dic):
for key,val in dic.items():
self.__dict__[key]=self[key]=dic2class(val) if isinstance(val,dict) else val
student_class=dic2class(student_dic)
#one way to edit:
student_class.class1.rank1['sex']='male'
student_class.class1.rank1['name']='Nan Xiang'
#two ways to query:
print student_class.class1.rank1
print student_class.class1['rank1']
print '-'*50
for rank in student_class.class1:
print getattr(student_class.class1,rank)
Results:
{'score': '', 'sex': 'male', 'name': 'Nan Xiang', 'weight': '', 'height': ''}
{'score': '', 'sex': 'male', 'name': 'Nan Xiang', 'weight': '', 'height': ''}
--------------------------------------------------
{'score': '', 'sex': '', 'name': '', 'weight': '', 'height': ''}
{'score': '', 'sex': '', 'name': '', 'weight': '', 'height': ''}
{'score': '', 'sex': 'male', 'name': 'Nan Xiang', 'weight': '', 'height': ''}
{'score': '', 'sex': '', 'name': '', 'weight': '', 'height': ''}
{'score': '', 'sex': '', 'name': '', 'weight': '', 'height': ''}

Categories