We Keep Coding

Python build dict from a mixture of dict keys and list values

Input body is:
{'columns': ['site_ref', 'site_name', 'region'], 'data': [['R005000003192', 'AIRTH DSR NS896876', 'WEST'], ['R005000003195', 'AIRTHREY DSR NS814971', 'WEST']]}
How could I build a new dict that will take the column values as keys and then populate a new dict for every list item within the data values for 1 to n?
Desired output would be:
{
{
"site_ref": "R005000003192",
"site_name": "AIRTH DSR NS896876",
"region": "WEST"
},
{
"site_ref": "R005000003195",
"site_name": "AIRTH DSR NS896876",
"region": "WEST"
}
}
I have attempted to iterate over with:
for i in range(len(result["data"])):
new_dict = []
new_dict.append(dict(zip(result["columns"], result["data"][i])))
But cannot seem to get it to complete the iteration

Note that you would require keys if the output should be a dictionary, which are currently missing in the provided desired output. However, you could create a list of dictionaries as follows:
d = {'columns': ['site_ref', 'site_name', 'region'], 'data': [['R005000003192', 'AIRTH DSR NS896876', 'WEST'], ['R005000003195', 'AIRTHREY DSR NS814971', 'WEST']]}
res = [{k: v for k, v in zip(d['columns'], datum)} for datum in d['data']]
print(res)
prints
[{'site_ref': 'R005000003192',
'site_name': 'AIRTH DSR NS896876',
'region': 'WEST'},
{'site_ref': 'R005000003195',
'site_name': 'AIRTHREY DSR NS814971',
'region': 'WEST'}]
If you want keys after all, you could e.g. use the numbering (i.e. 1 for the first, n for the n-th datum) for the keys, as follows:
res = {i+1: {k: v for k, v in zip(d['columns'], datum)} for i, datum in enumerate(d['data'])}
print(res)
prints
{1: {'site_ref': 'R005000003192',
'site_name': 'AIRTH DSR NS896876',
'region': 'WEST'},
2: {'site_ref': 'R005000003195',
'site_name': 'AIRTHREY DSR NS814971',
'region': 'WEST'}}

How to append a new key and value into a dictionary of lists using a function

{'Brazil': [19000000, 810000.00], 'Japan': [128000000, 350000.0]}
If I have dict1 which holds data about a country name, population and area, and I want to use a function to add a new country, population and area in the same format as my current dictionary (where the values are enclosed in a list), how would I do that?

Use below code:
dict1 = {'Brazil': [19000000, 810000.00], 'Japan': [128000000, 350000.0]} #initial dict
def addToDict(c, p, a):
#add new values to dict1, here we are adding country(c) as key, [population(p), area(a)] as value.
dict1[c] = [p, a] #dict[c] = [p,a] <-- [p,a] are values assigned to the key identified-in/new-key in dict1
return dict1 #optional - I used it just to see the final value.
print (addToDict('India', 1000000000, 10098978.778)) #main call
#result --> {'Brazil': [19000000, 810000.0], 'Japan': [128000000, 350000.0], 'India': [1000000000, 10098978.778]}

See the addNew function:
dict1 = {'Brazil': [19000000, 810000.00], 'Japan': [128000000, 350000.0]}
def addNew(dict, country, population, area):
dict.update({country: [population, area]})
addNew(dict1, 'countryTest', 200000, 1000.00)
print(dict1)
Output:
{'Brazil': [19000000, 810000.0], 'Japan': [128000000, 350000.0], 'countryTest': [200000, 1000.0]}

pop = 10
area = 20
name = "A"
dict1[name] = [pop, area]

dict1 = {}
def appendDict(country, population, area):
dict1[country] = [population, area]
appendDict("Brazil", "19000000", "810000.00")
appendDict("Japan", "128000000", "350000.0")
print(dict1)

Here is a good example:
travel_log = [
{
"country": "France",
"visits": 12,
"cities": ["Paris", "Lille", "Dijon"]
},
{
"country": "Germany",
"visits": 5,
"cities": ["Berlin", "Hamburg", "Stuttgart"]
},
]
dict1 = {}
def add_new_country(country,visits,cities):
dict1 = {}
dict1['country']= country
dict1['visits'] = visits
dict1['cities'] = cities
travel_log.append(dict1)
add_new_country("Russia", 2, ["Moscow", "Saint Petersburg"])
print(travel_log)

How to get the parent keypath in json file?

In python can we get the parent key path for a key, My json looks like following..
For example if i input a1: it should give: PROJ1/LOB1. All keys inside list[] are unique
{
'PROJ1': {
u'LOB1': [u'a1', u'a2'],
u'LOb2': [u'v1', u'v2'],
u'LOBA': [u'o1', u'oa', u'o2', u'o3'],
u'LOBX': [u'n1', u'n2'],
u'LOB': [u'b1', u'b2']
},
'PROJ12': {
u'LOBa': [u'aa1', u'aa2'],
u'LOBX': [u'vx1', u'vx2']
},
}
I found this on a different thread but its not working for the above data. failing with : unhashable type: 'list'
def keypaths(myDict):
for key, value in myDict.items():
if isinstance(value, collections.Mapping):
for subkey, subvalue in self.keypaths(value):
yield [key] + subkey, subvalue
else:
yield [key], value
reverse_dict = {value: keypath for keypath, value in keypaths(example_dict)}

You need to drill down the lists of values returned to get each of the a1, a2... etc. You can make a minor modification to the keypaths function to flip the keys/values being returned::
import collections
sample = {
'PROJ1': {
u'LOB1': [u'a1', u'a2'],
u'LOb2': [u'v1', u'v2'],
u'LOBA': [u'o1', u'oa', u'o2', u'o3'],
u'LOBX': [u'n1', u'n2'],
u'LOB': [u'b1', u'b2']
},
'PROJ12': {
u'LOBa': [u'aa1', u'aa2'],
u'LOBX': [u'vx1', u'vx2']
},
}
def keypaths(myDict):
for key, value in myDict.items():
if isinstance(value, collections.Mapping):
for subvalue, subkey in keypaths(value):
yield subvalue, [key] + subkey
elif isinstance(value,list):
for i in value:
yield i, [key]
else:
yield value, [key]
my_dict = {k:i for k, i in keypaths(sample)}
This assumes all your values are unique, otherwise the paths get overwritten.
Alternatively, you could have done the same without modifying keypaths function (if it's needed for other purposes):
def keypaths(myDict):
for key, value in myDict.items():
if isinstance(value, collections.Mapping):
for subkey, subvalue in keypaths(value):
yield [key] + subkey, subvalue
else:
yield [key], value
my_dict = {i: keypath for keypath, value in keypaths(sample) for i in value}
Note this alternative only works assuming all your values are lists (otherwise the dictionary comprehension would fail).
Output:
{'a1': ['PROJ1', 'LOB1'],
'a2': ['PROJ1', 'LOB1'],
'aa1': ['PROJ12', 'LOBa'],
'aa2': ['PROJ12', 'LOBa'],
'b1': ['PROJ1', 'LOB'],
'b2': ['PROJ1', 'LOB'],
'n1': ['PROJ1', 'LOBX'],
'n2': ['PROJ1', 'LOBX'],
'o1': ['PROJ1', 'LOBA'],
'o2': ['PROJ1', 'LOBA'],
'o3': ['PROJ1', 'LOBA'],
'oa': ['PROJ1', 'LOBA'],
'v1': ['PROJ1', 'LOb2'],
'v2': ['PROJ1', 'LOb2'],
'vx1': ['PROJ12', 'LOBX'],
'vx2': ['PROJ12', 'LOBX']}
Note this only works assuming all your values are lists (otherwise the dictionary comprehension would fail), and with unique values (otherwise the keypaths get overwritten for duplicate values).
Now all you have to do to get the keypath is:
my_dict.get('a1')
# ['PROJ1', 'LOB1']
my_dict.get('o1')
# ['PROJ1', 'LOBA']
my_dict.get('not exist')
# None

rebuilding arrays with nested defaultdict

This question is an extension of a previous question: rebuild python array based on common elements
- but different enough to warrant a new question:
I've been struggling with this for a bit now. My data is an array of dictionaries from an sql query. Each element in the array represents a shipment, and there are common values based on the keys.
data = [
{"CustName":"customer1", "PartNum":"part1", "delKey":"0001", "qty":"10", "memo":"blah1"},
{"CustName":"customer1", "PartNum":"part1", "delKey":"0002", "qty":"10", "memo":"blah2"},
{"CustName":"customer1", "PartNum":"part1", "delKey":"0003", "qty":"10", "memo":"blah3"},
{"CustName":"customer2", "PartNum":"part3", "delKey":"0004", "qty":"20", "memo":"blah4"},
{"CustName":"customer2", "PartNum":"part3", "delKey":"0005", "qty":"20", "memo":"blah5"},
{"CustName":"customer3", "PartNum":"partXYZ", "delKey":"0006", "qty":"50", "memo":"blah6"},
{"CustName":"customer3", "PartNum":"partABC", "delKey":"0007", "qty":"100", "memo":"blah7"}]
The output I want is grouped according to specific keys
dataOut = [
{"CustName":"customer1", "Parts":[
{"PartNum":"part1", "deliveries":[
{"delKey":"0001", "qty":"10", "memo":"blah1"},
{"delKey":"0002", "qty":"10", "memo":"blah2"},
{"delKey":"0003", "qty":"10", "memo":"blah3"}]}]},
{"CustName":"customer2", "Parts":[
{"PartNum":"part3", "deliveries":[
{"delKey":"0004", "qty":"20", "memo":"blah4"},
{"delKey":"0005", "qty":"20", "memo":"blah5"}]}]},
{"CustName":"customer3", "Parts":[
{"PartNum":"partXYZ", "deliveries":[
{"delKey":"0006", "qty":"50", "memo":"blah6"}]},
{"PartNum":"partABC", "deliveries":[
{"delKey":"0007", "qty":"100", "memo":"blah7"}]}]}]
I can get the grouping with a single level using defaultdict and list comprehension as provided by the previous question and modified slightly
d = defaultdict(list)
for item in data:
d[item['CustName']].append(item)
print([{'CustName': key, 'parts': value} for key, value in d.items()])
But I can't seem to get the second level in the output array - the grouping b the PartNum key. Through some research, I think what I need to do is use defaultdict as the type of the outer `defaultdict' like so:
d = defaultdict(defaultdict(list))
which throws errors because defaultdict returns a function, so I need to use lambda (yes?)
d = defaultdict(lambda:defaultdict(list))
for item in data:
d[item['CustName']].append(item) <----this?
My question is how to "access" the second level array in the loop and tell the "inner" defaultdict what to group on (PartNum)? The data comes to me from the database programmer and the project keeps evolving to add more and more data (keys), so I'd like this solution to be as general as possible in case more data gets thrown my way. I was hoping to be able to "chain" the defaultdicts depending on how many levels I need to go. I'm learning as I'm going, so I'm struggling trying to understand the lambda and the basics of the defaultdict type and where to go from here.

Using groupby as suggested by #Pynchia and using sorted for unordered data as suggested by #hege_hegedus:
from itertools import groupby
dataOut = []
dataSorted = sorted(data, key=lambda x: (x["CustName"], x["PartNum"]))
for cust_name, cust_group in groupby(dataSorted, lambda x: x["CustName"]):
dataOut.append({
"CustName": cust_name,
"Parts": [],
})
for part_num, part_group in groupby(cust_group, lambda x: x["PartNum"]):
dataOut[-1]["Parts"].append({
"PartNum": part_num,
"deliveries": [{
"delKey": delivery["delKey"],
"memo": delivery["memo"],
"qty": delivery["qty"],
} for delivery in part_group]
})
If you look at the second for loop, this will hopefully answer your question about accessing the second level array in the loop.

You could use a tree-like data structure based on an OrderedDefaultdict instead of a defaultdict(list). (The definition's from an unrelated answer of mine.)
from collections import OrderedDict
class OrderedDefaultdict(OrderedDict):
def __init__(self, *args, **kwargs):
if not args:
self.default_factory = None
else:
if not (args[0] is None or callable(args[0])):
raise TypeError('first argument must be callable or None')
self.default_factory = args[0]
args = args[1:]
super(OrderedDefaultdict, self).__init__(*args, **kwargs)
def __missing__ (self, key):
if self.default_factory is None:
raise KeyError(key)
self[key] = default = self.default_factory()
return default
Tree = lambda: OrderedDefaultdict(Tree)
d = Tree()
for rec in data:
custName, partNum, delKey = rec['CustName'], rec['PartNum'], rec['delKey']
details = {"qty": rec["qty"], "memo": rec["memo"]}
d[custName]['Parts'][partNum]['deliveries'][delKey] = details
So, for the data shown in your question, d would end up containing:
d = {
"customer1": {
"Parts": {
"part1": {
"deliveries": {"0001": {"memo": "blah1", "qty": "10"},
"0002": {"memo": "blah2", "qty": "10"},
"0003": {"memo": "blah3", "qty": "10"}}}}},
"customer2": {
"Parts": {
"part3": {
"deliveries": {"0004": {"memo": "blah4", "qty": "20"},
"0005": {"memo": "blah5", "qty": "20"}}}}},
"customer3": {
"Parts": {
"partXYZ": {
"deliveries": {"0006": {"memo": "blah6", "qty": "50"}}},
"partABC": {
"deliveries": {"0007": {"memo": "blah7", "qty": "100"}}}}}
}
Which could just simply be printed out since it's now grouped the way you want.

Sort by "CustName", "PartNum", "delKey". Iterate over the delivery items for each part, for each customer and accumulate to match your output spec.
I like to use operator.itemgetter - for me it makes things clearer.
import collections, itertools, operator
cust_name = operator.itemgetter('CustName')
part_num = operator.itemgetter('PartNum')
group_sort = operator.itemgetter('CustName', 'PartNum', 'delKey')
del_key = operator.itemgetter('delKey')
qty = operator.itemgetter('qty')
memo = operator.itemgetter('memo')
# sort on the relavent keys
data.sort(key = group_sort)
result = []
# iterate over customers
for custname, group1 in itertools.groupby(data, cust_name):
cust_dict = {'CustName' : custname, 'Parts': []}
# iterate over parts for this customer
for partnum, group2 in itertools.groupby(group1, part_num):
part_dict = {"PartNum" : partnum, 'deliveries' : []}
# iterate over delivery items for this part
for thing in group2:
part_dict['deliveries'].append({'delKey':del_key(thing),
'qty':qty(thing),
'memo':memo(thing)})
cust_dict['Parts'].append(part_dict)
result.append(cust_dict)
This clearly iterates over the items in the original data multiple times which may be a performance hit -- but I don't see a way around multiple iteration for what you need to do.

This is the prettiest way I could do it. It uses the same defaultdict idea to implement proper grouping, as python's builtin groupby function only works on ordered data.
Note that this version will mutate the items in the input dataset, so the leaf items in the result are the same dict instances as the input, but with "CustName" and "PartNum" entries deleted.
from collections import defaultdict
def groupby_mutate(seq, key):
d = defaultdict(list)
for item in seq:
d[item[key]].append(item)
del item[key]
return d
def your_operation(data):
return [ {
'CustName': CustName,
'Parts': [ {
'PartNum': PartNum,
'deliveries': deliveries
} for PartNum,deliveries in groupby_mutate(custItems, 'PartNum').items() ]
} for CustName,custItems in groupby_mutate(data, 'CustName').items() ]
# try it
from pprint import *
data = [
{"CustName":"customer1", "PartNum":"part1", "delKey":"0001", "qty":"10", "memo":"blah1"},
{"CustName":"customer1", "PartNum":"part1", "delKey":"0002", "qty":"10", "memo":"blah2"},
{"CustName":"customer1", "PartNum":"part1", "delKey":"0003", "qty":"10", "memo":"blah3"},
{"CustName":"customer2", "PartNum":"part3", "delKey":"0004", "qty":"20", "memo":"blah4"},
{"CustName":"customer2", "PartNum":"part3", "delKey":"0005", "qty":"20", "memo":"blah5"},
{"CustName":"customer3", "PartNum":"partXYZ", "delKey":"0006", "qty":"50", "memo":"blah6"},
{"CustName":"customer3", "PartNum":"partABC", "delKey":"0007", "qty":"100", "memo":"blah7"}
]
pprint(your_operation(data))
EDIT:
Just in the case somebody needs it in the future, here is a version that does not mutate the original data:
from collections import defaultdict
def groupby_getitem(seq, key):
d = defaultdict(list)
for item in seq:
d[item[key]].append(item)
return d
def your_operation(data):
return [ {
'CustName': CustName,
'Parts': [ {
'PartNum': PartNum,
'deliveries': [ dict(
(k,v) for k,v in delivery.items() if not k in ['CustName', 'PartNum']
) for delivery in deliveries ]
} for PartNum,deliveries in groupby_getitem(custItems, 'PartNum').items() ]
} for CustName,custItems in groupby_getitem(data, 'CustName').items() ]

Create list of tuples from dictionary?

I have a simple dictionary:
{"keyy":{"key": "value", "cey": "value"}, "kaye":{"key": "value"}}
That I want to encode as a "context-aware" tuple:
[("keyy","key","value"), ("keyy","cey","value"), ("kaye","key","value")]
My attempt with ValueError: too many values to unpack:
if __name__=='__main__':
mydict={"keyy":{"key": "value", "cey": "value"}, "kaye":{"key", "value"}}
mytuplelist=[tuple(k,v,val) for (k,v) in [(key,val) for (key,val) in mydict]]
print mytuplelist
How would I go about creating this list of tuples?

using list comprehension :
mydict={"keyy":{"key": "value", "cey": "value"}, "kaye":{"key": "value"}}
mytuplelist=[(key,y,mydict[key][y]) for key in mydict for y in mydict[key]]
print (mytuplelist)
output:
[('keyy', 'cey', 'value'), ('keyy', 'key', 'value'), ('kaye', 'key', 'value')]
using Simple For-In loops:
dic={"keyy":{"key": "value", "cey": "value"}, "kaye":{"key": "value"}}
lis=[]
for x in dic:
for y in dic[x]:
lis.append((x,y,dic[x][y]))
print(lis)
output:
[('keyy', 'cey', 'value'), ('keyy', 'key', 'value'), ('kaye', 'key', 'value')]

I like a generator for this:
mydict = {"keyy":{"key": "value", "cey": "value"}, "kaye":{"key": "value"}}
def unnest(d):
for outerkey in d.iterkeys():
for innerkey, value in d[outerkey].iteritems():
yield outerkey, innerkey, value
mytuplelist = list(unnest(mydict))
Or, you can certainly put it in a list comprehension:
mytuplelist = [(outerkey, innerkey, value) for outerkey in mydict.iterkeys()
for (innerkey, value) in mydict[outerkey].iteritems()]

The following example will create the list of tuples as you indicated:
mydict = {"keyy":{"key": "value", "cey": "value"}, "kaye":{"key": "value"}}
tuplelist = []
for outer, dct in mydict.iteritems():
tuplelist.extend((outer, key, val) for key, val in dct.iteritems())

By correcting your typo (missing : in the last nested dict, one of possible causes of "too many items to unpack") you can get back your list comprehension:
dict1 = {"keyy":{"key": "value", "cey": "value"}, "kaye":{"key": "value"}}
print [(key1, key2, dict1[key1][key2]) for key1 in dict1 for key2 in dict1[key1]]

Using a list comprehension
>>> D={"keyy":{"key": "value", "cey": "value"}, "kaye":{"key": "value"}}
>>> [(i,)+j for i in D for j in D[i].iteritems()]
[('keyy', 'cey', 'value'), ('keyy', 'key', 'value'), ('kaye', 'key', 'value')]

To fix your code:
mydict={"keyy":{"key": "value", "cey": "value"}, "kaye":{"key" : "value"}}
mytuplelist = [(outer, inner, val) for outer, d in mydict.iteritems()
for inner, val in d.iteritems()]
print mytuplelist
See kindall's answer on how to do this the easy way :)