We Keep Coding

python how to search a string, count values and group by in json

I have a python program calling an API that receives the result as below:
{
"result": [
{
"company" : "BMW",
"model" : "5"
},
{
"company" : "BMW",
"model" : "5"
},
{
"company" : "BMW",
"model" : "5"
},
{
"company" : "BMW",
"model" : "3"
},
{
"company" : "BMW",
"model" : "7"
},
{
"company" : "AUDI",
"model" : "A3"
},
{
"company" : "AUDI",
"model" : "A7"
},
]
}
Now my task is to identify the number of occurrences of elements from the list in JSON output and group them. The expected output should look like this:
{
"BMW" :
{
"5series" : 3,
"3series" : 1,
"7series" : 1,
},
"AUDI" :
{
"A3" : 1,
"A7" : 1,
},
"MERCEDES":
{
"EClass" : 0,
"SClass" : 0
}
}
I need to find the "company" from list of elements. This will include names that may not be in JSON response sometimes, then the expected output should include that as 0. The "model" names (3,5,7,A3 etc..,) are fixed, so we know that's those are only ones that may or may not be in json api response.
For ex: The List has 3 company names in below code. - companyname = ["BMW,"AUDI","MERCEDES"] . However, sometimes, the JSON API response may not have one or more elements. In this case, "MERCEDES" is missing, but the final output should include "MERCEDES" as well with value as 0.
Here is what i have tried so far:
def modelcount():
companyname= ["BMW","AUDI","MERCEDES"]
url = apiurl
#Send Request
apiresponse = requests.get(url, auth=(user, password), headers=headers, proxies=proxies)
# Decode the JSON response into a dictionary and use the data
data = apiresponse.json()
print(len(data['result']))
3series= 0
5series= 0
7series= 0
A3=0
A7=0
EClass = 0
SClass = 0
modelcountjson = {}
for name in companyname:
for item in data['result']:
models= {}
if item['company'] == name:
if item['model'] == 3:
3series = 3series + 1
elif item['model'] == 5:
5series = 5series + 1
elif item['model'] == 7:
7series = 7series + 1
models['3series'] = 3series
models['5series'] = 5series
models['7series'] = 7series
#I still haven't written AUDI, MERCEDES above. This is where i feel i am writing inefficiently.
modelcountjson[name] = models
return jsonify(modelcountjson)
```
As the number of models grow, I am worried of code getting redundant with many for loops and may cause performance overhead. I am looking for help on achieving the end result in most efficient way.
Thank you so much for your help.

A useful package for working directly with JSON-style dictionaries and lists is toolz (see documentation for more details). This way you can concisely group the data and count occurrences of each model while handling potentially missing data separately:
from toolz import itertoolz
result = {
"result": [
{
"company" : "BMW",
"model" : "5"
},
{
"company" : "BMW",
"model" : "5"
},
{
"company" : "BMW",
"model" : "5"
},
{
"company" : "BMW",
"model" : "3"
},
{
"company" : "BMW",
"model" : "7"
},
{
"company" : "AUDI",
"model" : "A3"
},
{
"company" : "AUDI",
"model" : "A7"
},
]
}
final_output = {}
grouped_result = itertoolz.groupby('company', result['result'])
if 'MERCEDES' not in grouped_result:
final_output['MERCEDES'] = {
'EClass': 0,
'SClass': 0
}
for key, value in grouped_result.items():
models = itertoolz.pluck('model', value)
final_output[key] = itertoolz.frequencies(models)
The output results in:
{'AUDI': {'A3': 1, 'A7': 1}, 'BMW': {'3': 1, '5': 3, '7': 1}, 'MERCEDES': {'EClass': 0, 'SClass': 0}}

You could go for a bit of a separation of code and config:
conf = {
'BMW': {'format': '{}series', 'keys': ['3', '5', '7']},
'AUDI': {'format': '{}', 'keys': ['A3', 'A7']},
'MERCEDES': {'format': '{}Class', 'keys': ['E', 'S']},
}
def modelcount():
# retrieve `data`
# ...
result = {
k: {
v['format'].format(key): 0 for key in v['keys']
} for k, v in conf.items()
}
for car in data['result']:
com = car['company']
mod = car['model']
key = conf[com]['format'].format(mod)
result[com][key] += 1
for com in result:
result[com]['Total'] = sum(result[com].values())
return result
>>> modelcount()
{'BMW': {'3series': 1, '5series': 3, '7series': 1},
'AUDI': {'A3': 1, 'A7': 1},
'MERCEDES': {'EClass': 0, 'SClass': 0}}
This way, for more companies and models, you will only have to touch the conf, not the code. The time complexity of this is O(m+n) with m the total number of distinct models and n the number of cars in the API response.

Error while adding key/value to Python Dict in nested loop

I have a Json Structure as Follows:
{
"_id" : ObjectId("asdasda156121s"),
"Hp" : {
"bermud" : [
{
"abc" : {
"gfh" : 1,
"fgh" : 0.0,
"xyz" : [
{
"kjl" : "0",
"bnv" : 0,
}
],
"xvc" : "bv",
"hgth" : "INnn",
"sdf" : 0,
}
}
},
{
"abc" : {
"gfh" : 1,
"fgh" : 0.0,
"xyz" : [
{
"kjl" : "0",
"bnv" : 0,
}
],
"xvc" : "bv",
"hgth" : "INnn",
"sdf" : 0,
}
}
},
..
I am trying to parse this json and add a new value with key ['cat'] inside the object 'xyz',below is my py code.
data = []
for x in a:
for y in x['Hp'].values():
for z in y:
for k in z['abc']['xyz']:
for m in data:
det = m['response']
// Some processing with det whose output is stored in s
k['cat'] = s
print x
However when x is printed only the last value is being appended onto the whole dictionary, wheras there are different values for s. Its obvious that the 'cat' key is being overwritten everytime the loop rounds,but can't find a way to make it right.What mistake am I making?

How to add new key into dictionary like this [{ {]. This looks more like a dictionary inside a list

I would like to add new key into the dictionary list. Example:
"label" : [] (with empty list)
[
{
"Next" : {
"seed" : [
{
"Argument" : [
{
"id" : 4,
"label" : "org"
},
{
"id" : "I"
},
{
"word" : "He",
"seed" : 2,
"id" : 3,
"label" : "object"
},
{
"word" : "Gets",
"seed" : 9,
"id" : 2,
"label" : "verb"
}
]
}
],
"Next" : "he,get",
"time" : ""
}
}
]
I tried to use loop into "seed" and then to "argument" then use .update("label":[]) in the loop but it won't work. Can anyone please give me an example of using for loop to loop from beginning then to add these new "label"?
My prefered goal: ( to have extra "label" within the dictionary according to my input)
Example:
[
{
"Next" : {
"seed" : [
{
"Argument" : [
{
"id" : 4,
"label" : "org"
},
{
"id" : "I"
},
{
"word" : "He",
"seed" : 2,
"id" : 3,
"label" : "object"
},
{
"word" : "Gets",
"seed" : 9,
"id" : 2,
"label" : "verb"
},
{
"id" : 5,
"label" : "EXTRA"
},
{
"id" : 6,
"label" : "EXTRA"
},
{
"id" : 7,
"label" : "EXTRA"
}
]
}
],
"Next" : "he,get",
"time" : ""
}
}
]
I am new to dictionary so really need help with this

If I understand your problem correctly, you want to add 'label' to dict in Argument where there is no label. You could do it like so -
for i in x[0]['Next']['seed'][0]['Argument']:
if not 'label' in i.keys():
i['label'] = []
Where x is your dict. But what's x[0]['Next']['seed'][0]['Argument']:?
Let's simplify your dict -
x = [{'Next': {'Next': 'he,get',
'seed': [{'Argument': [{these}, {are}, {your}, {dicts}]}],
'time': ''}}]
How did we reach here?
Let's see-
x = [{'Next'(parent dict): {'Next'(child of previous 'Next'):{},
'seed(child of previous 'Next')':[{these}, {are}, {your}, {dicts}](a list of dicts)}]
I hope that makes sense. And to add more dictionaries in Argument
# create a function that returns a dict
import random # I don't know how you want ids, so this
def create_dicts():
return {"id": random.randint(1, 10), "label": ""}
for i in range(3): # 3 is how many dicts you want to push in Argument
x[0]['Next']['seed'][0]['Argument'].append(create_dicts())
Now your dict will become -
[{'Next': {'Next': 'he,get',
'seed': [{'Argument': [{'id': 4, 'label': 'org'},
{'id': 'I'},
{'id': 3, 'label': 'object', 'seed': 2, 'word': 'He'},
{'id': 2, 'label': 'verb', 'seed': 9, 'word': 'Gets'},
{'id': 1, 'label': ''},
{'id': 4, 'label': ''},
{'id': 4, 'label': ''}]}],
'time': ''}}]

First things first: access the list of dict that need to be updated.
according to your given structure that's l[0]["Next"]["seed"][0]["Argument"]
Then iterate that list and check if label already exists, if it does not then add it as an empty list.
This can be done by explicit checking:
if "label" not in i:
i["label"] = []
or by re-assigning:
i["label"] = i.get("label", [])
Full Code:
import pprint
l = [ {
"Next" : {
"seed" : [ {
"Argument" : [ {
"id" : 4,
"label" : "org"
}, {
"id" : "I"
}, {
"word" : "He",
"seed" : 2,
"id" : 3,
"label" : "object"
}, {
"word" : "Gets",
"seed" : 9,
"id" : 2,
"label" : "verb"
} ]
} ],
"Next" : "he,get",
"time" : ""
} }]
# access the list of dict that needs to be updated
l2 = l[0]["Next"]["seed"][0]["Argument"]
for i in l2:
i["label"] = i.get("label", []) # use the existing label or add an empty list
pprint.pprint(l)
Output:
[{'Next': {'Next': 'he,get',
'seed': [{'Argument': [{'id': 4, 'label': 'org'},
{'id': 'I', 'label': []},
{'id': 3,
'label': 'object',
'seed': 2,
'word': 'He'},
{'id': 2,
'label': 'verb',
'seed': 9,
'word': 'Gets'}]}],
'time': ''}}]

You have a list with one nested dictionary. Get the list of the inner dicts, and iterate. Assuming your initial data structure is named data
dict_list = data[0]['Next']['seed'][0]['Argument']
for item in dict_list:
item['label'] = input()

Creating Python dictionaries based on a condition

I am trying to create dictionaries that will be order requests for a portfolio of funds. The dictionaries will be exported as JSON files, just to give a bit of context. This is what the dictionaries look like.
{
"accountID":"02e57c7d-d071-4c63-b491-1194a9939ea5.1452548617456",
"accountNo":"DWTE000005",
"userID":"02e57c7d-d071-4c63-b491-1194a9939ea5",
"accountType":2,
"ordType":"1",
"side":"B",
"instrumentID":"06926627-e950-48f3-9c53-b679f61120ec",
"orderQty":1.4312,
"comment":""
}
The condition that the dictionaries will be created upon are if the target weightings are above 0, as follows.
targetportfolio = {
'VXUS' : 0.2,
'GOVT' : 0,
'IVW' : 0.2,
'EEM' : 0.2,
'JNK' : 0,
'VDE' : 0.15,
'LQD' : 0,
'IJR' : 0.1,
'BIL' : 0,
'AAXJ' : 0.15
}
Where each fund (for example VXUS) will have an order generated if the amount in the target portfolio dict is greater than 0. I was thinking of something along the lines of the code below, but I am quite confused. Any ideas?
def ordergen(portfolio):
for i in portfolio:
{
"accountID" : "02e57c7d-d071-4c63-b491-1194a9939ea5.1452548617456",
"accountNo" : "DWTE000005",
"userID" : "02e57c7d-d071-4c63-b491-1194a9939ea5",
"accountType" : 2,
"ordType" : "1",
"side" : "B",
"instrumentID" :fundID[i],
"orderQty" : ,
"comment":""
}

targetPortfolio = {
'VXUS' : 0.2,
'GOVT' : 0,
'IVW' : 0.2,
'EEM' : 0.2,
'JNK' : 0,
'VDE' : 0.15,
'LQD' : 0,
'IJR' : 0.1,
'BIL' : 0,
'AAXJ' : 0.15
}
orders = []
for k in targetPortfolio.keys():
if targetPortfolio[k] > 0:
orders.append({
"accountID" : "02e57c7d-d071-4c63-b491-1194a9939ea5.1452548617456",
"accountNo" : "DWTE000005",
"userID" : "02e57c7d-d071-4c63-b491-1194a9939ea5",
"accountType" : 2,
"ordType" : "1",
"side" : "B",
"instrumentID" : "",
"orderQty" : "",
"comment":""
})
print(orders)
.keys() is your friend. You can now use 'k' to fill in your order details under the condition that their weight is above 0.

How can I write a recursive python function that splits a dictionary into an array of dictionaries?

I am looking to write a recursive function:
arguments: d, dictionary
result: list of dictionaries
def expand_dictionary(d):
return []
The function recursively goes through a dictionary and flattens nested objects using an _, in addition it expands out nested lists into the array, and includes the parent label.
Think of creating a relational model from a document.
Here is an example input and output:
original_object = {
"id" : 1,
"name" : {
"first" : "Alice",
"last" : "Sample"
},
"cities" : [
{
"id" : 55,
"name" : "New York"
},
{
"id" : 60,
"name" : "Chicago"
}
],
"teachers" : [
{
"id" : 2
"name" : "Bob",
"classes" : [
{
"id" : 13,
"name" : "math"
},
{
"id" : 16,
"name" : "spanish"
}
]
}
]
}
expected_output = [
{
"id" : 1,
"name_first" : "Alice",
"name_last" : "Sample"
},
{
"_parent_object" : "cities",
"id" : 55,
"name" : "New York"
},
{
"_parent_object" : "cities",
"id" : 60,
"name" : "Chicago"
},
{
"parent_object" :"teachers",
"id" : 2,
"name" : "Bob"
},
{
"parent_object" :"teachers_classes",
"id" : 13,
"name" : "math"
},
{
"parent_object" :"teachers_classes",
"id" : 16,
"name" : "spanish"
}
]
the code currently being used for flattening is:
def flatten_dictionary(d):
def expand(key, value):
if isinstance(value, dict):
return [ (key + '_' + k, v) for k, v in flatten_dictionary(value).items() ]
else:
#If value is null or empty array don't include it
if value is None or value == [] or value == '':
return []
return [ (key, value) ]
items = [ item for k, v in d.items() for item in expand(k, v) ]
return dict(items)

That will do
def expand_dictionary(d,name=None,l=None):
obj = {}
if l == None:
l = [obj]
else:
l.append(obj)
prefix = (name+'_'if name else '')
if prefix: obj['_parent_object'] = name
for i, v in d.iteritems():
if isinstance(v, list):
map(lambda x:expand_dictionary(x,prefix+i,l),v)
elif isinstance(v, dict):
obj.update(flatten_dictionary({i: v}))
else:
obj[i] = v
return l

After working through it a bit here is what I have come up with. Probably can be significantly optimized. Based on #paulo-scardine's comment I added the parent primary key to keep the relational model. Would love to hear optimization thoughts.
def expand_dictionary(original_object, object_name, objects=None):
if objects is None:
objects = []
def flatten_dictionary(dictionary):
def expand(key, value):
if isinstance(value, dict):
return [ (key + '_' + k, v) for k, v in flatten_dictionary(value).items() ]
else:
#If value is null or empty array don't include it
if value is None or value == [] or value == '':
return []
return [ (key, value) ]
items = [ item for k, v in dictionary.items() for item in expand(k, v) ]
return dict(items)
original_object_root = flatten_dictionary(original_object).copy()
original_object_root['_meta_object_name'] = object_name
for key,value in original_object_root.copy().items():
if isinstance(value, dict):
flatten_dictionary(value, objects)
if isinstance(value, list):
original_object_root.pop(key)
for nested_object in value:
nested_object['_meta_parent_foreign_key'] = original_object_root['id']
nested_object['_meta_object_name'] = object_name + "_" + key
expand_dictionary(nested_object, object_name + "_" + key, objects)
objects.append(original_object_root)
return objects