We Keep Coding

How to groupby columns by value and make json from them? Python3 Pandas

I have a dataset containing all the professors in Turkey. I need to change the shape of this data structure, but I couldn't find a solution. In this data, there is information about the university, faculty, department and title of approximately 44 thousand academicians.
[ { "name": "XX", "title": "PROFESÖR", "university": "GEBZE TEKNİK ÜNİVERSİTESİ", "faculty": "MÜHENDİSLİK FAKÜLTESİ", "department": "BİLGİSAYAR MÜHENDİSLİĞİ BÖLÜMÜ" }, { "name": "YY", "title": "PROFESÖR", "university": "GEBZE TEKNİK ÜNİVERSİTESİ", "faculty": "MÜHENDİSLİK FAKÜLTESİ", "department": "ELEKTRONİK MÜHENDİSLİĞİ BÖLÜMÜ" } ]
I have 44000 yields as above and I want to process them. For example, there are nearly 200 universities, I want to separate them.
{ "universities": [ { "id": 1, "name": "GEBZE TEKNİK ÜNİVERSİTESİ", "faculties": [ { "id" : 1, "name": "MÜHENDİSLİK FAKÜLTESİ", "departments" : [ { "id" : 1, "name" : "ELEKTRONİK MÜHENDİSLİĞİ BÖLÜMÜ", "academicians" : [ { "id" : 1, "name":"AA", "title" : "PROFESÖR" }, { "id" : 2, "name":"BB", "title" : "PROFESÖR" }, { "id" : 3, "name":"CC", "title" : "PROFESÖR" } ] }, { "id" : 2, "name" : "HARİTA MÜHENDİSLİĞİ BÖLÜMÜ", "academicians" : [ { "id" : 1, "name":"DD", "title" : "PROFESÖR" }, { "id" : 2, "name":"EE", "title" : "PROFESÖR" } ] } ] } ] } ] }
I want it as in the above format but I couldn't get it done. Can anyone help?

1.) get json datas
js_output = """{'universities': [{'id': 1,
'name': 'GEBZE TEKNİK ÜNİVERSİTESİ',
'faculties': [{'id': 1,
'name': 'MÜHENDİSLİK FAKÜLTESİ',
'departments': [{'id': 1,
'name': 'ELEKTRONİK MÜHENDİSLİĞİ BÖLÜMÜ',
'academicians': [{'id': 1, 'name': 'AA', 'title': 'PROFESÖR'},
{'id': 2, 'name': 'BB', 'title': 'PROFESÖR'},
{'id': 3, 'name': 'CC', 'title': 'PROFESÖR'}]},
{'id': 2,
'name': 'HARİTA MÜHENDİSLİĞİ BÖLÜMÜ',
'academicians': [{'id': 1, 'name': 'DD', 'title': 'PROFESÖR'},
{'id': 2, 'name': 'EE', 'title': 'PROFESÖR'}]}]}]}]}"""
js_input = """[{'name': 'XX',
'title': 'PROFESÖR',
'university': 'GEBZE TEKNİK ÜNİVERSİTESİ',
'faculty': 'MÜHENDİSLİK FAKÜLTESİ',
'department': 'BİLGİSAYAR MÜHENDİSLİĞİ BÖLÜMÜ'},
{'name': 'YY',
'title': 'PROFESÖR',
'university': 'GEBZE TEKNİK ÜNİVERSİTESİ',
'faculty': 'MÜHENDİSLİK FAKÜLTESİ',
'department': 'ELEKTRONİK MÜHENDİSLİĞİ BÖLÜMÜ'}]"""
2.) set json normalize
# get record_path with json keys and get frame list
df_list = []
def get_frames(l,j):
for i in range(len(l)):
df_list.append(pd.json_normalize(j, l[:i+1]))
records = ["universities", "faculties", "departments", "academicians"]
jdo = json.loads(js_output.replace("'",'"'))
get_frames(records, jdo)
3.) concatenate all frames
con = pd.DataFrame()
for df in df_list[:-1]: # because last item is dict and must be opened next step
con = pd.concat([con, df.iloc[:,:-1]], axis=1)
con = pd.concat([con, df_list[-1]], axis=1)
4.) drop na because of example frame is output template
df = con.dropna().copy()
5.) design columns and match input keys for next concatenates
df.columns = [
"uni_id",
"university",
"faculty_id",
"faculty",
"department_id",
"department",
"aca_id",
"name",
"title"
]
6.) refix id sections and join input frame with template
def input_join_to_get_desired_template(jdi):
jdf = pd.DataFrame(jdi)
con_df = pd.concat([df,jdf], ignore_index=True, sort=False)
# enumerate ids with unique counts ↓ =================================================
unique_uni = list(con_df["university"].unique())
unique_fac = list(con_df["faculty"].unique())
unique_dep = list(con_df["department"].unique())
con_df["uni_id"] = con_df["university"].apply(lambda x: unique_uni.index(x)+1)
con_df["faculty_id"] = con_df["faculty"].apply(lambda x: unique_fac.index(x)+1)
con_df["department_id"] = con_df["department"].apply(lambda x: unique_dep.index(x)+1)
# set academicians indexes
l = ["uni_id","faculty_id","department_id","aca_id"]
con_df["aca_id"] = 1
con_df["aca_id"] = con_df.groupby(l)["aca_id"].cumsum().to_frame()
# enumerate ids with unique counts ↑ =================================================
return con_df
jd_input = json.loads(js_input.replace("'",'"'))
result_df = input_join_to_get_desired_template(jd_input)
result_df
7.) get other inputs and test
js_input_test = """[{'name': 'hl',
'title': 'doc',
'university': 'GEBZE ÜNİVERSİTESİ',
'faculty': 'FAKÜLTESİ',
'department': 'BİLGİSAYAR BÖLÜMÜ'},
{'name': 'hl',
'title': 'PROFESÖR',
'university': 'GEBZE ÜNİVERSİTESİ',
'faculty': 'FAKÜLTESİ',
'department': 'BİLGİSAYAR BÖLÜMÜ'},
{'name': 'yz',
'title': 'yrddoc',
'university': 'TEKNİK ÜNİVERSİTESİ',
'faculty': 'other',
'department': 'BİLGİSAYAR MÜHENDİSLİĞİ BÖLÜMÜ'},
{'name': 'zz',
'title': 'doc',
'university': 'TEKNİK ÜNİVERSİTESİ',
'faculty': 'other',
'department': 'ELEKTRONİK MÜHENDİSLİĞİ BÖLÜMÜ'},
{'name': 'hl',
'title': 'PROFESÖR',
'university': 'TEKNİK ÜNİVERSİTESİ',
'faculty': 'other',
'department': 'BİLGİSAYAR MÜHENDİSLİĞİ BÖLÜMÜ'},
{'name': 'abc',
'title': 'prof',
'university': 'TEKNİK ÜNİVERSİTESİ',
'faculty': 'other',
'department': 'HARİTA MÜHENDİSLİĞİ BÖLÜMÜ'},
{'name': 'aaa',
'title': 'PROFESÖR',
'university': 'ÜNİVERSİTESİ',
'faculty': 'MÜHENDİSLİK',
'department': 'BİLGİSAYAR BÖLÜMÜ'},
{'name': 'bbb',
'title': 'PROFESÖR',
'university': 'ÜNİVERSİTESİ',
'faculty': 'MÜHENDİSLİK',
'department': 'BİLGİSAYAR BÖLÜMÜ'},
{'name': 'ccc',
'title': 'PROFESÖR',
'university': 'ÜNİVERSİTESİ',
'faculty': 'MÜHENDİSLİK',
'department': 'BİLGİSAYAR BÖLÜMÜ'},
{'name': 'ddd',
'title': 'PROFESÖR',
'university': 'ÜNİVERSİTESİ',
'faculty': 'MÜHENDİSLİK FAKÜLTESİ',
'department': 'ELEKTRONİK MÜHENDİSLİĞİ BÖLÜMÜ'}]"""
8.) and get results
jd_input = json.loads(js_input_test.replace("'",'"'))
result_df = input_join_to_get_desired_template(jd_input)
result_df

Generate Dictionary from nested List, Python 3.6

I have below List:
dimensionList = [{'key': 2109290, 'id': 'R', 'name': 'Reporter', 'isGeo': True, 'geoType': 'region'},
{'key': 2109300, 'id': 'C', 'name': 'Commodity', 'isGeo': False, 'geoType': None},
{'key': 2109310, 'id': 'P', 'name': 'Partner', 'isGeo': True, 'geoType': 'region'},
{'key': 2109320, 'id': 'TF', 'name': 'Trade Flow', 'isGeo': False, 'geoType': None},
{'key': 2109330, 'id': 'I', 'name': 'Measure', 'isGeo': False, 'geoType': None}]
I want to create dictionary from this list
Need Values of 'id' as Id of dictionary & 'name' as Values of dictionary
Expected Results:-
ResultsDict = {'R':'Reporter', 'C':'Commodity', 'P':'Partner', 'TF':'Trade Flow', 'I':'Measure'}

Use dict comprehension:
d = {x['id']:x['name'] for x in dimensionList}
print (d)
{'R': 'Reporter', 'C': 'Commodity', 'P': 'Partner', 'TF': 'Trade Flow', 'I': 'Measure'}

You need to loop through the list of dictionaries, pulling out the bits you want and adding them to your new dictionary.
ResultsDict = {}
for dict_item in dimensionList:
id = dict_item ['id']
name = dict_item ['name']
ResultsDict[id] = name
print(ResultsDict)

how to create a list that will store many values from a list of dictionaries

I have a list of Dictionaries in which airbnb[0] is
{
'room_id': '1133718',
'survey_id': '1280',
'host_id': '6219420',
'room_type': 'Shared room',
'country': '',
'city': 'Singapore',
'borough': '',
'neighborhood': 'MK03',
'reviews': 9.0,
'overall_satisfaction': 4.5,
'accommodates': '12',
'bedrooms': '1.0',
'bathrooms': '',
'price': 74.0,
'minstay': '',
'last_modified': '2017-05-17 09:10:25.431659',
'latitude': 1.293354,
'longitude': 103.769226,
'location': '0101000020E6100000E84EB0FF3AF159409C69C2F693B1F43F'
}
how do I go about it if I want to get a list consisting of only the room_id Value and the price for each dictionary in my list of dictionaries so that I can compile those lists in my new_list?

Not sure if this is what you're after but you can make a dictionary where the key is the room_id and the value the price for each property like so:
room_prices = { room['room_id'] : room['price'] for room in airbnb }
Then you access the price for a given room like so:
room_id = '1133718'
room_price = room_prices[room_id]

If you want them as tuples:
new_list = [(x['room_id'], x['price']) for x in airbnb]
# returns
[('1133718', 74.0)]
or a dict:
new_list = [{'room_id': x['room_id'], 'price': x['price']} for x in airbnb]
# returns
[{'room_id': '1133718', 'price': 74.0}]

A list comprehension selecting target keys in your list of dictionaries should do the job, assuming your list contains multiple dictionaries.
room_info =[{
'room_id': '1133718',
'survey_id': '1280',
'host_id': '6219420',
'room_type': 'Shared room',
'country': '',
'city': 'Singapore',
'borough': '',
'neighborhood': 'MK03',
'reviews': 9.0,
'overall_satisfaction': 4.5,
'accommodates': '12',
'bedrooms': '1.0',
'bathrooms': '',
'price': 74.0,
'minstay': '',
'last_modified': '2017-05-17 09:10:25.431659',
'latitude': 1.293354,
'longitude': 103.769226,
'location': '0101000020E6100000E84EB0FF3AF159409C69C2F693B1F43F'
},
{
'room_id': '1133718',
'survey_id': '1280',
'host_id': '6219420',
'room_type': 'Shared room',
'country': '',
'city': 'Singapore',
'borough': '',
'neighborhood': 'MK03',
'reviews': 9.0,
'overall_satisfaction': 4.5,
'accommodates': '12',
'bedrooms': '1.0',
'bathrooms': '',
'price': 74.0,
'minstay': '',
'last_modified': '2017-05-17 09:10:25.431659',
'latitude': 1.293354,
'longitude': 103.769226,
'location': '0101000020E6100000E84EB0FF3AF159409C69C2F693B1F43F'
}]
[[i['room_id'],i['price']] for i in room_info]
>>[['1133718', 74.0], ['1133718', 74.0]]
The result will return a nested list where each individual list contains the room_id and price detail.

It's easy to extract one element of the dict into a new list:
room_ids = [item.get('room_id') for item in airbnb]
Do that for all interesting ones and generate a new list of dicts, if you don't want separate lists. Or you can do all that in one loop
newlist = [{'room_id': item.get('room_id'), 'price': item.get('price')} for item in airbnb]
EDIT: Or a bit more verbose but more general:
mylist = [{'a': 1, 'b':2, 'c':1}, {'a': 2, 'b': 2, 'c':1}, {'a': 5, 'b': 2, 'c':1}, {'b': 5}]
interesting_keys = ['a', 'b']
newlist = []
for item in mylist:
d = dict()
for i in interesting_keys:
d[i] = item.get(i)
newlist.append(d)
print(nl)
will output:
[{'a': 1, 'b': 2}, {'a': 2, 'b': 2}, {'a': 5, 'b': 2}, {'a': None, 'b': 5}]

Python sort list of dictionaries

I have a list of dictionaries:
AccountValues = [
{'portfolio_ref': 1, 'tag': 'FullInit', 'value': '20642.95', 'currency': 'USD', 'percent': 0.0},
{'portfolio_ref': 1, 'tag': 'FullMaint', 'value': '21350.54', 'currency': 'USD', 'percent': 0.0},
{'portfolio_ref': 1, 'tag': 'NetLiq', 'value': '70976.05', 'currency': 'USD', 'percent': 100.0} ]
Simple mission per SQL description: Order by portfolio_ref ASC, percent DESC
What I tried unsuccessfully:
sorted(AccountsValues, key=lambda x: (x[1],-x[4]))
which gives me
KeyError: 1
Second attempt:
import operator
result = sorted(myAccountsValues, key=itemgetter('percent'))
which fails to sort on percentage.

You can use dict.__getitem__ or its syntactic sugar []:
res = sorted(AccountValues, key=lambda x: (x['portfolio_ref'], -x['percent']))
Remember that dictionaries are not indexable by integers. Historically (pre-3.6), they are not even ordered. Even in Python 3.7, you cannot directly extract the nth key or value.
Result:
print(res)
[{'portfolio_ref': 1, 'tag': 'NetLiq', 'value': '70976.05', 'currency': 'USD', 'percent': 100.0},
{'portfolio_ref': 1, 'tag': 'FullInit', 'value': '20642.95', 'currency': 'USD', 'percent': 0.0},
{'portfolio_ref': 1, 'tag': 'FullMaint', 'value': '21350.54', 'currency': 'USD', 'percent': 0.0}]

You just have to combine all the things you did correctly: sort keys as a tuple and the proper way of referencing a dict entry:
>>> sorted(AccountValues, key=lambda x: (x["portfolio_ref"], -x["percent"]))
[{'tag': 'NetLiq', 'portfolio_ref': 1, 'value': '70976.05', 'percent': 100.0, 'currency': 'USD'},
{'tag': 'FullInit', 'portfolio_ref': 1, 'value': '20642.95', 'percent': 0.0, 'currency': 'USD'},
{'tag': 'FullMaint', 'portfolio_ref': 1, 'value': '21350.54', 'percent': 0.0, 'currency': 'USD'}]
Better yet, use
sorted(AccountValues, key=itemgetter("portfolio_ref", "percent"))
Your first attempt failed because x[1] and x[4] are not valid references into the dictionaries: you have to use the labels you originally gave, not relative positions.
Your second attempt is deficient only because you don't have the secondary sort key.

How to convert/update the key-values information in defaultdict?

How do I convert the following defaultdict()?
defaultdict(<class 'dict'>, {
'key1_A': {
'id': 'key1',
'length': '663',
'type': 'A'},
'key1_B': {
'id': 'key1',
'length': '389',
'type': 'B'},
'key2_A': {
'id': 'key2',
'length': '865',
'type': 'A'},
'key2_B': {
'id': 'key2',
'length': '553',
'type': 'B' ........}})
the value of the id i.e key1 becomes the key, and the key called length is changed to length_A or B with corresponding values belonging in the earlier type.
defaultdict(<class 'dict'>, {
'key1': {
'length_A': '663',
'length_B': '389'},
'key2': {
'length_A': '865',
'length_B': '553'}})
Thanks,

I think this does what you want:
from collections import defaultdict
import pprint
d = {
'key1_A': {
'id': 'key1',
'length': '663',
'type': 'A',
},
'key1_B': {
'id': 'key1',
'length': '389',
'type': 'B',
},
'key2_A': {
'id': 'key2',
'length': '865',
'type': 'A',
},
'key2_B': {
'id': 'key2',
'length': '553',
'type': 'B',
},
}
transformed = defaultdict(dict)
for v in d.values():
transformed[v["id"]]["length_{}".format(v["type"])] = v["length"]
pprint.pprint(transformed)
# Output:
# defaultdict(<class 'dict'>,
# {'key1': {'length_A': '663', 'length_B': '389'},
# 'key2': {'length_A': '865', 'length_B': '553'}})