We Keep Coding

Converting incoming data in the form of "multipart/form-data" into a Querydict object

I want to convert a data like below to QueryDict object. Is there a ready-made class that can do this job? I could not parse the data that came in the form of the "multipart/form-data" as I wanted. So I need help
data = {
'name': 'erhan',
'last_name': 'koçlar',
'gender.code': 1,
'gender1.gender2.gender3.gender4.code': 1,
'tags[5]': 'TAG_2',
'tags[0]': 'TAG_1',
'tags[1]': 'TAG_2',
'persons[0].name': 'erhan',
'persons[1].name': 'ercan',
'files[0].file': 'file1',
'files[0].name': 'file_name_1',
'section_files[0]': 'file2'
'rows[0][0]': 'col_1',
'rows[0][1]': 'col_2',
'rows[1][0]': 'row_1',
'rows[1][1]': 'row_2'
}
#after convert
data = {
'name':'erhan',
'last_name': 'koçlar',
'gender': { 'code': 1 },
'gender1': { 'gender2': { 'gender3': { 'gender4': { 'code': 1 } } } },
'tags': [ 'TAG_1', 'TAG_2', 'TAG_2' ],
'persons': [ {'name':'erhan'}, {'name':'ercan'} ],
'files': [ {'file': 'file1', 'name':'file_name_1' }],
'section_files': [ 'file2' ],
'rows': [ [ 'col_1', 'col_2' ], [ 'row_1', 'row_2' ] ],
}

Something like this can do the Job. Note the modification I changed files[0].name to files[1].name as it would overwrite the field 0.
import re
data = {
'name': 'erhan',
'last_name': 'koçlar',
'gender.code': 1,
'gender1.gender2.gender3.gender4.code': 1,
'tags[5]': 'TAG_2',
'tags[0]': 'TAG_1',
'tags[1]': 'TAG_2',
'persons[0].name': 'erhan',
'persons[1].name': 'ercan',
'files[0].file': 'file1',
'files[1].name': 'file_name_1',
'section_files[0]': 'file2',
'rows[0][0]': 'col_1',
'rows[0][1]': 'col_2',
'rows[1][0]': 'row_1',
'rows[1][1]': 'row_2'
}
def buildSubDict(dkey,value):
keys = dkey.split('.')
while keys:
cdict = {keys.pop():value}
value = cdict
return cdict
def buildArray(keys, uniqueName, data):
aSize = None
for key in keys:
match =re.findall('[(\d+)]',key)
m = [[int(m)] for m in match]
if not aSize:
aSize = m
else:
aSize = [a + b for a,b in zip(aSize,m)]
aSize = [max(i)+1 for i in aSize]
res = ''
while aSize:
res = [res if isinstance(res, str) else res.copy() for n in range(aSize.pop())]
for key in keys:
match =re.findall('[(\d+)]',key)
ndim = [int(m) for m in match]
templst = []
citem = res
for idx in ndim:
new = citem[idx]
if not isinstance(new, list):
citem[idx] = data[key]
data.pop(key)
break
citem = new
data[uniqueName]=res
newKeys = {}
for key in dict(data):
subkey = None
if '.' in key:
sDict = buildSubDict(key,data[key])
data[list(sDict.keys())[0]] = sDict[list(sDict.keys())[0]]
data.pop(key)
uniqueKeys = set([i.split('[')[0] for i in data.keys() if '[' in i ])
for ukey in uniqueKeys:
subkeys = []
for key in data:
if ukey+'[' in key:
subkeys.append(key)
buildArray(subkeys, ukey, data)
After processing, data will look like this:
{'name': 'erhan',
'last_name': 'koçlar',
'gender': {'code': 1},
'gender1': {'gender2': {'gender3': {'gender4': {'code': 1}}}},
'section_files': ['file2'],
'persons': [{'name': 'erhan'}, {'name': 'ercan'}],
'rows': [['col_1', 'col_2'], ['row_1', 'row_2']],
'tags': ['TAG_1', 'TAG_2', '', '', '', 'TAG_2'],
'files': [{'file': 'file1'}, {'name': 'file_name_1'}]}
Also note the different implementation of tags, compared to your question. You can filter out the '' by setting them to None instead and then do a list comprehension [i for i in myList if i] to filter them out.

problem with iterating over non existing indexes python

I have extracted data from woocommerce webshop with api. Part of the top structure is like this:
{'id': 12345,
'attributes': [{'id': 1,
'name': 'kleur',
'position': 0,
'visible': True,
'variation': False,
'options': ['blauw']},
{'id': 2,
'name': 'maat',
'position': 1,
'visible': True,
'variation': True,
'options': ['s',
'm',
'l']}],
..................
}
try to make a list of dicts
all_webshop_skus = []
for item in all_data:
product= {}
product = {
'id':item['id'],
'sku': item['sku'],
'name' : item['name'],
'date_created': item['date_created'],
'brands': item['brands'][0]['name'],
'attributes': item['attributes'][1]['name']
}
all_webshop_skus.append(product)
iteration has index issues
---> 16 'attributes': item['attributes'][1]['name']
17 }
18 all_webshop_skus.append(product)
IndexError: list index out of range
IndexError: list index out of range
I think because not every item has a second element in the 'attributes' list of dicts. How can I extract 'name'_values from 'attributes' with 'attribute_id'_value = 2?

Loop through the attributes until you find the one you want, and use its name.
all_webshop_skus = []
for item in all_data:
for attr in item['attributes']:
if attr['id'] == 2:
name = attr['name']
break
else: # default if not found
name = ''
product = {
'id': item['id'],
'sku': item['sku'],
'name' : item['name'],
'date_created': item['date_created'],
'brands': item['brands'][0]['name'],
'attributes': name
}
all_webshop_skus.append(product)

How can I remove nested keys and create a new dict and link both with an ID?

I have a problem. I have a dict my_Dict. This is somewhat nested. However, I would like to 'clean up' the dict my_Dict, by this I mean that I would like to separate all nested ones and also generate a unique ID so that I can later find the corresponding object again.
For example, I have detail: {...}, this nested, should later map an independent dict my_Detail_Dict and in addition, detail should receive a unique ID within my_Dict. Unfortunately, my list that I give out is empty. How can I remove my slaughtered keys and give them an ID?
my_Dict = {
'_key': '1',
'group': 'test',
'data': {},
'type': '',
'code': '007',
'conType': '1',
'flag': None,
'createdAt': '2021',
'currency': 'EUR',
'detail': {
'selector': {
'number': '12312',
'isTrue': True,
'requirements': [{
'type': 'customer',
'requirement': '1'}]
}
}
}
def nested_dict(my_Dict):
my_new_dict_list = []
for key in my_Dict.keys():
#print(f"Looking for {key}")
if isinstance(my_Dict[key], dict):
print(f"{key} is nested")
# Add id to nested stuff
my_Dict[key]["__id"] = 1
my_nested_Dict = my_Dict[key]
# Delete all nested from the key
del my_Dict[key]
# Add id to key, but not the nested stuff
my_Dict[key] = 1
my_new_dict_list.append(my_Dict[key])
my_new_dict_list.append(my_Dict)
return my_new_dict_list
nested_dict(my_Dict)
[OUT] []
# What I want
[my_Dict, my_Details_Dict, my_Data_Dict]
What I have
{'_key': '1',
'group': 'test',
'data': {},
'type': '',
'code': '007',
'conType': '1',
'flag': None,
'createdAt': '2021',
'currency': 'EUR',
'detail': {'selector': {'number': '12312',
'isTrue': True,
'requirements': [{'type': 'customer', 'requirement': '1'}]}}}
What I want
my_Dict = {'_key': '1',
'group': 'test',
'data': 18,
'type': '',
'code': '007',
'conType': '1',
'flag': None,
'createdAt': '2021',
'currency': 'EUR',
'detail': 22}
my_Data_Dict = {'__id': 18}
my_Detail_Dict = {'selector': {'number': '12312',
'isTrue': True,
'requirements': [{'type': 'customer', 'requirement': '1'}]}, '__id': 22}

The following code snippet will solve what you are trying to do:
my_Dict = {
'_key': '1',
'group': 'test',
'data': {},
'type': '',
'code': '007',
'conType': '1',
'flag': None,
'createdAt': '2021',
'currency': 'EUR',
'detail': {
'selector': {
'number': '12312',
'isTrue': True,
'requirements': [{
'type': 'customer',
'requirement': '1'}]
}
}
}
def nested_dict(my_Dict):
# Initializing a dictionary that will store all the nested dictionaries
my_new_dict = {}
idx = 0
for key in my_Dict.keys():
# Checking which keys are nested i.e are dictionaries
if isinstance(my_Dict[key], dict):
# Generating ID
idx += 1
# Adding generated ID as another key
my_Dict[key]["__id"] = idx
# Adding nested key with the ID to the new dictionary
my_new_dict[key] = my_Dict[key]
# Replacing nested key value with the generated ID
my_Dict[key] = idx
# Returning new dictionary containing all nested dictionaries with ID
return my_new_dict
result = nested_dict(my_Dict)
print(my_Dict)
# Iterating through dictionary to get all nested dictionaries
for item in result.items():
print(item)

If I understand you correctly, you wish to automatically make each nested dictionary it's own variable, and remove it from the main dictionary.
Finding the nested dictionaries and removing them from the main dictionary is not so difficult. However, automatically assigning them to a variable is not recommended for various reasons. Instead, what I would do is store all these dictionaries in a list, and then assign them manually to a variable.
# Prepare a list to store data in
inidividual_dicts = []
id_index = 1
for key in my_Dict.keys():
# For each key, we get the current value
value = my_Dict[key]
# Determine if the current value is a dictionary. If so, then it's a nested dict
if isinstance(value, dict):
print(key + " is a nested dict")
# Get the nested dictionary, and replace it with the ID
dict_value = my_Dict[key]
my_Dict[key] = id_index
# Add the id to previously nested dictionary
dict_value['__id'] = id_index
id_index = id_index + 1 # increase for next nested dic
inidividual_dicts.append(dict_value) # store it as a new dictionary
# Manually write out variables names, and assign the nested dictionaries to it.
[my_Details_Dict, my_Data_Dict] = inidividual_dicts

create new list and append to dictionary

I'm trying to learn python. Assuming I have the below two dict().
In the 1st dict, it includes user info and the reporting line structures.
In the 2nd dict, it includes item counts belong to each individual.
I want to compare again these two dict and sum up the total item counts then display the result under name_S. The outcome is shown as follow:
data_set_1 = { 'id': 'mid',
'name': 'name_S',
'directory': [
{
'id': 'eid_A',
'name': 'name_A',
'directory': []
},
{ 'id': 'eid_B',
'name': 'name_B',
'directory': []
},
{ 'id': 'eid_C',
'name': 'name_C',
'directory': [
{'id': 'eid_C1',
'name': 'name_C1',
'directory': []},
{'id': 'eid_C2',
'name': 'name_C2',
'directory': []}]
}]}
data_set_2 = { 'eid_A': 5,
'eid_F': 3,
'eid_G': 0,
'eid_C': 1,
'eid_C1': 10,
'eid_C2': 20
}
Result:
{'name_S': 36}
I'm able to get the result if I did this way:
def combine_compare(data_set_1, data_set_2):
combine_result = dict()
combine_id = data_set_1['id']
combine_name = data_set_1['name']
combine_directory = data_set_1['directory']
if combine_directory:
combine_item_sum = 0
combine_item_count = data_set_2.get(combine_id, 0)
for combine_user in combine_directory:
# Recursion starts
for i in [combine_compare(combine_user, data_set_2)]:
for key, value in i.items():
combine_item_sum += value
combine_result[combine_name] = combine_item_sum + combine_item_count
else:
combine_result[combine_name] = data_set_2.get(combine_id, 0)
return combine_result
Now if I want to include the ids that have item counts in the final result, something like this:
#if there is result and directory is not None under name_S
{'name_S': [36, ('eid_A', 'eid_C', eid_C1', 'eid_C2')]}
#if there is no result and directory is not None under name_S, display a default str
{'name_S': [0, ('Null')]}
#if there is result and directory is None under name_S, display name_S id
{'name_S': [1, ('mid')]}
My original idea is to create a list and append the counts and ids but I'm struggling how I can accomplish this. Here is what I try but the list is not returning the outcome I'm expecting and I'm not sure how I can append the count in the list. Any help would be greatly appreciated.
def combine_compare(data_set_1, data_set_2):
combine_result = dict()
# Creating a new list
combine_list = list()
combine_id = data_set_1['id']
combine_name = data_set_1['name']
combine_directory = data_set_1['directory']
if combine_directory:
combine_item_sum = 0
combine_item_count = data_set_2.get(combine_id, 0)
for combine_user in combine_directory:
# Recursion starts
for i in [combine_compare(combine_user, data_set_2)]:
for key, value in i.items():
combine_item_sum += value
# Trying to append the ids where count > 0
if data_set_2.get(combine_user['id'], 0) != 0:
combine_list.append(combine_user['id'])
combine_result[combine_name] = combine_item_sum + combine_item_count
else:
combine_result[combine_name] = data_set_2.get(combine_id, 0)
# Trying to print the list to see the results
print(combine_list)
return combine_result

So what i understood from your question is that you want to know which ids mentioned in data_set_2 are there in data_set_1 and what is there sum. The following code is my version of solving the above mentioned problem.
data_set_1 = { 'id': 'mid',
'name': 'name_S',
'directory': [
{
'id': 'eid_A',
'name': 'name_A',
'directory': []
},
{ 'id': 'eid_B',
'name': 'name_B',
'directory': []
},
{ 'id': 'eid_C',
'name': 'name_C',
'directory': [
{'id': 'eid_C1',
'name': 'name_C1',
'directory': []},
{'id': 'eid_C2',
'name': 'name_C2',
'directory': []
}
]
}
]
}
data_set_2 = { 'eid_A': 5,
'eid_F': 3,
'eid_G': 0,
'eid_C': 1,
'eid_C1': 10,
'eid_C2': 20
}
value = 0
final_result={}
def compare(d1,d2):
global value
temp_result={}
if d1['name'] not in temp_result:
temp_result[d1['name']]=[]
for items in d1['directory']:
if items['directory']:
temp_value=compare(items,d2)
temp_result[d1['name']].append(temp_value)
result=check_for_value(items,d2)
if result:
value+=result
temp_result[d1['name']].append(items['id'])
return temp_result
def check_for_value(d,d2):
if d['id'] in d2:
return d2[d['id']]
final_result=compare(data_set_1,data_set_2)
final_result['value']=value
print("final_result:",final_result)
Which gives you an output dict which tells you exactly under which name does those ids come under.
The output is as follows:
final_result: {'value': 36, 'name_S': ['eid_A', {'name_C': ['eid_C1', 'eid_C2']}, 'eid_C']}
You can change the structure of the final result to make it exactly how you want it to be. Let me know if you have any trouble understanding the program.

Find item in a list of dictionaries

I have this data
data = [
{
'id': 'abcd738asdwe',
'name': 'John',
'mail': 'test#test.com',
},
{
'id': 'ieow83janx',
'name': 'Jane',
'mail': 'test#foobar.com',
}
]
The id's are unique, it's impossible that multiple dictonaries have the same id.
For example I want to get the item with the id "ieow83janx".
My current solution looks like this:
search_id = 'ieow83janx'
item = [x for x in data if x['id'] == search_id][0]
Do you think that's the be solution or does anyone know an alternative solution?

Since the ids are unique, you can store the items in a dictionary to achieve O(1) lookup.
lookup = {ele['id']: ele for ele in data}
then you can do
user_info = lookup[user_id]
to retrieve it

If you are going to get this kind of operations more than once on this particular object, I would recommend to translate it into a dictionary with id as a key.
data = [
{
'id': 'abcd738asdwe',
'name': 'John',
'mail': 'test#test.com',
},
{
'id': 'ieow83janx',
'name': 'Jane',
'mail': 'test#foobar.com',
}
]
data_dict = {item['id']: item for item in data}
#=> {'ieow83janx': {'mail': 'test#foobar.com', 'id': 'ieow83janx', 'name': 'Jane'}, 'abcd738asdwe': {'mail': 'test#test.com', 'id': 'abcd738asdwe', 'name': 'John'}}
data_dict['ieow83janx']
#=> {'mail': 'test#foobar.com', 'id': 'ieow83janx', 'name': 'Jane'}
In this case, this lookup operation will cost you some constant* O(1) time instead of O(N).

How about the next built-in function (docs):
>>> data = [
... {
... 'id': 'abcd738asdwe',
... 'name': 'John',
... 'mail': 'test#test.com',
... },
... {
... 'id': 'ieow83janx',
... 'name': 'Jane',
... 'mail': 'test#foobar.com',
... }
... ]
>>> search_id = 'ieow83janx'
>>> next(x for x in data if x['id'] == search_id)
{'id': 'ieow83janx', 'name': 'Jane', 'mail': 'test#foobar.com'}
EDIT:
It raises StopIteration if no match is found, which is a beautiful way to handle absence:
>>> search_id = 'does_not_exist'
>>> try:
... next(x for x in data if x['id'] == search_id)
... except StopIteration:
... print('Handled absence!')
...
Handled absence!

Without creating a new dictionary or without writing several lines of code, you can simply use the built-in filter function to get the item lazily, not checking after it finds the match.
next(filter(lambda d: d['id']==search_id, data))
should for just fine.

Would this not achieve your goal?
for i in data:
if i.get('id') == 'ieow83janx':
print(i)
(xenial)vash#localhost:~/python$ python3.7 split.py
{'id': 'ieow83janx', 'name': 'Jane', 'mail': 'test#foobar.com'}
Using comprehension:
[i for i in data if i.get('id') == 'ieow83janx']

if any(item['id']=='ieow83janx' for item in data):
#return item
As any function returns true if iterable (List of dictionaries in your case) has value present.
While using Generator Expression there will not be need of creating internal List. As there will not be duplicate values for the id in List of dictionaries, any will stop the iteration until the condition returns true. i.e the generator expression with any will stop iterating on shortcircuiting. Using List comprehension will create a entire List in the memory where as GE creates the element on the fly which will be better if you are having large items as it uses less memory.