Let say, I have a list data for example:
data = [
{'id': 1, 'name': 'brad', 'color': 'red', 'tags': [], 'author': {'name': 'admin'}},
{'id': 2, 'name': 'sylvia', 'color': 'blue', 'tags': [], 'author': {'name': 'user'}},
{'id': 3, 'name': 'sylwia', 'color': 'green', 'tags': [], 'author': {'name': 'admin'}},
{'id': 4, 'name': 'shane', 'color': 'red', 'tags': [], 'author': {'name': 'admin'}},
{'id': 5, 'name': 'shane', 'color': 'red', 'tags': ['python', 'django'], 'author': {'name': 'user'}}
]
and I want to make it ORM'able, such as what Django has doing:
ModelName.objects.filter(color__icontains="gree")
And this what I have do;
import operator
from collections import namedtuple
from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist
class DataQuerySet:
"""
Custom ORM for List dict data,
https://stackoverflow.com/a/58351973/6396981
"""
allowed_operations = {
'gt': operator.gt,
'lt': operator.lt,
'eq': operator.eq,
'icontains': operator.contains
}
def __init__(self, data):
self.data = data
def all(self):
return self.data
def filter(self, **kwargs):
"""
>>> kwargs = {'name': 'sylwia', 'id__gt': 1}
>>> DataQuerySet().filter(**kwargs)
[{'id': 3, 'name': 'sylwia', 'color': 'green'}]
"""
operation = namedtuple('Q', 'op key value')
def parse_filter(item):
"""item is expected to be a tuple with exactly two elements
>>> parse_filter(('id__gt', 2))
Q(op=<built-in function gt>, key='id', value=2)
>>> parse_filter(('id__ ', 2))
Q(op=<built-in function eq>, key='id', value=2)
>>> parse_filter(('color__bad', 'red'))
Traceback (most recent call last):
...
AssertionError: 'bad' operation is not allowed
"""
key, *op = item[0].split('__')
# no value after __ means exact value query, e.g. name='sylvia'
op = ''.join(op).strip() or 'eq'
assert op in self.allowed_operations, f'{repr(op)} operation is not allowed'
return operation(self.allowed_operations[op], key, item[1])
filtered_data = self.data.copy()
for item in map(parse_filter, kwargs.items()):
filtered_data = [
entry for entry in filtered_data
if item.op(entry[item.key], item.value)
]
return filtered_data
def get(self, **kwargs):
"""
>>> DataQuerySet().get(id=3)
[{'id': 3, 'name': 'sylwia', 'color': 'green'}]
"""
operation = namedtuple('Q', 'op key value')
def parse_get(item):
key, *op = item[0].split('__')
return operation(self.allowed_operations['eq'], key, item[1])
filtered_data = self.data.copy()
for item in map(parse_get, kwargs.items()):
filtered_data = [
entry for entry in filtered_data
if item.op(entry[item.key], item.value)
]
if len(filtered_data) > 1:
raise MultipleObjectsReturned(filtered_data)
elif len(filtered_data) < 1:
raise ObjectDoesNotExist(kwargs)
return filtered_data[0]
And to use it:
class DataModel:
def __init__(self, data):
self._data = DataQuerySet(data)
#property
def objects(self):
return self._data
data = [
{'id': 1, 'name': 'brad', 'color': 'red', 'tags': [], 'author': {'name': 'admin'}},
{'id': 2, 'name': 'sylvia', 'color': 'blue', 'tags': [], 'author': {'name': 'user'}},
{'id': 3, 'name': 'sylwia', 'color': 'green', 'tags': [], 'author': {'name': 'admin'}},
{'id': 4, 'name': 'shane', 'color': 'red', 'tags': [], 'author': {'name': 'admin'}},
{'id': 5, 'name': 'shane', 'color': 'red', 'tags': ['python', 'django'], 'author': {'name': 'user'}}
]
d = DataModel(data)
print(d.objects.filter(id__gt=2))
print(d.objects.filter(color='green'))
print(d.objects.filter(color__icontains='gree'))
print(d.objects.get(id=1))
Above tests is just work properly, but seems we have a problem when we want to do more:
print(d.objects.filter(tags__in=['python']))
print(d.objects.filter(author__name='admin'))
print(d.objects.filter(author__name__icontains='use'))
Finally, I found a nice module to handle that case, it called with reobject, and here is the test:
from reobject.models import Model, Field
from reobject.query.parser import Q as Query
data = [
{'name': 'brad', 'color': 'red', 'tags': [], 'author': {'name': 'admin'}},
{'name': 'sylvia', 'color': 'blue', 'tags': [], 'author': {'name': 'user'}},
{'name': 'sylwia', 'color': 'green', 'tags': [], 'author': {'name': 'admin'}},
{'name': 'shane', 'color': 'red', 'tags': [], 'author': {'name': 'admin'}},
{'name': 'shane', 'color': 'red', 'tags': ['python', 'django'], 'author': {'name': 'user'}}
]
class Book(Model):
name = Field()
color = Field()
tags = Field()
author = Field()
for item in data:
Book(**item)
Book.objects.all()
Book.objects.get(name='brad')
Book.objects.filter(name='brad')
Book.objects.filter(author__name='admin')
Book.objects.filter(tags__contains='python')
Book.objects.filter(Query(author__name='admin') | Query(author__name='user'))
Meanwhile, it still doesn't support with id or pk fields.
Mybe because it already taken.
Related
I am working on a coding challenge for self-development and I came across a question where I am given an input like this:
add {"id":1,"last":"Doe","first":"John","location":{"city":"Oakland","state":"CA","postalCode":"94607"},"active":true}
add {"id":2,"last":"Doe","first":"Jane","location":{"city":"San Francisco","state":"CA","postalCode":"94105"},"active":true}
add {"id":3,"last":"Black","first":"Jim","location":{"city":"Spokane","state":"WA","postalCode":"99207"},"active":true}
add {"id":4,"last":"Frost","first":"Jack","location":{"city":"Seattle","state":"WA","postalCode":"98204"},"active":false}
get {"location":{"state":"WA"},"active":true}
get {"id":1}
get {"active":true}
delete {"active":true}
get {}
And what I am doing is adding the entries that start with add to a list called database = []:
json_input = []
database = []
for line in sys.stdin:
json_input.append(line.split("', "))
for i in range(0, len(json_input)):
if json_input[i][0] == 'add':
database.append(json_input[i][1])
What I want to do is to print out every entry that matches what follows get and delete every entry that matches what follows delete. This is where I am stuck. Currently, this is what json_input() looks like. database is empty:
[
['add {"id":1,"last":"Doe","first":"John","location":{"city":"Oakland","state":"CA","postalCode":"94607"},"active":true}\n'],
['add {"id":2,"last":"Doe","first":"Jane","location":{"city":"San Francisco","state":"CA","postalCode":"94105"},"active":true}\n'],
['add {"id":3,"last":"Black","first":"Jim","location":{"city":"Spokane","state":"WA","postalCode":"99207"},"active":true}\n'],
['add {"id":4,"last":"Frost","first":"Jack","location":{"city":"Seattle","state":"WA","postalCode":"98204"},"active":false}\n'],
['get {"location":{"state":"WA"},"active":true}\n'], ['get {"id":1}\n'],
['get {"active":true}\n'], ['delete {"active":true}\n'],
['get {}']
]
Perhaps an easy-to-read way to handle this would be a simple class that maintains a list of records. You can add methods for the various commands you want to handle. Then it's just a matter of defining the methods and processing the input to pass to the methods. Here's a possible way (without any frills like error checking):
import json
raw_data = '''add {"id":1,"last":"Doe","first":"John","location":{"city":"Oakland","state":"CA","postalCode":"94607"},"active":true}
add {"id":2,"last":"Doe","first":"Jane","location":{"city":"San Francisco","state":"CA","postalCode":"94105"},"active":true}
add {"id":3,"last":"Black","first":"Jim","location":{"city":"Spokane","state":"WA","postalCode":"99207"},"active":true}
add {"id":4,"last":"Frost","first":"Jack","location":{"city":"Seattle","state":"WA","postalCode":"98204"},"active":false}
get {"location":{"state":"WA"},"active":true}
get {"id":1}
get {"active":true}
delete {"active":true}
get {}'''
class Data:
#staticmethod
def matches(obj, query):
if not isinstance(query, dict):
return obj == query
return all(Data.matches(obj.get(key), q) for key, q in query.items())
def __init__(self):
self.data = []
def add(self, record):
self.data.append(record)
def get(self, query):
for item in self.data:
if (Data.matches(item, query)):
print(item)
def delete(self, query):
self.data = [record for record in self.data if not Data.matches(record, query)]
data = Data()
for line in raw_data.split('\n'):
command, line = line.split(None, 1)
command = getattr(data, command)
command(json.loads(line))
This will print the records from WA then the active:True records. Then after deleting the True records it will print everything (the result of the {} query), which is the only one left -- the active:False record:
{'id': 3, 'last': 'Black', 'first': 'Jim', 'location': {'city': 'Spokane', 'state': 'WA', 'postalCode': '99207'}, 'active': True}
{'id': 1, 'last': 'Doe', 'first': 'John', 'location': {'city': 'Oakland', 'state': 'CA', 'postalCode': '94607'}, 'active': True}
{'id': 1, 'last': 'Doe', 'first': 'John', 'location': {'city': 'Oakland', 'state': 'CA', 'postalCode': '94607'}, 'active': True}
{'id': 2, 'last': 'Doe', 'first': 'Jane', 'location': {'city': 'San Francisco', 'state': 'CA', 'postalCode': '94105'}, 'active': True}
{'id': 3, 'last': 'Black', 'first': 'Jim', 'location': {'city': 'Spokane', 'state': 'WA', 'postalCode': '99207'}, 'active': True}
{'id': 4, 'last': 'Frost', 'first': 'Jack', 'location': {'city': 'Seattle', 'state': 'WA', 'postalCode': '98204'}, 'active': False}
If this were a test or a serious coding challenge, you would probably want to look carefully at matches() to make sure it properly handles edge cases (I didn't do that).
Using Python requests I want to grab a piece of JSON from one source and post it to a destination. The structure of the JSON received and the one required by the destination, however, differs a bit so my question is, how do I best map the items from the source structure onto the destination structure?
To illustrate, imagine we get a list of all purchases made by John and Mary. And now we want to post the individual items purchased linking these to the individuals who purchased them (NOTE: The actual use case involves thousands of entries so I am looking for an approach that would scale accordingly):
Source JSON:
{
'Total Results': 2,
'Results': [
{
'Name': 'John',
'Age': 25,
'Purchases': [
{
'Fruits': {
'Type': 'Apple',
'Quantity': 3,
'Color': 'Red'}
},
{
'Veggie': {
'Type': 'Salad',
'Quantity': 2,
'Color': 'Green'
}
}
]
},
{
'Name': 'Mary',
'Age': 20,
'Purchases': [
{
'Fruits': {
'Type': 'Orange',
'Quantity': 2,
'Color': 'Orange'
}
}
]
}
]
}
Destination JSON:
{
[
{
'Purchase': 'Apple',
'Purchased by': 'John',
'Quantity': 3,
'Type': 'Red',
},
{
'Purchase': 'Salad',
'Purchased by': 'John',
'Quantity': 2,
'Type': 'Green',
},
{
'Purchase': 'Orange',
'Purchased by': 'Mary',
'Quantity': 2,
'Type': 'Orange',
}
]
}
Any help on this would be greatly appreciated! Cheers!
Just consider loop through the dict.
res = []
for result in d['Results']:
value = {}
for purchase in result['Purchases']:
item = list(purchase.values())[0]
value['Purchase'] = item['Type']
value['Purchased by'] = result['Name']
value['Quantity'] = item['Quantity']
value['Type'] = item['Color']
res.append(value)
pprint(res)
[{'Purchase': 'Apple', 'Purchased by': 'John', 'Quantity': 3, 'Type': 'Red'},
{'Purchase': 'Salad', 'Purchased by': 'John', 'Quantity': 2, 'Type': 'Green'},
{'Purchase': 'Orange', 'Purchased by': 'Mary', 'Quantity': 2, 'Type': 'Orange'}]
I have the following dict
{'returnData': [{'eMail': None,
'firstName': 'Peter',
'id': '1234',
'name': 'Parker'},
{'eMail': 'lucky#mail.example',
'firstName': 'Lucky',
'id': '123',
'name': 'Luke'},
{'eMail': 'micky#mail.example',
'firstName': 'Micky',
'id': '3456',
'name': 'Mouse'}],
'status': {'errorCode': 0,
'message': None,
'subErrorCode': None,
'success': True}}
How would i search the dict for the values of eMail, firstName and name and return all found matches in a new dict.
For example, i search for mail.example it should return only two entries.
Python3
Solution 1:
list(filter(lambda profile: '#mail.example' in str(profile['eMail']), data['returnData']))
to search in all values of dict
list(filter(lambda profile: '#mail.example' in str(
profile.items()), data['returnData']))
Solution 2:
Create a search function.
data = {'returnData': [{'eMail': None,
'firstName': 'Peter',
'id': '1234',
'name': 'Parker'},
{'eMail': 'lucky#mail.example',
'firstName': 'Lucky',
'id': '123',
'name': 'Luke'},
{'eMail': 'micky#mail.example',
'firstName': 'Micky',
'id': '3456',
'name': 'Mouse'}],
'status': {'errorCode': 0,
'message': None,
'subErrorCode': None,
'success': True}}
def search(search_term, field, data):
result = []
for item in data:
if search_term in str(item[field]):
result.append(item)
return result
print(search("#mail.example", "eMail", data['returnData']))
You may want to use re module here
import re
d = {'returnData': [{'eMail': None,
'firstName': 'Peter',
'id': '1234',
'name': 'Parker'},
{'eMail': 'lucky#mail.example',
'firstName': 'Lucky',
'id': '123',
'name': 'Luke'},
{'eMail': 'micky#mail.example',
'firstName': 'Micky',
'id': '3456',
'name': 'Mouse'}],
'status': {'errorCode': 0,
'message': None,
'subErrorCode': None,
'success': True}}
out = []
for entry in d['returnData']:
pattern = 'mail.example'
email = entry.get('eMail', None)
if email and re.search(pattern, email):
out.append(email)
print(out)
Iterate over dict_['returnData'] to look for the eMail:
dict_ = {'returnData': [{'eMail': None,
'firstName': 'Peter',
'id': '1234',
'name': 'Parker'},
{'eMail': 'lucky#mail.example',
'firstName': 'Lucky',
'id': '123',
'name': 'Luke'},
{'eMail': 'micky#mail.example',
'firstName': 'Micky',
'id': '3456',
'name': 'Mouse'}],
'status': {'errorCode': 0,
'message': None,
'subErrorCode': None,
'success': True}}
for elem in dict_['returnData']:
print(elem['eMail'])
Note: use if elem['firstName']: print(elem['eMail']) for only two
enteries
OUTPUT:
None
lucky#mail.example
micky#mail.example
EDIT:
even better, using lists:
firstNames = []
names = []
emails = []
for elem in dict_['returnData']:
if elem['firstName']:
firstNames.append(elem['firstName'])
if elem['name']:
names.append(elem['name'])
if elem['eMail']:
emails.append(elem['eMail'])
print("First Names: {}".format(firstNames))
print("Names: {}".format(names))
print("Emails: {}".format(emails))
OUTPUT:
First Names: ['Peter', 'Lucky', 'Micky']
Names: ['Parker', 'Luke', 'Mouse']
Emails: ['lucky#mail.example', 'micky#mail.example']
I have 3 lists:
names = ["john", "paul", "george", "ringo"]
job = ["guitar", "bass", "guitar", "drums"]
status = ["dead", "alive", "dead", "alive"]
I am trying to figure out the best way to combine these lists into a dict like the following:
{"person":{"Name":"john", "Job":"guitar", "Status":"dead"}, "person":{"Name":"paul", "Job":"bass", "Status":"alive"}, "person":{"Name":"george", "Job":"guitar", "Status":"dead"}, "person":{"Name":"ringo", "Job":"drums", "Status":"alive"}}
I have tried using dict(zip) but cannot get it to format like above.
Thanks in advance!
I think what you want is a list of dictionaries. You can zip your three lists together and use a list comprehension. Here's an example:
[
{'name': name, 'job': job, 'status': status}
for name, job, status in zip(names, jobs, statuses)
]
(also renaming your job to jobs and status to statuses)
Which will give you:
[
{'name': 'john', 'job': 'guitar', 'status': 'dead'},
{'name': 'paul', 'job': 'bass', 'status': 'alive'},
{'name': 'george', 'job': 'guitar', 'status': 'dead'},
{'name': 'ringo', 'job': 'drums', 'status': 'alive'}
]
I think this is what you are looking for:
>>> names = ["john", "paul", "george", "ringo"]
>>> job = ["guitar", "bass", "guitar", "drums"]
>>> status = ["dead", "alive", "dead", "alive"]
>>> persons = []
>>> for n, j, s in zip(names, job, status):
... person = { 'name': n, 'job': j, 'status': s }
... persons.append(person)
...
>>> persons
[{'status': 'dead', 'job': 'guitar', 'name': 'john'}, {'status': 'alive', 'job': 'bass', 'name': 'paul'}, {'status': 'dead', 'job': 'guitar', 'name': 'george'}, {'status': 'alive', 'job': 'drums', 'name': 'ringo'}]
>>>
Try this approach:
names = ["john", "paul", "george", "ringo"]
job = ["guitar", "bass", "guitar", "drums"]
status = ["dead", "alive", "dead", "alive"]
two step process:
description =[{'job': j, 'status': s} for j,s in zip(job,status)]
artist ={n: i for n,i in zip(names,description)}}
final output:
print(artist)
{'john': {'job': 'guitar', 'status': 'dead'},
'paul': {'job': 'bass', 'status': 'alive'},
'george': {'job': 'guitar', 'status': 'dead'},
'ringo': {'job': 'drums', 'status': 'alive'}}
you can then do something like this:
artist['ringo']['job']
output:
'drums'
Here‘s a simplified example of some data I have:
{"id": "1234565", "fields": {"name": "john", "email":"john#example.com", "country": "uk"}}
The wholeo nested dictionary is a bigger list of address data. The goal is to create pairs of people from the list with randomized partners where partners from the same country should be preferd. So my first real issue is to find a good way to group them by that country value.
I‘m sure there‘s a smarter way to do this than iterating through the dict and writing all records out to some new list/dict?
I think this is close to what you need:
result = {key:[i for i in value] for key, value in itertools.groupby(people, lambda item: item["fields"]["country"])}
What this does is use itertools.groupby to group all people in the people list by their specified country. The resulting dictionary has countries as keys, and the unpacked groupings (matching people) as values. Input is expected as a list of dictionaries like the one in your example:
people = [{"id": "1234565", "fields": {"name": "john", "email":"john#example.com", "country": "uk"}},
{"id": "654321", "fields": {"name": "sam", "email":"sam#example.com", "country": "uk"}}]
Sample output:
>>> print(result)
>>> {'uk': [{'fields': {'name': 'john', 'email': 'john#example.com', 'country': 'uk'}, 'id': '1234565'}, {'fields': {'name': 'sam', 'email': 'sam#example.com', 'country': 'uk'}, 'id': '654321'}]}
For a cleaner result, the looping construct can be tweaked so that only the ID of each person is included in the result dict:
result = {key:[i["id"] for i in value] for key, value in itertools.groupby(people, lambda item: item["fields"]["country"])}
>>> print(result)
>>> {'uk': ['1234565', '654321']}
EDIT: Sorry, I forgot about the sorting. Simply sort the list of people by country before putting it through groupby. It should now work properly:
sort = sorted(people, key=lambda item: item["fields"]["country"])
Here is another one that uses defaultdict:
import collections
def make_groups(nested_dicts, nested_key):
default = collections.defaultdict(list)
for nested_dict in nested_dicts:
for value in nested_dict.values():
try:
default[value[nested_key]].append(nested_dict)
except TypeError:
pass
return default
To test the results:
import random
COUNTRY = {'af', 'br', 'fr', 'mx', 'uk'}
people = [{'id': i, 'fields': {
'name': 'name'+str(i),
'email': str(i)+'#email',
'country': random.sample(COUNTRY, 1)[0]}}
for i in range(10)]
country_groups = make_groups(people, 'country')
for country, persons in country_groups.items():
print(country, persons)
Random output:
fr [{'id': 0, 'fields': {'name': 'name0', 'email': '0#email', 'country': 'fr'}}, {'id': 1, 'fields': {'name': 'name1', 'email': '1#email', 'country': 'fr'}}, {'id': 4, 'fields': {'name': 'name4', 'email': '4#email', 'country': 'fr'}}]
br [{'id': 2, 'fields': {'name': 'name2', 'email': '2#email', 'country': 'br'}}, {'id': 8, 'fields': {'name': 'name8', 'email': '8#email', 'country': 'br'}}]
uk [{'id': 3, 'fields': {'name': 'name3', 'email': '3#email', 'country': 'uk'}}, {'id': 7, 'fields': {'name': 'name7', 'email': '7#email', 'country': 'uk'}}]
af [{'id': 5, 'fields': {'name': 'name5', 'email': '5#email', 'country': 'af'}}, {'id': 9, 'fields': {'name': 'name9', 'email': '9#email', 'country': 'af'}}]
mx [{'id': 6, 'fields': {'name': 'name6', 'email': '6#email', 'country': 'mx'}}]