I write a simple json api, I use one base class, and I mostly write one api view per one model class. What I want is to combine the output of few views into one url endpoint, with as least as possible additional code.
code:
# base class
class JsonView(View):
def get(self, request):
return JsonResponse(self.get_json())
def get_json(self):
return {}
class DerivedView(JsonView):
param = None
def get_json(self):
# .. use param..
return {'data': []}
urls.py:
url('/endpoint1', DerivedView.as_view(param=1))
url('/endpoint2', DerivedView2.as_view())
# What I want:
url('/combined', combine_json_views({
'output1': DerivedView.as_view(param=1),
'output2': DerivedView2.as_view()
}))
So /combined would give me the following json response:
{'output1': {'data': []}, 'output2': output of DerivedView2}
This is how combine_json_views could be implemented:
def combine_json_views(views_dict):
d = {}
for key, view in views_dict.items():
d[key] = view() # The problem is here
return json.dumps(d)
The problem is that calling view() give me the encoded json, so calling json.dumps again gives invalid json. I could call json.loads(view()), but that looks bad to decode the json that I just encoded.
How can I modify the code (maybe a better base class) here, while keeping it elegant and short? without adding too much code. Is there any way to access the data (dict) which is used to construct JsonResponse?
You can create a combined view that calls the get_json() methods and combines them:
class CombinedView(JsonView):
def get_json(self):
view1 = DerivedView(param=1)
view2 = DerivedView2()
d = view1.get_json()
d.update(view2.get_json())
return d
then:
url('/combined', CombinedView.as_view()),
Related
I am using Django DRF's CursorPagination for lazy loading of my data, and currently my goal is to sort the data by more than one field.
This is how my code looks like now:
class EndlessPagination(CursorPagination):
ordering_param = ''
def set_ordering_param(self, request):
self.ordering = request.query_params.get(self.ordering_param, None)
if not self.ordering:
raise ValueError('Url must contain a parameter named ' +
self.ordering_param)
if self.ordering.startswith("\"") or self.ordering.endswith("\""):
raise ValueError('Ordering parameter should not include quotation marks'
def paginate_queryset(self, queryset, request, view=None):
# This function is designed to set sorting param right in the URL
self.set_ordering_param(request)
return super(EndlessPagination, self).paginate_queryset(queryset, request, view)
This code works fine for urls like my_url/sms/270380?order_by=-timestamp, but what if I want to sort by several fields ?
Use str.split() to split the url params
class EndlessPagination(CursorPagination):
ordering_param = 'order_by'
def set_ordering_param(self, request):
ordering_param_list = request.query_params.get(self.ordering_param, None)
self.ordering = ordering_param_list.split(',')
# here, "self.ordering" will be a "list", so, you should update the validation logic
"""
if not self.ordering:
raise ValueError('Url must contain a parameter named ' +
self.ordering_param)
if self.ordering.startswith("\"") or self.ordering.endswith("\""):
raise ValueError('Ordering parameter should not include quotation marks'
"""
def paginate_queryset(self, queryset, request, view=None):
# This function is designed to set sorting param right in the URL
self.set_ordering_param(request)
return super(EndlessPagination, self).paginate_queryset(queryset, request, view)
Example URLs
1. my_url/sms/270380?order_by=-timestamp
2. my_url/sms/270380?order_by=-timestamp,name
3. my_url/sms/270380?order_by=-name,foo,-bar
UPDATE-1
First of all thanks to you for giving a chance to dig deep :)
As you said, me too didn't see comma seperated query_params in popular APIs. So, Change the url format to something like,my_url/sms/270380??order_by=-name&order_by=foo&order_by=-bar
At this time, the request.query_params['order_by'] will be a list equal to ['-name','foo','-bar']. So, you don't want to use the split() function, hence your set_ordering_param() method become,
def set_ordering_param(self, request):
self.ordering = request.query_params.get(self.ordering_param, None)
#...... your other validations
Basically, I realize that I am writing the same test case (test_update_with_only_1_field) for a similar URL for multiple models
from django.test import RequestFactory, TestCase
class BaseApiTest(TestCase):
def setUp(self):
superuser = User.objects.create_superuser('test', 'test#api.com', 'testpassword')
self.factory = RequestFactory()
self.user = superuser
self.client.login(username=superuser.username, password='testpassword')
class SomeModelApiTests(base_tests.BaseApiTest):
def test_update_with_only_1_field(self):
"""
Tests for update only 1 field
GIVEN the following shape and related are valid
WHEN we update only with just 1 field
THEN we expect the update to be successful
"""
shape_data = {
'name': 'test shape',
'name_en': 'test shape en',
'name_zh_hans': 'test shape zh hans',
'serial_number': 'test shape serial number',
'model_name': {
'some_field': '123'
}
}
data = json.dumps(shape_data)
response = self.client.post(reverse('shape-list-create'), data, 'application/json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
some_model = response.data['some_model']
new_some_field = '12345'
data = json.dumps({'some_field': new_some_field, 'id': response.data['some_model']['id']})
response = self.client.put(reverse('some-model', args=[some_model['id']]), data, 'application/json')
self.assertEqual(response.status_code, status.HTTP_200_OK)
self.assertEqual(new_some_field, response.data['some_field'])
I need to do this for more than 10 times. Which I have already done so.
the only difference each time, is the following phrases "some_model", "some-model", and "some_field"
I was wondering if there's a faster way to do this.
I can think abstractly two ways:
create a template in a text editor that somehow can generate the final test case which I then copy and paste. I am using sublime text 3 though I am okay to switch to another text editor
There's a way I can write slightly more code in the form of converting this test case into a behavior class that the individual test class can call. aka composition.
Which one makes more sense or there's a different way to do this?
Please note that BaseApi class is also inherited by other test class that do NOT have that repetitive test case method.
I guess what you want is "parameterized tests", standard unittest could do this with parameterized package:
import unittest
from parameterized import parameterized
class SomeModelApiTests(unittest.TestCase):
#parameterized.expand([
('case1', 'm1', 'f1', 'nf1'),
('case1', 'm2', 'f2', 'nf2'),
])
def test_update_with_only_1_field(self, dummy_subtest_name, model_name, field_name, new_field_value):
print(model_name, field_name, new_field_value)
will yields:
test_update_with_only_1_field_0_case1 (t.SomeModelApiTests) ... m1 f1 nf1
ok
test_update_with_only_1_field_1_case1 (t.SomeModelApiTests) ... m2 f2 nf2
ok
pytest testing framework has better support builtin on parameterized tests, worth looking at.
You could create a list / dict of "some_model" to test, and use subtest (https://docs.python.org/3/library/unittest.html#distinguishing-test-iterations-using-subtests) for each of your "some_model" items.
my_list_of_model = [FirstModel, SecondModel]
for my_model in my_list_of_model:
with subTest(model=mymodel):
# Testing model here
If you want a different TestCase for each of your model, I think the multiple inheritance is the way to go:
class BaseApiTestCase(TestCase):
def setUp():
# Setup stuff
class RepetitiveTestCaseMixin:
# Class to do the repetitive stuff
def test_update_should_work(self):
# Do some thing with self.model and self.field here
class ModelTestCase(BaseApiTestCase, RepetitiveTestCaseMixin):
#classmethod
def setUpClass(cls):
super().setUpClass()
cls.model = MyModel
cls.field = 'some_field'
Projects I work on we sometimes use mixin + "customization hooks" when a test needs to repeated. (and endpoints like the "shape-list-create" is subject to change/refactored)
Example for question:
class TestUpdateWithOnly1FieldMixin(object):
some_model = None
some_field = None
some_model2 = None
def get_some_model(self):
return self.some_model
def get_some_field(self):
return self.some_field
def get_some_model2(self):
return self.some_model2
def test_update_with_only_1_field(self):
some_model = self.get_some_model()
# represents some-model in example
some_model2 = self.get_some_model2()
some_field = self.get_some_field()
shape_data = {
'name': 'test shape',
'name_en': 'test shape en',
'name_zh_hans': 'test shape zh hans',
'serial_number': 'test shape serial number',
'model_name': {
some_field: '123'
}
}
data = json.dumps(shape_data)
response = self.client.post(reverse('shape-list-create'), data, 'application/json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
some_model_data = response.data[some_model]
class SomeModelApiTests(base_tests.BaseApiTest, TestUpdateWithOnly1FieldMixin):
some_model = 'choose your model'
some_field = 'some_field'
some_model2 = 'some-model'
def get_some_field(self):
# Do customization
return 'some-field after customize'
How to split the customization hooks and what to put in mixin etc is based on the situation.
In my opinion the goal is to have the actual test case easy to follow. (Maybe move the "post shape-list-create" into a separate function as it might not really be relevant for that test case)
Another example, going a bit overboard with customizations but just to give an idea.
class TestWithGoodNameMixin(object):
some_model = None
some_field = None
# "Customization hooks"
def get_shape_data(self):
return {self.some_field: 'x'}
def create_model(self, shape_data):
response = self.client.post(reverse('shape-list-create'), shape_data,
'application/json')
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
return response[self.some_model]
def create_put_data(self, some_model_data):
# Add default implementation
pass
# .....
def test_update_with_only_1_field(self):
shape_data = self.get_shape_data()
some_model_data = self.create_model(shape_data)
data = self.create_put_data(some_model_data)
response = self.put_data(data)
self.assert_put_response(response)
You can use pytest package for unit testing.
It is very simple and easy to use.
#pytest.mark.parametrize() decorator can be used to achieve that functionality.
An example for parametrized test cases is as follows:
import pytest
class SampleTesting(object):
data_for_test = [
('{inputdata1:value1}','output1'),
('{inputdata1:value2}','output2')
]
#pytest.mark.parametrized('input_data, expected_output', data_for_test)
def test_sample_function(self, input_data, expected_output):
response = function_to_be_tested(input_data)
assert response == expected_output
You can read more about this decorator in the docs'
You can also use the #pytest.fixture() decorator to setup the test function.
I've been searching the internet and couldn't find a simple example to encode and decode a custom object using JSON in python.
Let's say I have the following class:
class Test:
def __init__(self, name=None, grade=None):
self.name = name
self.grade = grade
and also have a list of Test objects:
t1 = Test("course1", 80)
t2 = Test("course2", 90)
list_of_tests = [t1, t2]
How can I serialize the class Test and the object list_of_tests
using JSON? I want to be able to write it to a file and read it from a file, using python.
To be honest the easiest thing to do here is to manually create a list of dictionaries from your objects. Then you can pass that directly to the JSON functions.
data = [{'name': x.name, 'grade': x.grade} for x in list_of_tests]
with open('output.json', 'w') as out:
json.dump(data, out)
and read it back:
with open('output.json') as inp:
data = json.load(inp)
list_of_tests = [Test(x['name'], x['grade']) for x in data]
You can control how an unrecognised object is serialised by dumps(default=converter_function). For it to be valid JSON you'd have to return a plain dict with the fields you want plus some tag field identifying that it is to be treated specially by loads.
Then have another converter function to reverse the process passed to loads() as object_hook.
Playing with new Google App Engine MapReduce library filters for input_reader I would like to know how can I filter by ndb.Key.
I read this post and I've played with datetime, string, int, float, in filters tuples, but How I can filter by ndb.Key?
When I try to filter by a ndb.Key I get this error:
BadReaderParamsError: Expected Key, got u"Key('Clients', 406)"
Or this error:
TypeError: Key('Clients', 406) is not JSON serializable
I tried to pass a ndb.Key object and string representation of the ndb.Key.
Here are my two filters tuples:
Sample 1:
input_reader': {
'input_reader': 'mapreduce.input_readers.DatastoreInputReader',
'entity_kind': 'model.Sales',
'filters': [("client","=", ndb.Key('Clients', 406))]
}
Sample 2:
input_reader': {
'input_reader': 'mapreduce.input_readers.DatastoreInputReader',
'entity_kind': 'model.Sales',
'filters': [("client","=", "%s" % ndb.Key('Clients', 406))]
}
This is a bit tricky.
If you look at the code on Google Code you can see that mapreduce.model defines a JSON_DEFAULTS dict which determines the classes that get special-case handling in JSON serialization/deserialization: by default, just datetime. So, you can monkey-patch the ndb.Key class into there, and provide it with functions to do that serialization/deserialization - something like:
from mapreduce import model
def _JsonEncodeKey(o):
"""Json encode an ndb.Key object."""
return {'key_string': o.urlsafe()}
def _JsonDecodeKey(d):
"""Json decode a ndb.Key object."""
return ndb.Key(urlsafe=d['key_string'])
model.JSON_DEFAULTS[ndb.Key] = (_JsonEncodeKey, _JsonDecodeKey)
model._TYPE_IDS['Key'] = ndb.Key
You may also need to repeat those last two lines to patch mapreduce.lib.pipeline.util as well.
Also note if you do this, you'll need to ensure that this gets run on any instance that runs any part of a mapreduce: the easiest way to do this is to write a wrapper script that imports the above registration code, as well as mapreduce.main.APP, and override the mapreduce URL in your app.yaml to point to your wrapper.
Make your own input reader based on DatastoreInputReader, which knows how to decode key-based filters:
class DatastoreKeyInputReader(input_readers.DatastoreKeyInputReader):
"""Augment the base input reader to accommodate ReferenceProperty filters"""
def __init__(self, *args, **kwargs):
try:
filters = kwargs['filters']
decoded = []
for f in filters:
value = f[2]
if isinstance(value, list):
value = db.Key.from_path(*value)
decoded.append((f[0], f[1], value))
kwargs['filters'] = decoded
except KeyError:
pass
super(DatastoreKeyInputReader, self).__init__(*args, **kwargs)
Run this function on your filters before passing them in as options:
def encode_filters(filters):
if filters is not None:
encoded = []
for f in filters:
value = f[2]
if isinstance(value, db.Model):
value = value.key()
if isinstance(value, db.Key):
value = value.to_path()
entry = (f[0], f[1], value)
encoded.append(entry)
filters = encoded
return filters
Are you aware of the to_old_key() and from_old_key() methods?
I had the same problem and came up with a workaround with computed properties.
You can add to your Sales model a new ndb.ComputedProperty with the Key id. Ids are just strings, so you wont have any JSON problems.
client_id = ndb.ComputedProperty(lambda self: self.client.id())
And then add that condition to your mapreduce query filters
input_reader': {
'input_reader': 'mapreduce.input_readers.DatastoreInputReader',
'entity_kind': 'model.Sales',
'filters': [("client_id","=", '406']
}
The only drawback is that Computed properties are not indexed and stored until you call the put() parameter, so you will have to traverse all the Sales entities and save them:
for sale in Sales.query().fetch():
sale.put()
I use this code to define my class in GAE Python:
class Pair(db.Model):
find = db.StringProperty()
replace = db.StringProperty()
rule = db.StringProperty()
tags = db.StringListProperty()
created = db.DateTimeProperty()
updated = db.DateTimeProperty(auto_now=True)
Then I use this code to serialize objects of that class with simplejson:
class PairEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Pair):
return [str(obj.created), str(obj.updated), obj.find, obj.replace, obj.tags, obj.rule]
Finally I use this code to output the result as the response:
pairsquery = GqlQuery("SELECT * FROM Pair")
pairs = pairsquery.fetch(1000)
pairsList = []
for pair in pairs:
pairsList.append(json.dumps(pair, cls=PairEncoder))
serialized = json.dumps({
'pairs': pairsList,
'count': pairsquery.count()
})
self.response.out.write(serialized)
Here is a sample result I get:
{"count": 2, "pairs": ["[\"2010-12-06 12:32:48.140000\", \"2010-12-06 12:32:48.140000\", \"random string\", \"replacement\", [\"ort\", \"common\", \"movies\"], \"remove\"]", "[\"2010-12-06 12:37:07.765000\", \"2010-12-06 12:37:07.765000\", \"random string\", \"replacement\", [\"ort\", \"common\", \"movies\"], \"remove\"]"]}
All seems to be fine, except one thing - I need the fields in the response to have names from the class Pair, so there won't be just values but the names of the corresponding fields too. How can I do that?
class PairEncoder(json.JSONEncoder):
def default(self, obj):
if isinstance(obj, Pair):
return {"created": str(obj.created), "updated:": str(obj.updated), "find": obj.find, "replace": obj.replace, "tags": obj.tags, "rule": obj.rule}
return json.JSONEncoder.default(self, obj)
But you are 'double encoding' here - i.e. encoding the pairs, adding that string to an object and encoding that too. If you 'double decode' on the other end it should work - but it's not the 'proper' way to do things.
I supposed I found a better simple solution for this, instead of serializing it with simplejson, I just created a method inside Pair class that looks like this:
def return_dict(self):
return {'find':self.find, 'replace':self.replace, 'rule':self.rule, 'tags':self.tags}
and does all I need. Thanks!