django timeseries postgres beginner - python

I'm writing small application in django in order to learn databases and web etc.
The application should display players data and record statistics over time.
To answer questions such as:
How many random battles palyer has played in last month/week/day? - graph
What players left/entered clan? - etc.
I've been searching and reading and I found these suggestions:
What is a good combination of technology to use?
MongoDB, InfluxDB, PostgreSQL, RedisDB, or from packages django-timeseries, django-reversion.
I did not find my approach to be good and I probably, can anybody suggest me which database to use and how models should look like then a bit?
Critics or advice with database design is very appreciated.
Data is being downloaded from 3rd party API as jsons.
Structures which are downloaded:
Player.json
{
"501435906": { # this is PlayerID
"last_battle_time": 1484160229,
"statistics": {
"all": {
"battles": 70555
},
"random": {
"battles": 67361
}
}
}, # then next players continue
}
clan.json
{
"500004323": { # clan ID
"members": [
{
"account_id": 501435906, # PlayerID same as in Player.json
"account_name": "Player1",
"joined_at": 1447589992,
"role": "private",
"role_i18n": "Private"
},]
"name": "Full Clan Name",
"tag": "TAG",
}
stronghold.json
{
"500323931": { # PlayerID
"stronghold_skirmish": null,
"total_resources_earned": 0,
"week_resources_earned": 0
}, # next player follows
}
My approach:
Merge data together
{
"500004323": {
'name': 'Full Clan Name',
'tag': 'TAG'
"members": {
"500012979": {
"account_id": "500012979",
"account_name": "Player1",
"joined_clan_date": 1415990768,
"last_battle_time": 1484160229,
"role": "Commander",
"statistics": {
"all": {
"battles": 70555
},
"random": {
"battles": 67361
}
},
"stronghold": {
"stronghold_skirmish": {
"battles": 2223
},
"total_resources_earned": 32582,
"week_resources_earned": 80
}
}, # next members
}, # next clan
}
And import this data into following models:
class Clan(models.Model):
"""Clan model"""
clan_id = models.IntegerField(primary_key=True)
name = models.CharField(max_length=100)
tag = models.CharField(max_length=5)
#property
def members(self):
return Player.objects.filter(clan=self)
def kick_player(self, player):
player.leave_clan()
class PlayerLeaversManager(models.Manager):
def leave_clan(self, players):
"""Take list of players and remove them from their clan
:param players:
"""
for player in players:
player.leave_clan()
class Player(models.Model):
"""Player model"""
account_id = models.IntegerField(primary_key=True)
access_token = models.TextField(blank=True,
null=True)
access_token_expires_at = models.CharField(max_length=10,
blank=True,
null=True)
account_name = models.CharField(max_length=250)
clan = models.ForeignKey('Clan',
on_delete=models.CASCADE,
blank=True,
null=True,
related_name='current_clan')
previous_clan = models.ForeignKey('Clan',
on_delete=models.CASCADE,
blank=True,
null=True,
related_name='previous_clan')
# objects = models.Manager()
objects = PlayerLeaversManager()
def __str__(self):
return '{0} - {1}'.format(self.account_name, self.account_id)
def get_absolute_url(self):
return reverse('wot:player_detail',
args=[self.account_name])
def leave_clan(self):
self.previous_clan = self.clan
self.clan = None
self.save()
class PlayerData(models.Model):
"""Players data daily tracked"""
created = models.DateTimeField(auto_now_add=True)
player = models.ForeignKey('Player',
on_delete=models.CASCADE,
null=True,
blank=True
)
joined_clan_date = models.DateTimeField(blank=True,
null=True)
role_in_clan = models.CharField(max_length=250,
blank=True,
null=True)
battles_on_random = models.IntegerField(blank=True,
null=True)
battles_all = models.IntegerField(blank=True,
null=True)
battles_stronghold = models.IntegerField(blank=True,
null=True)
tank = models.ManyToManyField('Vehicle',
related_name='tanks',
blank=True,)
last_battle_time = models.DateTimeField(blank=True,
null=True)
# stronghold stats
total_resources_earned = models.IntegerField(blank=True, null=True)
week_resources_earned = models.IntegerField(blank=True, null=True)
Whole code can be found on my github https://github.com/1oglop1/anv_wot
Thank you for all suggestions.

After some more research and asking at python FB community I will try following:
1) Normalize the models
2) Use PostgreSQL as backend.

Related

How to retrieve all Variants of a single Product in a single JSON object per Product- Django Rest Framework

The way I am doing it is like in getProductList function in my views.py and it is giving me each product with its variants in different objects for eg the same product with different size is put into a different object even if the product is the same like so:
JSON:
[
{
"prod_id": {
"id": 3,
"prod_name": "CRUISER-149S-DEERCALF",
"category": 2
},
"size_id": {
"id": 2,
"name": "26"
},
"color_id": 2,
"image_id": {
"id": 10,
"prod_image": "/media/products/IMG_7617_ftgqoa.jpg"
}
},
{
"prod_id": {
"id": 3,
"prod_name": "CRUISER-149S-DEERCALF",
"category": 2
},
"size_id": {
"id": 3,
"name": "27"
},
"color_id": 2,
"image_id": {
"id": 10,
"prod_image": "/media/products/IMG_7617_ftgqoa.jpg"
}
}
]
Is there a way to maybe make it more optimized? like have only one product object that has all the info of a single product like size for example can be a property with values of all sizes in it like size: [26, 27] instead of a different object if a size is different.
models.py
class Products(models.Model):
prod_name = models.CharField(max_length=200)
category = models.ForeignKey(Categories,null=True, on_delete=models.SET_NULL)
def __str__(self):
return '%s'%(self.prod_name)
class Variants(models.Model):
prod_id = models.ForeignKey(Products, on_delete=models.CASCADE)
size_id = models.ForeignKey(Size, null=True, on_delete=models.SET_NULL, related_name='variants')
color_id = models.ForeignKey(Color, null=True, on_delete=models.SET_NULL, default='')
image_id = models.ForeignKey(Image,null=True, on_delete=models.SET_NULL, default='')
def __str__(self):
return '%s %s %s %s %s %s %s'%(self.id , self.prod_id, self.size_id, self.image_id, self.color_id, self.quantity, self.price)
serializer.py
class ProductsSerializer(serializers.ModelSerializer):
# variants = serializers.StringRelatedField(many=True)
class Meta:
model = Products
fields = "__all__"
class SizeSerializer(serializers.ModelSerializer):
class Meta:
model = Size
fields = "__all__"
class VariantsSerializer(serializers.ModelSerializer):
prod_id = ProductsSerializer(many=False, read_only=True)
image_id = ImageSerializer(many=False, read_only=True)
size_id = SizeSerializer(many=False, read_only=True)
class Meta:
model = Variants
fields = ['prod_id', 'size_id', 'quantity', 'price', 'color_id', 'image_id']
views.py
def getProductList(request):
print(request.GET.get('category'))
if(request.method == 'GET'):
if request.GET.get('category') == 'all' :
productList = Variants.objects.all().select_related()
serializer = VariantsSerializer(productList, many=True)
return JsonResponse(serializer.data, safe=False)
return JsonResponse({'details': 'Error in getting product list'})

Django rest framework - Foreign key - one to many relation

For my project I need to have a simple process
a form => many questions => for each questions a response type => if the response type is a multiple choice, then possible response are in a uniq group.
My goal is to be able to push though DRF/Django rest framework the whole initial form.
Here are my models:
class FormModels(models.Model):
class Status(models.IntegerChoices):
DRAFT = 1, _("DRAFT")
PUBLISHED = 2, _("PUBLISHED")
ARCHIVED = 5, _("ARCHIVED")
form_name = models.CharField(unique=True, blank=False, null=False, max_length=255)
form_model = models.CharField(unique=True, blank=False, null=False, max_length=255)
status = models.IntegerField(choices=Status.choices, blank=False, null=False, default=1)
def __str__(self):
return self.form_name
class FormResponseTypeModels(models.Model):
class Type(models.IntegerChoices):
TEXT = 1, _('TEXT')
MULTICHOICE = 2, _('MULTI')
type = models.IntegerField(choices=Type.choices, blank=False, null=False, default=1)
group = models.IntegerField(blank=False, null=True, default=1)
def __str__(self):
return str(self.type)
class Meta:
constraints = [
UniqueConstraint(fields=['group'], name='unique_group')
]
class MultipleChoiceDataModels(models.Model):
text = models.CharField(blank=False, null=False, max_length=255)
value = models.CharField(blank=False, null=False, max_length=255)
order = models.IntegerField(blank=False, null=False, default=1)
group_refid = models.ForeignKey(blank=False,
null=False,
default=1,
to=FormResponseTypeModels,
to_field="group",
related_name="groups",
on_delete=models.PROTECT)
def __str__(self):
return self.text
class FormQuestionModels(models.Model):
form_id = models.ForeignKey(to=FormModels, on_delete=models.PROTECT)
form_response_type = models.ForeignKey(to=FormResponseTypeModels, on_delete=models.PROTECT, related_name="test")
form_question = models.TextField(max_length=2000)
def __str__(self):
return self.form_question
Here is my serializers.py
class FormMultipleChoiceDataSerializer(serializers.ModelSerializer):
class Meta:
model = MultipleChoiceDataModels
fields = "__all__"
depth = 10
class FormResponseTypeSerializer(serializers.ModelSerializer):
groups = FormMultipleChoiceDataSerializer(many=True,
read_only=False,
required=False,
)
class Meta:
model = FormResponseTypeModels
fields = "__all__"
depth = 10
class FormQuestionSerializer(serializers.ModelSerializer):
form_response_type_set = FormResponseTypeSerializer(many=False,
read_only=False,
required=False
)
class Meta:
model = FormQuestionModels
fields = "__all__"
depth = 10
class FormSerializer(serializers.HyperlinkedModelSerializer):
questions = FormQuestionSerializer(many=True,
read_only=False,
source='formquestionmodels_set',
required=False)
class Meta:
model = FormModels
fields = "__all__"
def validate(self, attrs):
pass
return super().validate(attrs=attrs)
def create(self, validated_data):
# https://stackoverflow.com/questions/58023503/drf-serializer-field-renamed-to-its-source-in-validated-data
# question is 'formquestionmodels_set'
questions_data = validated_data.pop('formquestionmodels_set')
form = FormModels.objects.create(**validated_data)
for question_data in questions_data:
if "form_response_type_set" in questions_data:
form_response_type_set = question_data.pop("form_response_type_set")
max_group_id = None
if "formmultiplechoicedatamodel_set" in form_response_type_set:
form_response_type_set_groups = form_response_type_set.pop("formmultiplechoicedatamodel_set")
# find max group_id + 1
max_group_id = 1 + MultipleChoiceDataModels.objects.aggregate(Max("group_refid"))
for form_response_type_set_group in form_response_type_set_groups:
MultipleChoiceDataModels.objects.create(group_refid=max_group_id,
**form_response_type_set_group)
if max_group_id:
form_response_type = FormResponseTypeModels.objects.create(group=max_group_id,
**form_response_type_set)
else:
form_response_type = FormResponseTypeModels.objects.create(**form_response_type_set)
FormQuestionModels.objects.create(form_id=form, form_response_type=form_response_type, **question_data)
return form
Expected result would be:
{
"form_name": "test7",
"form_model": "test7",
"status": 1,
"questions": [
{
"form_question": "q1",
"form_response_type_set": {
"type": 1,
"group": [
{
"text": "a",
"value": "1",
"order": 1,
"group_refid": 1
}
]
}
},
{
"form_question": "q2",
"form_id": {
"form_name": "form_name",
"form_model": "form_model",
"status": 1
},
"form_response_type_set": {
"type": 1,
"group": [
{
"text": "a",
"value": "1",
"order": 1,
"group_refid": 1
}
]
}
}
]
}
But for the group part, I can't get it display. I got:
"questions": [
{
"id": 1,
"form_question": "q1",
"form_id": {
"id": 1,
"form_name": "t1",
"form_model": "t1",
"status": 2
},
"form_response_type": {
"id": 1,
"type": 1,
"group": null
}
},
{
"id": 2,
"form_question": "q2",
"form_id": {
"id": 1,
"form_name": "t1",
"form_model": "t1",
"status": 2
},
"form_response_type": {
"id": 2,
"type": 2,
"group": 1
}
}
],
"form_name": "t1",
"form_model": "t1",
"status": 2
}
]
With django shell I get this result:
FormResponseTypeSerializer():
id = IntegerField(label='ID', read_only=True)
groups = FormMultipleChoiceDataSerializer(many=True, read_only=False, required=False):
id = IntegerField(label='ID', read_only=True)
text = CharField(max_length=255)
value = CharField(max_length=255)
order = IntegerField(max_value=2147483647, min_value=-2147483648, required=False)
group_refid = NestedSerializer(read_only=True):
id = IntegerField(label='ID', read_only=True)
type = ChoiceField(choices=[(1, 'TEXT'), (2, 'MULTI')], required=False, validators=[<django.core.validators.MinValueValidator object>, <django.core.validators.MaxValueValidator object>])
group = IntegerField(allow_null=True, max_value=2147483647, min_value=-2147483648, required=False)
type = ChoiceField(choices=[(1, 'TEXT'), (2, 'MULTI')], required=False, validators=[<django.core.validators.MinValueValidator object>, <django.core.validators.MaxValueValidator object>])
group = IntegerField(allow_null=True, max_value=2147483647, min_value=-2147483648, required=False)
I've tried source and related name, many type of serializers, _set syntax for varialbe source ... , read 40+ stack overflow threads, but each time I'm stuck with this (also, if it matters, I would like to be able to post a new form at once with same endpoint)
Can you kindly point me to my error ?
I'm not sure what I did for it to work, but after lots of tries
using related_name in models,
changing the fields name to avoid using text, value and order,
deleting the database and apply migrations again,
set depth=1 to each parent serializer.
Now it works!
If someone have a good explanation, I would be glad to read it & learn!

How to clean the json output of a #property variable in Django Rest Framework

The issues is, when I do a get request for this model, the JSON response is not clean making it difficult to work with in Retrofit.
Here is an example JSON output:
[
{
"id": 1,
"restName": "Buddy's",
"banner": "http://127.0.0.1:8000/media/buddysImg.jpg",
"zipcode": 48212,
"restRating": {
"rating__avg": 4.0
},
"website": "Buddys.com",
"phone": 3138929001,
"restPrice": 0.0
}
]
And this is how I'd like it to look:
[
{
"id": 1,
"restName": "Buddy's",
"banner": "http://127.0.0.1:8000/media/buddysImg.jpg",
"zipcode": 48212,
"restRating": 4.0,
"website": "Buddys.com",
"phone": 3138929001,
"restPrice": 0.0
}
]
Here is my model :
class Rest(models.Model):
restName = models.CharField(max_length=50, null=False, default = " ")
zipcode = models.PositiveIntegerField( null=False, default = 0)
#restRating = models.FloatField( null=False, default = 0)
banner = models.ImageField( null=True)
website = models.CharField(max_length=50, null=False, default = " ")
phone = models.PositiveIntegerField( null=False, default = 0)
restPrice = models.FloatField( null=False, default = 0)
#property
def restRating(self):
avg = Rating.objects.filter(restId=self.pk).aggregate(Avg('rating'))
return avg
def __str__(self):
return self.restName
And here is my serializer :
class restSerializer(serializers.ModelSerializer):
restRating = serializers.FloatField
class Meta:
model = Rest
fields = ['id', 'restName', 'banner', 'zipcode', 'restRating', 'website', 'phone', 'restPrice']
Thank you for the help.
you can return only the value from your property. You need to change your property to this:
#property
def restRating(self):
avg = Rating.objects.filter(restId=self.pk).aggregate(Avg('rating'))['rating__avg']
return avg
from the aggregate docs:
aggregate() is a terminal clause for a QuerySet that, when invoked,
returns a dictionary of name-value pairs. The name is an identifier
for the aggregate value; the value is the computed aggregate. The name
is automatically generated from the name of the field and the
aggregate function.

iteration through lookups table in Django Rest Framework

I have a lookups table that contains course categories and subcategories separate:
{
"id": 138,
"lookup": "CRS_CTGRY",
"attr1": "Arts and Humanities",
"attr2": "الفنون والعلوم الإنسانية",
"attr3": null,
"attr4": null,
"attr5": null,
"editable": 1
},
{
"id": 155,
"lookup": "CRS_SB_CTGRY",
"attr1": "Photography",
"attr2": "النصوير",
"attr3": "138",
"attr4": null,
"attr5": null,
"editable": 1
},
The relation between them is that attr3 = id_of_the_category && attr1 = CRS_SB_CTGRY
I want to merge them together in one list like:
{"id":138,"
"lookup":"CRS_CTRGY",
"name":"Arts and Humanities",
"subcategories":{"id": 154,
"lookup": "CRS_SB_CTGRY",
"attr1": "Music",
"attr2": "الموسيقي",
"attr3": "138",
"attr4": null,
"attr5": null,
"editable": 1
}}
This is my models.py:
class Lookups(models.Model):
lookup = models.CharField(max_length=45)
attr1 = models.CharField(max_length=100)
attr2 = models.CharField(max_length=100, blank=True, null=True)
attr3 = models.CharField(max_length=100, blank=True, null=True)
attr4 = models.CharField(max_length=100, blank=True, null=True)
attr5 = models.CharField(max_length=100, blank=True, null=True)
editable = models.IntegerField(blank=True, null=True)
class Meta:
managed = True
db_table = 'lookups'
unique_together = (('lookup', 'attr1', 'attr2', 'attr3', 'attr4', 'attr5'),)
How can i do it? and where to put the code? in the serializers class?
I really have no idea what you are trying to do, because your example (text) doesn't match the example (code screenshots).
From your code screenshot it seems that if lookup is CRS_CTGRY, this is the main category, and if lookup is CRS_SUB_CTGRY, then this is a sub category, and the parent category id is stored in attr3.
Now you want you get a dictionary object for this structure. Assuming you have all the data in your model:
combined = []
lookup_indices = {} # stores parents lookup for easier access.
for obj in Lookups.objects.filter(lookup='CRS_CTGY').values():
obj['subcategories'] = []
combined.append(obj)
lookup_indicies[obj.pk] = len(combined) - 1
for obj in Lookups.objects.filter(lookup='CRS_SUB_CTGRY').values():
try:
combined[lookup_indicies[int(obj['attr3'])]]['subcategories'].append(obj)
except KeyError:
print('No parent id {} found for sub cat {}'.format(obj['attr3'], obj))
print(combined)

Ways to save django model using nested dictionary

I'll try to make this as simple as I can:
I have 2 models
from django.db import models
class OrderDetail(models.Model):
product = models.CharField(max_length=100)
price = models.CharField(max_length=50)
class Order(models.Model):
url = models.CharField(max_length=255, unique=True)
loaded_info = models.BooleanField(default=False)
status = models.CharField(max_length=100, null=True)
details = models.OneToOneField(OrderDetail)
And I whish to save using a dict like this:
data = {
"order": {
"url": "http://stackoverflow.com/",
"loaded_info": True,
"status": "complete",
"details": {
"product": "Fresh answer",
"price": "50 points"
}
}
}
I'd like to do something close to:
order = Order(**data).save()
And get Order and OrderDetail saved using a single line.
Have a look at https://docs.djangoproject.com/en/1.9/topics/serialization/
In this case, you would do something like:
qs = Order.objects.select_related('details').get(pk=1)
data = serializers.serialize("json", qs)

Categories