The problem I have is that I don't seem to be able to filter my data based on timestamp, i.e. both date and hour.
My model looks as follows:
# Create your models here.
class HourlyTick(models.Model):
id = models.IntegerField(primary_key=True)
timestamp = models.DateTimeField(blank=True, null=True)
symbol = models.TextField(blank=True, null=True)
open = models.IntegerField(blank=True, null=True)
high = models.IntegerField(blank=True, null=True)
low = models.IntegerField(blank=True, null=True)
close = models.IntegerField(blank=True, null=True)
trades = models.IntegerField(blank=True, null=True)
volume = models.IntegerField(blank=True, null=True)
vwap = models.FloatField(blank=True, null=True)
class Meta:
managed = False
db_table = 'xbtusd_hourly'
My view:
class HourlyTickList(ListAPIView):
serializer_class = HourlyTickSerializer
def get(self, request):
start = request.GET.get('start', None)
end = request.GET.get('end', None)
tz = pytz.timezone("Europe/Paris")
start_dt = datetime.datetime.fromtimestamp(int(start) / 1000, tz)
end_dt = datetime.datetime.fromtimestamp(int(end) / 1000, tz)
qs = HourlyTick.objects.filter(timestamp__range = (start_dt, end_dt))
rawData = serializers.serialize('python', qs)
fields = [d['fields'] for d in rawData]
fieldsJson = json.dumps(fields, indent=4, sort_keys=True, default=str)
return HttpResponse(fieldsJson, content_type='application/json')
The message I receive is:
RuntimeWarning: DateTimeField HourlyTick.timestamp received a naive
datetime (2017-01-15 06:00:00) while time zone support is active.
RuntimeWarning)
However, when I use make_aware to fix this error, I get the error:
ValueError: Not naive datetime (tzinfo is already set)
My database contains data that looks like this:
2017-01-06T12:00:00.000Z
For some reason, the first option returns results, but it totally ignores the time.
How do I fix this?
The problem was because Python couldn't interpret the format I had stored in the database. Two solutions possible:
Writing a raw query with string transformation in Django
Storing the datetime fields in a different format
I went with option 2 since it was an already automated script for retrieving the data and now it works fine.
Related
I am trying to calculate the difference between consecutive numeric values in the odometer_reading field of my model.
My Models.py has fields like below:
class Refuel(models.Model):
vehicle = models.ForeignKey(Vehicle, blank=True, null=True, on_delete=models.SET_NULL)
gaz_station = models.ForeignKey(
GazStation, related_name=_("Station"), blank=True, null=True, on_delete=models.SET_NULL
)
odometer_reading = models.PositiveIntegerField(_("Compteur KM"), blank=True, null=True)
snitch = models.PositiveIntegerField(_("Mouchard KM"), blank=True, null=True)
fuel_quantity = models.DecimalField(_("Quantitée en Litres"), max_digits=5, decimal_places=1)
fuel_unit_price = models.DecimalField(_("Prix en DH"), max_digits=6, decimal_places=2)
note = models.CharField(_("Remarque"), max_length=255, blank=True, null=True)
created_at = models.DateTimeField(_("Created at"), auto_now_add=True, editable=False)
updated_at = models.DateTimeField(_("Updated at"), auto_now=True)
is_active = models.BooleanField(default=True)
#property
def total_price(self):
total_price = self.fuel_quantity * self.fuel_unit_price
return total_price
class Meta:
ordering = ["gaz_station", "-created_at"]
def __str__(self):
return self.vehicle.serie
I want to use a CBV to get the distance between every two refuel for the same vehicle so I can calculate fuel consumption per km.
is there a way to do it?
EDITED:
I want to return with every refuel the fuel consumption per km using the precedent refuel.
In your views you can create a view function that pulls two refuel objects then you can take the difference and use it as your templates context. You then can access it in the template using whatever you call it in this example we just used the same name as the variable "difference".
from myapp.models import Refuel
from django.template import loader
from django.shortcuts import render
from django.http import HttpResponse
def odometer_difference(request):
# get the two refuel objects with the odometer readings you needed to compare
refuel_1 = Refuel.objects.filter('your_filter')
refuel_2 = Refuel.objects.filter('your_filter')
#define template
template = loader.get_template('my_template.html')
difference = refuel_2.odometer_reading - refuel_1.odometer_reading
context = { 'difference':difference}
return HttpResponse(template.render(context, request))
You can use Window functions that uses Lag to get a previous row's value like this:
from django.db.models import Window
from django.db.models.functions import Lag
last_odometer_reading = Window(
expression=Lag('odometer_reading', default=0),
partition_by=F('vehicle')
order_by=F('created_at').asc(),
)
Refuel.objects.annotate(
last_odometer_reading=last_odometer_reading
).annotate(
odometer_difference=F('odometer_reading') - F('last_odometer_reading')
)
Each refuel row will be annotated with the last odometer reading (based on the refuel of the same vehicle) and will also be annotated with the difference between the current reading and the last reading.
I'm Using the form_valid **RefuelCreatView** when validation the new record to get the oldest data and perform my calculation and store the RefuelConsumption Model.
class RefuelCreationView(LoginRequiredMixin, CreateView):
model = Refuel
form_class = RefuelCreationForm
template_name = "refuel/refuel_form.html"
success_url = reverse_lazy("refuel:controlor-refuel-list")
def form_valid(self, form):
form.instance.user_id = self.request.user.id
form.instance.gaz_station = GazStation.objects.get(Controlor_id=self.request.user.id)
old_refuel_data = Refuel.objects.order_by().distinct("vehicle")
for refuel in old_refuel_data:
if refuel.vehicle == form.instance.vehicle and form.instance.odometer_reading:
consumption = (refuel.fuel_quantity / (form.instance.odometer_reading - refuel.odometer_reading)) * 100
FuelConsumption.objects.create(
vehicle=refuel.vehicle,
gaz_station=form.instance.gaz_station,
Controlor_id=self.request.user,
driver=refuel.vehicle.driver,
consumption=consumption,
is_active=True,
)
elif refuel.vehicle == form.instance.vehicle and form.instance.snitch:
consumption = (refuel.fuel_quantity / (form.instance.snitch - refuel.snitch)) * 100
FuelConsumption.objects.create(
vehicle=refuel.vehicle,
gaz_station=form.instance.gaz_station,
Controlor_id=self.request.user,
driver=refuel.vehicle.driver,
consumption=consumption,
is_active=True,
)
return super().form_valid(form)
def get_queryset(self, *args, **kwargs):
return Refuel.objects.select_related("vehicle__gaz_station__Controlor_id").filter(
vehicle__gaz_station__Controlor_id=self.request.user
)
and this work very well. I don't know if it's respecting the best practices in programming or not, I'm open to any advise.
I'm creating a Hotel table reservation web app in Django 3.x.x
I have a Table Model with isBooked field that is set to either False or True depending on a table status. Code for this model:
class Table(models.Model):
title = models.CharField(max_length=50, null=True)
t_id = models.IntegerField("Table ID", null=True, unique=True)
isBooked = models.BooleanField('Is Booked', default=False)
chairs_booked = models.IntegerField("Chairs Booked", default=0)
max_chairs = models.IntegerField("Total Chairs", default=0)
max_book = models.IntegerField("Least Bookable", default=0)
chairs_left = models.IntegerField("empty chairs", default=0)
Now, I have another Model called Reservation that stores data for booked Tables. Here is its code:
class Reservation(models.Model):
t_username = models.CharField("Booked By", max_length=100, null=True)
t_email = models.EmailField("Email Address", null=True)
t_phone = models.IntegerField("Phone Number", default=0, null=True)
t_id = models.IntegerField("Table Id", null=True)
booked_date = models.CharField("Check In Date", null=True, max_length=50) # '2020-10-27'
booked_time = models.CharField("Check in Time", null=True, max_length=50) # '7:00'
def _get_checkout_date(self):
t = self.booked_date
return t
checkout_date = property(_get_checkout_date)
def _get_checkout_time(self):
the_time = dt.datetime.strptime(self.booked_time, '%H:%M')
new_time = the_time + dt.timedelta(hours=3)
return new_time.strftime('%H:%M')
checkout_time = property(_get_checkout_time)
at this point, I have a table with following information stored in variables:
booked_time this holds a string for time showing when table was booked e.g. 7:00
checkout_time this holds a string for time showing when table should be vacated/checked-out e.g 10:00
so based on the above variables, I want to automatically mark a table's isBooked property to False after certain time has passed (e.g. 3 hours) since it was booked. How do I pull off such a thing?
if this is bad design ( It smells like one..) how do I automatically mark a table either booked/Free using this information/fields available in their respective models?
If you really need to update such data in database level, a solution would be using cronjobs, but this means to leave Django's domain.
It's not usual such a need, as often the meaning of the information is useful on application level, rather than database level. This means that you should evaluate how much time has passed, or if the passed time is above/below a threshold within a view function or model function.
An elegant solution for this would be to add a property to your model, and remove the is_booked field:
#property
def is_booked(self):
time_treshold_in_hours = 3
booked_datetime = datetime.strptime('{} {}'.format(self.booked_date, self.booked_time), '%Y-%m-%d %H:%M')
booked_deltatime = datetime.now() - booked_datetime
return booked_deltatime.seconds // 3600 > time_treshold_in_hours
and within the view you are evaluating if a table is booked, you can do:
def book_table(request):
# <get table>
if table.is_booked:
return HttpResponse('This table is booked already')
# <do your booking>
I'd also recommend changing the fields booked_date and booked_time to:
booked_on = models.DecimalField()
I want to export a report from the available data into a CSV file. I wrote the following code and it works fine. What do you suggest to improve the query?
Models:
class shareholder(models.Model):
title = models.CharField(max_length=100)
code = models.IntegerField(null=False)
class Company(models.Model):
isin = models.CharField(max_length=20, null=False)
cisin = models.CharField(max_length=20)
name_fa = models.CharField(max_length=100)
name_en = models.CharField(max_length=100)
class company_shareholder(models.Model):
company = models.ManyToManyField(Company)
shareholder = models.ForeignKey(shareholder, on_delete=models.SET_NULL, null=True)
share = models.IntegerField(null = True) # TODO: *1000000
percentage = models.DecimalField(max_digits=8, decimal_places=2, null=True)
difference = models.DecimalField(max_digits=11, decimal_places=2, null=True)
update_datetime = models.DateTimeField(null=True)
View:
def ExportAllShare(request):
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="shares.csv"'
response.write(u'\ufeff'.encode('utf8'))
writer = csv.writer(response)
writer.writerow(['date','company','shareholder title','shareholder code','difference','share'])
results = company_shareholder.objects.all()
for result in results:
row = (
result.update_datetime,
result.company.first().name_fa,
result.shareholder.title,
result.shareholder.code,
result.difference,
result.share,
)
writer.writerow(row)
return (response)
First of all if it's working fine for you, then it's working fine, don't optimize prematurely.
But, in a query like this you are running into n+1 problem. In Django you avoid it using select_related and prefetch_related. Like this:
results = company_shareholder.objects.select_related('shareholder').prefetch_related('company').all()
This should reduce the number of queries you are generating. If you need a little bit more performance and since you are not using percentage I would defer it.
Also, I would highly suggest you follow PEP8 styling guide and name your classes in CapWords convention like Shareholder and CompanyShareholder.
Using Django 1.11.6, MySql
I`m importing uniq only data rows from CSV file (~530 rows).
After 1st import - all 530 records updated to the DB.
If I import this file 2d time and ~30 last records will be updated to DB.
Get data:
obj.account = int(ready_item[0].replace("\"","").replace("*",""))
pai_obj.reporting_mask = str(ready_item[0].replace("\"","").replace("*",""))
pai_obj.group = ready_item[1].replace("\"","")
pai_obj.location = ready_item[2].replace("\"","")
pai_obj.terminal = ready_item[4].replace("\"","")
pai_obj.settlement_type = ready_item[5].replace("\"","")
pai_obj.settlement_date = datetime_or_none(report_data)
pai_obj.amount = float_or_none(ready_item[6].replace("\"","").replace("$","").replace(",",""))
data.append(pai_obj)
Import vie get_or_create():
for record in data:
Accountmode.objects.get_or_create(
account=record.account,
reporting_mask=record.reporting_mask,
group=record.group,
location=record.location,
terminal=record.terminal,
settlement_type=record.settlement_type,
amount=record.amount,
defaults={'settlement_date': record.settlement_date})
The Model:
class Accountmode(models.Model):
account = models.IntegerField(blank=True, default=0)
reporting_mask = models.IntegerField(blank=False, default=0)
group = models.CharField(max_length=1024, blank=True, null=True)
location = models.CharField(max_length=1024, blank=True, null=True)
settlement_date = models.DateField(null=True)
terminal = models.CharField(max_length=1024, blank=False, null=True)
settlement_type = models.CharField(max_length=1024, blank=False, null=True)
amount = models.DecimalField(max_digits=25, decimal_places=2)
created_date = models.DateTimeField(default=datetime.now, blank=True)
As I know, get_or_create() should check if data already exist first and create new record if Not. Why get_or_create() pass some records?
The case was about Flout values with +3 symbols after come (12,012).
Those values were duplicating each time the user import same file.
Next solution was found:
1. Save amount and other values at str during file rows parsing.
obj.account = int(ready_item[0].replace("\"","").replace("*",""))
pai_obj.reporting_mask = str(ready_item[0].replace("\"","").replace("*",""))
pai_obj.group = ready_item[1].replace("\"","")
pai_obj.location = ready_item[2].replace("\"","")
pai_obj.terminal = ready_item[4].replace("\"","")
pai_obj.settlement_type = ready_item[5].replace("\"","")
pai_obj.settlement_date = datetime_or_none(report_data)
pai_obj.amount = *str*(ready_item[6].replace("\"","").replace("$","").replace(",",""))
data.append(pai_obj)
The SQL I want to accomplish is this -
SELECT jobmst_id, jobmst_name, jobdtl_cmd, jobdtl_params FROM jobmst
INNER JOIN jobdtl ON jobmst.jobdtl_id = jobdtl.jobdtl_id
WHERE jobmst_id = 3296
I've only had success once with an inner join in django off of a annote and order_by but I can't seem to get it to work doing either prefetch_related() or select_related()
My models are as so -
class Jobdtl(models.Model):
jobdtl_id = models.IntegerField(primary_key=True)
jobdtl_cmd = models.TextField(blank=True)
jobdtl_fromdt = models.DateTimeField(blank=True, null=True)
jobdtl_untildt = models.DateTimeField(blank=True, null=True)
jobdtl_fromtm = models.DateTimeField(blank=True, null=True)
jobdtl_untiltm = models.DateTimeField(blank=True, null=True)
jobdtl_priority = models.SmallIntegerField(blank=True, null=True)
jobdtl_params = models.TextField(blank=True) # This field type is a guess.
class Meta:
managed = False
db_table = 'jobdtl'
class Jobmst(MPTTModel):
jobmst_id = models.IntegerField(primary_key=True)
jobmst_type = models.SmallIntegerField()
jobmst_prntid = TreeForeignKey('self', null=True, blank=True, related_name='children', db_column='jobmst_prntid')
jobmst_name = models.TextField(db_column='jobmst_name', blank=True)
# jobmst_owner = models.IntegerField(blank=True, null=True)
jobmst_owner = models.ForeignKey('Owner', db_column='jobmst_owner', related_name = 'Jobmst_Jobmst_owner', blank=True, null=True)
jobmst_crttm = models.DateTimeField()
jobdtl_id = models.ForeignKey('Jobdtl', db_column='jobdtl_id', blank=True, null=True)
jobmst_prntname = models.TextField(blank=True)
class MPTTMeta:
order_insertion_by = ['jobmst_id']
class Meta:
managed = True
db_table = 'jobmst'
I have a really simple view like so -
# Test Query with Join
def test_queryjoin(request):
queryset = Jobmst.objects.filter(jobmst_id=3296).order_by('jobdtl_id')
queryresults = serializers.serialize("python", queryset, fields=('jobmst_prntid', 'jobmst_id', 'jobmst_prntname', 'jobmst_name', 'jobmst_owner', 'jobdtl_cmd', 'jobdtl_params'))
t = get_template('test_queryjoin.html')
html = t.render(Context({'query_output': queryresults}))
return HttpResponse(html)
I've tried doing a bunch of things -
queryset = Jobmst.objects.all().prefetch_related()
queryset = Jobmst.objects.all().select_related()
queryset = jobmst.objects.filter(jobmst_id=3296).order_by('jobdtl_id')
a few others as well I forget.
Each time the json I'm getting is only from the jobmst table with no mention of the jobdtl results which I want. If I go the other way and do Jobdtl.objects.xxxxxxxxx same thing it's not giving me the results from the other model.
To recap I want to display fields from both tables where a certain clause is met.
What gives?
Seems that I was constantly looking in the wrong place. Coming from SQL I kept thinking in terms of inner joining tables which is not how this works. I'm joining the results from models.
Hence, rethinking my search I came across itertools and the chain function.
I now have 2 queries under a def in my views.py
from itertools import chain
jobmstquery = Jobmst.objects.filter(jobmst_id=3296)
jobdtlquery = Jobdtl.objects.filter(jobdtl_id=3296)
queryset = chain(jobmstquery, jobdtlquery)
queryresults = serializers.serialize("python", queryset)
That shows me the results from each table "joined" like I would want in SQL. Now I can focus on filtering down the results to give me what I want.
Remember folks, the information you need is almost always there, it's just a matter of knowing how to look for it :)
What you are looking for might be this
queryset = Jobmst.objects.filter(id=3296).values_list(
'id', 'name', 'jobmst_owner__cmd', 'jobmst_owner__params')
You would get your results with only one query and you should be able to use sort with this.
P.S. Coming from SQL you might find some great insights playing with queryset.query (the SQL generated by django) in a django shell.