We Keep Coding

Alternative to using `regroup` in Django template on `ListView` using a lot of memory

I have tried using the following to regroup a ListView queryset in my template so that objects are grouped by a related field's value:
<div class="grid grid-cols-4">
{% regroup object_list by related_field.sort_value as grouped_list %}
{% for group in grouped_list %}
<div class="border-2 border-black">
<span>... {{group.grouper}} ...</span>
{% for item in group.list %}
<p>{{item.related_field.name}}</p>
<p>{{item.description}}</p>
{% endfor %}
</div>
{% endfor %}
</div>
The model has about a dozen fields, two of them relational (first related model has a couple dozen fields, the other has just a few) and there's several hundred total objects in the ListView queryset results.
View:
class BigListView(ListView):
model = MyModel
template_name = 'my_model/big_list_view.html'
def get_queryset(self):
return MyModel.objects.order_by('related_model_b.sort_value')
Models:
class MyModel(models.Model):
code = models.CharField(max_length=30, unique=True)
price = models.DecimalField(max_digits=8, decimal_places=2)
sale_price = models.DecimalField(max_digits=8, decimal_places=2)
sale_price_quantity_3_5 = models.DecimalField(max_digits=8, decimal_places=2, blank=True, null=True)
sale_price_quantity_6 = models.DecimalField(max_digits=8, decimal_places=2, blank=True, null=True)
description = models.CharField(max_length=40)
location = models.CharField(max_length=20)
quantity_on_hand = models.IntegerField()
size = models.CharField(max_length=20)
tag_description = models.CharField(max_length=100)
color = ColorField(help_text='Theme Color')
image = models.ImageField(upload_to=assign_product_sku, blank=True, null=True)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
related_model_a = models.ForeignKey(RelatedModelA, related_name='mymodels', on_delete=models.CASCADE)
related_model_b = models.ForeignKey('RelatedModelB', related_name='mymodels', on_delete=models.CASCADE, blank=True, null=True)
class RelatedModelA(index.Indexed, models.Model):
sku = models.CharField(max_length=20, unique=True)
name = models.CharField(max_length=100)
alt_name = models.CharField(max_length=100, blank=True)
description = models.TextField(blank=True)
age = models.CharField(max_length=3, blank=True)
size = models.CharField(max_length=100, blank=True)
weight = models.CharField(max_length=100, blank=True)
shape = models.CharField(max_length=100, blank=True)
style = models.CharField(max_length=100, blank=True)
rate = models.CharField(max_length=100, blank=True)
attribute_a = models.CharField(max_length=100, blank=True)
attribute_b = models.CharField(max_length=100, blank=True)
attribute_c = models.CharField(max_length=100, blank=True)
attribute_d = models.CharField(max_length=100, blank=True)
attribute_e = models.CharField(max_length=100, blank=True)
attribute_f = models.CharField(max_length=100, blank=True)
attribute_g = models.CharField(max_length=100, blank=True)
attribute_h = models.CharField(max_length=100, blank=True)
attribute_i = models.CharField(max_length=100, blank=True)
attribute_j = models.CharField(max_length=100, blank=True)
attribute_k = models.CharField(max_length=100, blank=True)
attribute_l = models.CharField(max_length=100, blank=True)
attribute_m = models.CharField(max_length=100, blank=True)
attribute_n = models.CharField(max_length=100, blank=True)
attribute_o = models.CharField(max_length=100, blank=True)
attribute_p = models.CharField(max_length=100, blank=True)
attribute_q = models.CharField(max_length=40, blank=True)
attribute_r = models.CharField(max_length=100, blank=True)
attribute_s = models.CharField(max_length=100, blank=True)
comment = models.CharField(max_length=100, blank=True)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
image_url = models.URLField(blank=True)
slug = models.SlugField(max_length=100, blank=True)
tags = TaggableManager(blank=True)
class RelatedModelB(models.model):
code = models.CharField(max_length=30, unique=True)
sort_value = models.IntegerField()
last_retrieved = models.DateTimeField(auto_now=True)
json_data = models.JSONField(blank=True, null=True)
Viewing this on local/dev machine has no problems, but hitting the view on production Heroku it crashes the app because of excess memory usage (~2GB). The app is currently running on Hobby (512MB) and while I will likely bump this up to Standard 1x/2x, that's still well short on memory unless I go up to a higher performance dyno which is overkill with the exception of this single view.
What can I do to reduce this view's memory usage?

For starters you access related_field (related_model_b/related_model_a?) for each item in your queryset which is performing a DB query each time, use select_related() to fetch the related data in a single query
MyModel.objects.select_related('related_model_b', 'related_model_a').order_by(...)
Because you already sort by related_field.sort_value you can get away with using the ifchanged tag instead of regroup if regroup is a massive problem
{% for item in object_list %}
{% ifchanged item.related_field.sort_value %}<span>{{ item.related_field.sort_value }}</span>{% endifchanged %}
<p>{{item.related_field.name}}</p>
<p>{{item.description}}</p>
{% endfor %}

I ended up with a combination of Iain's suggestion to use selected_related and also using defer() to exclude some of the fields in my related models that aren't necessary for this view.
Specifically, I believe (but haven't tested) that the JSONField attached to a related model is the culprit, as this field normally holds just a couple kB of data, but in some cases it's holding ~400kB.
I changed the queryset in my view to the following:
class BigListView(ListView):
model = MyModel
template_name = 'my_model/big_list.html'
def get_queryset(self):
return MyModel.objects.select_related('related_model_a','related_model_b').order_by('related_model_b__sort_value').defer(
'related_model_b__json_data',
... additional fields ...
)
Original memory usage was nearly 2gB.
Using select_related only reduced that to ~800mB.
Using select_related and defer() reduced it down to ~200mB, which is no different than the app typically uses for any given operation.

Why is my django template tag not able to iterate my object across a model

I'm having an issue getting my template to produce a table that includes the models name, a count of the number of child models to that parent, and then a sum of a field in that child model. I've gotten the first two to return the desired output, but the last two only yield an empty space in the cell. Can't figure out where I'm missing it.
MODELS.PY FILE
class Employee(models.Model):
user = models.OneToOneField(User,null=True, blank=True, on_delete=models.CASCADE)
username = models.CharField(max_length=200, null=True)
firstname = models.CharField(max_length=200, null=True)
lastname = models.CharField(max_length=200, null=True)
email = models.CharField(max_length=200, null=True)
team = models.ForeignKey(Team, null=True, on_delete=models.SET_NULL)
profile_pic = models.ImageField(default="hulkhoganicon.jpg", null=True, blank=True)
date_created = models.DateTimeField(auto_now_add=True, null=True)
def __str__(self):
return self.firstname
class Event(models.Model):
STATUS = (
('Appointment Set', 'Appointment Set'),
('Hot Lead', 'Hot Lead'),
('Booked', 'Booked'),
('Lost Sale', 'Lost Sale'),
('X-Dated', 'X-Dated'),
)
PACKAGE = (
('1st Package', '1st Package'),
('2nd Package', '2nd Package'),
('3rd Package', '3rd Package'),
('4th Package', '4th Package'),
)
ADDONS = (
('None','None'),
('2nd Setup', '2nd Setup'),
('Uplighting', 'Uplighting'),
('Monogram', 'Monogram'),
('Intelligent Lighting', 'Intelligent Lighting'),
('Other', 'Other'),
)
employee = models.ForeignKey(Employee, null=True, on_delete=models.SET_NULL)
eventType = models.ForeignKey(EventType, null=True, on_delete=models.SET_NULL)
teamAssigned = models.ForeignKey(Team, null=True, on_delete=models.SET_NULL)
eventName = models.CharField(max_length=1000, null=True)
date_of_event = models.DateField(auto_now=False, null=True)
date_of_appt = models.DateField(auto_now=False, null=True)
date_booked = models.DateField(auto_now=False, null=True)
date_created = models.DateTimeField(auto_now_add=True, null=True)
package = models.CharField(max_length=200, null=True, choices=PACKAGE)
price = models.FloatField(null=True)
addons = models.CharField(max_length=200, null=True, choices=ADDONS)
addonPrice = models.FloatField(null=True)
addonOverage = models.FloatField(null=True)
status = models.CharField(max_length=200, null=True, choices=STATUS)
notes = models.CharField(max_length=1000, null=True)
tags = models.ManyToManyField(Tag)
def __str__(self):
return self.eventName
VIEWS.PY FILE
def teamDashboard(request):
events = Event.objects.all()
employees = Employee.objects.all()
total_employees = employees.count()
total_events = events.count()
events_sum = events.aggregate(Sum('price'))['price__sum']
booked_sum = events.filter(status='Booked').aggregate(Sum('price'))['price__sum']
Green_events = events.filter(teamAssigned__name='Green').count()
Green_events_sum = events.filter(teamAssigned__name='Green').aggregate(Sum('price'))['price__sum']
Green_booked_sum = events.filter(teamAssigned__name='Green', status='Booked').aggregate(Sum('price'))['price__sum']
Blue_events = events.filter(teamAssigned__name='Blue').count()
Blue_events_sum = events.filter(teamAssigned__name='Blue').aggregate(Sum('price'))['price__sum']
Orange_events = events.filter(teamAssigned__name='Orange').count()
Orange_events_sum = events.filter(teamAssigned__name='Orange').aggregate(Sum('price'))['price__sum']
Purple_events = events.filter(teamAssigned__name='Purple').count()
Purple_events_sum = events.filter(teamAssigned__name='Purple').aggregate(Sum('price'))['price__sum']
Booked = events.filter(status='Booked').count()
Appointment_Set = events.filter(status='Appointment Set').count()
Appt_set_sum = events.filter(status='Appointment Set').aggregate(Sum('price'))
Date = datetime.now()
context = {'Date':Date, 'total_employees':total_employees, 'events':events, 'employees':employees,
'total_events':total_events, 'Booked':Booked, 'Appointment_Set':Appointment_Set,
'Appt_set_sum':Appt_set_sum, 'Green_events':Green_events, 'Blue_events':Blue_events,
'Orange_events':Orange_events,'Purple_events':Purple_events, 'Green_events_sum':Green_events_sum,
'Blue_events_sum':Blue_events_sum, 'Orange_events_sum':Orange_events_sum,
'Purple_events_sum':Purple_events_sum,'Green_booked_sum':Green_booked_sum, 'events_sum':events_sum,
'booked_sum':booked_sum,
}
context['segment'] = 'teamDashboard'
return render(request, 'accounts/team-dashboard.html', context)
TEAM-DASHBOARD.HTML FILE
<tbody>
{% for employee in employees %}
{% if employee.team.name == "Green" %}
<tr>
<td></td>
<td>{{employee.firstname}}</td>
<td>{{employee.event_set.count}}</td>
<td>{{events_sum.employee_set}}</td>
<td>{{employee.events.booked_sum}}</td>
</tr>
{% endif %}
</tbody>
{% endfor %}
I'm just starting out with Django so thank you in advance if I'm making a really dumb mistake!

POST 400 Bad Request Django

My branch model has 4 primary keys which are unique together: ('branch_short_name', 'partner_short_name', 'tenant', 'contributor'). I would like to be able to create/ POST some objects with similar
'branch_short_name', 'partner_short_name', 'tenant' but then with a unique 'contributor'.
When doing that I get a 400 bad request error in postman. I don't know where the problem lies, in the model or in the serializer.
models.py
class Branch(models.Model):
additional_address1 = models.CharField(max_length=255, blank=True, null=True)
additional_address2 = models.CharField(max_length=255, blank=True, null=True)
branch_number = models.CharField(max_length=255, blank=True, null=True)
branch_short_name = models.CharField(primary_key=True, max_length=255)
building = models.CharField(max_length=255, blank=True, null=True)
city = models.CharField(max_length=255, blank=True, null=True)
end_date = models.DateTimeField(blank=True, null=True)
country_code = models.CharField(max_length=255, blank=True, null=True)
created = models.DateTimeField(blank=True, null=True)
display_name = models.CharField(max_length=255, blank=True, null=True)
display_web = models.BooleanField(blank=True, null=True)
district = models.CharField(max_length=255, blank=True, null=True)
file_origin_date = models.DateTimeField(blank=True, null=True)
filename = models.CharField(max_length=255, blank=True, null=True)
floor = models.CharField(max_length=255, blank=True, null=True)
business_name = models.CharField(max_length=255, blank=True, null=True)
house_number = models.CharField(max_length=255, blank=True, null=True)
import_file_number = models.BigIntegerField(blank=True, null=True)
contributor = models.CharField(max_length=255)
job_id = models.CharField(max_length=255, blank=True, null=True)
latitude = models.FloatField(blank=True, null=True)
longitude = models.FloatField(blank=True, null=True)
start_date = models.DateTimeField(blank=True, null=True)
branch_hours = models.CharField(max_length=255, blank=True, null=True)
po_box = models.CharField(max_length=255, blank=True, null=True)
province = models.CharField(max_length=255, blank=True, null=True)
region = models.CharField(max_length=255, blank=True, null=True)
remarks = models.CharField(max_length=255, blank=True, null=True)
street = models.CharField(max_length=255, blank=True, null=True)
tenant = models.CharField(max_length=255)
updated = models.DateTimeField(blank=True, null=True)
zip_code = models.CharField(max_length=255, blank=True, null=True)
ranking = models.FloatField()
branch_type = models.IntegerField(blank=True, null=True)
branch_contact_text1 = models.CharField(max_length=255, blank=True, null=True)
branch_contact_text2 = models.CharField(max_length=255, blank=True, null=True)
phone_number = models.CharField(max_length=255, blank=True, null=True)
email_address = models.CharField(max_length=255, blank=True, null=True)
website = models.CharField(max_length=500, blank=True, null=True)
branch_hours_osm = models.CharField(max_length=400, blank=True, null=True)
partner_short_name = models.ForeignKey('Partner', models.DO_NOTHING, db_column='partner_short_name')
features = models.TextField(blank=True, null=True) # This field type is a guess.
marketing_attribute_short_names = models.TextField(blank=True, null=True) # This field type is a guess.
marketing_attribute_values = models.TextField(blank=True, null=True) # This field type is a guess.
class Meta:
managed = False
db_table = 'branch'
unique_together = (('branch_short_name', 'partner_short_name', 'tenant', 'contributor'),)
serializers.py
class BranchSerializer(serializers.ModelSerializer):
class Meta:
model = Branch
fields = '__all__'
views.py
#api_view(['GET', 'POST', 'DELETE'])
def branch_list(request):
# GET list of branchs, POST a new branch, DELETE all branchs
if request.method == 'GET':
branchlist = Branch.objects.all()[:10]
branchlist_serializer = BranchSerializer(branchlist, many=True)
return JsonResponse(branchlist_serializer.data, safe=False)
# 'safe=False' for objects serialization
elif request.method == 'POST':
branch_data = JSONParser().parse(request)
branch_serializer = BranchSerializer(data=branch_data)
if branch_serializer.is_valid():
branch_serializer.save()
return JsonResponse(branch_serializer.data, status=status.HTTP_201_CREATED)
return JsonResponse(branch_serializer.errors, status=status.HTTP_400_BAD_REQUEST)
urls.py
urlpatterns = [
url(r'^api/places$', views.branch_list),
url(r'^api/placescount$', views.branch_count),
url(r'^api/latestimport/(?P<contributor>\D+)$', views.latest_import),
url(r'^api/partners$', views.partner_list),
url(r'^api/partners/(?P<partner_short_name>[0-9]+)$', views.partner_detail),
url(r'^api/places/(?P<branch_short_name>[0-9]+)$', views.branch_detail),
url(r'^api/places/(?P<branch_short_name>[0-9]+)/(?P<contributor>\D+)$' , views.branch_from_contributor),
]

Django {% for loop - showing 1 out of 3 values

I added for loop to my html, after I added more data to the Queryset. For some reason it's showing 1 out of 3 values I passed. - I could print them, it's just not showing up on the html.
The result page - client_name (in blue), missing product_name + waiting_time (in red)
Any ideas? thank you
models.py
class Orders(models.Model):
created_date = models.DateTimeField(auto_now_add=True)
client_id = models.IntegerField()
client_name = models.CharField(max_length=50, blank=True, null=True)
delivery_city = models.CharField(max_length=50, blank=True, null=True)
delivery_address = models.CharField(max_length=50, blank=True, null=True)
product_id = models.IntegerField()
sold_by = models.CharField(max_length=20)
sale_type = models.CharField(max_length=20, blank=True, null=True)
units_to_buy = models.IntegerField(blank=True, null=True)
order_cost = models.DecimalField(decimal_places=2, max_digits=100000, blank=True, null=True)
payment_method = models.CharField(max_length=20, blank=True, null=True)
payment_channel = models.CharField(max_length=20, blank=True, null=True)
invoice = models.CharField(max_length=20, blank=True, null=True)
delivery_cost = models.DecimalField(decimal_places=2, max_digits=100000, blank=True, null=True)
delivery_type = models.CharField(max_length=20, blank=True, null=True)
delivery_status = models.CharField(max_length=20, default='ממתין למשלוח')
delivery_person_name = models.CharField(max_length=20, blank=True, null=True)
notes = models.TextField(blank=True, null=True)
modified_date = models.DateTimeField(auto_now=True)
def __str__(self):
return '{}'.format(self.client_name + ' - (' + self.sold_by + ')')
class Clients(models.Model):
first_name = models.CharField(max_length=20)
last_name = models.CharField(max_length=20, blank=True, null=True)
client_type = models.CharField(max_length=15, blank=True, null=True)
company_name = models.CharField(max_length=30, blank=True, null=True)
address = models.CharField(max_length=30, blank=True, null=True)
city = models.CharField(max_length=15, blank=True, null=True)
phone_number = models.CharField(max_length=10)
additional_phone_number = models.CharField(max_length=10, blank=True, null=True)
email = models.EmailField(max_length=30, blank=True, null=True)
card_type = models.CharField(max_length=15, blank=True, null=True)
four_digits = models.IntegerField(blank=True, null=True)
notes = models.TextField(blank=True, null=True)
added = models.DateTimeField(auto_now=True)
created = models.DateTimeField(auto_now=True)
def __str__(self):
return '{}'.format(self.first_name)
class Products(models.Model):
product_name = models.CharField(max_length=50, verbose_name=u'שם המוצר')
supplier_name = models.CharField(max_length=20, blank=True, null=True)
purchase_price = models.DecimalField(decimal_places=2, max_digits=100000, blank=True, null=True)
purchase_price_before_fees = models.DecimalField(decimal_places=2, max_digits=100000, blank=True, null=True)
final_price = models.DecimalField(decimal_places=2, max_digits=100000, blank=True, null=True)
final_price_before_fees = models.DecimalField(decimal_places=2, max_digits=100000, blank=True, null=True)
inventory_in = models.IntegerField(blank=True, null=True)
inventory_out = models.IntegerField(blank=True, null=True)
inventory_total = models.IntegerField(blank=True, null=True)
notes = models.TextField(blank=True, null=True)
created = models.DateTimeField(auto_now=True)
def __str__(self):
return '{}'.format(self.product_name)
URLs.py
...
path('orders/', views.orders_filter_view, name='orders'),
...
Views.py
def orders_filter_view(request):
qs = models.Orders.objects.all()
...
for order in qs:
client = models.Clients.objects.get(pk=order.client_id)
client_name = client.first_name
qs.client_name = client_name <<<<<<<<<<<<<<<<<<<<<<<<<<<<<
product = models.Products.objects.get(pk=order.product_id)
product_name = product.product_name
qs.product_name = product_name <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
tz_info = order.created_date.tzinfo
qs.waiting_time = order.created_date - datetime.datetime.now(tz_info) <<<<<<<<<<<<<<<<<<<<<<<
total_matches = qs.count()
context = {
'orders': qs, <<<<<<<<<<<<<<<<<<<<<<<<<<<<<
'view': 'הזמנות',
'total_matches': total_matches,
'titles': orders_titles
}
return render(request, 'adolim/orders.html', context)
Orders.html
{% for order in orders %}
<tr>
<td class="center">{{ order.client_name }}</td> <<<<<<<<<<<<<<<<<<<<<<<<
<td class="center">{{ order.product_name }}</td> <<<<<<<<<<<<<<<<<<<<<<<<
...
<td class="center">{{ order.delivery_person_name }}</td>
{% if order.delivery_status == 'סופק' %}
<td class="center green-text">{{ order.delivery_status }}</td>
{% else %}
<td class="center">{{ order.delivery_status }}</td>
{% endif %}
<td class="center yellow-text">{{ order.waiting_time }}</td> <<<<<<<<<<<<<<<<<
The result page - client_name (in blue), missing product_name + waiting_time (in red)

One thing you are doing wrong is you run query in a for loop which is a big problem instead I suggest you to create a query with joining tables, one way to do it is using select_related or another that I like is using values: main difference is select_related will return you objects with all the fields while values will return an array of dictionaries with specified fields.
More info regarding select_related https://docs.djangoproject.com/en/3.0/ref/models/querysets/#select-related
# I am not sure how your Models looks like but I assume it is One-to-Many Clients-Orders
def orders_filter_view(request):
# client, product in quotes is the foreign key field name
qs = models.Orders.objects.select_related('client', 'product').all()
...
# I am not sure what are you trying to do with dates
# But now you can easily access any data from foreign key objects without hitting DB again
for order in qs:
print(order.client.first_name)
print(order.product.product_name)
tz_info = order.created_date.tzinfo
qs.waiting_time = order.created_date - datetime.datetime.now(tz_info)

Column closingevent.id doesn't exist LINE 1: SELECT "closingevent"."id", "closingevent"."owner", "closing

I've a problem with my templates. I try to render the following template
{% if liste_column %}
<table>
<tr><th>Owner</th>
<th>RegimeID</th>
<th>ClosingeventID</th>
<th>Category</th>
<th>Transaction or Closing</th>
<th>**** Accounting Field</th>
<th>Data Source</th>
</tr>
{% for item in liste_column %}
<tr><td>{{item.owner}}</td>
<td>{{item.regimeid}}</td>
<td>{{item.closingeventid}}</td>
<td>{{item.category}}</td>
<td>{{item.transactionorclosing}}</td>
<td>{{item.****accountingfield}}</td>
<td>{{item.datasource}}</td>
</tr>
{% endfor %}
</table>
{% else %}
<p>No Data are available.</p>
{% endif %}
And get the error:
Column closingevent.id doesn't exist LINE 1: SELECT "closingevent"."id", "closingevent"."owner", "closing...
My table has no column which is called "closingevent"."id" and I even didn't try to get this. I get it for other templates too.
Can you help me? I think it could be the problem that my model doesn't have a primary key and just a foreign key.
class Closingevent(models.Model):
owner = models.CharField(max_length=800, blank=True, null=True)
regimeid = models.ForeignKey('Regime', models.DO_NOTHING,
db_column='regimeid')
closingeventid = models.FloatField()
category = models.CharField(max_length=800, blank=True, null=True)
closingevent = models.CharField(max_length=800, blank=True, null=True)
transactionorclosing = models.CharField(max_length=1, blank=True,
null=True)
****accountingfield = models.CharField(max_length=1, blank=True,
null=True)
datasource = models.CharField(max_length=800, blank=True, null=True)
debitacoountnumber = models.CharField(max_length=800, blank=True,
null=True)
debitsubitem = models.CharField(max_length=1, blank=True, null=True)
debitaccountname = models.ForeignKey('Lineitemaccounting',
models.DO_NOTHING,db_column='debitaccountname',
blank=True,null=True,related_name='daccountname2')
debitbalancesheet = models.CharField(max_length=1, blank=True, null=True)
debitprofitandlose = models.CharField(max_length=1, blank=True,
null=True)
debitreconciliation = models.CharField(max_length=1, blank=True,
null=True)
creditaccountnumber = models.CharField(max_length=1, blank=True,
null=True)
creditsubitem = models.CharField(max_length=1, blank=True, null=True)
creditaccountname = models.ForeignKey('Lineitemaccounting',
models.DO_NOTHING, db_column='creditaccountname', blank=True,
null=True,related_name='caccountname2')
creditbalancesheet = models.CharField(max_length=1, blank=True,
null=True)
creditprofitandloss = models.CharField(max_length=1, blank=True,
null=True)
creditreconciliation = models.CharField(max_length=1, blank=True,
null=True)
class Meta:
managed = False
db_table = 'closingevent'
unique_together = (('regimeid', 'closingeventid'),)