We Keep Coding

How to save/create thousands of records to postgres using queryset in Django?

I want to store thousands of records in the Student table which has id_users foreign key field. I tried without having foreign key it works smooth but when I added fk constraint it becomes very very slow and took more than 20mins to store 500 records and same without fk took only few seconds.
models - Below models are from different apps
./users/models.py
class Users(models.Model):
id = models.IntegerField(primary_key=True)
cts = models.DateTimeField(blank=True, null=True)
uts = models.DateTimeField(blank=True, null=True)
id_states = models.ForeignKey(States, models.DO_NOTHING, db_column='id_states', blank=True, null=True)
id_user_type = models.ForeignKey(UserType, models.DO_NOTHING, db_column='id_user_type')
firstname = models.TextField(blank=True, null=True)
lastname = models.TextField(blank=True, null=True)
class Meta:
managed = False
db_table = 'users'
./student/models.py
class Student(models.Model):
id = models.BigAutoField(primary_key=True)
cts = models.DateTimeField()
uts = models.DateTimeField()
id_users = models.ForeignKey('users.Users', models.DO_NOTHING, db_column='id_users')
num_subjects = models.IntegerField()
num_teachers = models.IntegerField()
class Meta:
managed = False
db_table = 'students'
bulk_create() method -
def saveCounts(cumulative_counts: pd.DataFrame, model: django.db.models.base.ModelBase) -> None:
records = cumulative_counts.to_dict('records')
model_instances = [model(
cts=timezone.now(),
uts=timezone.now(),
id_users=Users.objects.get(
id=record['id_users']),
num_subjects=record['num_subjects'],
num_teachers=record['num_teachers'],
) for record in records]
model.objects.bulk_create(model_instances)

Format Column with Django-Tables2 based on another boolean column

I have a table whith a few columns amongst them one is called commune and has a string type and another is confiance_commune which is a boolean value.
I would like commune to be rendered bold and green if confiance_commune is true.
Here is my code :
models.py
class Mrae(models.Model):
titre = models.TextField(blank=True, null=True)
lien = models.TextField(blank=True, null=True)
description = models.TextField(blank=True, null=True)
type_projet = models.TextField(blank=True, null=True)
pv_mentionne = models.BooleanField(blank=True, null=True)
commune = models.TextField(blank=True, null=True)
commune_m = models.TextField(blank=True, null=True)
departement = models.IntegerField(blank=True, null=True)
date = models.DateField(blank=True, null=True)
confiance_commune = models.BooleanField(blank=True, null=True)
index = models.AutoField(primary_key=True)
class Meta:
managed = False
db_table = 'mrae'
tables.py
from django_tables2 import tables
from django_tables2.columns import URLColumn
from .models import Mrae
class MraeTable(tables.Table):
lien = URLColumn("Lien")
class Meta:
model = Mrae
attrs = {"class": "table table-responsive"}
fields = ['titre', 'lien', 'pv_mentionne', 'date', 'commune', 'departement']
tamplate_name = "django_tables2/bootstrap-responsive.html"

You can use the custom render function calling with the record, then just create a simple if statement that checks the confiance_commune attribute to set the style for the commune data.
https://django-tables2.readthedocs.io/en/latest/pages/custom-data.html#table-render-foo-methods
class MraeTable(tables.Table):
lien = URLColumn("Lien")
class Meta:
model = Mrae
attrs = {"class": "table table-responsive"}
fields = ['titre', 'lien', 'pv_mentionne', 'date', 'commune', 'departement']
template_name = "django_tables2/bootstrap-responsive.html"
def render_commune(self, record):
if record.confiance_commune:
return format_html(
"<span style='color:green;font-weight:bold;'>{}</span>",
mark_safe(record.commune)
)
return mark_safe(record.commune)

django - Runpython function to turn charfield into foreignkey

I've been strugglin to relate a csv imported data model with a spatial data model based on a CharField.
I've created both models and now im trying to transfer the data from one field to a new one to be the ForeignKey field. I made a Runpython funtion to apply on the migration but it goives the an error:
ValueError: Cannot assign "'921-5'":
"D2015ccccccc.rol_fk" must be a "D_Base_Roles" instance.
Here are the models:
class D_Base_Roles(models.Model):
predio = models.CharField(max_length=254)
dest = models.CharField(max_length=254)
dir = models.CharField(max_length=254)
rol = models.CharField(primary_key=True, max_length=254)
vlr_tot = models.FloatField()
ub_x2 = models.FloatField()
ub_y2 = models.FloatField()
instrum = models.CharField(max_length=254)
codzona = models.CharField(max_length=254)
nomzona = models.CharField(max_length=254)
geom = models.MultiPointField(srid=32719)
def __str__(self):
return str(self.rol)
class Meta():
verbose_name_plural = "Roles"
class D2015ccccccc(models.Model):
id = models.CharField(primary_key=True, max_length=80)
nombre_archivo = models.CharField(max_length=180, blank=True, null=True)
derechos = models.CharField(max_length=120, blank=True, null=True)
dir_calle = models.CharField(max_length=120, blank=True, null=True)
dir_numero = models.CharField(max_length=120, blank=True, null=True)
fecha_certificado = models.CharField(max_length=50, blank=True, null=True)
numero_certificado = models.CharField(max_length=50, blank=True, null=True)
numero_solicitud = models.CharField(max_length=50, blank=True, null=True)
rol_sii = models.CharField(max_length=50, blank=True, null=True)
zona_prc = models.CharField(max_length=120, blank=True, null=True)
##NEW EMPTY FOREIGNKEY FIELD
rol_fk = models.ForeignKey(D_Base_Roles, on_delete=models.CASCADE, blank=True, null=True)
def __str__(self):
return str(self.numero_certificado)
class Meta:
managed = True
#db_table = 'domperm2015cip'
verbose_name_plural = "2015 Certificados Informaciones Previas"
ordering = ['numero_certificado']
The Runpython function:
def pop_rol(apps, schema_editor):
roles = apps.get_model('b_dom_edificacion', 'D2015ccccccc')
for r in roles.objects.all():
rol = roles
r.rol_fk = r.rol_sii
r.save()
D_Base_Roles.rol values are all unique, and 921-5 is one of those values.
What am I missing?

You probably need to assign an object, not a string. Change the line
r.rol_fk = r.rol_sii
to
r.rol_fk = D_Base_Roles.objects.get(rol=r.rol_sii)
Maybe adjust to whatever the correct field for looking up D_Base_Roles instances is.
Note: this will make a database query for every iteration of the for-loop

DRF - filter across 2 models

I'm working with a legacy database where I have a serializer setup on Table A like so -
class TblapplicationsSerializer(serializers.ModelSerializer):
class Meta:
model = Tblapplications
fields = ('applicationid', 'applicationname', 'description', 'drtierid', 'saglink', 'supportinstructions',
'defaultincidentpriorityid', 'applicationorigintypeid', 'installationtypeid', 'comments',
'lastmodifieddate', 'lastmodifiedby', 'assetstatusid', 'recordownerid', 'adl_app')
depth = 2
I'm using a standard filter -
class TblapplicationsFilter(django_filters.FilterSet):
name = django_filters.CharFilter(name="applicationname", lookup_type="exact")
env = django_filters.CharFilter(name="adl_app__environmentid__domain")
class Meta:
model = Tblapplications
fields = ['applicationname', 'name', 'env']
Here's where it goes sideways. What I want to be able to do is filter on my URL like /api/applications/?name=xxx&env=DEV. It would then return the application and any databases that are linked with the environment of DEV. The name was understandably easy, but the only was I figured out the environment was to make the api point for applications touch the middle table that links the two but it returns multiple values because it's grabbing each time application is referenced with a separate database.
I've updated the Serializer and Filter based on comments given and the serializer, without the &env=DEV returns all the appropriate data (domain is nested in a reverse relationship). I then want my filter to filter the results based on that. Which means that it needs to somehow know to limit the results on the reverse relationship to only what's provided from the nested value.
If you see my models -
class Tblapplicationdatabaselinks(models.Model):
id = models.AutoField(db_column='ID', primary_key=True)
applicationid = models.ForeignKey('Tblapplications', db_column='applicationId', to_field='applicationid',
related_name='adl_app')
dbid = models.ForeignKey('Tbldatabases', db_column='dbId', to_field='id', related_name='adl_db')
environmentid = models.ForeignKey('Tbldomaincodes', db_column='environmentId', to_field='id',
related_name='adl_envlink')
comments = models.TextField(blank=True)
lastmodifieddate = models.DateTimeField(db_column='lastModifiedDate', blank=True, null=True)
lastmodifiedby = models.CharField(db_column='lastModifiedBy', max_length=255, blank=True)
# upsize_ts = models.TextField(blank=True) # This field type is a guess.
class Meta:
managed = False
db_table = 'tblApplicationDatabaseLinks'
class Tblapplications(models.Model):
applicationid = models.AutoField(db_column='applicationId', primary_key=True)
applicationname = models.CharField(db_column='applicationName', max_length=255)
description = models.TextField(blank=True)
drtierid = models.ForeignKey(Tbldomaincodes, db_column='drTierID', blank=True, null=True, to_field='id',
related_name='app_drtier')
saglink = models.TextField(db_column='sagLink', blank=True)
supportinstructions = models.TextField(db_column='supportInstructions', blank=True)
defaultincidentpriorityid = models.IntegerField(db_column='defaultIncidentPriorityId', blank=True, null=True)
applicationorigintypeid = models.IntegerField(db_column='applicationOriginTypeId')
installationtypeid = models.ForeignKey(Tbldomaincodes, db_column='installationTypeId', to_field='id',
related_name='app_insttype')
comments = models.TextField(blank=True)
assetstatusid = models.ForeignKey(Tbldomaincodes, db_column='assetStatusId', to_field='id',
related_name='app_status')
recordownerid = models.ForeignKey(Tblusergroups, db_column='recordOwnerId', blank=True, null=True,
to_field='groupid', related_name='app_owner')
lastmodifieddate = models.DateTimeField(db_column='lastModifiedDate', blank=True, null=True)
lastmodifiedby = models.CharField(db_column='lastModifiedBy', max_length=255, blank=True)
# upsize_ts = models.TextField(blank=True) # This field type is a guess.
class Meta:
managed = False
db_table = 'tblApplications'
class Tbldatabases(models.Model):
dbid = models.AutoField(db_column='dbId', primary_key=True)
dbname = models.CharField(db_column='dbName', max_length=255)
serverid = models.ForeignKey('Tblservers', db_column='serverId', to_field='serverid', related_name='db_serv')
servicename = models.CharField(db_column='serviceName', max_length=255, blank=True)
dbtypeid = models.IntegerField(db_column='dbTypeId', blank=True, null=True)
inceptiondate = models.DateTimeField(db_column='inceptionDate', blank=True, null=True)
comments = models.TextField(blank=True)
assetstatusid = models.IntegerField(db_column='assetStatusId')
recordownerid = models.IntegerField(db_column='recordOwnerId', blank=True, null=True)
lastmodifieddate = models.DateTimeField(db_column='lastModifiedDate', blank=True, null=True)
lastmodifiedby = models.CharField(db_column='lastModifiedBy', max_length=255, blank=True)
# upsize_ts = models.TextField(blank=True) # This field type is a guess.
class Meta:
managed = False
db_table = 'tblDatabases'
class Tbldomaincodes(models.Model):
id = models.IntegerField(db_column='ID', primary_key=True)
domain = models.CharField(primary_key=True, max_length=255)
displayname = models.CharField(db_column='displayName', primary_key=True, max_length=255)
displayorder = models.IntegerField(db_column='displayOrder', blank=True, null=True)
comments = models.TextField(blank=True)
lastmodifieddate = models.DateTimeField(db_column='lastModifiedDate', blank=True, null=True)
lastmodifiedby = models.CharField(db_column='lastModifiedBy', max_length=255, blank=True)
# upsize_ts = models.TextField(blank=True) # This field type is a guess.
class Meta:
managed = False
db_table = 'tblDomainCodes'

Extend your filter set and reference the field in the other model:
class TblapplicationsFilter(django_filters.FilterSet):
name = django_filters.CharFilter(name="applicationname", lookup_type="exact")
env = django_filters.CharFilter(name="environmentid__name")
# ^^^^^^^^^^^^^^^^^^^
class Meta:
model = Tblapplications
fields = ['applicationname', 'name', 'env']
Also, you may wish to name your ForeignKey fields without the id suffix, which is the Django convention. In Django, when you access Tblapplications.environmentid, it is normally a model instance, not the actual id integer itself.

Django model query

I have question about Django query models. I know how to write simple query, but Im not familiar with LEFT JOIN on two tables. So can you give me some advice on his query for better understanding DJango ORM.
query
select
count(ips.category_id_id) as how_many,
ic.name
from
izibizi_category ic
left join
izibizi_product_service ips
on
ips.category_id_id = ic.id
where ic.type_id_id = 1
group by ic.name, ips.category_id_id
From this query I get results:
How many | name
0;"fghjjh"
0;"Papir"
0;"asdasdas"
0;"hhhh"
0;"Boljka"
0;"ako"
0;"asd"
0;"Čokoladne pahuljice"
0;"Mobitel"
2;"Čokolada"
And I have also try with his Django query:
a = Category.objects.all().annotate(Count('id__category',distinct=True)).filter(type_id=1)
But no results.
My models:
models.py
class Category(models.Model):
id = models.AutoField(primary_key=True)
type_id = models.ForeignKey('CategoryType')
name = models.CharField(max_length=255)
def __str__(self):
return str(self.name)
class Product_service(models.Model):
id = models.AutoField(primary_key=True)
name = models.CharField(max_length=255, blank=True, null=True)
selling_price = models.DecimalField(decimal_places=5, max_digits=255, blank=True, null=True)
purchase_price = models.DecimalField(decimal_places=5, max_digits=255, blank=True, null=True)
description = models.CharField(max_length=255, blank=True, null=True)
image = models.FileField(upload_to="/", blank=True, null=True)
product_code = models.CharField(max_length=255, blank=True, null=True)
product_code_supplier = models.CharField(max_length=255, blank=True, null=True)
product_code_buyer = models.CharField(max_length=255, blank=True, null=True)
min_unit_state = models.CharField(max_length=255, blank=True, null=True)
state = models.CharField(max_length=255, blank=True, null=True)
vat_id = models.ForeignKey('VatRate')
unit_id = models.ForeignKey('Units')
category_id = models.ForeignKey('Category')
If you culd help me on this problem.

You should add a related name on the category_id field like:
category_id = models.ForeignKey('Category', related_name="product_services")
so that in your query you can do:
a = Category.objects.all().annotate(Count('product_services',distinct=True)).filter(type_id=1)
and then you can access the individual counts as:
a[0].product_services__count