We Keep Coding

Django migration with uuid field generates duplicated values

I have a uuid field (not a primary key). The generated migration is:
from __future__ import unicode_literals
from django.db import migrations, models
import uuid
class Migration(migrations.Migration):
dependencies = [
....
]
operations = [
...
migrations.AddField(
model_name='device',
name='uuid',
field=models.UUIDField(default=uuid.uuid4, unique=True),
),
...
]
But when doing python manage.py migrate it is crashing with:
django.db.utils.IntegrityError: could not create unique index
"restaurants_device_uuid_key" DETAIL: Key
(uuid)=(f3858ded-b8e0-4ac0-8436-8a61b10efc73) is duplicated.
Strangely enough, the problem does not seem to occur with primary keys (which are maybe created by the database, and not internally by django?)
How can I add a uuid field, and make sure that migrations work?

Here is an example doing everything in one single migration thanks to a RunPython call.
# -*- coding: utf-8 -*
from __future__ import unicode_literals
from django.db import migrations, models
import uuid
def create_uuid(apps, schema_editor):
Device = apps.get_model('device_app', 'Device')
for device in Device.objects.all():
device.uuid = uuid.uuid4()
device.save()
class Migration(migrations.Migration):
dependencies = [
('device_app', 'XXXX'),
]
operations = [
migrations.AddField(
model_name='device',
name='uuid',
field=models.UUIDField(blank=True, null=True),
),
migrations.RunPython(create_uuid),
migrations.AlterField(
model_name='device',
name='uuid',
field=models.UUIDField(unique=True)
)
]

(Answer taken from the first comment)
See the django docs - Migrations that add unique fields
They recommend changing your single migration into three separate migrations:
Create field, set to null but not unique
Generate unique UUIDs
Alter the field to be unique

In the mode, you have configured, that you want unique values for the uuid fields, but with default values(the same for all). So if you have two 'device' objects in the database, the migrations add 'uuid' field to them with the default 'uuid.uuid4' value and when it tries to set it to the second one, it crashes because of the unique constrains.
If you drop your db and create new objects probably there will be not problems but thats not a solution for production db obviously :D.
A better solution is to create a data migration which sets different uuid value (generated by the default 'uuid' library) to every existing object in the database. You can read more about data migrations here:
https://docs.djangoproject.com/en/1.10/topics/migrations/#data-migrations
Then, when you create new objects, django will generate different uuid automatically. ;)
For the primary keys: Django adds it to the model by default.

You can provide a management command to populate the uuid field after uu_id column is created in the model but this has to be done after migrating the model and setting the field default as None:
from django.core.management.base import BaseCommand
from django.apps import apps
import uuid
class Command(BaseCommand):
def handle(self, *args, **options):
classes()
def classes():
app_models = apps.get_app_config('appname').get_models()
for model in app_models:
field = None
try:
field = model._meta.get_field('uu_id')
except:
pass
if field:
uu_id_list = list(model.objects.all().values_list('uu_id',flat=True))
if None in uu_id_list:
for row in model.objects.all():
row.uu_id = uuid.uuid4()
row.save()

Data migrations for OneToOneField in django

I have a Product model and I want to extend by using OneToOneField.
For example
class Product:
name = models.CharField(..)
price = models.FloatField(...)
I want to do like this
class MyProduct:
product = models.OneToOneField(myapp.Product, on_delete=models.CASCADE)
location = models.CharField(...)
and using signal
def create_myproduct(sender, instance, created, **kwargs):
"""Create MyProduct class for every new Product"""
if created:
MyProduct.objects.create(product=instance)
signals.post_save.connect(create_myproduct, sender=Product, weak=False,
dispatch_uid='models.create_myproduct')
This works for newly created Product, so I can do like this in template.
{{ product.myproduct.location }}
But Old products that created before adding this OneToOneRelation,has no field 'myproduct' and that template code didn't work.
I heard I need a data migrations for old product using RunPython or manage.py shell. Can you teach me how to do? I read a documentation from django, but still don't fully understand.

you can add new migration. and apply it.
something like this code:
# -*- coding: utf-8 -*-
# Generated by Django 1.11.2 on 2017-07-22 06:04
from __future__ import unicode_literals
from django.db import migrations, models
def create_myproducts(apps, schema_editor):
Product = apps.get_model('myapp', 'Product')
MyProduct = apps.get_model('myapp', 'MyProduct')
for prod in Product.objects.all():
MyProduct.objects.create(product=prod)
class Migration(migrations.Migration):
dependencies = [
('myapp', 'your last migration'),
]
operations = [
migrations.RunPython(create_myproducts)
]

I just found out.
Like Rohit Jain said
product.myproduct is None.
When I tried to access product.myproduct, I got an exception that object does not exist. It has a relation to myproduct but the actual object doesn't exist.
What I really want was creating MyProduct object and add it to Product class.
So I did it in python manage.py shell
products = Product.objects.all()
for prod in products:
if not hasattr(prod, 'myproduct'):
prod.myproduct = MyProduct.objects.create(product=prod)
prod.save()
I think it works for me now.
Thank you guys

You should just migrate your models in a normal way
python manage.py makemigrations
python manage.py migrate
During making migrations you will be asked how to fill new fields for existing data
Please notice that when you are using Django under 1.7 version you do not have migrations (and syncdb will not do the job for existing tables) - consider using the 3rd part tool like south

Django: loaddata in migrations errors

Something really annoying is happening to me since using Django migrations (not south) and using loaddata for fixtures inside of them.
Here is a simple way to reproduce my problem:
create a new model Testmodel with 1 field field1 (CharField or whatever)
create an associated migration (let's say 0001) with makemigrations
run the migration
and add some data in the new table
dump the data in a fixture testmodel.json
create a migration with call_command('loaddata', 'testmodel.json'): migration 0002
add some a new field to the model: field2
create an associated migration (0003)
Now, commit that, and put your db in the state just before the changes: ./manage.py migrate myapp zero. So you are in the same state as your teammate that didn't get your changes yet.
If you try to run ./manage.py migrate again you will get a ProgrammingError at migration 0002 saying that "column field2 does not exist".
It seems it's because loaddata is looking into your model (which is already having field2), and not just applying the fixture to the db.
This can happen in multiple cases when working in a team, and also making the test runner fail.
Did I get something wrong? Is it a bug? What should be done is those cases?
--
I am using django 1.7

loaddata command will simply call serializers. Serializers will work on models state from your models.py file, not from current migration, but there is little trick to fool default serializer.
First, you don't want to use that serializer by call_command but rather directly:
from django.core import serializers
def load_fixture(apps, schema_editor):
fixture_file = '/full/path/to/testmodel.json'
fixture = open(fixture_file)
objects = serializers.deserialize('json', fixture, ignorenonexistent=True)
for obj in objects:
obj.save()
fixture.close()
Second, monkey-patch apps registry used by serializers:
from django.core import serializers
def load_fixture(apps, schema_editor):
original_apps = serializers.python.apps
serializers.python.apps = apps
fixture_file = '/full/path/to/testmodel.json'
fixture = open(fixture_file)
objects = serializers.deserialize('json', fixture, ignorenonexistent=True)
for obj in objects:
obj.save()
fixture.close()
serializers.python.apps = original_apps
Now serializer will use models state from apps instead of default one and whole migration process will succeed.

To expand on the answer from GwynBleidD and mix in this issue since Postgres won't reset the primary key sequences when loaded this way (https://stackoverflow.com/a/14589706/401636)
I think I now have a failsafe migration for loading fixture data.
utils.py:
import os
from io import StringIO
import django.apps
from django.conf import settings
from django.core import serializers
from django.core.management import call_command
from django.db import connection
os.environ['DJANGO_COLORS'] = 'nocolor'
def reset_sqlsequence(apps=None, schema_editor=None):
"""Suitable for use in migrations.RunPython"""
commands = StringIO()
cursor = connection.cursor()
patched = False
if apps:
# Monkey patch django.apps
original_apps = django.apps.apps
django.apps.apps = apps
patched = True
else:
# If not in a migration, use the normal apps registry
apps = django.apps.apps
for app in apps.get_app_configs():
# Generate the sequence reset queries
label = app.label
if patched and app.models_module is None:
# Defeat strange test in the mangement command
app.models_module = True
call_command('sqlsequencereset', label, stdout=commands)
if patched and app.models_module is True:
app.models_module = None
if patched:
# Cleanup monkey patch
django.apps.apps = original_apps
sql = commands.getvalue()
print(sql)
if sql:
# avoid DB error if sql is empty
cursor.execute(commands.getvalue())
class LoadFixtureData(object):
def __init__(self, *files):
self.files = files
def __call__(self, apps=None, schema_editor=None):
if apps:
# If in a migration Monkey patch the app registry
original_apps = serializers.python.apps
serializers.python.apps = apps
for fixture_file in self.files:
with open(fixture_file) as fixture:
objects = serializers.deserialize('json', fixture)
for obj in objects:
obj.save()
if apps:
# Cleanup monkey patch
serializers.python.apps = original_apps
And now my data migrations look like:
# -*- coding: utf-8 -*-
# Generated by Django 1.11.1 on foo
from __future__ import unicode_literals
import os
from django.conf import settings
from django.db import migrations
from .utils import LoadFixtureData, reset_sqlsequence
class Migration(migrations.Migration):
dependencies = [
('app_name', '0002_auto_foo'),
]
operations = [
migrations.RunPython(
code=LoadFixtureData(*[
os.path.join(settings.BASE_DIR, 'app_name', 'fixtures', fixture) + ".json"
for fixture in ('fixture_one', 'fixture_two',)
]),
# Reverse will NOT remove the fixture data
reverse_code=migrations.RunPython.noop,
),
migrations.RunPython(
code=reset_sqlsequence,
reverse_code=migrations.RunPython.noop,
),
]

When you run python manage.py migrate it's trying to load your testmodel.json in fixtures folder, but your model (after updated) does not match with data in testmodel.json. You could try this:
Change your directory from fixture to _fixture.
Run python manage.py migrate
Optional, you now can change _fixture by fixture and load your data as before with migrate command or load data with python manage.py loaddata app/_fixtures/testmodel.json

Django 1.7 datamigration and user groups

I'm trying to implement a datamigration using django 1.7 native migration system. Here is what I've done.
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
def create_basic_user_group(apps, schema_editor):
"""Forward data migration that create the basic_user group
"""
Group = apps.get_model('auth', 'Group')
Permission = apps.get_model('auth', 'Permission')
group = Group(name='basic_user')
group.save()
perm_codenames = (
'add_stuff',
'...',
)
# we prefere looping over all these in order to be sure to fetch them all
perms = [Permission.objects.get(codename=codename)
for codename in perm_codenames]
group.permissions.add(*perms)
group.save()
def remove_basic_user_group(apps, schema_editor):
"""Backward data migration that remove the basic_user group
"""
group = Group.objects.get(name='basic_user')
group.delete()
class Migration(migrations.Migration):
"""This migrations automatically create the basic_user group.
"""
dependencies = [
]
operations = [
migrations.RunPython(create_basic_user_group, remove_basic_user_group),
]
But when I try to run the migration, I got a LookupError exception telling me that no app with label 'auth' could be found.
How can I create my groups in a clean way that could also be used in unit tests ?

I've done what you are trying to do. The problems are:
The documentation for 1.7 and 1.8 is quite clear: If you want to access a model from another app, you must list this app as a dependency:
When writing a RunPython function that uses models from apps other than the one in which the migration is located, the migration’s dependencies attribute should include the latest migration of each app that is involved, otherwise you may get an error similar to: LookupError: No installed app with label 'myappname' when you try to retrieve the model in the RunPython function using apps.get_model().
So you should have a dependency on the latest migration in auth.
As you mentioned in a comment you will run into an issue whereby the permissions you want to use are not created yet. The problem is that the permissions are created by signal handler attached to the post_migrate signal. So the permissions associated with any new model created in a migration are not available until the migration is finished.
You can fix this by doing this at the start of create_basic_user_group:
from django.contrib.contenttypes.management import update_contenttypes
from django.apps import apps as configured_apps
from django.contrib.auth.management import create_permissions
for app in configured_apps.get_app_configs():
update_contenttypes(app, interactive=True, verbosity=0)
for app in configured_apps.get_app_configs():
create_permissions(app, verbosity=0)
This will also create the content types for each model (which are also created after the migration), see below as to why you should care about that.
Perhaps you could be more selective than I am in the code above: update just some key apps rather than update all apps. I've not tried to be selective. Also, it is possible that both loop could be merged into one. I've not tried it with a single loop.
You get your Permission objects by searching by codename but codename is not guaranteed to be unique. Two apps can have models called Stuff and so you could have an add_stuff permission associated with two different apps. If this happens, your code will fail. What you should do is search by codename and content_type, which are guaranteed to be unique together. A unique content_type is associated with each model in the project: two models with the same name but in different apps will get two different content types.
This means adding a dependency on the contenttypes app, and using the ContentType model: ContentType = apps.get_model("contenttypes", "ContentType").

As said in https://code.djangoproject.com/ticket/23422, the signal post_migrate should be sent before dealing with Permission objects.
But there is a helper function already on Django to sent the needed signal: django.core.management.sql.emit_post_migrate_signal
Here, it worked this way:
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
from django.core.management.sql import emit_post_migrate_signal
PERMISSIONS_TO_ADD = [
'view_my_stuff',
...
]
def create_group(apps, schema_editor):
# Workarounds a Django bug: https://code.djangoproject.com/ticket/23422
db_alias = schema_editor.connection.alias
try:
emit_post_migrate_signal(2, False, 'default', db_alias)
except TypeError: # Django < 1.8
emit_post_migrate_signal([], 2, False, 'default', db_alias)
Group = apps.get_model('auth', 'Group')
Permission = apps.get_model('auth', 'Permission')
group, created = Group.objects.get_or_create(name='MyGroup')
permissions = [Permission.objects.get(codename=i) for i in PERMISSIONS_TO_ADD]
group.permissions.add(*permissions)
class Migration(migrations.Migration):
dependencies = [
('auth', '0001_initial'),
('myapp', '0002_mymigration'),
]
operations = [
migrations.RunPython(create_group),
]

So, I figure out how to solve this problem and I get the following exit: get_model will only fetch Your model apps. I don't have sure about if this would be a good pratice, but it worked for me.
I just invoked the model Directly and made the changes.
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import models, migrations
from django.contrib.auth.models import Group
def create_groups(apps, schema_editor):
g = Group(name='My New Group')
g.save()
class Migration(migrations.Migration):
operations = [
migrations.RunPython(create_groups)
]
And then, just apply a /manage.py migrate to finish.
I hope it helps.

Django 1.7 where to put the code to add Groups programmatically?

I have been trying to find the answer in the Django Auth docs, but can not seem to find what I am looking for.
The problem I am having is, when I define the code for adding Groups (same as Groups in the admin page):
#read_only
group, created = Group.objects.get_or_create(name='read_only')
if created:
group.permissions.add(can_read_campaign)
logger.info('read_only_user Group created')
#standard
group, created = Group.objects.get_or_create(name='standard_user')
if created:
group.permissions.add(can_edit_users)
logger.info('standard_user Group created')
#admin
group, created = Group.objects.get_or_create(name='admin_user')
if created:
group.permissions.add(can_edit_campaign, can_edit_users)
logger.info('admin_user Group created')
When I have run this code in models.py and init.py and they both give me this error:
django.core.exceptions.AppRegistryNotReady
I presume this is due to the Model/init trying to insert things into the django app/admin too early?
How can I add these Groups programmatically?
EDIT:
This is not a duplicate question, this was actually adding permission and groups within the models during setup of the project, rather than through the shell.
I have solved this issues, by using signals and receivers (django modules).
I added the code to create the permissions/groups into it's own function and decorated this with a receiver (post_migrate), which will run this function after migrations are complete, removing this error.
#receiver(post_migrate)
def init_groups(sender, **kwargs):
#permission and group code goes here

I was recommended this way to do it:
Create a fake migration in the appropriate module:
python manage.py makemigrations --empty yourappname
Open up the file that was created, which should look like this:
# -*- coding: utf-8 -*-
from django.db import models, migrations
class Migration(migrations.Migration):
dependencies = [
('yourappname', '0001_initial'),
]
operations = [
]
And add your code:
# -*- coding: utf-8 -*-
from django.db import models, migrations
def add_group_permissions():
#read_only
group, created = Group.objects.get_or_create(name='read_only')
if created:
group.permissions.add(can_read_campaign)
logger.info('read_only_user Group created')
#standard
group, created = Group.objects.get_or_create(name='standard_user')
if created:
group.permissions.add(can_edit_users)
logger.info('standard_user Group created')
#admin
group, created = Group.objects.get_or_create(name='admin_user')
if created:
group.permissions.add(can_edit_campaign, can_edit_users)
logger.info('admin_user Group created')
class Migration(migrations.Migration):
dependencies = [
('yourappname', '0001_initial'),
]
operations = [
migrations.RunPython(add_group_permissions),
]
Finally, run the migration:
python manage.py migrate
This is nice because you can deploy to Heroku or wherever and be sure it'll be applied, as it's just another migration.

Combining #Robert Grant and this I was able to do it like:
python manage.py makemigrations --empty yourappname
And then:
from django.contrib.auth.models import Group, Permission
from django.db import models, migrations
import logging
logger = logging.getLogger(__name__)
campaign_group_permissions = {
"Campaign Manager": [
"add_campaign",
"change_campaign",
"delete_campaign",
"view_campaign",
"add_campaignsms",
"add_sending",
"change_sending",
"view_sending"
]
}
def add_group_permissions():
# See https://code.djangoproject.com/ticket/23422
db_alias = schema_editor.connection.alias
try:
emit_post_migrate_signal(2, False, 'default')
except TypeError: # Django < 1.8
emit_post_migrate_signal([], 2, False, 'default', db_alias)
for group in campaign_group_permissions:
role, created = Group.objects.get_or_create(name=group)
logger.info(f'{group} Group created')
for perm in campaign_group_permissions[group]:
role.permissions.add(Permission.objects.get(codename=perm))
logger.info(f'Permitting {group} to {perm}')
role.save()
class Migration(migrations.Migration):
dependencies = [
('yourappname', '0001_initial'),
]
operations = [
migrations.RunPython(add_group_permissions),
]
Note: this works on Django 3.x, but I'm pretty sure it will work for Django 1.7 as well.

#Ruloweb's fantastic response almost worked for me, but I had to make a couple tweaks to get it to work in Django 3.1 with multiple apps.
First, I needed to add arguments to the add_group_permissions() function. I also needed to import the emit_post_migration_signal:
from django.contrib.auth.models import Group, Permission
from django.core.management.sql import emit_post_migrate_signal # <-- Added this
from django.db import models, migrations
import logging
logger = logging.getLogger(__name__)
public_group_permissions = {
"Your permission group name here": ['your permissions here']
}
def add_group_permissions(apps, schema_editor): # <-- updated this
# See https://code.djangoproject.com/ticket/23422
db_alias = schema_editor.connection.alias
try:
emit_post_migrate_signal(2, False, 'default')
except TypeError: # Django < 1.8
emit_post_migrate_signal([], 2, False, 'default', db_alias)
for group in public_group_permissions:
role, created = Group.objects.get_or_create(name=group)
logger.info(f'{group} Group created')
for perm in public_group_permissions[group]:
role.permissions.add(Permission.objects.get(codename=perm))
logger.info(f'Permitting {group} to {perm}')
role.save()
class Migration(migrations.Migration):
dependencies = [
('your_app_name', '0001_initial'),
]
operations = [
migrations.RunPython(add_group_permissions),
]