How to use SQLAlchemy's events in Pyramid - python

views.py
def add_post(topic, request):
post_form = PostForm(request.POST)
if 'submit' in request.POST and post_form.validate():
post = Post(body=post_form.body.data)
post.user = request.user
post.topic = topic
DBSession.add(post)
request.session.flash(_('Post was added'))
transaction.commit()
raise HTTPFound(location=request.route_url('topic',id=topic.id))
return {'post_form':post_form}
models.py
class Topic(Base):
__tablename__ = 'topics'
id = Column(Integer, primary_key=True)
...
post_count = Column(Integer, default=0)
posts = relationship('Post', primaryjoin="Post.topic_id==Topic.id", backref='topic', lazy='dynamic')
class Post(Base):
__tablename__ = 'posts'
id = Column(Integer, primary_key=True)
...
topic_id = Column(Integer, ForeignKey('topics.id'))
def post_inserted(mapper, conn, post):
topic = post.topic
topic.post_count = topic.posts.count()
event.listen(Post, "after_insert", post_inserted)
I want in my Pyramid app to use SQLAchemy event 'after_insert', to update Topic model with number of posts belong to it. But I get exeption:
Traceback (most recent call last):
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/waitress/channel.py", line 329, in service
task.service()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/waitress/task.py", line 173, in service
self.execute()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/waitress/task.py", line 380, in execute
app_iter = self.channel.server.application(env, start_response)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/pyramid/router.py", line 251, in __call__
response = self.invoke_subrequest(request, use_tweens=True)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/pyramid/router.py", line 227, in invoke_subrequest
response = handle_request(request)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/pyramid_tm/__init__.py", line 107, in tm_tween
return response
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/transaction/_manager.py", line 116, in __exit__
self.commit()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/transaction/_manager.py", line 107, in commit
return self.get().commit()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/transaction/_transaction.py", line 354, in commit
reraise(t, v, tb)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/transaction/_transaction.py", line 345, in commit
self._commitResources()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/transaction/_transaction.py", line 493, in _commitResources
reraise(t, v, tb)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/transaction/_transaction.py", line 465, in _commitResources
rm.tpc_begin(self)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/zope/sqlalchemy/datamanager.py", line 86, in tpc_begin
self.session.flush()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/session.py", line 1583, in flush
self._flush(objects)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/session.py", line 1654, in _flush
flush_context.execute()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/unitofwork.py", line 331, in execute
rec.execute(self)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/unitofwork.py", line 475, in execute
uow
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/persistence.py", line 67, in save_obj
states_to_insert, states_to_update)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/persistence.py", line 702, in _finalize_insert_update_commands
mapper.dispatch.after_insert(mapper, connection, state)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/event.py", line 291, in __call__
fn(*args, **kw)
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/events.py", line 360, in wrap
wrapped_fn(*arg, **kw)
File "/home/user/workspace/myforum/cube_forum/models.py", line 165, in post_saved
topic.post_count = topic.posts.count()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/dynamic.py", line 249, in count
sess = self.__session()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/dynamic.py", line 219, in __session
sess.flush()
File "/home/user/workspace/myforum/env/lib/python2.6/site-packages/sqlalchemy/orm/session.py", line 1577, in flush
raise sa_exc.InvalidRequestError("Session is already flushing")
InvalidRequestError: Session is already flushing
How to do it right in Pyramid/SQLalchemy?
EDIT: Question is actually how to use SQLAlchemy's events in Pyramid.

SQLAlchemy event "after_insert" not suitable for this task.
The answer is using Pyramid's custom events as described here http://dannynavarro.net/2011/06/12/using-custom-events-in-pyramid/

I see that there is a problem on the database level - there no guarantee to get valid posts count value. Imagine if two or more posts were added to topic at the same time within multiple transactions - there is no way to tell for sure valid count of associated posts unless you will lock topic table or record every time you are adding new post.
Possible solutions:
1) Add method to Topic class. Always valid number, but it will execute query into db every time.
class Topic(Base):
...
def posts_count(self):
return self.posts.count()
2) Add post to topic and update post_count in your code without using after_insert event. You'll need to update topic.post_count every time when you change topic for post. Also this method won't resolve issue I described when several posts were added to topic at the same time.
def add_post(topic, request):
post_form = PostForm(request.POST)
if 'submit' in request.POST and post_form.validate():
post = Post(body=post_form.body.data)
post.user = request.user
topic.posts.append(post)
topic.post_count = topic.posts.count()
DBSession.add(post)
DBSession.add(topic)
request.session.flash(_('Post was added'))
transaction.commit()
raise HTTPFound(location=request.route_url('topic',id=topic.id))
return {'post_form':post_form}

Related

Test of Django ProfileListView fails with ValueError: Cannot assign "<SimpleLazyObject:....>": "Profile.user" must be a "User" instance

I am a Django beginner and a SOF newbie, sorry if this question sounds a silly to some.
I am struggling with my integration tests.
In my app I have a one-to-one User/Profile relationship. I have a list view to show registered users' profile data:
class ProfileListView(views.ListView, LoginRequiredMixin):
model = Profile
template_name = 'registration/profile_list.html'
paginate_by = 8
def get_context_data(self, *, object_list=None, **kwargs):
context = super(ProfileListView, self).get_context_data()
# superuser raises DoesNotExist at /accounts/profiles/ as they are created with createsuperuser in manage.py
# hence are not assigned a profile automatically => create profile for them here
try:
context['current_profile'] = Profile.objects.get(pk=self.request.user.pk)
except ObjectDoesNotExist:
Profile.objects.create(user=self.request.user)
context['current_profile'] = Profile.objects.get(pk=self.request.user.pk)
# get all other users' profiles apart from staff and current user
regular_users = User.objects \
.filter(is_superuser=False, is_staff=False, is_active=True) \
.exclude(pk=self.request.user.pk)
context['non_staff_active_profiles'] = Profile.objects.filter(user__in=regular_users)
return context
I want to test the get_context_data() method to ensure it returns:
correct logged in user
correct queryset of non-staff profiles
My test breaks as soon as I try something like:
response = self.client.get('/accounts/profiles/')
I understand I need to pass user/profile data to the client but I could not figure out how to do that. It looks like it fails because of context['current_profile'] = Profile.objects.get(pk=self.request.user.pk) and I have no idea why.
The whole 'test' is below:
def test_view_get_context_data__should_return_correct_context(self):
new_user = User.objects.create_user(**self.VALID_USER_DATA_1)
# create profile
new_profile = Profile.objects.create(user=new_user)
# test profile
self.assertEqual(new_profile.user_id, new_user.pk)
response = self.client.get('/accounts/profiles/')
It fails with:
/home/kk/Documents/Github/Phonotheque/venv/bin/python /snap/pycharm-professional/280/plugins/python/helpers/pycharm/django_test_manage.py test Phonotheque.accounts_app.tests.views.test_ProfileListView.ProfilesListViewTests.test_view_get_context_data__should_return_correct_context /home/kk/Documents/Github/Phonotheque
Testing started at 15:54 ...
Found 1 test(s).
Creating test database for alias 'default'...
System check identified no issues (0 silenced).
/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/views/generic/list.py:91: UnorderedObjectListWarning: Pagination may yield inconsistent results with an unordered object_list: <class 'Phonotheque.accounts_app.models.Profile'> QuerySet.
return self.paginator_class(
Destroying test database for alias 'default'...
Error
Traceback (most recent call last):
File "/home/kk/Documents/Github/Phonotheque/Phonotheque/accounts_app/views.py", line 138, in get_context_data
context['current_profile'] = Profile.objects.get(pk=self.request.user.pk)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/db/models/query.py", line 496, in get
raise self.model.DoesNotExist(
Phonotheque.accounts_app.models.Profile.DoesNotExist: Profile matching query does not exist.
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/kk/Documents/Github/Phonotheque/Phonotheque/accounts_app/tests/views/test_ProfileListView.py", line 65, in test_view_get_context_data__should_return_correct_context
response = self.client.get('/accounts/profiles/')
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/test/client.py", line 836, in get
response = super().get(path, data=data, secure=secure, **extra)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/test/client.py", line 424, in get
return self.generic(
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/test/client.py", line 541, in generic
return self.request(**r)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/test/client.py", line 810, in request
self.check_exception(response)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/test/client.py", line 663, in check_exception
raise exc_value
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/core/handlers/exception.py", line 55, in inner
response = get_response(request)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/core/handlers/base.py", line 197, in _get_response
response = wrapped_callback(request, *callback_args, **callback_kwargs)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/views/generic/base.py", line 84, in view
return self.dispatch(request, *args, **kwargs)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/views/generic/base.py", line 119, in dispatch
return handler(request, *args, **kwargs)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/views/generic/list.py", line 174, in get
context = self.get_context_data()
File "/home/kk/Documents/Github/Phonotheque/Phonotheque/accounts_app/views.py", line 140, in get_context_data
Profile.objects.create(user=self.request.user)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/db/models/query.py", line 512, in create
obj = self.model(**kwargs)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/db/models/base.py", line 541, in __init__
_setattr(self, field.name, rel_obj)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/db/models/fields/related_descriptors.py", line 338, in __set__
super().__set__(instance, value)
File "/home/kk/Documents/Github/Phonotheque/venv/lib/python3.8/site-packages/django/db/models/fields/related_descriptors.py", line 235, in __set__
raise ValueError(
ValueError: Cannot assign "<SimpleLazyObject: <django.contrib.auth.models.AnonymousUser object at 0x7f682f11e220>>": "Profile.user" must be a "User" instance.
Process finished with exit code 1
How do I simulate creation of multiple users/profiles, logging in of one of them and obtaining the relevant data?
Thanks a lot in advance.
It was quite a specific question and I very much doubt anyone ever will be looking at this answer but just in case:
def test_view_get_context_data__with__logged_in_user_should_return_correct_context(self):
user_data = {'username': 'BayHuy', 'password': '11111111', }
new_user = User.objects.create_user(**user_data)
new_profile = Profile.objects.create(user=new_user)
self.assertEqual(len(User.objects.all()), 1)
self.assertEqual(new_profile.user_id, new_user.pk)
self.assertEqual(len(Profile.objects.all()), 1)
self.client.login(**user_data)
response = self.client.get(reverse('profiles-list'))
self.assertEqual(
new_profile,
response.context_data['current_profile'])
self.assertEqual(
new_profile, response.context_data['current_profile'])
self.assertEqual(len(response.context_data['profile_list']), 1)

Random NoTransaction in Pyramid

I'm having trouble identifying the source of transaction.interfaces.NoTransaction errors within my Pyramid App. I don't see any patterns to when the error happens, so to me it's quite random.
This app is a (semi-) RESTful API and uses SQLAlchemy and MySQL. I'm currently running within a docker container that connects to an external (bare metal) MySQL instance on the same host OS.
Here's the stack trace for a login attempt within the App. This error happened right after another login attempt that was actually successful.
2020-06-15 03:57:18,982 DEBUG [txn.140501728405248:108][waitress-1] new transaction
2020-06-15 03:57:18,984 INFO [sqlalchemy.engine.base.Engine:730][waitress-1] BEGIN (implicit)
2020-06-15 03:57:18,984 DEBUG [txn.140501728405248:576][waitress-1] abort
2020-06-15 03:57:18,985 ERROR [waitress:357][waitress-1] Exception while serving /auth
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/waitress/channel.py", line 350, in service
task.service()
File "/usr/local/lib/python3.8/site-packages/waitress/task.py", line 171, in service
self.execute()
File "/usr/local/lib/python3.8/site-packages/waitress/task.py", line 441, in execute
app_iter = self.channel.server.application(environ, start_response)
File "/usr/local/lib/python3.8/site-packages/pyramid/router.py", line 270, in __call__
response = self.execution_policy(environ, self)
File "/usr/local/lib/python3.8/site-packages/pyramid_retry/__init__.py", line 127, in retry_policy
response = router.invoke_request(request)
File "/usr/local/lib/python3.8/site-packages/pyramid/router.py", line 249, in invoke_request
response = handle_request(request)
File "/usr/local/lib/python3.8/site-packages/pyramid_tm/__init__.py", line 178, in tm_tween
reraise(*exc_info)
File "/usr/local/lib/python3.8/site-packages/pyramid_tm/compat.py", line 36, in reraise
raise value
File "/usr/local/lib/python3.8/site-packages/pyramid_tm/__init__.py", line 135, in tm_tween
userid = request.authenticated_userid
File "/usr/local/lib/python3.8/site-packages/pyramid/security.py", line 381, in authenticated_userid
return policy.authenticated_userid(self)
File "/opt/REDACTED-api/REDACTED_api/auth/policy.py", line 208, in authenticated_userid
result = self._authenticate(request)
File "/opt/REDACTED-api/REDACTED_api/auth/policy.py", line 199, in _authenticate
session = self._get_session_from_token(token)
File "/opt/REDACTED-api/REDACTED_api/auth/policy.py", line 320, in _get_session_from_token
session = service.get(session_id)
File "/opt/REDACTED-api/REDACTED_api/service/__init__.py", line 122, in get
entity = self.queryset.filter(self.Meta.model.id == entity_id).first()
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3375, in first
ret = list(self[0:1])
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3149, in __getitem__
return list(res)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3481, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3502, in _execute_and_instances
conn = self._get_bind_args(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3517, in _get_bind_args
return fn(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3496, in _connection_from_session
conn = self.session.connection(**kw)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1138, in connection
return self._connection_for_bind(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1146, in _connection_for_bind
return self.transaction._connection_for_bind(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 458, in _connection_for_bind
self.session.dispatch.after_begin(self.session, self, conn)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/event/attr.py", line 322, in __call__
fn(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/zope/sqlalchemy/datamanager.py", line 268, in after_begin
join_transaction(
File "/usr/local/lib/python3.8/site-packages/zope/sqlalchemy/datamanager.py", line 233, in join_transaction
DataManager(
File "/usr/local/lib/python3.8/site-packages/zope/sqlalchemy/datamanager.py", line 89, in __init__
transaction_manager.get().join(self)
File "/usr/local/lib/python3.8/site-packages/transaction/_manager.py", line 91, in get
raise NoTransaction()
transaction.interfaces.NoTransaction
The trace shows that the execution eventually reaches my project, but only my custom authentication policy. And it fails right where the database should be queried for the user.
What intrigues me here is the third line on the stack trace. It seems Waitress somehow aborted the transaction it created? Any clue why?
EDIT: Here's the code where that happens: policy.py:320
def _get_session_from_token(self, token) -> UserSession:
try:
session_id, session_secret = self.parse_token(token)
except InvalidToken as e:
raise SessionNotFound(e)
service = AuthService(self.dbsession, None)
try:
session = service.get(session_id) # <---- Service Class called here
except NoResultsFound:
raise SessionNotFound("Invalid session found Request headers. "
"Session id: %s".format(session_id))
if not service.check_session(session, session_secret):
raise SessionNotFound("Session signature does not match")
now = datetime.now(tz=pytz.UTC)
if session.validity < now:
raise SessionNotFound(
"Current session ID {session_id} is expired".format(
session_id=session.id
)
)
return session
And here is an a view on the that service class method:
class AuthService(ModelService):
class Meta:
model = UserSession
queryset = Query(UserSession)
search_fields = []
order_fields = [UserSession.created_at.desc()]
# These below are from the generic ModelClass father class
def __init__(self, dbsession: Session, user_id: str):
self.user_id = user_id
self.dbsession = dbsession
self.Meta.queryset = self.Meta.queryset.with_session(dbsession)
self.logger = logging.getLogger("REDACTED")
#property
def queryset(self):
return self.Meta.queryset
def get(self, entity_id) -> Base:
entity = self.queryset.filter(self.Meta.model.id == entity_id).first()
if not entity:
raise NoResultsFound(f"Could not find requested ID {entity_id}")
As you can see, the there's already some exception treatment. I really don't see what other exception I could try to catch on AuthService.get
I found the solution to be much simpler than tinkering inside Pyramid or SQLAlchemy.
Debugging my Authentication Policy closely, I found out that my it was keeping a sticky reference for the dbsession. It was stored on the first request ever who used it, and never released.
The first request works as expected, the following one fails: My understanding is that the object is still in memory while the app is running, and after the initial transaction is closed. The second request has a new connection, and a new transaction, but the object in memory still points to the previous one, that when used ultimately causes this.
What I don't understand is why the exception didn't happen sometimes. As I mentioned initially, it was seemingly random.
Another thing that I struggled with was in writing a test case to expose the issue. On my tests, the issue never happens because I have (and I've never seen it done differently) a single connection and a single transaction throughout the entire testing session, as opposed of a new connection/transaction per request, so I have not found no way to actually reproduce.
Please let me know if that makes sense, and if you can shed a light on how to expose the bug on a test case.

Django Foreign Key of Auth.User - column "user_id" does not exist

UPDATE:
This is probably a better representation of my question:
Using loadtestdata, how do you populate the auth.User database? I just want to populate the database with bogus users, and simulations that are linked to those bogus users.
I have looked at all relevant resources but I'm unable to make any headway.
Situation:
I am building a simulation model using Django and am looking to store simulation data as well as sets of parameter data. Many sets of simulation data should be linked to each user, and many sets of parameter data can be linked to each simulation. Thus, I have tried to model this under 'models.py' of my Django app.
from django.db import models
from django.conf import settings
# Create your models here.
class Simulation(models.Model):
user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, )
date = models.DateTimeField()
# Each simulation has only one graph
# Graphing parameters
hill_num = models.CharField(max_length=3)
div_type = models.CharField(max_length=10)
s3_url = models.CharField(max_length=256)
def __str__(self):
return str(self.sim_id)
class Parameter(models.Model):
# Each simulation can have many sets of simulation parameters
simulation = models.ForeignKey('Simulation', on_delete=models.CASCADE)
lsp = models.PositiveIntegerField()
plots = models.PositiveIntegerField()
pioneer = models.BooleanField()
neutral = models.BooleanField()
# for pioneers
p_max = models.PositiveIntegerField()
p_num = models.PositiveIntegerField()
p_start = models.PositiveIntegerField()
# for non-pioneers
np_max = models.PositiveIntegerField()
np_num = models.PositiveIntegerField()
np_start = models.PositiveIntegerField()
def __str__(self):
return str(self.param_id)
./manage.py makemigrations works but when I try to populate the database with python manage.py loadtestdata auth.User:10 divexplorer.Simulation:40 divexplorer.Parameter:300, it throws this error:
auth.User(pk=72): JshtkqSzw3
auth.User(pk=73): QwPfxJc_KS1k5sgH5BN98J
auth.User(pk=74): fuEhnZ
auth.User(pk=75): a
auth.User(pk=76): XjVXXLYGz3MJSfmZ54wGxXo
auth.User(pk=77): fhOWIp
auth.User(pk=78): 5tkEhKOjX2UUbFe
auth.User(pk=79): JgG4Y4PqkcapNJJOlFW1LOQ
auth.User(pk=80): fhRmfQHNim4zM8hGPzpYdkwaHI7
auth.User(pk=81): cEPQtyByKdUs8Gw58DrfNtpsCRB_
Traceback (most recent call last):
File "manage.py", line 22, in <module>
execute_from_command_line(sys.argv)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/core/management/__init__.py", line 363, in execute_from_command_line
utility.execute()
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/core/management/__init__.py", line 355, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/core/management/base.py", line 283, in run_from_argv
self.execute(*args, **cmd_options)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/core/management/base.py", line 330, in execute
output = self.handle(*args, **options)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/utils/decorators.py", line 185, in inner
return func(*args, **kwargs)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/autofixture/management/commands/loadtestdata.py", line 225, in handle
autofixture.create(model, count, **kwargs)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/autofixture/__init__.py", line 136, in create
return autofixture.create(count, **create_kwargs)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/autofixture/base.py", line 554, in create
instance = self.create_one(commit=commit, **kwargs)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/autofixture/base.py", line 519, in create_one
instance.save()
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/models/base.py", line 807, in save
force_update=force_update, update_fields=update_fields)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/models/base.py", line 837, in save_base
updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/models/base.py", line 923, in _save_table
result = self._do_insert(cls._base_manager, using, fields, update_pk, raw)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/models/base.py", line 962, in _do_insert
using=using, raw=raw)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/models/manager.py", line 85, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/models/query.py", line 1076, in _insert
return query.get_compiler(using=using).execute_sql(return_id)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 1099, in execute_sql
cursor.execute(sql, params)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/backends/utils.py", line 80, in execute
return super(CursorDebugWrapper, self).execute(sql, params)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/Users/evanma/anaconda/lib/python2.7/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
django.db.utils.ProgrammingError: column "user_id" of relation "divexplorer_simulation" does not exist
LINE 1: INSERT INTO "divexplorer_simulation" ("user_id", "date", "hi...
I have spent hours trying to work through this error but to no avail. Any ideas?
I have tried renaming the argument db_column in the ForeignKey function, applying default values, but none of them work. Would appreciate some input thank you very much!
I believe you need to specify if related models should also be created with random values.
Please check the docs: http://django-autofixture.readthedocs.io/en/latest/loadtestdata.html
Where it is stated:
There are a few command line options available. Mainly to control the behavior of related fields. If foreingkey or many to many fields should be populated with existing data or if the related models are also generated on the fly. Please have a look at the help page of the command for more information:
django-admin.py help loadtestdata
Unfortunately I can't check a running instance of Django know in order to point you to the exact option and it's value but checking the docs I would say that you have to use this option from loadtestdata:
...
make_option('--generate-fk', action='store', dest='generate_fk', default=None,
help=u'Do not use already existing instances for ForeignKey relations. ' +
'Create new instances instead. You can specify a comma sperated list of ' +
'field names or ALL to indicate that all foreignkeys should be generated ' +
'automatically.'),
...

IntegrityError: column user_id is not unique in django tastypie test unit

I have an estrange error testing a resourcemodel in tastypie.
I have this test
class LocationResourceTest(ResourceTestCase):
# Use ``fixtures`` & ``urls`` as normal. See Django's ``TestCase``
# documentation for the gory details.
fixtures = ['admin_user.json']
def runTest(self):
super(LocationResourceTest, self).runTest()
def setUp(self):
super(LocationResourceTest, self).setUp()
# Create a user.
self.user = User.objects.create_user(username='johndoe',email='johndoe#example.com',password='password')
# Create an API key for the user:
ApiKey.objects.create(user=self.user)
def get_credentials(self):
return self.create_apikey(username=self.user.username, api_key=self.user.api_key.key)
def test_create_apikey(self):
# Try api key authentication using ResourceTestCase.create_apikey().
credentials = self.get_credentials()
resp = self.api_client.get('/api/v1/location/',authentication=credentials,format='json')
self.assertHttpOK(resp)
def test_get_list_unauthorzied(self):
pass
When I execute the test,I get the following error
======================================================================
ERROR: test_get_list_unauthorzied (geolocation.tests.api.LocationResourceTest)
----------------------------------------------------------------------
Traceback (most recent call last):
File "/Users/phantomis/Memoria/smartaxi_server/geolocation/tests/api.py", line 24, in setUp
ApiKey.objects.create(user=self.user)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/models/manager.py", line 137, in create
return self.get_query_set().create(**kwargs)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/models/query.py", line 377, in create
obj.save(force_insert=True, using=self.db)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django_tastypie-0.9.12_alpha-py2.7.egg/tastypie/models.py", line 47, in save
return super(ApiKey, self).save(*args, **kwargs)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/models/base.py", line 463, in save
self.save_base(using=using, force_insert=force_insert, force_update=force_update)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/models/base.py", line 551, in save_base
result = manager._insert([self], fields=fields, return_id=update_pk, using=using, raw=raw)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/models/manager.py", line 203, in _insert
return insert_query(self.model, objs, fields, **kwargs)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/models/query.py", line 1593, in insert_query
return query.get_compiler(using=using).execute_sql(return_id)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/models/sql/compiler.py", line 912, in execute_sql
cursor.execute(sql, params)
File "/Users/phantomis/Virtualenvs/django-memoria/lib/python2.7/site-packages/django/db/backends/sqlite3/base.py", line 344, in execute
return Database.Cursor.execute(self, query, params)
IntegrityError: column user_id is not unique
The error is in the method "test_get_list_unauthorzied" , is weird because if i comment that method, my test pass (that method is completely empty)
setUp is run for each of the test_* methods you have. When you comment out the empty test case, setUp is only run once. With the other test_* method not commented out, setUp is run twice, and this is how the uniqueness constraint gets violated.
I would create a tearDown method that deletes the user and api key you created in the other test case.

Add an object associating two existing objects

I have two tables, testInstance and bugzilla that are associated by a third one, bzCheck, like this:
class Instance(Base):
__tablename__ = "testInstance"
id = Column(Integer, primary_key=True)
bz_checks = relation(BZCheck, backref="instance")
class BZCheck(Base):
__tablename__ = "bzCheck"
instance_id = Column(Integer, ForeignKey("testInstance.id"), primary_key=True)
bz_id = Column(Integer, ForeignKey("bugzilla.id"), primary_key=True)
status = Column(String, nullable=False)
bug = relation(Bugzilla, backref="checks")
class Bugzilla(Base):
__tablename__ = "bugzilla"
id = Column(Integer, primary_key=True)
The backend is a postgresql server ; I'm using SQLalchemy 0.5
If I create the Instance, Bugzilla and BZCheck ojects, then do
bzcheck.bug = bugzilla
instance.bz_checks.append(bzcheck)
and then add and commit them ; everything is fine.
But now, let's assume I have an existing instance and an existing bugzilla and want to associate them:
instance = session.query(Instance).filter(Instance.id == 31).one()
bugzilla = session.query(Bugzilla).filter(Bugzilla.id == 19876).one()
check = BZCheck(status="OK")
check.bug = bugzilla
instance.bz_checks.append(check)
It fails:
In [6]: instance.bz_checks.append(check)
2012-01-09 18:43:50,713 INFO sqlalchemy.engine.base.Engine.0x...3bd0 select nextval('"bzCheck_instance_id_seq"')
2012-01-09 18:43:50,713 INFO sqlalchemy.engine.base.Engine.0x...3bd0 None
2012-01-09 18:43:50,713 INFO sqlalchemy.engine.base.Engine.0x...3bd0 ROLLBACK
It tries to get a new ID from an unexisting sequence instead of using the foreign key "testInstance.id"... I don't understand why.
I have had similar problems when trying to modify objects after commiting them ; I should have missed something fundamental but what ?
the part you're missing here is the stack trace. Always look at the stack trace - what is critical here is that it's autoflush, produced by the access of instance.bz_checks:
Traceback (most recent call last):
File "test.py", line 44, in <module>
instance.bz_checks.append(check)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/attributes.py", line 168, in __get__
return self.impl.get(instance_state(instance),dict_)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/attributes.py", line 453, in get
value = self.callable_(state, passive)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/strategies.py", line 563, in _load_for_state
result = q.all()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/query.py", line 1983, in all
return list(self)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/query.py", line 2092, in __iter__
self.session._autoflush()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 973, in _autoflush
self.flush()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 1547, in flush
self._flush(objects)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 1616, in _flush
flush_context.execute()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/unitofwork.py", line 328, in execute
rec.execute(self)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/unitofwork.py", line 472, in execute
uow
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/mapper.py", line 2291, in _save_obj
execute(statement, params)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1405, in execute
params)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1538, in _execute_clauseelement
compiled_sql, distilled_params
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1646, in _execute_context
context)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1639, in _execute_context
context)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/default.py", line 330, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (IntegrityError) null value in column "instance_id" violates not-null constraint
'INSERT INTO "bzCheck" (bz_id, status) VALUES (%(bz_id)s, %(status)s) RETURNING "bzCheck".instance_id' {'status': 'OK', 'bz_id': 19876}
you can see this because the line of code is:
instance.bz_checks.append(check)
then autoflush:
self.session._autoflush()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 973, in _autoflush
Three solutions:
a. temporarily disable autoflush (see http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DisableAutoflush)
b. ensure that the BZCheck association object is always created with it's full state needed before accessing any collections:
BZState(bug=bugzilla, instance=instance)
(this is usually a good idea for association objects - they represent the association between two points so it's most appropriate that they be instantiated with this state)
c. change the cascade rules so that the operation of check.bug = somebug doesn't actually place check into the session just yet. You can do this with cascade_backrefs, described at http://www.sqlalchemy.org/docs/orm/session.html#controlling-cascade-on-backrefs. (but you'd need to be on 0.6 or 0.7)

Categories