I'm trying to work my way through the example given in the sqlalchemy tutorial but I'm getting errors. As far as I can tell I'm following the example to the letter. Here's the code that I have from it so far. It fails when I .first() after I query the DB.
I'm on version 0.7.5 and python 2.7
from sqlalchemy import Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
engine = create_engine('sqlite:///:memory:', echo=True)
engine.execute("select 1").scalar() # works fine
Session = sessionmaker(bind=engine)
session = Session()
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String)
fullname = Column(String)
password = Column(String)
def __init__(self, name, fullname, password):
self.name = name
self.fullname = fullname
self.password = password
def __repr__(self):
return "<User('%s','%s', '%s')>" % (self.name, self.fullname, self.password)
jeff_user = User("jeff", "Jeff", "foo")
session.add(jeff_user)
our_user = session.query(User).filter_by(name='jeff').first() # fails here
jeff_user.password = "foobar"
session.add_all([
User('wendy', 'Wendy Williams', 'foobar'),
User('mary', 'Mary Contrary', 'xxg527'),
User('fred', 'Fred Flinstone', 'blah')])
session.dirty # shows nothing as dirty
session.new # shows nothing as new
Here is the error message
2012-02-25 17:48:33,879 INFO sqlalchemy.engine.base.Engine BEGIN (implicit)
2012-02-25 17:48:33,886 INFO sqlalchemy.engine.base.Engine INSERT INTO users (name, fullname, password) VALUES (?, ?, ?)
2012-02-25 17:48:33,886 INFO sqlalchemy.engine.base.Engine ('jeff', 'Jeff', 'foo')
2012-02-25 17:48:33,887 INFO sqlalchemy.engine.base.Engine ROLLBACK
Traceback (most recent call last):
File "learning_sql.py", line 35, in <module>
our_user = session.query(User).filter_by(name='ed').first() # fails here
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2024, in first
ret = list(self[0:1])
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 1918, in __getitem__
return list(res)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/query.py", line 2092, in __iter__
self.session._autoflush()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 983, in _autoflush
self.flush()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1559, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 1630, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 331, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 475, in execute
uow
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/mapper.py", line 2291, in _save_obj
execute(statement, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1405, in execute
params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1538, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1646, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1639, in _execute_context
context)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 330, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (OperationalError) no such table: users u'INSERT INTO users (name, fullname, password) VALUES (?, ?, ?)' ('jeff', 'Jeff', 'foo')
The expected print out is this
>>> our_user = session.query(User).filter_by(name='ed').first()
BEGIN (implicit)
INSERT INTO users (name, fullname, password) VALUES (?, ?, ?)
('ed', 'Ed Jones', 'edspassword')
SELECT users.id AS users_id, users.name AS users_name, users.fullname AS users_fullname, users.password AS users_password
FROM users
WHERE users.name = ?
LIMIT 1 OFFSET 0
('ed',)
>>> our_user
<User('ed','Ed Jones', 'edspassword')>
For some reason my code is causing a ROLLBACK when it should be SELECT.
You are getting this error because database is missing the table structure (you apparently missed this line in the tutorial, it's in the Declare Mapping subsection). Create the schema by adding following just after your table/model definitions:
Class User(Base)
...
# Initialize database schema (create tables)
Base.metadata.create_all(engine)
Related
I always get an excessive traceback error, when querying the database finds no result. Why does this query not simply return None?
userExpireDate = session.query(PayingUser.expireDate).filter(PayingUser.mail == mailaddress).first()
This gives this horror error message when the mailaddress can't be found:
Traceback (most recent call last):
File "/Volumes/Mac HD/Benutzer/oliver/PycharmProjects/ErsterTest/Praeprocessor.py", line 274, in <module>
main()
File "/Volumes/Mac HD/Benutzer/oliver/PycharmProjects/ErsterTest/Praeprocessor.py", line 263, in main
userExpireDate = checkUser(mail)
File "/Volumes/Mac HD/Benutzer/oliver/PycharmProjects/ErsterTest/Praeprocessor.py", line 57, in checkUser
userExpireDate = session.query(PayingUser.expireDate).filter(PayingUser.mail == sender).first()
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3240, in first
ret = list(self[0:1])
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3018, in __getitem__
return list(res)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3341, in __iter__
self.session._autoflush()
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1588, in _autoflush
util.raise_from_cause(e)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 1577, in _autoflush
self.flush()
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2470, in flush
self._flush(objects)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2608, in _flush
transaction.rollback(_capture_exception=True)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py", line 68, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 153, in reraise
raise value
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/session.py", line 2568, in _flush
flush_context.execute()
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 422, in execute
rec.execute(self)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/unitofwork.py", line 589, in execute
uow,
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 245, in save_obj
insert,
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/orm/persistence.py", line 1137, in _emit_insert_statements
statement, params
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 988, in execute
return meth(self, multiparams, params)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 287, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1107, in _execute_clauseelement
distilled_params,
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1253, in _execute_context
e, statement, parameters, cursor, context
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1473, in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
raise value.with_traceback(tb)
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1249, in _execute_context
cursor, statement, parameters, context
File "/Volumes/Mac HD/Benutzer/oliver/ErsterTest/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 580, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (raised as a result of Query-invoked autoflush; consider using a session.no_autoflush block if this flush is occurring prematurely)
(sqlite3.OperationalError) no such table: mails
[SQL: INSERT INTO mails (sender, subject, received, answer, text, html, category, count, danger) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)]
[parameters: ('anemailaddress#gmx.at', 'An subject', '2019-10-15 23:59:11.219968', 2, '\r\n\r\n> Anfang der weitergeleiteten Nachricht:\r\n>\r\n> Von: Name <anemailaddress#gmx.at>\r\n> Betreff: An subject (87 characters truncated) ... > Kopie: MrX <anemailaddress#gmx.at>\r\n> Antwort an: MrX <anemailaddress#gmx.at>\r\n>\r\n> Ein weiteres Mail\r\n\r\n', '<html><head><meta http-equiv="Content-Type" content="text/html; charset=us-ascii"></head><body style="word-wrap: break-word; -webkit-nbsp-mode: space ... (2865 characters truncated) ... ass=""></span></div><br class=""><div class=""><div class="">Ein weiteres Mail<br class=""></div></div></blockquote></div><br class=""></body></html>', '', 0, 0)]
(Background on this error at: http://sqlalche.me/e/e3q8)
Why?
Interestingly enough it gives no Error when I do this:
for userExpireDate in session.query(PayingUser.expireDate).filter(PayingUser.mail == sender):
But I'd like to detect if there was no result. And if I try it this way (I know that's not elegant but nevertheless):
i = 0
for userExpireDate in session.query(PayingUser.expireDate).filter(PayingUser.mail == sender):
i += 1
...
if i == 0:
print('No user in database')
then I again get a horribly long traceback error.
Here is the model definition in another file called: payingUser.py
from sqlalchemy import Column, String, Integer, DateTime
from base import Base
from datetime import datetime
class PayingUser(Base):
__tablename__ = 'payingUsers'
mailaddress = Column(String, primary_key=True, unique=True)
firstName = Column(String)
lastName = Column(String)
expireDate = Column(DateTime)
def __init__(self, mailaddress, firstName, lastName):
self.mailaddress = mailaddress
self.firstName = firstName
self.lastName = lastName
self.expireDate = datetime.now()
What's happening here? Can anybody help? Yes, I'm new to sqlalchemy.
There is another table that's not used in this query, but kind of seems to appear in the traceback.
from sqlalchemy import Column, String, Integer, DateTime, Text, Table
from datetime import datetime
from base import Base
class Mail(Base):
__tablename__ = 'mails'
mail_id = Column(Integer, primary_key=True)
sender = Column(String(100))
subject = Column(String)
received = Column(DateTime, nullable=True)
answer = Column(Integer)
text = Column(Text)
html = Column(Text)
def __init__(self):
self.sender = ''
self.subject = ''
self.received = 0
self.answer = 0
self.text = ''
self.html = ''
Thanks!!
I am an idiot!
The reason for my error was, that I had the model definitions in separated files and I imported one of them AFTER the create_all(engine) statement. So I practically did this:
from testingUser import TestingUser
Base.metadata.create_all(engine)
session = Session()
from mail import Mail
Making it correct solved all my problems.
from testingUser import TestingUser
from mail import Mail
Base.metadata.create_all(engine)
session = Session()
Me butthead!!! This took me 3 days. Arghh.
I have a Flask application with sqlalchemy in it. Running manage db init, manage db migrate and manage db upgrade worked perfectly. So the tables are created on db2 without any problems. When i try to do an insert, i get all kinds of errors.
Traceback (most recent call last):
File "*path*/lib/python3.6/site-packages/ibm_db_dbi.py", line 1258, in _execute_helper
return_value = ibm_db.execute(self.stmt_handler, parameters)
Exception: Binding Error:
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/ibm_db_sa/ibm_db.py", line 107, in do_execute
cursor.execute(statement, parameters)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/ibm_db_dbi.py", line 1356, in execute
self._execute_helper(parameters)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/ibm_db_dbi.py", line 1268, in _execute_helper
raise self.messages[len(self.messages) - 1]
ibm_db_dbi.DatabaseError: ibm_db_dbi::DatabaseError: Binding Error:
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/flask/app.py", line 1813, in full_dispatch_request
rv = self.dispatch_request()
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/flask/app.py", line 1799, in dispatch_request
return self.view_functions[rule.endpoint](**req.view_args)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/flask_restplus/api.py", line 319, in wrapper
resp = resource(*args, **kwargs)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/flask/views.py", line 88, in view
return self.dispatch_request(*args, **kwargs)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/flask_restplus/resource.py", line 44, in dispatch_request
resp = meth(*args, **kwargs)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api/app/api.py", line 37, in get
snp.sensor_info()
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api/app/network.py", line 123, in sensor_info
db_session.commit()
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/scoping.py", line 153, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 943, in commit
self.transaction.commit()
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 467, in commit
self._prepare_impl()
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 447, in _prepare_impl
self.session.flush()
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2254, in flush
self._flush(objects)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2380, in _flush
transaction.rollback(_capture_exception=True)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 187, in reraise
raise value
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/session.py", line 2344, in _flush
flush_context.execute()
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 391, in execute
rec.execute(self)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/unitofwork.py", line 556, in execute
uow
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 181, in save_obj
mapper, table, insert)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/orm/persistence.py", line 830, in _emit_insert_statements
execute(statement, multiparams)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
context)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/util/compat.py", line 186, in reraise
raise value.with_traceback(tb)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/ibm_db_sa/ibm_db.py", line 107, in do_execute
cursor.execute(statement, parameters)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/ibm_db_dbi.py", line 1356, in execute
self._execute_helper(parameters)
File "/Users/sebastianlindblom/Documents/GrowSmarter/metrolive-api-env/lib/python3.6/site-packages/ibm_db_dbi.py", line 1268, in _execute_helper
raise self.messages[len(self.messages) - 1]
sqlalchemy.exc.DatabaseError: (ibm_db_dbi.DatabaseError) ibm_db_dbi::DatabaseError: Binding Error: [SQL: 'INSERT INTO "schema"."table" (sensor_id, code, lat, lon, x, y) VALUES (?, ?, ?, ?, ?, ?)'] [parameters: (2855, 30, 59.291658, 18.077241, 411.59730406100783, 769.5522643844229)] (Background on this error at: http://sqlalche.me/e/4xp6)
The model looks like:
class Sensor(Base):
__tablename__ = 'table'
__table_args__ = {'schema':'schema'}
sensor_id = db.Column(db.Integer, primary_key=True)
code = db.Column(db.Integer, unique=True, nullable=False)
lat = db.Column(db.Float, unique=True, nullable=False)
lon = db.Column(db.Float, unique=True, nullable=False)
x = db.Column(db.Float, unique=True, nullable=False)
y = db.Column(db.Float, unique=True, nullable=False)
def __repr__(self):
return '<Sensor %r, x=%r, y=%r>' % self.code % self.x % self.y
The insert code looks like:
from .models import Sensor
from .database import db_session
ins = Sensor(sensor_id = sensor_coords['sensor_code'][0],
code = sensor_coords['sensor'][0],
lat = sensor_coords['lat'][0],
lon = sensor_coords['long'][0],
x = sensor_coords['x'][0],
y = sensor_coords['y'][0])
db_session.add(ins)
db_session.commit()
Looking at the error message, this is how it's trying to bind the sql statement:
[SQL: 'INSERT INTO "schema"."table" (sensor_id, code, lat, lon, x, y) VALUES (?, ?, ?, ?, ?, ?)'] [parameters: (2855, 30, 59.291658, 18.077241, 411.59730406100783, 769.5522643844229)]
which looks correct? (schema and table names are replaced).
My database.py looks like:
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
import ibm_db_sa
from app.api import app
engine = create_engine(app.config['SQLALCHEMY_DATABASE_URI'], convert_unicode=True)
db_session = scoped_session(sessionmaker(autocommit=False,
autoflush=False,
bind=engine))
Base = declarative_base()
Base.query = db_session.query_property()
def init_db():
Base.metadata.create_all(bind=engine)
I can't figure out whats wrong.. The insert format looks correct?
Appreciate all the help I can get. Thanks!
Got it to work!
Had to cast the types in the insert code as:
ins = Sensor(sensor_id = int(sensor_coords['sensor_code'][0]),
code = int(sensor_coords['sensor'][0]),
lat = float(sensor_coords['lat'][0]),
lon = float(sensor_coords['long'][0]),
x = float(sensor_coords['x'][0]),
y = float(sensor_coords['y'][0]))
Didn't realize numpy used its own int and float types...
I'm writting application to store leaked DBs and hashes in MySQL. I want to save some space, so I figured out it may be a good idea to store only unique hashes in Hashes table and put a reference from DBList table. In other words, if I want to get all hashes that belongs to db_id=1 I should ask for them using association table.
For testing purposes I've created these files:
create_fresh_db.py
mydb.py
regex_test.txt
My problem is that association table called dbtohash is filled properly only when I'm inserting unique hashes. If I'm trying to create different name for DB and using the same hashlist (regex_test.txt) an IntegrityError exception is raised. How can I fix that?
EDIT: Following suggestion given in comments. Here are relevant parts of my code. This is how I create many-to-many relationship and association table:
dbtohash = db.Table(
'dbtohash',
db.Column(
'hash_id',
db.Integer,
db.ForeignKey('Hashes.hash_id')),
db.Column(
'db_id',
db.Integer,
db.ForeignKey('DBList.db_id')))
class DBList(db.Model):
__tablename__ = 'DBList'
db_id = db.Column(db.Integer, primary_key=True)
db_name = db.Column(db.String(128), index=True, unique=True)
class Hashes(db.Model):
__tablename__ = 'Hashes'
hash_id = db.Column(db.Integer, primary_key=True)
hash_val = db.Column(db.String(1024), unique=True)
hash_salt = db.Column(db.String(256))
hash_plain = db.Column(db.String(256))
subscriptions = db.relationship(
'DBList', secondary=dbtohash, backref=db.backref(
'dbtohashref', lazy='dynamic'))
And this is how I insert data to it:
new list
# if we have some matched hashes...
if len(hash_pass) > 0:
# create new list
newlist = DBList(
db_name = '{}_{}'.format('db', datetime.utcnow()),
)
print('newlist: {}'.format(newlist))
# try to insert DB name to DBList table, raise exception if name is taken
try:
db.session.add(newlist)
db.session.commit()
db.session.flush()
except IntegrityError:
db.session.rollback()
print('this DB name is taken, choose another one', 'danger')
insert hashes to DB
failed=[]
for hash in hash_pass:
inserthash = Hashes (
hash_val = hash,
)
try:
db.session.add(inserthash)
db.session.commit()
db.session.flush()
hashid=inserthash.hash_id
except IntegrityError:
db.session.rollback()
failed.append(hash)
hashid=db.session.query(Hashes.hash_id).filter_by(hash_val=hash).one()[0]
# fill up association table
newlist.dbtohashref.append(inserthash)
db.session.commit()
if failed:
print('failed: {}'.format(failed))
problematic line is: newlist.dbtohashref.append(inserthash)
EDIT 2: full trace attached
Traceback (most recent call last):
File "mydb.py", line 88, in <module>
db.session.commit()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/scoping.py", line 153, in do
return getattr(self.registry(), name)(*args, **kwargs)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 943, in commit
self.transaction.commit()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 467, in commit
self._prepare_impl()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 447, in _prepare_impl
self.session.flush()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2254, in flush
self._flush(objects)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2380, in _flush
transaction.rollback(_capture_exception=True)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2344, in _flush
flush_context.execute()
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 391, in execute
rec.execute(self)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 556, in execute
uow
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 181, in save_obj
mapper, table, insert)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 866, in _emit_insert_statements
execute(statement, params)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 269, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1060, in _execute_clauseelement
compiled_sql, distilled_params
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
context)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/home/user/envs/project/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 507, in do_execute
cursor.execute(statement, parameters)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/cursors.py", line 165, in execute
result = self._query(query)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/cursors.py", line 321, in _query
conn.query(q)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 860, in query
self._affected_rows = self._read_query_result(unbuffered=unbuffered)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 1061, in _read_query_result
result.read()
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 1349, in read
first_packet = self.connection._read_packet()
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 1018, in _read_packet
packet.check_error()
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/connections.py", line 384, in check_error
err.raise_mysql_exception(self._data)
File "/home/user/envs/project/local/lib/python2.7/site-packages/pymysql/err.py", line 107, in raise_mysql_exception
raise errorclass(errno, errval)
sqlalchemy.exc.IntegrityError: (pymysql.err.IntegrityError) (1062, u"Duplicate entry 'd38b70e33d7ae06c1696910eb6f93ca4' for key 'hash_val'") [SQL: u'INSERT INTO `Hashes` (hash_val, hash_salt, hash_plain) VALUES (%(hash_val)s, %(hash_salt)s, %(hash_plain)s)'] [parameters: {'hash_salt': None, 'hash_val': 'd38b70e33d7ae06c1696910eb6f93ca4', 'hash_plain': None}] (Background on this error at: http://sqlalche.me/e/gkpj)
Using: postgres (PostgreSQL) 9.4.5
I just migrated a sqlite3 db onto a postgresql db. For some reason since this migration, when I try to create a user, an error regarding the user_id (which is a primary key) is being raised. This was not an issue before with sqlite3. I have spent time looking through the docs and stack questions, but remain confused.
Inside api.create_user():
api.create_user(username ='lola ', firstname ='cats ', lastname ='lcatk', email='cags#falc.com')
sqlalchemy db Model:
class User(Base):
__tablename__ = 'users'
#user_id = Column(Integer, primary_key=True)
#changed to:
id = Column(Integer, primary_key=True)
username = Column(String(50))
firstname = Column(String(50))
lastname = Column(String(50))
email = Column(String(300))
password = Column(String(12))
institution = Column(String(50))
def __init__(self, username, firstname, lastname, email):
self.username = username
self.firstname = firstname
self.lastname = lastname
self.email = email
def __repr__(self):
return "<User(username ='%s', firstname ='%s', lastname ='%s', email='%s')>" % (self.username, self.firstname, self.lastname, self.email)
pyramid views.py:
#view_config(#code supplying the template and etc.)
def save_assessment_result(request):
with transaction.manager:
username = request.params['username']
firstname = request.params['firstname']
lastname = request.params['lastname']
email = request.params['email']
user = api.create_user(username, firstname, lastname, email)
#mode code to commit and etc.
transaction.commit()
return HTTPCreated()
postgres Server.log:
ERROR: null value in column "user_id" violates not-null constraint
DETAIL: Failing row contains (null, lola , cats , lcatk, cags#falc.com, null, null, 2015-10-19 23:02:21.560395).
STATEMENT: INSERT INTO users (username, firstname, lastname, email, password, institution, created_on) VALUES ('lola ', 'cats ', 'lcatk', 'cags#falc.com', NULL, NULL, '2015-10-19T23:02:21.560395'::timestamp) RETURNING users.user_id
Traceback:
015-10-19 19:02:21,563 ERROR [pyramid_debugtoolbar][Dummy-3] Exception at http://0.0.0.0:6432/save_assessment_result
File "/Users/ack/code/venv/NotssWEB/notssweb/views/default.py", line 61, in save_assessment_result
assessment = api.retrieve_assessment(assessment_id)
File "/usr/local/lib/python2.7/site-packages/notssdb/api/object.py", line 112, in retrieve_assessment
filter(Assessment.assessment_id == something_unique).one()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2473, in one
ret = list(self)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2515, in __iter__
self.session._autoflush()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1292, in _autoflush
util.raise_from_cause(e)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1282, in _autoflush
self.flush()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2004, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2122, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2086, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 373, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 532, in execute
uow
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 174, in save_obj
mapper, table, insert)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 781, in _emit_insert_statements
execute(statement, params)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
context)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1341, in _handle_dbapi_exception
exc_info
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
context)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 450, in do_execute
cursor.execute(statement, parameters)
IntegrityError: (raised as a result of Query-invoked autoflush; consider using a session.no_autoflush block if this flush is occurring prematurely) (psycopg2.IntegrityError) null value in column "user_id" violates not-null constraint
DETAIL: Failing row contains (null, lola , cats , lcatk, cags#falc.com, null, null, 2015-10-19 23:02:21.560395).
[SQL: 'INSERT INTO users (username, firstname, lastname, email, password, institution, created_on) VALUES (%(username)s, %(firstname)s, %(lastname)s, %(email)s, %(password)s, %(institution)s, %(created_on)s) RETURNING users.user_id'] [parameters: {'username': u'lola ', 'firstname': u'cats ', 'lastname': u'lcatk', 'institution': None, 'created_on': datetime.datetime(2015, 10, 19, 23, 2, 21, 560395), 'password': None, 'email': u'cags#falc.com'}]
2015-10-19 19:02:21,564 DEBUG [notssweb][Dummy-3] route matched for url http://0.0.0.0:6432/_debug_toolbar/exception?token=a30c0989db02aeff9cd2&tb=4459323984; route_name: 'debugtoolbar', path_info: u'/_debug_toolbar/exception', pattern: '/_debug_toolbar/*subpath', matchdict: {'subpath': (u'exception',)}, predicates: '
First.
I changed the id naming convention from user_id to simply id in the database (this is true for all others).
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
Second.
The issue was found in dumping the original sqlite db file into postgres (from sqlite3 --> postgres) instead of creating the db in postgres.
So, I ran the original code in SQLAlchemy, but this time pointing to the postgres db:
engine = create_engine('postgresql://localhost/some_db')
This created the needed relationships and sequences (shown via \ds in postgres, psql). Issue resolved.
This question already has an answer here:
IntegrityError when inserting data in an association table using SQLAlchemy
(1 answer)
Closed 5 years ago.
I just switched from MySQL to SQLalchemy and I have to say that it is more difficult to get my head around sqlalchemy than I thought. Currently I have some trouble with a many-to-many relationship. I have users and queries in my model. A user can have many queries and a query offers a new article to read every day. I want to store which user read which query on what date. My models.py looks like this
class User(db.Model):
id = db.Column(db.Integer, primary_key=True)
read_dates = db.relationship("ReadIndex", backref="user")
def __repr__(self):
return '<User %r>' % (self.id)
class Queries(db.Model):
id = db.Column(db.Integer, primary_key=True)
user_id = db.Column(db.Integer, db.ForeignKey('user.id'))
def __repr__(self):
return '<Queries %r>' % (self.id)
class ReadIndex(db.Model):
user_id = db.Column(db.Integer, db.ForeignKey('user.id'), primary_key=True)
query_id = db.Column(db.Integer, db.ForeignKey('queries.id'), primary_key=True)
read_datetime = db.Column(db.DateTime)
read_query = db.relationship("Queries", backref="user_assocs")
def __repr__(self):
return '<ReadIndex>'
so I have users and queries. To store which user read which query on what date I have the class ReadIndex, which has an extra field called read_datetime, where I store the date of a user-query association. I than want to add such an association like
user = models.User.query.filter_by(id=user_id).first()
query = models.Queries.query.filter_by(id=query_id).first()
a = models.ReadIndex(read_datetime=dt.utcnow())
a.read_query = query
user.read_dates.append(a)
db.session.commit()
where query_id and user_id are the corresponding ids to select my objects. If I run this I get an error
sqlalchemy.exc.IntegrityError
IntegrityError: (sqlite3.IntegrityError) NOT NULL constraint failed:
read_index.query_id [SQL: u'INSERT INTO read_index (user_id, read_datetime)
VALUES (?, ?)'] [parameters: (1, '2015-07-29 20:55:50.898366')]
I am not sure what the problem is? I don't seem to violate any NOT NULL
edit: I noticed that I assigned query=None to a.read_query. Correcting this slightly changed the error
IntegrityError: (raised as a result of Query-invoked autoflush;
consider using a session.no_autoflush block if this flush is occurring prematurely)
(sqlite3.IntegrityError) NOT NULL constraint failed: read_index.user_id
[SQL: u'INSERT INTO read_index (query_id, read_datetime) VALUES (?, ?)']
[parameters: (1, '2015-07-29 23:11:11.934038')]
here is exactly what happens:
>>> from app import app, db, models
>>> user = models.User.query.filter_by(id=1).first()
>>> user
<User u'Florian Beutler'>
>>> query = models.Queries.query.filter_by(id=1).first()
>>> query
<Queries 1>
>>> import datetime
>>> a = models.ReadIndex(read_datetime=datetime.datetime.utcnow())
>>> a.read_query = query
>>> user.read_dates.append(a)
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/usr/local/lib/python2.7/site- packages/sqlalchemy/orm/attributes.py", line 237, in __get__
return self.impl.get(instance_state(instance), dict_)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/attributes.py", line 578, in get
value = self.callable_(state, passive)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/strategies.py", line 529, in _load_for_state
return self._emit_lazyload(session, state, ident_key, passive)
File "<string>", line 1, in <lambda>
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/strategies.py", line 599, in _emit_lazyload
result = q.all()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2399, in all
return list(self)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/query.py", line 2515, in __iter__
self.session._autoflush()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1292, in _autoflush
util.raise_from_cause(e)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 1282, in _autoflush
self.flush()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2004, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2122, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/langhelpers.py", line 60, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/session.py", line 2086, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 373, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/unitofwork.py", line 532, in execute
uow
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 174, in save_obj
mapper, table, insert)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/orm/persistence.py", line 781, in _emit_insert_statements
execute(statement, params)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 914, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1010, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1146, in _execute_context
context)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1341, in _handle_dbapi_exception
exc_info
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 199, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1139, in _execute_context
context)
File "/usr/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 450, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (raised as a result of Query-invoked autoflush; consider using a session.no_autoflush block if this flush is occurring prematurely) (sqlite3.IntegrityError) NOT NULL constraint failed: read_index.user_id [SQL: u'INSERT INTO read_index (query_id, read_datetime) VALUES (?, ?)'] [parameters: (1, '2015-07-29 23:17:06.013485')]
constraint?
thanks
carl
The .first() method either returns the first result or None if there isn't one, are you sure you're not assigning a.read_query to None?
You are getting this error because the user_id and query_id fields of the ReadIndex table are None. So, when you commit, ReadIndex object is trying to get created without having values in its primary key fields. So, do this,
user = models.User.query.filter_by(id=user_id).first()
query = models.Queries.query.filter_by(id=query_id).first()
a=models.ReadIndex(user_id=user.id,query_id=query.id,read_datetime=dt.utcnow())
db.session.add(a)
db.session.commit()
You dont need to manually append anything to the relationship. SQLAlchemy does that for you automatically. You can crosscheck if you want,
for ud in user.read_dates:
print(ud)
Hope this helps!