I have Postgres DB with a table of pending operations. One column in the operation in an enum with the status of the enum. I used the standard python (2.7) enum, with AutoNumber (myenum.py):
class AutoNumber(enum.Enum):
def __new__(cls):
value = len(cls.__members__) + 1
obj = object.__new__(cls)
obj._value_ = value
return obj
class MyStatus(AutoNumber):
INITIAL = ()
ACCEPTED = ()
DENIED = ()
ACK_PENDING = ()
AUTHORIZED = ()
ACTIVE = ()
END = ()
DELETED = ()
# end enum
The table looks like (also in myenum.py):
Base = declarative_base()
class MyOperation(Base):
__tablename__ = 'operations'
id = Column( Integer, primary_key=True )
status = Column( Enum(MyStatus) )
status_message = Column( String )
status_time = Column( DateTime )
def __repr__(self):
return "<MyOperation(%s, %s, %s, %s)>" % \
( self.id, self.status, self.status_time, self.status_message )
# end class
Generally this works fine. In the SAME FILE that defines MyStatus (myoper.py), I can change the status and save it back to the DB and it works fine:
def checkOper( oper ):
oper.status = MyStatus.DENIED
oper.status_message = "failed check (internal)"
oper.status_time = datetime.datetime.utcnow()
Here's how I call it (within myoper.py)
checkOper( oper )
session.add(oper)
session.commit()
This is all in the same file (myoper.py).
However, if I pass an oper object to an external function, and IT changes the status, then I get a sqlalchemy.exc.StatementError.
Here's the external function (myoper_test.py):
import datetime
from myoper import MyStatus
def extCheckOper( oper ):
oper.status = MyStatus.DENIED
oper.status_message = "failed check (external)"
oper.status_time = datetime.datetime.utcnow()
Here's how I call it (from myoper.py):
from myoper_test import extCheckOper
extCheckOper( oper )
session.add(oper)
session.commit()
Here's the stack trace:
Traceback (most recent call last):
File "./myoper.py", line 120, in <module>
session.commit()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 906, in commit
self.transaction.commit()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 461, in commit
self._prepare_impl()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 441, in _prepare_impl
self.session.flush()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2177, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2297, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2261, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 389, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 548, in execute
uow
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 177, in save_obj
mapper, table, update)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 737, in _emit_update_statements
execute(statement, multiparams)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 945, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1121, in _execute_context
None, None)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
exc_info
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1116, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 639, in _init_compiled
for key in compiled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 639, in <genexpr>
for key in compiled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/sqltypes.py", line 1446, in process
value = self._db_value_for_elem(value)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/sqltypes.py", line 1354, in _db_value_for_elem
'"%s" is not among the defined enum values' % elem)
sqlalchemy.exc.StatementError: (exceptions.LookupError) "MyStatus.DENIED" is not among the defined enum values [SQL: u'UPDATE operations SET status=%(status)s, status_message=%(status_message)s, status_time=%(status_time)s WHERE operations.id = %(operations_id)s'] [parameters: [{'status': <MyStatus.DENIED: 6>, 'status_time': datetime.datetime(2017, 10, 18, 20, 22, 44, 350035), 'status_message': 'failed check (external)', 'operations_id': 3}]]
I've tried inspecting the type both in the internal file, and external file, but it's ways listed as <enum 'MyStatus'>.
I have found, that if I assign the oper.status to the enum .name, then that DOES work:
def extCheckOper( oper ):
oper.status = MyStatus.AUTHORIZED.name
oper.status_message = "authorized check (external)"
oper.status_time = datetime.datetime.utcnow()
But that's obviously pretty ugly.
So - what am I doing wrong? What is different about MyStatus in the file it's defined in, vs an external file that screws up SQL Alchemy?
I posted this question to the SQL Alchemy mailing list and got an answer. Link to thread
Turns out this one of those "gotcha's" about python and has nothing really to do with SQL Alchemy. Here's a reference: Executing Main Module Twice.
In this particular case, when I executed my script, the MyStatus was created with a particular id (python handle on the type). But when myoper_test imported MyStatus from myoper, it was created AGAIN with a different id.
So when the extCheckOper assigned a MyStatus value to the status field, it was a different MyStatus than SQL Alchemy created the DB mapping with, so when SQL Alchemy tried to save it to the DB, the " is " operator failed, since the (external) MyStatus was different than (original) MyStatus.
There are a couple of different workarounds. One way is to not run the code as main (after moving exiting main code into main() function):
$ python -c "from myoper import main; import sys; main(*sys.argv[1:])" ext_check 1
The better solution is to avoid this problem entirely - move the code that calls out externally to an internal test script. The code in main stays within mainly within the main script (sorry, couldn't resist... :-) ).
Related
I'm trying to reflect a table, while loading only specific columns using include_columns. However, omitting a column that has a ForeignKeyConstraint causes an exception:
Traceback (most recent call last):
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 3825, in _set_parent
ColumnCollectionConstraint._set_parent(self, table)
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 3414, in _set_parent
ColumnCollectionMixin._set_parent(self, table)
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 3371, in _set_parent
for col in self._col_expressions(table):
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 3365, in _col_expressions
return [
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 3366, in <listcomp>
table.c[col] if isinstance(col, util.string_types) else col
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/base.py", line 1213, in __getitem__
return self._index[key]
KeyError: 'DiscountDefinition_FK'
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File ".../script.py", line 62, in <module>
POSBillItem = Table('POSBillItem', metadata, include_columns=POSBillItem_columns, autoload_with=engine)
File "<string>", line 2, in __new__
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/util/deprecations.py", line 309, in warned
return fn(*args, **kwargs)
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 607, in __new__
metadata._remove_table(name, schema)
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 602, in __new__
table._init(name, metadata, *args, **kw)
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 677, in _init
self._autoload(
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 712, in _autoload
conn_insp.reflect_table(
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/engine/reflection.py", line 795, in reflect_table
self._reflect_fk(
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/engine/reflection.py", line 997, in _reflect_fk
table.append_constraint(
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 913, in append_constraint
constraint._set_parent_with_dispatch(self)
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/base.py", line 1046, in _set_parent_with_dispatch
self._set_parent(parent, **kw)
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/sql/schema.py", line 3827, in _set_parent
util.raise_(
File "/usr/local/lib/python3.9/dist-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
sqlalchemy.exc.ArgumentError: Can't create ForeignKeyConstraint on table 'POSBillItem': no column named 'DiscountDefinition_FK' is present.
from sqlalchemy import create_engine, engine
from sqlalchemy import MetaData, Table
conn_string = engine.URL.create(
"mssql+pyodbc",
username,
password,
host,
port,
db,
query={"driver": "ODBC Driver 18 for SQL Server", "TrustServerCertificate": "YES"}
)
engine = create_engine(conn_string)
metadata = MetaData()
Table1_columns = [
"PosBill_FK",
"Quantity",
"UnitPrice",
#"DiscountDefinition_FK",
#"originID"
]
POSBillItem = Table('POSBillItem', metadata, include_columns=Table1_columns, autoload_with=engine)
If i uncomment "DiscountDefinition_FK", exception no longer occurs, though i get a warning:
.../script.py:62: SAWarning: Omitting index key for (originID), key covers omitted columns.
POSBillItem = Table('POSBillItem', metadata, include_columns=POSBillItem_columns, autoload_with=engine)
If i uncomment "originID", warning goes away as well.
Is there a way to avoid loading these columns without manually adding columns?
The include_columns option is not, in fact, honored by foreign key reflection - in a discussion on GitHub it was confirmed to be a bug.
As for warnings on index keys, they would probably be phased out in the future, but right now the options are: either including them or supressing the warning with warnings library.
UPD: Fixed in SQLAlchemy 1.4.38.
I have noticed a very weird behavior when running sqlachemy (version 1.4) with a context manager that handles the session. I am trying to validate catching the relevant exception when inserting a duplicate primary key. Instead, the flow of the program breaks (exit) and an "except" declaration is never reached.
Here is a simplified version of the code that reproduces that behavior:
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from contextlib import contextmanager
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, Integer
# Define the Engine and the Base
DB_PATH = 'sqlite:///:memory:'
engine = create_engine(DB_PATH, echo=False)
Base = declarative_base()
# Define a basic table called 'labels'
class Label(Base):
__tablename__ = 'labels'
id = Column(Integer, primary_key=True)
# Create all tables
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
#contextmanager
def session_scope(session_cls):
"""Provide a transactional scope around a series of operations."""
session = session_cls()
try:
yield session
session.commit()
except Exception as e:
session.rollback()
raise e
finally:
session.close()
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
with session_scope(session_cls=Session) as session:
try:
session.add(label2)
except Exception as e:
# Unfortunately, this code is never reached as I would expect....
print("Cannot insert another row with existing id")
print(e)
Instead of printing "Cannot insert another row with existing id", the code exits:
Traceback (most recent call last):
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1771, in _execute_context
self.dialect.do_execute(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlite3.IntegrityError: UNIQUE constraint failed: labels.id
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 49, in <module>
print(e)
File "/Users/rbahumi/homebrew/Cellar/python#3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py", line 124, in __exit__
next(self.gen)
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 30, in session_scope
raise e
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 27, in session_scope
session.commit()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1428, in commit
self._transaction.commit(_to_root=self.future)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 829, in commit
self._prepare_impl()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 808, in _prepare_impl
self.session.flush()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3298, in flush
self._flush(objects)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3438, in _flush
transaction.rollback(_capture_exception=True)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3398, in _flush
flush_context.execute()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/unitofwork.py", line 456, in execute
rec.execute(self)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/unitofwork.py", line 630, in execute
util.preloaded.orm_persistence.save_obj(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/persistence.py", line 242, in save_obj
_emit_insert_statements(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/persistence.py", line 1094, in _emit_insert_statements
c = connection._execute_20(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1583, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1452, in _execute_clauseelement
ret = self._execute_context(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1814, in _execute_context
self._handle_dbapi_exception(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1995, in _handle_dbapi_exception
util.raise_(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1771, in _execute_context
self.dialect.do_execute(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) UNIQUE constraint failed: labels.id
[SQL: INSERT INTO labels (id) VALUES (?)]
[parameters: (1,)]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
In order to debug it, I tried to replace the Session class with a "mock" that will raise the same exception:
from sqlalchemy.exc import IntegrityError
class MockSession(object):
def __init__(self, *args, **kwargs):
print("Session:: __init__")
def add(self, *args, **kwargs):
raise IntegrityError("Session::add Exception", orig=Exception(), params={})
def commit(self, *args, **kwargs):
print("Session:: commit")
def rollback(self, *args, **kwargs):
print("Session:: rollback")
def close(self, *args, **kwargs):
print("Session:: close")
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
# This time, call with the MockSession
with session_scope(session_cls=MockSession) as session:
try:
session.add(label2)
except Exception as e:
# This time, call with the MockSession reached this block as expected
print("Cannot insert another row with existing id")
print(e)
When I do that, the flow behaves as expected:
Session:: __init__
Cannot insert another row with existing id
(builtins.Exception)
[SQL: Session::add Exception]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
Session:: commit
Session:: close
Any idea what can cause this sort of behavior?
The issue was solved by extracting the try/except clause outside the 'with' statement:
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
try:
with session_scope(session_cls=Session) as session:
session.add(label2)
except:
print("We have reached this point :-)")
This is the correct way of implementing this, because the context manager's exit() function, which can also raise an exception, should also be inside the try/except clause.
I'm using Doubles to track time in my database (as Unix timestamps with fractional portions). For that, I've created the following user type (that takes pytz-aware datetimes from the application and converts them to UTC timestamps).
from sqlalchemy.types import TypeDecorator, Float
class DoubleDatetime(TypeDecorator):
impl = Float
def __init__(self):
TypeDecorator.__init__(self, as_decimal=False)
def process_bind_param(self, value, dialect):
return value.timestamp()
def process_result_value(self, value, dialect):
return datetime.datetime.utcfromtimestamp(value)
This works fine for all datetimes in my system. I'm using the following column definition to have the database (in my case, SQLite3) insert the transaction time:
from sqlalchemy import Column
from sqlalchemy.sql.functions import now
column = Column("transaction_begin", DoubleDatetime, server_default=now())
Despite defining the transaction_begin column as a DoubleDatetime, my database still stores the records with a string.
How can I force the server transaction timestamp to use fractional Unix epochs?
What have I tried?
Reading the SQLite3 docs, I know I need to do some Julian Day trickery to do this.
SELECT (julianday('now') - 2440587.5) * 86400.0;
But I can't find julianday in sqlalchemy.sql.functions, to wrap it in a zero-argument lambda (to pass as the server_default arg to Column).
I tried the following, but it doesn't seem to work.
from sqlalchemy import text
server_default = text("SELECT (julianday('now') - 2440587.5) * 86400.0;")
Nor does the following:
from sqlalchemy.sql import select, text
server_default = select([text("(julianday('now') - 2440587.5) * 86400.0;")])
Traceback:
Traceback (most recent call last):
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 509, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: near "SELECT": syntax error
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "tasks.py", line 81, in <module>
metadata.create_all(engine)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/sql/schema.py", line 4005, in create_all
tables=tables)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1940, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1549, in _run_visitor
**kwargs).traverse_single(element)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 121, in traverse_single
return meth(obj, **kw)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 757, in visit_metadata
_is_metadata_operation=True)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/sql/visitors.py", line 121, in traverse_single
return meth(obj, **kw)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 791, in visit_table
include_foreign_key_constraints=include_foreign_key_constraints
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 948, in execute
return meth(self, multiparams, params)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/sql/ddl.py", line 68, in _execute_on_connection
return connection._execute_ddl(self, multiparams, params)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1009, in _execute_ddl
compiled
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1200, in _execute_context
context)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1413, in _handle_dbapi_exception
exc_info
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 265, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 248, in reraise
raise value.with_traceback(tb)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1193, in _execute_context
context)
File "/home/randm/Libraries/anaconda3/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 509, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) near "SELECT": syntax error [SQL: "\nCREATE TABLE barchartbarmessage (\n\tsymbol VARCHAR NOT NULL, \n\tdate DATE NOT NULL, \n\topen NUMERIC, \n\thigh NUMERIC, \n\tlow NUMERIC, \n\tclose NUMERIC, \n\tvolume INTEGER, \n\tprevious_day_open_interest INTEGER, \n\tprevious_day_volume INTEGER, \n\tmessage_time FLOAT, \n\ttransaction_begin FLOAT DEFAULT SELECT (julianday('now') - 2440587.5) * 86400.0; NOT NULL, \n\ttransaction_end FLOAT, \n\tPRIMARY KEY (symbol, date, transaction_begin)\n)\n\n"] (Background on this error at: http://sqlalche.me/e/e3q8)
I had this same question, and found that this worked. The expression here is specific to PostgreSQL. I originally was calculating this value on my application server, which led to unexpected results. See this post for more information. Note that in PostgreSQL, at least, when setting defaults using expressions you don't include SELECT at the start.
import sqlalchemy as sa
logged_ts_ms = sa.Column(sa.BigInteger, server_default=sa.text("""(extract(epoch from now()) * 1000)::bigint;"""))
I'm currently learning SQLAlchemy, and i found this strange thing. I was experimenting with a table which stores a person's name and address, and to get them i use this:
session.query(User)
And to get the first item, i tried:
session.query(User).first()
Which throws a DatabaseError:
Traceback (most recent call last):
File "<pyshell#4>", line 1, in <module>
session.query(User).first()
File "build\bdist.win32\egg\sqlalchemy\orm\query.py", line 2275, in first
ret = list(self[0:1])
File "build\bdist.win32\egg\sqlalchemy\orm\query.py", line 2142, in __getitem__
return list(res)
File "build\bdist.win32\egg\sqlalchemy\orm\query.py", line 2346, in __iter__
return self._execute_and_instances(context)
File "build\bdist.win32\egg\sqlalchemy\orm\query.py", line 2361, in _execute_and_instances
result = conn.execute(querycontext.statement, self._params)
File "build\bdist.win32\egg\sqlalchemy\engine\base.py", line 664, in execute
return meth(self, multiparams, params)
File "build\bdist.win32\egg\sqlalchemy\sql\elements.py", line 272, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "build\bdist.win32\egg\sqlalchemy\engine\base.py", line 761, in _execute_clauseelement
compiled_sql, distilled_params
File "build\bdist.win32\egg\sqlalchemy\engine\base.py", line 874, in _execute_context
context)
File "build\bdist.win32\egg\sqlalchemy\engine\base.py", line 1023, in _handle_dbapi_exception
exc_info
File "build\bdist.win32\egg\sqlalchemy\util\compat.py", line 185, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "build\bdist.win32\egg\sqlalchemy\engine\base.py", line 867, in _execute_context
context)
File "build\bdist.win32\egg\sqlalchemy\engine\default.py", line 376, in do_execute
cursor.execute(statement, parameters)
DatabaseError: (DatabaseError) ORA-01036: illegal variable name/number
'SELECT test_user_uid, test_user_name, test_user_address \nFROM (SELECT test_user."uid" AS test_user_uid, test_user.name AS test_user_name, test_user.address AS test_user_address \nFROM test_user) \nWHERE ROWNUM <= :ROWNUM_1' {'ROWNUM_1': 1}
However, i was able to retrieve what i wanted if i select all the rows, and loop through the query object:
users = [user for user in session.query(User)]
user1 = users[0]
That's all, i thought it's strange. Here's my mapping class:
class User(Base):
__tablename__ = 'test_user'
uid = Column(Integer, primary_key = True)
name = Column(String(50))
address = Column(String(100))
def __repr__(self):
return "<User (%s, %s)"%(self.name, self.address)
My best guess is that Session.query().first() is looking for the first row, with the generated query. However, the working method retrieves all the rows, and select the first one in Python. The problem is clearly from the generated query (invalid query). The main question is, what caused SQLAlchemy to create an invalid query?
Also, i noticed that SQLAlchemy makes things more difficult by making a query with sub-query. Is that behavior intended?
I hope i can get a satisfying answer, thanks!
Well, it didn't take me long to realize this. It turns out that it's a version problem, i was previously using cx_Oracle version 5.0.2 10g, i tried to upgrade it to version 5.1.2 10g, and things works fine.
This is probably an undocumented bug in SQLAlchemy, i can't find a place where they mention it.
Conclusion: If you want to use the latest version of SQLAlchemy (0.9.0b1) with Oracle 10g, you shouldn't use cx_Oracle older than version 5.1.2 10g.
Hope this helps, and thanks for reading the question!
I have two tables, testInstance and bugzilla that are associated by a third one, bzCheck, like this:
class Instance(Base):
__tablename__ = "testInstance"
id = Column(Integer, primary_key=True)
bz_checks = relation(BZCheck, backref="instance")
class BZCheck(Base):
__tablename__ = "bzCheck"
instance_id = Column(Integer, ForeignKey("testInstance.id"), primary_key=True)
bz_id = Column(Integer, ForeignKey("bugzilla.id"), primary_key=True)
status = Column(String, nullable=False)
bug = relation(Bugzilla, backref="checks")
class Bugzilla(Base):
__tablename__ = "bugzilla"
id = Column(Integer, primary_key=True)
The backend is a postgresql server ; I'm using SQLalchemy 0.5
If I create the Instance, Bugzilla and BZCheck ojects, then do
bzcheck.bug = bugzilla
instance.bz_checks.append(bzcheck)
and then add and commit them ; everything is fine.
But now, let's assume I have an existing instance and an existing bugzilla and want to associate them:
instance = session.query(Instance).filter(Instance.id == 31).one()
bugzilla = session.query(Bugzilla).filter(Bugzilla.id == 19876).one()
check = BZCheck(status="OK")
check.bug = bugzilla
instance.bz_checks.append(check)
It fails:
In [6]: instance.bz_checks.append(check)
2012-01-09 18:43:50,713 INFO sqlalchemy.engine.base.Engine.0x...3bd0 select nextval('"bzCheck_instance_id_seq"')
2012-01-09 18:43:50,713 INFO sqlalchemy.engine.base.Engine.0x...3bd0 None
2012-01-09 18:43:50,713 INFO sqlalchemy.engine.base.Engine.0x...3bd0 ROLLBACK
It tries to get a new ID from an unexisting sequence instead of using the foreign key "testInstance.id"... I don't understand why.
I have had similar problems when trying to modify objects after commiting them ; I should have missed something fundamental but what ?
the part you're missing here is the stack trace. Always look at the stack trace - what is critical here is that it's autoflush, produced by the access of instance.bz_checks:
Traceback (most recent call last):
File "test.py", line 44, in <module>
instance.bz_checks.append(check)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/attributes.py", line 168, in __get__
return self.impl.get(instance_state(instance),dict_)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/attributes.py", line 453, in get
value = self.callable_(state, passive)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/strategies.py", line 563, in _load_for_state
result = q.all()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/query.py", line 1983, in all
return list(self)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/query.py", line 2092, in __iter__
self.session._autoflush()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 973, in _autoflush
self.flush()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 1547, in flush
self._flush(objects)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 1616, in _flush
flush_context.execute()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/unitofwork.py", line 328, in execute
rec.execute(self)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/unitofwork.py", line 472, in execute
uow
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/mapper.py", line 2291, in _save_obj
execute(statement, params)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1405, in execute
params)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1538, in _execute_clauseelement
compiled_sql, distilled_params
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1646, in _execute_context
context)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/base.py", line 1639, in _execute_context
context)
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/engine/default.py", line 330, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (IntegrityError) null value in column "instance_id" violates not-null constraint
'INSERT INTO "bzCheck" (bz_id, status) VALUES (%(bz_id)s, %(status)s) RETURNING "bzCheck".instance_id' {'status': 'OK', 'bz_id': 19876}
you can see this because the line of code is:
instance.bz_checks.append(check)
then autoflush:
self.session._autoflush()
File "/Users/classic/dev/sqlalchemy/lib/sqlalchemy/orm/session.py", line 973, in _autoflush
Three solutions:
a. temporarily disable autoflush (see http://www.sqlalchemy.org/trac/wiki/UsageRecipes/DisableAutoflush)
b. ensure that the BZCheck association object is always created with it's full state needed before accessing any collections:
BZState(bug=bugzilla, instance=instance)
(this is usually a good idea for association objects - they represent the association between two points so it's most appropriate that they be instantiated with this state)
c. change the cascade rules so that the operation of check.bug = somebug doesn't actually place check into the session just yet. You can do this with cascade_backrefs, described at http://www.sqlalchemy.org/docs/orm/session.html#controlling-cascade-on-backrefs. (but you'd need to be on 0.6 or 0.7)