I am trying to build a server to sample a streaming price feed and update a postgres DB using SQLAlchemy. I am using threaded instances of a mapped class, which seems to work but is not stable.
There are no issues with 1 or 2 instances of the Stream class, but with say 10, the thread fails randomly and silently. Each time before it fails, SQLAlchemy gives an error message, so it seems this is what is killing the thread. There is nothing wrong with the stream, it is always stable.
Have I missed something with the SQLAlchemy setup? Is there a better way of feeding multiple realtime subscriptions into SQL?
The code:
import time
import json
from threading import Thread, Lock
import sqlalchemy as db
from sqlalchemy.orm import scoped_session, sessionmaker, relationship
from sqlalchemy.ext.declarative import declarative_base
# Setup SQLAlchemy
engine = db.create_engine('postgresql://localhost:5432/Project', echo=False)
metadata = db.MetaData(bind=engine)
Session = scoped_session(sessionmaker(bind=engine))
Base = declarative_base()
Base.metadata.create_all(engine)
session = Session()
#DB classes
#static data table
class StockMaster(Base):
__tablename__ = 'stock_master'
id = db.Column(db.Integer, primary_key=True)
ticker = db.Column(db.String)
stock_name = db.Column(db.String)
#classmethod
def find_by_ticker(cls,ticker):
return session.query(StockMaster).filter(StockMaster.ticker==ticker).first()
#live data table
class StockLive(Base):
__tablename__ = 'stock_live'
id = db.Column(db.Integer, primary_key=True)
quote = db.Column(db.Numeric)
timestamp = db.Column(db.Numeric)
ticker_id = db.Column(db.Integer, db.ForeignKey('stock_master.id'))
ticker = relationship("StockMaster", foreign_keys=[ticker_id])
def __init__(self, quote, ticker_id, timestamp):
self.quote=quote
self.ticker_id=ticker_id
self.timestamp=timestamp
def save_to_db(self):
session.add(self)
session.commit()
#classmethod
def find_by_ticker_id(cls,ticker_id):
return session.query(StockLive).filter(StockLive.ticker_id==ticker_id).first()
#classmethod
def find_by_ticker(cls,ticker):
ticker_id = StockMaster.find_by_ticker(ticker).id
return session.query(StockLive).filter(StockLive.ticker_id==ticker_id).first()
class Stream(Thread):
def __init__(self,ticker):
Thread.__init__(self)
self.ticker=ticker
self.quote=1
self.data_set = StockLive.find_by_ticker(self.ticker)
self.count=0
def run(self):
con.subscribe(self.ticker)
current_mid=1
while True:
new_data = json.loads(con.get_price(self.ticker).to_json())
new_mid = new_data['Mid']
if new_mid == current_mid:
pass
else:
current_mid = new_mid
self.data_set.quote = current_mid
self.data_set.timestamp = time.time()
try:
self.data_set.save_to_db()
self.count+=1
except:
self.data_set = StockLive.find_by_ticker(self.ticker)
print('error saving to db for '+self.ticker)
time.sleep(.1)
if __name__ == '__main__':
threads={}
for ticker in tickerlist:
try:
threads[ticker]=Stream(ticker)
threads[ticker].setName('Thread ' + ticker)
threads[ticker].start()
except:
print('Error setting up '+ticker)
while True:
for ticker in tickerlist:
if threads[ticker].isAlive()==False:
threads[ticker]=Stream(ticker)
SQLAlchemy error message:
/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py:2323:
SAWarning: Usage of the 'Session.add()' operation is not currently
supported within the execution stage of the flush process. Results may
not be consistent. Consider using alternative event listeners or
connection-level operations instead. % method)
/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py:2425:
SAWarning: Attribute history events accumulated on 1 previously clean
instances within inner-flush event handlers have been reset, and will
not result in database updates. Consider using set_committed_value()
within inner-flush event handlers to avoid this warning. % len_)
Exception in thread Thread MSFT: Traceback (most recent call last):
File
"/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2436, in _flush
transaction.commit() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 465, in commit
self._assert_active(prepared_ok=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg) sqlalchemy.exc.ResourceClosedError: This transaction is closed
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"", line 48, in run
self.data_set.save_to_db() File "", line 44, in save_to_db
session.commit() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 954, in commit
self.transaction.commit() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 467, in commit
self._prepare_impl() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 447, in _prepare_impl
self.session.flush() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2313, in flush
self._flush(objects) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2440, in _flush
transaction.rollback(_capture_exception=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 76, in exit
compat.reraise(type_, value, traceback) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/util/compat.py",
line 249, in reraise
raise value File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2440, in _flush
transaction.rollback(_capture_exception=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in rollback
self._assert_active(prepared_ok=True, rollback_ok=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg) sqlalchemy.exc.ResourceClosedError: This transaction is closed
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"/anaconda3/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run() File "", line 53, in run
self.data_set = StockLive.find_by_ticker(self.ccy) File "", line 52, in find_by_ticker
ticker_id = StockMaster.find_by_ticker(ticker).id File "", line 23, in find_by_ticker
return session.query(StockMaster).filter(StockMaster.ticker==ticker).first()
File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/query.py",
line 2895, in first
ret = list(self[0:1]) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line
2687, in getitem
return list(res) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line
2994, in iter
self.session._autoflush() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1493, in _autoflush
self.flush() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2313, in flush
self._flush(objects) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2400, in _flush
subtransactions=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 865, in begin
nested=nested) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 297, in _begin
self._assert_active() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 264, in _assert_active
"This session is in 'prepared' state; no further " sqlalchemy.exc.InvalidRequestError: This session is in 'prepared'
state; no further SQL can be emitted within this transaction.
There may be other issues with your code but an obvious one is that you share your session with multiple threads.
Instead of setting a global session, these should be created in each thread.
I am not able to run your code but you could try something like this:
Remove the global session variable completely. You do not need it. Then modify your methods and thread to include a local session:
#classmethod
def find_by_ticker(cls,ticker, session):
return session.query(StockMaster).filter(StockMaster.ticker==ticker).first()
...
class Stream(Thread):
def __init__(self,ticker):
Thread.__init__(self)
self.ticker=ticker
self.quote=1
self.session = Session()
self.data_set = StockLive.find_by_ticker(self.ticker, self.session)
self.count=0
Or something like that. This will now make each thread have their own session and your code will start behaving better.
Related
I have noticed a very weird behavior when running sqlachemy (version 1.4) with a context manager that handles the session. I am trying to validate catching the relevant exception when inserting a duplicate primary key. Instead, the flow of the program breaks (exit) and an "except" declaration is never reached.
Here is a simplified version of the code that reproduces that behavior:
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from contextlib import contextmanager
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, Integer
# Define the Engine and the Base
DB_PATH = 'sqlite:///:memory:'
engine = create_engine(DB_PATH, echo=False)
Base = declarative_base()
# Define a basic table called 'labels'
class Label(Base):
__tablename__ = 'labels'
id = Column(Integer, primary_key=True)
# Create all tables
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
#contextmanager
def session_scope(session_cls):
"""Provide a transactional scope around a series of operations."""
session = session_cls()
try:
yield session
session.commit()
except Exception as e:
session.rollback()
raise e
finally:
session.close()
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
with session_scope(session_cls=Session) as session:
try:
session.add(label2)
except Exception as e:
# Unfortunately, this code is never reached as I would expect....
print("Cannot insert another row with existing id")
print(e)
Instead of printing "Cannot insert another row with existing id", the code exits:
Traceback (most recent call last):
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1771, in _execute_context
self.dialect.do_execute(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlite3.IntegrityError: UNIQUE constraint failed: labels.id
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 49, in <module>
print(e)
File "/Users/rbahumi/homebrew/Cellar/python#3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py", line 124, in __exit__
next(self.gen)
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 30, in session_scope
raise e
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 27, in session_scope
session.commit()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1428, in commit
self._transaction.commit(_to_root=self.future)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 829, in commit
self._prepare_impl()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 808, in _prepare_impl
self.session.flush()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3298, in flush
self._flush(objects)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3438, in _flush
transaction.rollback(_capture_exception=True)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3398, in _flush
flush_context.execute()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/unitofwork.py", line 456, in execute
rec.execute(self)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/unitofwork.py", line 630, in execute
util.preloaded.orm_persistence.save_obj(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/persistence.py", line 242, in save_obj
_emit_insert_statements(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/persistence.py", line 1094, in _emit_insert_statements
c = connection._execute_20(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1583, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1452, in _execute_clauseelement
ret = self._execute_context(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1814, in _execute_context
self._handle_dbapi_exception(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1995, in _handle_dbapi_exception
util.raise_(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1771, in _execute_context
self.dialect.do_execute(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) UNIQUE constraint failed: labels.id
[SQL: INSERT INTO labels (id) VALUES (?)]
[parameters: (1,)]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
In order to debug it, I tried to replace the Session class with a "mock" that will raise the same exception:
from sqlalchemy.exc import IntegrityError
class MockSession(object):
def __init__(self, *args, **kwargs):
print("Session:: __init__")
def add(self, *args, **kwargs):
raise IntegrityError("Session::add Exception", orig=Exception(), params={})
def commit(self, *args, **kwargs):
print("Session:: commit")
def rollback(self, *args, **kwargs):
print("Session:: rollback")
def close(self, *args, **kwargs):
print("Session:: close")
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
# This time, call with the MockSession
with session_scope(session_cls=MockSession) as session:
try:
session.add(label2)
except Exception as e:
# This time, call with the MockSession reached this block as expected
print("Cannot insert another row with existing id")
print(e)
When I do that, the flow behaves as expected:
Session:: __init__
Cannot insert another row with existing id
(builtins.Exception)
[SQL: Session::add Exception]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
Session:: commit
Session:: close
Any idea what can cause this sort of behavior?
The issue was solved by extracting the try/except clause outside the 'with' statement:
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
try:
with session_scope(session_cls=Session) as session:
session.add(label2)
except:
print("We have reached this point :-)")
This is the correct way of implementing this, because the context manager's exit() function, which can also raise an exception, should also be inside the try/except clause.
I'm having trouble identifying the source of transaction.interfaces.NoTransaction errors within my Pyramid App. I don't see any patterns to when the error happens, so to me it's quite random.
This app is a (semi-) RESTful API and uses SQLAlchemy and MySQL. I'm currently running within a docker container that connects to an external (bare metal) MySQL instance on the same host OS.
Here's the stack trace for a login attempt within the App. This error happened right after another login attempt that was actually successful.
2020-06-15 03:57:18,982 DEBUG [txn.140501728405248:108][waitress-1] new transaction
2020-06-15 03:57:18,984 INFO [sqlalchemy.engine.base.Engine:730][waitress-1] BEGIN (implicit)
2020-06-15 03:57:18,984 DEBUG [txn.140501728405248:576][waitress-1] abort
2020-06-15 03:57:18,985 ERROR [waitress:357][waitress-1] Exception while serving /auth
Traceback (most recent call last):
File "/usr/local/lib/python3.8/site-packages/waitress/channel.py", line 350, in service
task.service()
File "/usr/local/lib/python3.8/site-packages/waitress/task.py", line 171, in service
self.execute()
File "/usr/local/lib/python3.8/site-packages/waitress/task.py", line 441, in execute
app_iter = self.channel.server.application(environ, start_response)
File "/usr/local/lib/python3.8/site-packages/pyramid/router.py", line 270, in __call__
response = self.execution_policy(environ, self)
File "/usr/local/lib/python3.8/site-packages/pyramid_retry/__init__.py", line 127, in retry_policy
response = router.invoke_request(request)
File "/usr/local/lib/python3.8/site-packages/pyramid/router.py", line 249, in invoke_request
response = handle_request(request)
File "/usr/local/lib/python3.8/site-packages/pyramid_tm/__init__.py", line 178, in tm_tween
reraise(*exc_info)
File "/usr/local/lib/python3.8/site-packages/pyramid_tm/compat.py", line 36, in reraise
raise value
File "/usr/local/lib/python3.8/site-packages/pyramid_tm/__init__.py", line 135, in tm_tween
userid = request.authenticated_userid
File "/usr/local/lib/python3.8/site-packages/pyramid/security.py", line 381, in authenticated_userid
return policy.authenticated_userid(self)
File "/opt/REDACTED-api/REDACTED_api/auth/policy.py", line 208, in authenticated_userid
result = self._authenticate(request)
File "/opt/REDACTED-api/REDACTED_api/auth/policy.py", line 199, in _authenticate
session = self._get_session_from_token(token)
File "/opt/REDACTED-api/REDACTED_api/auth/policy.py", line 320, in _get_session_from_token
session = service.get(session_id)
File "/opt/REDACTED-api/REDACTED_api/service/__init__.py", line 122, in get
entity = self.queryset.filter(self.Meta.model.id == entity_id).first()
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3375, in first
ret = list(self[0:1])
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3149, in __getitem__
return list(res)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3481, in __iter__
return self._execute_and_instances(context)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3502, in _execute_and_instances
conn = self._get_bind_args(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3517, in _get_bind_args
return fn(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/query.py", line 3496, in _connection_from_session
conn = self.session.connection(**kw)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1138, in connection
return self._connection_for_bind(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 1146, in _connection_for_bind
return self.transaction._connection_for_bind(
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/orm/session.py", line 458, in _connection_for_bind
self.session.dispatch.after_begin(self.session, self, conn)
File "/usr/local/lib/python3.8/site-packages/sqlalchemy/event/attr.py", line 322, in __call__
fn(*args, **kw)
File "/usr/local/lib/python3.8/site-packages/zope/sqlalchemy/datamanager.py", line 268, in after_begin
join_transaction(
File "/usr/local/lib/python3.8/site-packages/zope/sqlalchemy/datamanager.py", line 233, in join_transaction
DataManager(
File "/usr/local/lib/python3.8/site-packages/zope/sqlalchemy/datamanager.py", line 89, in __init__
transaction_manager.get().join(self)
File "/usr/local/lib/python3.8/site-packages/transaction/_manager.py", line 91, in get
raise NoTransaction()
transaction.interfaces.NoTransaction
The trace shows that the execution eventually reaches my project, but only my custom authentication policy. And it fails right where the database should be queried for the user.
What intrigues me here is the third line on the stack trace. It seems Waitress somehow aborted the transaction it created? Any clue why?
EDIT: Here's the code where that happens: policy.py:320
def _get_session_from_token(self, token) -> UserSession:
try:
session_id, session_secret = self.parse_token(token)
except InvalidToken as e:
raise SessionNotFound(e)
service = AuthService(self.dbsession, None)
try:
session = service.get(session_id) # <---- Service Class called here
except NoResultsFound:
raise SessionNotFound("Invalid session found Request headers. "
"Session id: %s".format(session_id))
if not service.check_session(session, session_secret):
raise SessionNotFound("Session signature does not match")
now = datetime.now(tz=pytz.UTC)
if session.validity < now:
raise SessionNotFound(
"Current session ID {session_id} is expired".format(
session_id=session.id
)
)
return session
And here is an a view on the that service class method:
class AuthService(ModelService):
class Meta:
model = UserSession
queryset = Query(UserSession)
search_fields = []
order_fields = [UserSession.created_at.desc()]
# These below are from the generic ModelClass father class
def __init__(self, dbsession: Session, user_id: str):
self.user_id = user_id
self.dbsession = dbsession
self.Meta.queryset = self.Meta.queryset.with_session(dbsession)
self.logger = logging.getLogger("REDACTED")
#property
def queryset(self):
return self.Meta.queryset
def get(self, entity_id) -> Base:
entity = self.queryset.filter(self.Meta.model.id == entity_id).first()
if not entity:
raise NoResultsFound(f"Could not find requested ID {entity_id}")
As you can see, the there's already some exception treatment. I really don't see what other exception I could try to catch on AuthService.get
I found the solution to be much simpler than tinkering inside Pyramid or SQLAlchemy.
Debugging my Authentication Policy closely, I found out that my it was keeping a sticky reference for the dbsession. It was stored on the first request ever who used it, and never released.
The first request works as expected, the following one fails: My understanding is that the object is still in memory while the app is running, and after the initial transaction is closed. The second request has a new connection, and a new transaction, but the object in memory still points to the previous one, that when used ultimately causes this.
What I don't understand is why the exception didn't happen sometimes. As I mentioned initially, it was seemingly random.
Another thing that I struggled with was in writing a test case to expose the issue. On my tests, the issue never happens because I have (and I've never seen it done differently) a single connection and a single transaction throughout the entire testing session, as opposed of a new connection/transaction per request, so I have not found no way to actually reproduce.
Please let me know if that makes sense, and if you can shed a light on how to expose the bug on a test case.
I have Postgres DB with a table of pending operations. One column in the operation in an enum with the status of the enum. I used the standard python (2.7) enum, with AutoNumber (myenum.py):
class AutoNumber(enum.Enum):
def __new__(cls):
value = len(cls.__members__) + 1
obj = object.__new__(cls)
obj._value_ = value
return obj
class MyStatus(AutoNumber):
INITIAL = ()
ACCEPTED = ()
DENIED = ()
ACK_PENDING = ()
AUTHORIZED = ()
ACTIVE = ()
END = ()
DELETED = ()
# end enum
The table looks like (also in myenum.py):
Base = declarative_base()
class MyOperation(Base):
__tablename__ = 'operations'
id = Column( Integer, primary_key=True )
status = Column( Enum(MyStatus) )
status_message = Column( String )
status_time = Column( DateTime )
def __repr__(self):
return "<MyOperation(%s, %s, %s, %s)>" % \
( self.id, self.status, self.status_time, self.status_message )
# end class
Generally this works fine. In the SAME FILE that defines MyStatus (myoper.py), I can change the status and save it back to the DB and it works fine:
def checkOper( oper ):
oper.status = MyStatus.DENIED
oper.status_message = "failed check (internal)"
oper.status_time = datetime.datetime.utcnow()
Here's how I call it (within myoper.py)
checkOper( oper )
session.add(oper)
session.commit()
This is all in the same file (myoper.py).
However, if I pass an oper object to an external function, and IT changes the status, then I get a sqlalchemy.exc.StatementError.
Here's the external function (myoper_test.py):
import datetime
from myoper import MyStatus
def extCheckOper( oper ):
oper.status = MyStatus.DENIED
oper.status_message = "failed check (external)"
oper.status_time = datetime.datetime.utcnow()
Here's how I call it (from myoper.py):
from myoper_test import extCheckOper
extCheckOper( oper )
session.add(oper)
session.commit()
Here's the stack trace:
Traceback (most recent call last):
File "./myoper.py", line 120, in <module>
session.commit()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 906, in commit
self.transaction.commit()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 461, in commit
self._prepare_impl()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 441, in _prepare_impl
self.session.flush()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2177, in flush
self._flush(objects)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2297, in _flush
transaction.rollback(_capture_exception=True)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/langhelpers.py", line 66, in __exit__
compat.reraise(exc_type, exc_value, exc_tb)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/session.py", line 2261, in _flush
flush_context.execute()
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 389, in execute
rec.execute(self)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/unitofwork.py", line 548, in execute
uow
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 177, in save_obj
mapper, table, update)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/orm/persistence.py", line 737, in _emit_update_statements
execute(statement, multiparams)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 945, in execute
return meth(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/elements.py", line 263, in _execute_on_connection
return connection._execute_clauseelement(self, multiparams, params)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1053, in _execute_clauseelement
compiled_sql, distilled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1121, in _execute_context
None, None)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1402, in _handle_dbapi_exception
exc_info
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/util/compat.py", line 203, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/base.py", line 1116, in _execute_context
context = constructor(dialect, self, conn, *args)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 639, in _init_compiled
for key in compiled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/engine/default.py", line 639, in <genexpr>
for key in compiled_params
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/sqltypes.py", line 1446, in process
value = self._db_value_for_elem(value)
File "/usr/local/lib/python2.7/dist-packages/sqlalchemy/sql/sqltypes.py", line 1354, in _db_value_for_elem
'"%s" is not among the defined enum values' % elem)
sqlalchemy.exc.StatementError: (exceptions.LookupError) "MyStatus.DENIED" is not among the defined enum values [SQL: u'UPDATE operations SET status=%(status)s, status_message=%(status_message)s, status_time=%(status_time)s WHERE operations.id = %(operations_id)s'] [parameters: [{'status': <MyStatus.DENIED: 6>, 'status_time': datetime.datetime(2017, 10, 18, 20, 22, 44, 350035), 'status_message': 'failed check (external)', 'operations_id': 3}]]
I've tried inspecting the type both in the internal file, and external file, but it's ways listed as <enum 'MyStatus'>.
I have found, that if I assign the oper.status to the enum .name, then that DOES work:
def extCheckOper( oper ):
oper.status = MyStatus.AUTHORIZED.name
oper.status_message = "authorized check (external)"
oper.status_time = datetime.datetime.utcnow()
But that's obviously pretty ugly.
So - what am I doing wrong? What is different about MyStatus in the file it's defined in, vs an external file that screws up SQL Alchemy?
I posted this question to the SQL Alchemy mailing list and got an answer. Link to thread
Turns out this one of those "gotcha's" about python and has nothing really to do with SQL Alchemy. Here's a reference: Executing Main Module Twice.
In this particular case, when I executed my script, the MyStatus was created with a particular id (python handle on the type). But when myoper_test imported MyStatus from myoper, it was created AGAIN with a different id.
So when the extCheckOper assigned a MyStatus value to the status field, it was a different MyStatus than SQL Alchemy created the DB mapping with, so when SQL Alchemy tried to save it to the DB, the " is " operator failed, since the (external) MyStatus was different than (original) MyStatus.
There are a couple of different workarounds. One way is to not run the code as main (after moving exiting main code into main() function):
$ python -c "from myoper import main; import sys; main(*sys.argv[1:])" ext_check 1
The better solution is to avoid this problem entirely - move the code that calls out externally to an internal test script. The code in main stays within mainly within the main script (sorry, couldn't resist... :-) ).
I am trying to step down MongoDB primary using python script.I see the below on my output. Is there a way I can mark the exit code as OK.
Code:
if 'primary' in isMaster:
primary =(isMaster['primary']).split(':')[0]
conn = pymongo.MongoClient('mongodb://'+ primary +':10000 replicaSet=test-ipe1')
stepdown = conn.admin.command("replSetStepDown",100)
if 'ismaster' in isMaster == 'true':
print("I am still the primary")
else:
print("I am no longer the primary")
else:
primary = "No primary currently elected."
Traceback:
Traceback (most recent call last):
File "./repldown.py", line 39, in <module>
stepdown = conn.admin.command("replSetStepDown",60)
File "/usr/local/lib64/python2.6/site-packages/pymongo/database.py", line 391, in command
result = self["$cmd"].find_one(command, **extra_opts)
File "/usr/local/lib64/python2.6/site-packages/pymongo/collection.py", line 604, in find_one
for result in self.find(spec_or_id, *args, **kwargs).limit(-1):
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 904, in next
if len(self.__data) or self._refresh():
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 848, in _refresh
self.__uuid_subtype))
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 782, in __send_message
res = client._send_message_with_response(message, **kwargs)
File "/usr/local/lib64/python2.6/site-packages/pymongo/mongo_client.py", line 1051, in _send_message_with_response
raise AutoReconnect(str(e))
pymongo.errors.AutoReconnect: connection closed
Yes, do:
from pymongo.errors import AutoReconnect
try:
stepdown = conn.admin.command("replSetStepDown",100)
except AutoReconnect:
pass
The primary deliberately closes all connections immediately when it steps down, so the command throws an exception. Just ignore it and continue.
I'm having a strange problem right now.
In my program I try to catch an exception inside a class, de class is used in a separate python file but the exception bubbles trough to the calling file.
How can I fix this?
Example:
main.py
#from sqlalchemy.exc import SQLAlchemyError, DatabaseError
from os import path
from helpers.db import DB
from models.group import Group
from models.entity import Entity
PROJECT_PATH = path.split(path.abspath(__file__))[0]
DATABASE_PATH = PROJECT_PATH + '/datastore/test.db'
class PasswordManager:
database = DB(DATABASE_PATH)
if __name__ == "__main__":
PasswordManager()
output
Traceback (most recent call last):
trying to build a SQLEngine
File "/home/tom/Projects/test/test/test.py", line 11, in <module>
class test:
File "/home/tom/Projects/test/test/test.py", line 23, in test
database = DB(DATABASE_PATH)
File "/home/tom/Projects/test/test/helpers/db.py", line 19, in __init__
self.buildDatabaseModels()
File "/home/tom/Projects/test/test/helpers/db.py", line 39, in buildDatabaseModels
Base.metadata.create_all(self.SQLEngine, checkfirst=True)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/schema.py", line 3308, in create_all
tables=tables)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1516, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1164, in _run_visitor
**kwargs).traverse_single(element)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/visitors.py", line 119, in traverse_single
return meth(obj, **kw)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/ddl.py", line 696, in visit_metadata
if self._can_create_table(t)]
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/ddl.py", line 674, in _can_create_table
table.name, schema=table.schema)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/dialects/sqlite/base.py", line 781, in has_table
cursor = _pragma_cursor(connection.execute(statement))
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 709, in execute
return self._execute_text(object, multiparams, params)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 858, in _execute_text
statement, parameters
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 927, in _execute_context
context)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1076, in _handle_dbapi_exception
exc_info
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 185, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 920, in _execute_context
context)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 425, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.DatabaseError: (DatabaseError) file is encrypted or is not a database u'PRAGMA table_info("groups")' ()
DB.py
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import DatabaseError, SQLAlchemyError
from models import *
from helpers import Base
from helpers.crypt import Crypt
class DB:
SQLEngine = None
SessionMaker = None
Session = None
BuildEngineErrorCount = 0
def __init__(self, db_path):
self.buildEngine(db_path)
self.buildDatabaseModels()
def buildEngine(self, db_path):
try:
self.SQLEngine = create_engine('sqlite:////' + db_path)
print "trying to build a SQLEngine"
except BaseException, er:
print "Errors %s" % er
for error in er.args:
if error == '(DatabaseError) file is encrypted or is not a database':
self.BuildEngineErrorCount += 1
c = Crypt()
c.setKey('Test')
c.decryptDB(db_path)
if self.BuildEngineErrorCount < 5:
self.buildEngine(db_path)
def buildDatabaseModels(self):
if self.SQLEngine is None:
raise Exception("No SQLEngine found")
Base.metadata.create_all(self.SQLEngine, checkfirst=True)
def createSession(self):
if self.SessionMaker is not None or self.Session is not None:
raise Exception("Session already mapped")
self.SessionMaker = sessionmaker(bind=self.SQLEngine)
self.Session = self.SessionMaker()
return self.Session
As you can see I try to catch that exception in the db class, but it isn't catching anything.
Does anybody know what I do wrong?
Your exception is coming from a different method than the one you put the try-except in. Specifically, the exception comes from
Base.metadata.create_all(self.SQLEngine, checkfirst=True)
in buildDatabaseModels, but you have your try-except in buildEngine.