I'm having a strange problem right now.
In my program I try to catch an exception inside a class, de class is used in a separate python file but the exception bubbles trough to the calling file.
How can I fix this?
Example:
main.py
#from sqlalchemy.exc import SQLAlchemyError, DatabaseError
from os import path
from helpers.db import DB
from models.group import Group
from models.entity import Entity
PROJECT_PATH = path.split(path.abspath(__file__))[0]
DATABASE_PATH = PROJECT_PATH + '/datastore/test.db'
class PasswordManager:
database = DB(DATABASE_PATH)
if __name__ == "__main__":
PasswordManager()
output
Traceback (most recent call last):
trying to build a SQLEngine
File "/home/tom/Projects/test/test/test.py", line 11, in <module>
class test:
File "/home/tom/Projects/test/test/test.py", line 23, in test
database = DB(DATABASE_PATH)
File "/home/tom/Projects/test/test/helpers/db.py", line 19, in __init__
self.buildDatabaseModels()
File "/home/tom/Projects/test/test/helpers/db.py", line 39, in buildDatabaseModels
Base.metadata.create_all(self.SQLEngine, checkfirst=True)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/schema.py", line 3308, in create_all
tables=tables)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1516, in _run_visitor
conn._run_visitor(visitorcallable, element, **kwargs)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1164, in _run_visitor
**kwargs).traverse_single(element)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/visitors.py", line 119, in traverse_single
return meth(obj, **kw)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/ddl.py", line 696, in visit_metadata
if self._can_create_table(t)]
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/sql/ddl.py", line 674, in _can_create_table
table.name, schema=table.schema)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/dialects/sqlite/base.py", line 781, in has_table
cursor = _pragma_cursor(connection.execute(statement))
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 709, in execute
return self._execute_text(object, multiparams, params)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 858, in _execute_text
statement, parameters
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 927, in _execute_context
context)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 1076, in _handle_dbapi_exception
exc_info
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/util/compat.py", line 185, in raise_from_cause
reraise(type(exception), exception, tb=exc_tb)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/base.py", line 920, in _execute_context
context)
File "/home/tom/Projects/test/local/lib/python2.7/site-packages/sqlalchemy/engine/default.py", line 425, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.DatabaseError: (DatabaseError) file is encrypted or is not a database u'PRAGMA table_info("groups")' ()
DB.py
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from sqlalchemy.exc import DatabaseError, SQLAlchemyError
from models import *
from helpers import Base
from helpers.crypt import Crypt
class DB:
SQLEngine = None
SessionMaker = None
Session = None
BuildEngineErrorCount = 0
def __init__(self, db_path):
self.buildEngine(db_path)
self.buildDatabaseModels()
def buildEngine(self, db_path):
try:
self.SQLEngine = create_engine('sqlite:////' + db_path)
print "trying to build a SQLEngine"
except BaseException, er:
print "Errors %s" % er
for error in er.args:
if error == '(DatabaseError) file is encrypted or is not a database':
self.BuildEngineErrorCount += 1
c = Crypt()
c.setKey('Test')
c.decryptDB(db_path)
if self.BuildEngineErrorCount < 5:
self.buildEngine(db_path)
def buildDatabaseModels(self):
if self.SQLEngine is None:
raise Exception("No SQLEngine found")
Base.metadata.create_all(self.SQLEngine, checkfirst=True)
def createSession(self):
if self.SessionMaker is not None or self.Session is not None:
raise Exception("Session already mapped")
self.SessionMaker = sessionmaker(bind=self.SQLEngine)
self.Session = self.SessionMaker()
return self.Session
As you can see I try to catch that exception in the db class, but it isn't catching anything.
Does anybody know what I do wrong?
Your exception is coming from a different method than the one you put the try-except in. Specifically, the exception comes from
Base.metadata.create_all(self.SQLEngine, checkfirst=True)
in buildDatabaseModels, but you have your try-except in buildEngine.
Related
I have noticed a very weird behavior when running sqlachemy (version 1.4) with a context manager that handles the session. I am trying to validate catching the relevant exception when inserting a duplicate primary key. Instead, the flow of the program breaks (exit) and an "except" declaration is never reached.
Here is a simplified version of the code that reproduces that behavior:
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from contextlib import contextmanager
from sqlalchemy.orm import sessionmaker
from sqlalchemy import Column, Integer
# Define the Engine and the Base
DB_PATH = 'sqlite:///:memory:'
engine = create_engine(DB_PATH, echo=False)
Base = declarative_base()
# Define a basic table called 'labels'
class Label(Base):
__tablename__ = 'labels'
id = Column(Integer, primary_key=True)
# Create all tables
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
#contextmanager
def session_scope(session_cls):
"""Provide a transactional scope around a series of operations."""
session = session_cls()
try:
yield session
session.commit()
except Exception as e:
session.rollback()
raise e
finally:
session.close()
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
with session_scope(session_cls=Session) as session:
try:
session.add(label2)
except Exception as e:
# Unfortunately, this code is never reached as I would expect....
print("Cannot insert another row with existing id")
print(e)
Instead of printing "Cannot insert another row with existing id", the code exits:
Traceback (most recent call last):
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1771, in _execute_context
self.dialect.do_execute(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlite3.IntegrityError: UNIQUE constraint failed: labels.id
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 49, in <module>
print(e)
File "/Users/rbahumi/homebrew/Cellar/python#3.9/3.9.2_1/Frameworks/Python.framework/Versions/3.9/lib/python3.9/contextlib.py", line 124, in __exit__
next(self.gen)
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 30, in session_scope
raise e
File "/Users/rbahumi/IdeaProjects/cafeteria/omri2.py", line 27, in session_scope
session.commit()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 1428, in commit
self._transaction.commit(_to_root=self.future)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 829, in commit
self._prepare_impl()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 808, in _prepare_impl
self.session.flush()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3298, in flush
self._flush(objects)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3438, in _flush
transaction.rollback(_capture_exception=True)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/langhelpers.py", line 70, in __exit__
compat.raise_(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/session.py", line 3398, in _flush
flush_context.execute()
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/unitofwork.py", line 456, in execute
rec.execute(self)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/unitofwork.py", line 630, in execute
util.preloaded.orm_persistence.save_obj(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/persistence.py", line 242, in save_obj
_emit_insert_statements(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/orm/persistence.py", line 1094, in _emit_insert_statements
c = connection._execute_20(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1583, in _execute_20
return meth(self, args_10style, kwargs_10style, execution_options)
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/sql/elements.py", line 323, in _execute_on_connection
return connection._execute_clauseelement(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1452, in _execute_clauseelement
ret = self._execute_context(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1814, in _execute_context
self._handle_dbapi_exception(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1995, in _handle_dbapi_exception
util.raise_(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/util/compat.py", line 207, in raise_
raise exception
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/base.py", line 1771, in _execute_context
self.dialect.do_execute(
File "/Users/rbahumi/venv/cafeteria/lib/python3.9/site-packages/sqlalchemy/engine/default.py", line 717, in do_execute
cursor.execute(statement, parameters)
sqlalchemy.exc.IntegrityError: (sqlite3.IntegrityError) UNIQUE constraint failed: labels.id
[SQL: INSERT INTO labels (id) VALUES (?)]
[parameters: (1,)]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
In order to debug it, I tried to replace the Session class with a "mock" that will raise the same exception:
from sqlalchemy.exc import IntegrityError
class MockSession(object):
def __init__(self, *args, **kwargs):
print("Session:: __init__")
def add(self, *args, **kwargs):
raise IntegrityError("Session::add Exception", orig=Exception(), params={})
def commit(self, *args, **kwargs):
print("Session:: commit")
def rollback(self, *args, **kwargs):
print("Session:: rollback")
def close(self, *args, **kwargs):
print("Session:: close")
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
# This time, call with the MockSession
with session_scope(session_cls=MockSession) as session:
try:
session.add(label2)
except Exception as e:
# This time, call with the MockSession reached this block as expected
print("Cannot insert another row with existing id")
print(e)
When I do that, the flow behaves as expected:
Session:: __init__
Cannot insert another row with existing id
(builtins.Exception)
[SQL: Session::add Exception]
(Background on this error at: https://sqlalche.me/e/14/gkpj)
Session:: commit
Session:: close
Any idea what can cause this sort of behavior?
The issue was solved by extracting the try/except clause outside the 'with' statement:
if __name__ == "__main__":
# Create a label with id=1 and insert it to the DB
label1 = Label(id=1)
with session_scope(session_cls=Session) as session:
session.add(label1)
# create a another label with the same id and try to insert it also
label2 = Label(id=1)
try:
with session_scope(session_cls=Session) as session:
session.add(label2)
except:
print("We have reached this point :-)")
This is the correct way of implementing this, because the context manager's exit() function, which can also raise an exception, should also be inside the try/except clause.
I am trying to build a server to sample a streaming price feed and update a postgres DB using SQLAlchemy. I am using threaded instances of a mapped class, which seems to work but is not stable.
There are no issues with 1 or 2 instances of the Stream class, but with say 10, the thread fails randomly and silently. Each time before it fails, SQLAlchemy gives an error message, so it seems this is what is killing the thread. There is nothing wrong with the stream, it is always stable.
Have I missed something with the SQLAlchemy setup? Is there a better way of feeding multiple realtime subscriptions into SQL?
The code:
import time
import json
from threading import Thread, Lock
import sqlalchemy as db
from sqlalchemy.orm import scoped_session, sessionmaker, relationship
from sqlalchemy.ext.declarative import declarative_base
# Setup SQLAlchemy
engine = db.create_engine('postgresql://localhost:5432/Project', echo=False)
metadata = db.MetaData(bind=engine)
Session = scoped_session(sessionmaker(bind=engine))
Base = declarative_base()
Base.metadata.create_all(engine)
session = Session()
#DB classes
#static data table
class StockMaster(Base):
__tablename__ = 'stock_master'
id = db.Column(db.Integer, primary_key=True)
ticker = db.Column(db.String)
stock_name = db.Column(db.String)
#classmethod
def find_by_ticker(cls,ticker):
return session.query(StockMaster).filter(StockMaster.ticker==ticker).first()
#live data table
class StockLive(Base):
__tablename__ = 'stock_live'
id = db.Column(db.Integer, primary_key=True)
quote = db.Column(db.Numeric)
timestamp = db.Column(db.Numeric)
ticker_id = db.Column(db.Integer, db.ForeignKey('stock_master.id'))
ticker = relationship("StockMaster", foreign_keys=[ticker_id])
def __init__(self, quote, ticker_id, timestamp):
self.quote=quote
self.ticker_id=ticker_id
self.timestamp=timestamp
def save_to_db(self):
session.add(self)
session.commit()
#classmethod
def find_by_ticker_id(cls,ticker_id):
return session.query(StockLive).filter(StockLive.ticker_id==ticker_id).first()
#classmethod
def find_by_ticker(cls,ticker):
ticker_id = StockMaster.find_by_ticker(ticker).id
return session.query(StockLive).filter(StockLive.ticker_id==ticker_id).first()
class Stream(Thread):
def __init__(self,ticker):
Thread.__init__(self)
self.ticker=ticker
self.quote=1
self.data_set = StockLive.find_by_ticker(self.ticker)
self.count=0
def run(self):
con.subscribe(self.ticker)
current_mid=1
while True:
new_data = json.loads(con.get_price(self.ticker).to_json())
new_mid = new_data['Mid']
if new_mid == current_mid:
pass
else:
current_mid = new_mid
self.data_set.quote = current_mid
self.data_set.timestamp = time.time()
try:
self.data_set.save_to_db()
self.count+=1
except:
self.data_set = StockLive.find_by_ticker(self.ticker)
print('error saving to db for '+self.ticker)
time.sleep(.1)
if __name__ == '__main__':
threads={}
for ticker in tickerlist:
try:
threads[ticker]=Stream(ticker)
threads[ticker].setName('Thread ' + ticker)
threads[ticker].start()
except:
print('Error setting up '+ticker)
while True:
for ticker in tickerlist:
if threads[ticker].isAlive()==False:
threads[ticker]=Stream(ticker)
SQLAlchemy error message:
/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py:2323:
SAWarning: Usage of the 'Session.add()' operation is not currently
supported within the execution stage of the flush process. Results may
not be consistent. Consider using alternative event listeners or
connection-level operations instead. % method)
/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py:2425:
SAWarning: Attribute history events accumulated on 1 previously clean
instances within inner-flush event handlers have been reset, and will
not result in database updates. Consider using set_committed_value()
within inner-flush event handlers to avoid this warning. % len_)
Exception in thread Thread MSFT: Traceback (most recent call last):
File
"/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2436, in _flush
transaction.commit() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 465, in commit
self._assert_active(prepared_ok=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg) sqlalchemy.exc.ResourceClosedError: This transaction is closed
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"", line 48, in run
self.data_set.save_to_db() File "", line 44, in save_to_db
session.commit() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 954, in commit
self.transaction.commit() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 467, in commit
self._prepare_impl() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 447, in _prepare_impl
self.session.flush() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2313, in flush
self._flush(objects) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2440, in _flush
transaction.rollback(_capture_exception=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/util/langhelpers.py",
line 76, in exit
compat.reraise(type_, value, traceback) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/util/compat.py",
line 249, in reraise
raise value File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2440, in _flush
transaction.rollback(_capture_exception=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 483, in rollback
self._assert_active(prepared_ok=True, rollback_ok=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 285, in _assert_active
raise sa_exc.ResourceClosedError(closed_msg) sqlalchemy.exc.ResourceClosedError: This transaction is closed
During handling of the above exception, another exception occurred:
Traceback (most recent call last): File
"/anaconda3/lib/python3.7/threading.py", line 917, in _bootstrap_inner
self.run() File "", line 53, in run
self.data_set = StockLive.find_by_ticker(self.ccy) File "", line 52, in find_by_ticker
ticker_id = StockMaster.find_by_ticker(ticker).id File "", line 23, in find_by_ticker
return session.query(StockMaster).filter(StockMaster.ticker==ticker).first()
File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/query.py",
line 2895, in first
ret = list(self[0:1]) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line
2687, in getitem
return list(res) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line
2994, in iter
self.session._autoflush() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 1493, in _autoflush
self.flush() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2313, in flush
self._flush(objects) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 2400, in _flush
subtransactions=True) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 865, in begin
nested=nested) File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 297, in _begin
self._assert_active() File "/anaconda3/lib/python3.7/site-packages/sqlalchemy/orm/session.py",
line 264, in _assert_active
"This session is in 'prepared' state; no further " sqlalchemy.exc.InvalidRequestError: This session is in 'prepared'
state; no further SQL can be emitted within this transaction.
There may be other issues with your code but an obvious one is that you share your session with multiple threads.
Instead of setting a global session, these should be created in each thread.
I am not able to run your code but you could try something like this:
Remove the global session variable completely. You do not need it. Then modify your methods and thread to include a local session:
#classmethod
def find_by_ticker(cls,ticker, session):
return session.query(StockMaster).filter(StockMaster.ticker==ticker).first()
...
class Stream(Thread):
def __init__(self,ticker):
Thread.__init__(self)
self.ticker=ticker
self.quote=1
self.session = Session()
self.data_set = StockLive.find_by_ticker(self.ticker, self.session)
self.count=0
Or something like that. This will now make each thread have their own session and your code will start behaving better.
I am trying to step down MongoDB primary using python script.I see the below on my output. Is there a way I can mark the exit code as OK.
Code:
if 'primary' in isMaster:
primary =(isMaster['primary']).split(':')[0]
conn = pymongo.MongoClient('mongodb://'+ primary +':10000 replicaSet=test-ipe1')
stepdown = conn.admin.command("replSetStepDown",100)
if 'ismaster' in isMaster == 'true':
print("I am still the primary")
else:
print("I am no longer the primary")
else:
primary = "No primary currently elected."
Traceback:
Traceback (most recent call last):
File "./repldown.py", line 39, in <module>
stepdown = conn.admin.command("replSetStepDown",60)
File "/usr/local/lib64/python2.6/site-packages/pymongo/database.py", line 391, in command
result = self["$cmd"].find_one(command, **extra_opts)
File "/usr/local/lib64/python2.6/site-packages/pymongo/collection.py", line 604, in find_one
for result in self.find(spec_or_id, *args, **kwargs).limit(-1):
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 904, in next
if len(self.__data) or self._refresh():
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 848, in _refresh
self.__uuid_subtype))
File "/usr/local/lib64/python2.6/site-packages/pymongo/cursor.py", line 782, in __send_message
res = client._send_message_with_response(message, **kwargs)
File "/usr/local/lib64/python2.6/site-packages/pymongo/mongo_client.py", line 1051, in _send_message_with_response
raise AutoReconnect(str(e))
pymongo.errors.AutoReconnect: connection closed
Yes, do:
from pymongo.errors import AutoReconnect
try:
stepdown = conn.admin.command("replSetStepDown",100)
except AutoReconnect:
pass
The primary deliberately closes all connections immediately when it steps down, so the command throws an exception. Just ignore it and continue.
I am new to django 1.7 and python3. I am using OSX. As I was following the django 1.7 documentation online,
I tried
python3 manage.py migrate
and it resulted
Operations to perform:
Apply all migrations: auth, contenttypes, sessions, admin
Running migrations:
No migrations to apply.
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/contrib/contenttypes/models.py", line 44, in get_for_model
ct = self._get_from_cache(opts)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/contrib/contenttypes/models.py", line 34, in _get_from_cache
return self.__class__._cache[self.db][key]
KeyError: 'default'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/backends/mysql/base.py", line 128, in execute
return self.cursor.execute(query, args)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 184, in execute
self.errorhandler(self, exc, value)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/connections.py", line 37, in defaulterrorhandler
raise errorvalue
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 171, in execute
r = self._query(query)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 330, in _query
rowcount = self._do_query(q)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 294, in _do_query
db.query(q)
_mysql_exceptions.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '%s AND `django_content_type`.`model` = %s) LIMIT 21' at line 1")
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/contrib/contenttypes/models.py", line 50, in get_for_model
defaults={'name': smart_text(opts.verbose_name_raw)},
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/manager.py", line 92, in manager_method
return getattr(self.get_queryset(), name)(*args, **kwargs)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/query.py", line 422, in get_or_create
return self.get(**lookup), False
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/query.py", line 351, in get
num = len(clone)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/query.py", line 122, in __len__
self._fetch_all()
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/query.py", line 966, in _fetch_all
self._result_cache = list(self.iterator())
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/query.py", line 265, in iterator
for row in compiler.results_iter():
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/sql/compiler.py", line 700, in results_iter
for rows in self.execute_sql(MULTI):
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/models/sql/compiler.py", line 786, in execute_sql
cursor.execute(sql, params)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/backends/utils.py", line 81, in execute
return super(CursorDebugWrapper, self).execute(sql, params)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/utils/six.py", line 549, in reraise
raise value.with_traceback(tb)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/backends/utils.py", line 65, in execute
return self.cursor.execute(sql, params)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/db/backends/mysql/base.py", line 128, in execute
return self.cursor.execute(query, args)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 184, in execute
self.errorhandler(self, exc, value)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/connections.py", line 37, in defaulterrorhandler
raise errorvalue
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 171, in execute
r = self._query(query)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 330, in _query
rowcount = self._do_query(q)
File "/Users/NAME/Library/Python/3.4/lib/python/site-packages/MySQLdb/cursors.py", line 294, in _do_query
db.query(q)
django.db.utils.ProgrammingError: (1064, "You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '%s AND `django_content_type`.`model` = %s) LIMIT 21' at line 1")
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "manage.py", line 10, in <module>
execute_from_command_line(sys.argv)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/core/management/__init__.py", line 385, in execute_from_command_line
utility.execute()
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/core/management/__init__.py", line 377, in execute
self.fetch_command(subcommand).run_from_argv(self.argv)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/core/management/base.py", line 288, in run_from_argv
self.execute(*args, **options.__dict__)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/core/management/base.py", line 338, in execute
output = self.handle(*args, **options)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/core/management/commands/migrate.py", line 164, in handle
emit_post_migrate_signal(created_models, self.verbosity, self.interactive, connection.alias)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/core/management/sql.py", line 268, in emit_post_migrate_signal
using=db)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/dispatch/dispatcher.py", line 198, in send
response = receiver(signal=self, sender=sender, **named)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/contrib/auth/management/__init__.py", line 83, in create_permissions
ctype = ContentType.objects.db_manager(using).get_for_model(klass)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/contrib/contenttypes/models.py", line 58, in get_for_model
" is migrated before trying to migrate apps individually."
RuntimeError: Error creating new content types. Please make sure contenttypes is migrated before trying to migrate apps individually.`
I don't know what it means and I am sure it is not right.
Is the migration successful?
Please help. Thanks
Here is my /Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/site-packages/django/contrib/contenttypes/models.py
from __future__ import unicode_literals
from django.apps import apps
from django.db import models
from django.db.utils import OperationalError, ProgrammingError
from django.utils.translation import ugettext_lazy as _
from django.utils.encoding import smart_text, force_text
from django.utils.encoding import python_2_unicode_compatible
class ContentTypeManager(models.Manager):
# Cache to avoid re-looking up ContentType objects all over the place.
# This cache is shared by all the get_for_* methods.
_cache = {}
def get_by_natural_key(self, app_label, model):
try:
ct = self.__class__._cache[self.db][(app_label, model)]
except KeyError:
ct = self.get(app_label=app_label, model=model)
self._add_to_cache(self.db, ct)
return ct
def _get_opts(self, model, for_concrete_model):
if for_concrete_model:
model = model._meta.concrete_model
elif model._deferred:
model = model._meta.proxy_for_model
return model._meta
def _get_from_cache(self, opts):
key = (opts.app_label, opts.model_name)
return self.__class__._cache[self.db][key]
def get_for_model(self, model, for_concrete_model=True):
"""
Returns the ContentType object for a given model, creating the
ContentType if necessary. Lookups are cached so that subsequent lookups
for the same model don't hit the database.
"""
opts = self._get_opts(model, for_concrete_model)
try:
ct = self._get_from_cache(opts)
except KeyError:
try:
ct, created = self.get_or_create(
app_label=opts.app_label,
model=opts.model_name,
defaults={'name': smart_text(opts.verbose_name_raw)},
)
except (OperationalError, ProgrammingError):
# It's possible to migrate a single app before contenttypes,
# as it's not a required initial dependency (it's contrib!)
# Have a nice error for this.
raise RuntimeError(
"Error creating new content types. Please make sure contenttypes" +
" is migrated before trying to migrate apps individually."
)
self._add_to_cache(self.db, ct)
return ct
def get_for_models(self, *models, **kwargs):
"""
Given *models, returns a dictionary mapping {model: content_type}.
"""
for_concrete_models = kwargs.pop('for_concrete_models', True)
# Final results
results = {}
# models that aren't already in the cache
needed_app_labels = set()
needed_models = set()
needed_opts = set()
for model in models:
opts = self._get_opts(model, for_concrete_models)
try:
ct = self._get_from_cache(opts)
except KeyError:
needed_app_labels.add(opts.app_label)
needed_models.add(opts.model_name)
needed_opts.add(opts)
else:
results[model] = ct
if needed_opts:
cts = self.filter(
app_label__in=needed_app_labels,
model__in=needed_models
)
for ct in cts:
model = ct.model_class()
if model._meta in needed_opts:
results[model] = ct
needed_opts.remove(model._meta)
self._add_to_cache(self.db, ct)
for opts in needed_opts:
# These weren't in the cache, or the DB, create them.
ct = self.create(
app_label=opts.app_label,
model=opts.model_name,
name=smart_text(opts.verbose_name_raw),
)
self._add_to_cache(self.db, ct)
results[ct.model_class()] = ct
return results
def get_for_id(self, id):
"""
Lookup a ContentType by ID. Uses the same shared cache as get_for_model
(though ContentTypes are obviously not created on-the-fly by get_by_id).
"""
try:
ct = self.__class__._cache[self.db][id]
except KeyError:
# This could raise a DoesNotExist; that's correct behavior and will
# make sure that only correct ctypes get stored in the cache dict.
ct = self.get(pk=id)
self._add_to_cache(self.db, ct)
return ct
def clear_cache(self):
"""
Clear out the content-type cache. This needs to happen during database
flushes to prevent caching of "stale" content type IDs (see
django.contrib.contenttypes.management.update_contenttypes for where
this gets called).
"""
self.__class__._cache.clear()
def _add_to_cache(self, using, ct):
"""Insert a ContentType into the cache."""
# Note it's possible for ContentType objects to be stale; model_class() will return None.
# Hence, there is no reliance on model._meta.app_label here, just using the model fields instead.
key = (ct.app_label, ct.model)
self.__class__._cache.setdefault(using, {})[key] = ct
self.__class__._cache.setdefault(using, {})[ct.id] = ct
#python_2_unicode_compatible
class ContentType(models.Model):
name = models.CharField(max_length=100)
app_label = models.CharField(max_length=100)
model = models.CharField(_('python model class name'), max_length=100)
objects = ContentTypeManager()
class Meta:
verbose_name = _('content type')
verbose_name_plural = _('content types')
db_table = 'django_content_type'
ordering = ('name',)
unique_together = (('app_label', 'model'),)
def __str__(self):
# self.name is deprecated in favor of using model's verbose_name, which
# can be translated. Formal deprecation is delayed until we have DB
# migration to be able to remove the field from the database along with
# the attribute.
#
# We return self.name only when users have changed its value from the
# initial verbose_name_raw and might rely on it.
model = self.model_class()
if not model or self.name != model._meta.verbose_name_raw:
return self.name
else:
return force_text(model._meta.verbose_name)
def model_class(self):
"Returns the Python model class for this type of content."
try:
return apps.get_model(self.app_label, self.model)
except LookupError:
return None
def get_object_for_this_type(self, **kwargs):
"""
Returns an object of this type for the keyword arguments given.
Basically, this is a proxy around this object_type's get_object() model
method. The ObjectNotExist exception, if thrown, will not be caught,
so code that calls this method should catch it.
"""
return self.model_class()._base_manager.using(self._state.db).get(**kwargs)
def get_all_objects_for_this_type(self, **kwargs):
"""
Returns all objects of this type for the keyword arguments given.
"""
return self.model_class()._base_manager.using(self._state.db).filter(**kwargs)
def natural_key(self):
return (self.app_label, self.model)
I believe it is the dependency with mysql. After I switched to postgresql, everything is solved. I found the connector of python with mysql is only up to python 3.3 and I am using python3.4. That is probably the reason and I could not find a connector for mysql and python 3.4.
I assume that there could be some problems with the dependencies.
Instead of running:
python3 manage.py migrate
try with:
python manage.py makemigrations yourapp
python manage.py migrate
if this doesn't work go with:
python manage.py syncdb --all
I have seen some questions about using SQLAlchemy on App Engine to connect to Google Cloud SQL. But I'm not sure if it is possible to develop using a local MySQL database and the existing SQLAlchemy dialect. On my first attempt, I added SQLAlchemy 0.8.0 to the app and defined a schema:
from sqlalchemy import create_engine, Column, Integer, Table
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
foo_table = Table('foo', Base.metadata,
Column('id', Integer, primary_key=True, autoincrement=True),
)
And when I tried to create the tables on the development server using:
url = 'mysql+gaerdbms:///%s?instance=%s' % ('database_name', 'instance_name')
engine = create_engine(url)
Base.metadata.create_all(engine)
...I got an error DBAPIError: (ImportError) No module named pwd None None, which means that SQLAlchemy is importing a module that is blacklisted by the development server.
Am I doing something wrong? Or, if not, what should I do to use SQLAlchemy on the development server? Or maybe the first question is: Can I use the SQLAlchemy's gaerdbms dialect to develop in a local MySql database using the dev server?
Edit: this error doesn't happen only when trying to create tables. I created the tables manually and tried to query them, and the same error occurs.
The full traceback is:
Traceback (most recent call last):
File "[...]/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1535, in __call__
rv = self.handle_exception(request, response, e)
File "[...]/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1529, in __call__
rv = self.router.dispatch(request, response)
File "[...]/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1278, in default_dispatcher
return route.handler_adapter(request, response)
File "[...]/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 1102, in __call__
return handler.dispatch()
File "[...]/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 572, in dispatch
return self.handle_exception(e, self.app.debug)
File "[...]/google_appengine/lib/webapp2-2.5.2/webapp2.py", line 570, in dispatch
return method(*args, **kwargs)
File "[...]/webapp/admin.py", line 12, in get
db.Base.metadata.create_all(engine)
File "[...]/webapp/sqlalchemy/schema.py", line 2784, in create_all
tables=tables)
File "[...]/webapp/sqlalchemy/engine/base.py", line 1486, in _run_visitor
with self._optional_conn_ctx_manager(connection) as conn:
File "/usr/lib/python2.7/contextlib.py", line 17, in __enter__
return self.gen.next()
File "[...]/webapp/sqlalchemy/engine/base.py", line 1479, in _optional_conn_ctx_manager
with self.contextual_connect() as conn:
File "[...]/webapp/sqlalchemy/engine/base.py", line 1669, in contextual_connect
self.pool.connect(),
File "[...]/webapp/sqlalchemy/pool.py", line 272, in connect
return _ConnectionFairy(self).checkout()
File "[...]/webapp/sqlalchemy/pool.py", line 425, in __init__
rec = self._connection_record = pool._do_get()
File "[...]/webapp/sqlalchemy/pool.py", line 855, in _do_get
return self._create_connection()
File "[...]/webapp/sqlalchemy/pool.py", line 225, in _create_connection
return _ConnectionRecord(self)
File "[...]/webapp/sqlalchemy/pool.py", line 318, in __init__
self.connection = self.__connect()
File "[...]/webapp/sqlalchemy/pool.py", line 368, in __connect
connection = self.__pool._creator()
File "[...]/webapp/sqlalchemy/engine/strategies.py", line 80, in connect
return dialect.connect(*cargs, **cparams)
File "[...]/webapp/sqlalchemy/engine/default.py", line 279, in connect
return self.dbapi.connect(*cargs, **cparams)
File "[...]/google_appengine/google/storage/speckle/python/api/rdbms_googleapi.py", line 183, in __init__
super(GoogleApiConnection, self).__init__(*args, **kwargs)
File "[...]/google_appengine/google/storage/speckle/python/api/rdbms.py", line 810, in __init__
self.OpenConnection()
File "[...]/google_appengine/google/storage/speckle/python/api/rdbms.py", line 832, in OpenConnection
self.SetupClient()
File "[...]/google_appengine/google/storage/speckle/python/api/rdbms_googleapi.py", line 193, in SetupClient
self._client = RdbmsGoogleApiClient(**kwargs)
File "[...]/google_appengine/google/storage/speckle/python/api/rdbms_googleapi.py", line 106, in __init__
rdbms.OAUTH_CREDENTIALS_PATH)
File "/usr/lib/python2.7/posixpath.py", line 259, in expanduser
import pwd
File "[...]/google_appengine/google/appengine/tools/devappserver2/python/sandbox.py", line 822, in load_module
raise ImportError('No module named %s' % fullname)
DBAPIError: (ImportError) No module named pwd None None
I found a workaround. As it is, SQLAlchemy's gaerdbms dialect can't connect to a local database. But with the dialect below it can. Folow the instructions from this answer but use this dialect instead:
# mysql/gaerdbms.py
# Copyright (C) 2005-2013 the SQLAlchemy authors and contributors <see AUTHORS file>
#
# This module is part of SQLAlchemy and is released under
# the MIT License: http://www.opensource.org/licenses/mit-license.php
"""
.. dialect:: mysql+gaerdbms
:name: Google Cloud SQL
:dbapi: rdbms
:connectstring: mysql+gaerdbms:///<dbname>?instance=<instancename>
:url: https://developers.google.com/appengine/docs/python/cloud-sql/developers-guide
This dialect is based primarily on the :mod:`.mysql.mysqldb` dialect with minimal
changes.
.. versionadded:: 0.7.8
Pooling
-------
Google App Engine connections appear to be randomly recycled,
so the dialect does not pool connections. The :class:`.NullPool`
implementation is installed within the :class:`.Engine` by
default.
"""
import os
import re
from sqlalchemy.dialects.mysql.mysqldb import MySQLDialect_mysqldb
from sqlalchemy.pool import NullPool
class MySQLDialect_gaerdbms(MySQLDialect_mysqldb):
#classmethod
def dbapi(cls):
# from django:
# http://code.google.com/p/googleappengine/source/
# browse/trunk/python/google/storage/speckle/
# python/django/backend/base.py#118
# see also [ticket:2649]
# see also https://stackoverflow.com/q/14224679/34549
if is_production():
# Production mode.
from google.storage.speckle.python.api import rdbms_apiproxy
return rdbms_apiproxy
elif is_remote_mode():
# Development mode with remote database.
from google.storage.speckle.python.api import rdbms_googleapi
return rdbms_googleapi
else:
# Development mode with local database.
from google.appengine.api import rdbms_mysqldb
return rdbms_mysqldb
#classmethod
def get_pool_class(cls, url):
# Cloud SQL connections die at any moment
return NullPool
def create_connect_args(self, url):
opts = url.translate_connect_args()
if is_production() or is_remote_mode():
# 'dsn' and 'instance' are because we are skipping
# the traditional google.api.rdbms wrapper.
# they are not needed in local mode; 'dns' even causes an error.
opts['dsn'] = ''
opts['instance'] = url.query['instance']
return [], opts
def _extract_error_code(self, exception):
match = re.compile(r"^(\d+):|^\((\d+),").match(str(exception))
# The rdbms api will wrap then re-raise some types of errors
# making this regex return no matches.
code = match.group(1) or match.group(2) if match else None
if code:
return int(code)
dialect = MySQLDialect_gaerdbms
def is_production():
return os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine')
def is_remote_mode():
return os.getenv('SETTINGS_MODE') == 'prod'
This dialect uses a local database by default when running on the development server. To use remote access to Google Cloud SQL during development, a variable must be set in the environment, following the pattern used by Django:
os.environ['SETTINGS_MODE'] = 'prod'