I want to write some py.test code to test 2 simple sqlalchemy ORM classes that were created based on this Tutorial. The problem is, how do I set a the database in py.test to a test database and rollback all changes when the tests are done? Is it possible to mock the database and run tests without actually connect to de database?
here is the code for my classes:
from sqlalchemy import create_engine, ForeignKey
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
from sqlalchemy.orm import sessionmaker, relationship
eng = create_engine('mssql+pymssql://user:pass#host/my_database')
Base = declarative_base(eng)
Session = sessionmaker(eng)
intern_session = Session()
class Author(Base):
__tablename__ = "Authors"
AuthorId = Column(Integer, primary_key=True)
Name = Column(String)
Books = relationship("Book")
def add_book(self, title):
b = Book(Title=title, AuthorId=self.AuthorId)
intern_session.add(b)
intern_session.commit()
class Book(Base):
__tablename__ = "Books"
BookId = Column(Integer, primary_key=True)
Title = Column(String)
AuthorId = Column(Integer, ForeignKey("Authors.AuthorId"))
Author = relationship("Author")
I usually do that this way:
I do not instantiate engine and session with the model declarations, instead I only declare a Base with no bind:
Base = declarative_base()
and I only create a session when needed with
engine = create_engine('<the db url>')
db_session = sessionmaker(bind=engine)
You can do the same by not using the intern_session in your add_book method but rather use a session parameter.
def add_book(self, session, title):
b = Book(Title=title, AuthorId=self.AuthorId)
session.add(b)
session.commit()
It makes your code more testable since you can now pass the session of your choice when you call the method.
And you are no more stuck with a session bound to a hardcoded database url.
I add a custom --dburl option to pytest using its pytest_addoption hook.
Simply add this to your top-level conftest.py:
def pytest_addoption(parser):
parser.addoption('--dburl',
action='store',
default='<if needed, whatever your want>',
help='url of the database to use for tests')
Now you can run pytest --dburl <url of the test database>
Then I can retrieve the dburl option from the request fixture
From a custom fixture:
#pytest.fixture()
def db_url(request):
return request.config.getoption("--dburl")
# ...
Inside a test:
def test_something(request):
db_url = request.config.getoption("--dburl")
# ...
At this point you are able to:
get the test db_url in any test or fixture
use it to create an engine
create a session bound to the engine
pass the session to a tested method
It is quite a mess to do this in every test, so you can make a usefull usage of pytest fixtures to ease the process.
Below are some fixtures I use:
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
#pytest.fixture(scope='session')
def db_engine(request):
"""yields a SQLAlchemy engine which is suppressed after the test session"""
db_url = request.config.getoption("--dburl")
engine_ = create_engine(db_url, echo=True)
yield engine_
engine_.dispose()
#pytest.fixture(scope='session')
def db_session_factory(db_engine):
"""returns a SQLAlchemy scoped session factory"""
return scoped_session(sessionmaker(bind=db_engine))
#pytest.fixture(scope='function')
def db_session(db_session_factory):
"""yields a SQLAlchemy connection which is rollbacked after the test"""
session_ = db_session_factory()
yield session_
session_.rollback()
session_.close()
Using the db_session fixture you can get a fresh and clean db_session for each single test.
When the test ends, the db_session is rollbacked, keeping the database clean.
Related
Making an API using FastAPI and SQLAlchemy I'm experiencing strange behaviour when database (SQLite) is in-memory which doesn't occur when stored as file.
Model:
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, String
Base = declarative_base()
class Thing(Base):
__tablename__ = "thing"
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(String)
I create two global engine objects. One with database as file, the other as in-memory database:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
args = dict(echo=True, connect_args={"check_same_thread": False})
engine1 = create_engine("sqlite:///db.sqlite", **args)
engine2 = create_engine("sqlite:///:memory:", **args)
Session1 = sessionmaker(bind=engine1)
Session2 = sessionmaker(bind=engine2)
I create my FastAPI app and a path to add an object to database:
from fastapi import FastAPI
app = FastAPI()
#app.get("/")
def foo(x: int):
with {1: Session1, 2: Session2}[x]() as session:
session.add(Thing(name="foo"))
session.commit()
My main to simulate requests and check everything is working:
from fastapi.testclient import TestClient
if __name__ == "__main__":
Base.metadata.create_all(engine1)
Base.metadata.create_all(engine2)
client = TestClient(app)
assert client.get("/1").status_code == 200
assert client.get("/2").status_code == 200
thing table is created in engine1 and committed, same with engine2. On first request "foo" was successfully inserted into engine1's database (stored as file) but second request raises "sqlite3.OperationalError" claiming "no such table: thing".
Why is there different behaviour between the two? Why does in-memory database claim the table doesn't exist even though SQLAlchemy logs show create table statement ran successfully and was committed?
The docs explain this in the following https://docs.sqlalchemy.org/en/14/dialects/sqlite.html#using-a-memory-database-in-multiple-threads
To use a :memory: database in a multithreaded scenario, the same connection object must be shared among threads, since the database exists only within the scope of that connection. The StaticPool implementation will maintain a single connection globally, and the check_same_thread flag can be passed to Pysqlite as False
It also shows how to get the intended behavior, so in your case
from sqlalchemy.pool import StaticPool
args = dict(echo=True, connect_args={"check_same_thread": False}, poolclass=StaticPool)
I am trying to setup a database just like in a tutorial but I am getting a programming error that a table doesn't exist when I'm trying to add a User
This is the file that errors (database.py):
from sqlalchemy import create_engine, MetaData
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.ext.declarative import declarative_base
engine = create_engine(
"mysql+pymysql://testuser:testpassword#localhost/test?charset=utf8",
connect_args = {
"port": 3306
},
echo="debug",
echo_pool=True
)
db_session = scoped_session(
sessionmaker(
bind=engine,
autocommit=False,
autoflush=False
)
)
Base = declarative_base()
def init_db():
import models
Base.metadata.create_all(bind=engine)
from models import User
db_session.add(
User(username="testuser", password_hash=b"", password_salt=b"", balance=1)
)
db_session.commit()
print("Initialized the db")
if __name__ == "__main__":
init_db()
To init the database (create the tables) I just run the file.
It errors when it creates the test user.
Here is models.py:
from sqlalchemy import Column, Integer, Numeric, Binary, String
from sqlalchemy.orm import relationship
from database import Base
class User(Base):
__tablename__ = "users"
id = Column(Integer, primary_key=True)
username = Column(String(16), unique=True)
password_hash = Column(Binary(32))
password_salt = Column(Binary(32))
balance = Column(Numeric(precision=65, scale=8))
def __repr__(self):
return "<User(balance={})>".format(balance)
I tried:
Committing before adding users (after create_all)
Drop existing tables from the database (although it seems like the table never gets committed)
from models import User instead of import models (before create_all)
Sorry if there are so many simillar questions, I promise I scavenged for answers, but it's always silly mistakes I made sure I didn't make (or atleast the ones I saw).
I am using MariaDB.
Sorry for long post, many thanks in advance.
The Base in database.py isn't the same Base that is imported into models.py.
A simple test is to put a print('creating Base') function call just above the Base = declarative_base() statement, and you'll see it is being created twice.
Python calls the module that is being executed '__main__', which you know as you have the if __name__ == '__main__' conditional at the bottom of your module. So the first Base that is created is __main__.Base. Then, in models.py, from database import Base causes the database module to be parsed again, creating database.Base in the namespace, and that is the Base from which User inherits. Then back in database.py, the Base.metadata.create_all(bind=engine) call is using the metadata from __main__.Base which has no tables in it, and as such creates nothing.
Don't execute out of the module that creates the Base instance. Create another module called main.py (or whatever), and move your init_db() function there and import Base, db_session and engine from database.py into main.py. That way, you are always using the same Base instance. This is example of main.py:
from database import Base, db_session, engine
from models import User
def init_db():
Base.metadata.create_all(bind=engine)
db_session.add(
User(username="testuser", password_hash=b"", password_salt=b"", balance=1)
)
db_session.commit()
print("Initialized the db")
if __name__ == "__main__":
init_db()
Declare Base class once(for each database) & import it to all modules which define table classes (inherited from Base)
For Base (a metaclass) to scan & find out all classes which are inherited from it, we need to import all the modules where such table classes (inherited from Base) are defined to module where we call Metadata.create_all(engine).
You need to import the relevant model where you call "Base.metadata.create_all". Example below to create user table
from ModelBase import Base
from UserModel import User
def create_db_schema(engine):
Base.metadata.create_all(engine,checkfirst=True)
I'm starting to write tests with Flask-SQLAlchemy, and I'd like to add some fixtures for those. I have plenty of good data for that in my development database and a lot of tables so writing data manually would get annoying. I'd really like to just sample data from the dev database into fixtures and then use those. What's a good way to do this?
i would use factory boy
to create a model factory you just do:
import factory
from . import models
class UserFactory(factory.Factory):
class Meta:
model = models.User
first_name = 'John'
last_name = 'Doe'
admin = False
then to create instances:
UserFactory.create()
to add static data just give as kwarg to create
UserFactory.create(name='hank')
so to seed a bunch of stuff throw that in a for loop. :)
If you need to handle fixtures with SQLAlchemy or another ORM/backend then the Fixture package may be of use: Flask-Fixtures 0.3.3
That is a simple library that allows you to add database fixtures for your unit tests using nothing but JSON or YAML.
While Kyle's answer is correct, we still need to provide the model factory with a database session, otherwise we would never actually commit to the db. Also, factory boy has a dedicated class SQLAlchemyModelFactory for interacting with SQLAlchemy.
https://factoryboy.readthedocs.io/en/stable/orms.html#sqlalchemy
The whole setup could look something like this:
import pytest
import os
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from factory.alchemy import SQLAlchemyModelFactory
engine = create_engine( os.getenv("SQLALCHEMY_DATABASE_URI"))
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
# this resets our tables in between each test
def _reset_schema():
db = SessionLocal()
for table in Base.metadata.sorted_tables:
db.execute(
'TRUNCATE {name} RESTART IDENTITY CASCADE;'.format(name=table.name)
)
db.commit()
#pytest.fixture
def test_db():
yield engine
engine.dispose()
_reset_schema()
#pytest.fixture
def session(test_db):
connection = test_db.connect()
transaction = connection.begin()
db = scoped_session(sessionmaker(bind=engine))
try:
yield db
finally:
db.close()
transaction.rollback()
connection.close()
db.remove()
class UserFactory(SQLAlchemyModelFactory):
class Meta:
model = models.User
first_name = 'John'
last_name = 'Doe'
admin = False
#pytest.fixture(autouse=True)
def provide_session_to_factories(session):
# usually you'd have one factory for each db table
for factory in [UserFactory, ...]:
factory._meta.sqlalchemy_session = session
I am working on a Pyramid app with SQLAlchemy as the ORM. I am trying to test a model with a class method:
# this is essentially a global used by all the models
Session = scoped_session(sessionmaker(autocommit=False))
class Role(Base):
__tablename__ = 'role'
id = sa.Column(sa.types.Integer, primary_key=True)
name = sa.Column(sa.types.Text, unique=True, nullable=False)
def __init__(self, **kwargs):
super(Role, self).__init__(**kwargs)
#classmethod
def find_all(self):
return Session.query(Role).order_by(Role.name).all()
I am using factory_boy to test and here is how I am trying to set up my testing factory:
import factory
from factory.alchemy import SQLAlchemyModelFactory
from sqlalchemy.orm import scoped_session, sessionmaker
from zk.model.meta import Base
from zk.model.role import Role
session = scoped_session(sessionmaker())
engine = create_engine('sqlite://')
session.configure(bind=engine)
Base.metadata.create_all(engine)
class RoleFactory(SQLAlchemyModelFactory):
FACTORY_FOR = Role
FACTORY_SESSION = session
However when I try to call RoleFactory.find_all() in a test, I get an error: E UnboundExecutionError: Could not locate a bind configured on mapper Mapper|Role|role, SQL expression or this Session
I tried monkeypatching meta and replacing that global Session with my session, but then I get this error: E AttributeError: type object 'RoleFactory' has no attribute 'find_all'
I tried calling RoleFactory.FACTORY_FOR.find_all() but then I get the same UnboundExecutionError.
Do I need to do something else for factory_boy to know about the class method?
This might be too obvious, but it seems that what you've got is a RoleFactory instance, when you need a Role instance, the factory wouldn't have access to any classmethods since it's not a child of the class. Try doing this and seeing what happens:
role = RoleFactory.build()
roles = role.find_all()
I’m working on a library where the user shall be able to simply declare a few classes which are automatically backed by the database. In short, somewhere hidden in the code, there is
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class LibraryBase(Base):
# important library stuff
and the user should then do
class MyStuff(LibraryBase):
# important personal stuff
class MyStuff_2(LibraryBase):
# important personal stuff
mystuff = MyStuff()
Library.register(mystuff)
mystuff.changeIt() # apply some changes to the instance
Library.save(mystuff) # and save it
# same for all other classes
In a static environment, e.g. the user has created one file with all personal classes and imports this file, this works pretty well. All class names are fixed and SQLAlchemy knows how to map each class.
In an interactive environment, things are different: Now, there is a chance of a class being defined twice. Both classes might have different modules; but still SQLAlchemy will complain:
SAWarning: The classname 'MyStuff' is already in the registry of this declarative base, mapped to < class 'OtherModule.MyStuff' >
Is there a way to deal with this? Can I somehow unload a class from its declarative_base so that I can exchange its definition with a new one?
You can use:
sqlalchemy.orm.instrumentation.unregister_class(cl)
del cl._decl_class_registry[cl.__name__]
The first line is to prevent accidental use of your unregisted class. The second unregisters and will prevent the warning.
It looks like, And I'm not really sure this even works, but I think what you want is
sqlalchemy.orm.instrumentation.unregister_class()
http://hg.sqlalchemy.org/sqlalchemy/file/762548ff8eef/lib/sqlalchemy/orm/instrumentation.py#l466
In my project I use this solution.
Where library specified columns defined as mixin by declared_attr and target mapper created by type call with bases, as result I have full functional mapper.
from sqlalchemy import create_engine, BigInteger, Column
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.ext.declarative import declared_attr
Base = declarative_base()
class LibraryBase(object):
__tablename__ = 'model'
#declared_attr
def library_field(self):
return Column(BigInteger)
class MyLibrary(object):
#classmethod
def register(cls, entity):
tablename = entity.__tablename__
Mapper = type('Entity_%s' % tablename, (Base, LibraryBase, entity), {
'__tablename__': tablename,
'id': Column(BigInteger, primary_key=True),
})
return Mapper
#classmethod
def setup(cls):
Base.metadata.create_all()
class MyStaff(object):
__tablename__ = 'sometable1'
#declared_attr
def staff_field(self):
return Column(BigInteger)
def mymethod(self):
print('My method:', self)
class MyStaff2(MyStaff):
__tablename__ = 'sometable2'
if __name__ == '__main__':
engine = create_engine('sqlite://', echo=True)
Base.metadata.bind = engine
Session = scoped_session(sessionmaker(bind=engine))
session = Session()
# register and install
MyStaffMapper = MyLibrary.register(MyStaff)
MyStaffMapper2 = MyLibrary.register(MyStaff2)
MyLibrary.setup()
MyStaffMapper().mymethod()
MyStaffMapper2().mymethod()
session.query(MyStaffMapper.library_field) \
.filter(MyStaffMapper.staff_field != None) \
.all()