We Keep Coding

Cannot use SQLAlchemy Session() outside Flask application context for unit testing

This is my inside flaskapp/__init__.py for creating my Flask app, with it I can access Session inside any module in the package, by just importing Session from flaskapp/db.py:
import os
from flask import Flask
def create_app(test_config=None):
app = Flask(__name__, instance_relative_config=True)
app.config.from_mapping(
SECRET_KEY=b'some_secret_key',
DEBUG=True,
SQLALCHEMY_DATABASE_URI=f'sqlite:///{os.path.join(app.instance_path, "flaskapp.sqlite")}',
)
if test_config is None:
app.config.from_pyfile('config.py', silent=True)
else:
app.config.from_mapping(test_config)
try:
os.makedirs(app.instance_path)
except OSError:
pass
with app.app_context():
from flaskapp.routes import home, auth
from flaskapp.db import init_db
init_db()
return app
This is my flaskapp/db.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from flask import current_app
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
Session = sessionmaker(bind=engine)
def init_db():
import flaskapp.models as models
models.Base.metadata.create_all(engine)
def drop_db():
import flaskapp.models as models
models.Base.metadata.drop_all(engine)
With that, I can access the database in any other module, for example, flaskapp/auth.py:
from flask import flash, session
from sqlalchemy import select
from flaskapp.db import Session
from flaskapp.models import User
def log_user(username: str, password: str) -> bool:
with Session() as db_session:
stmt1 = select(User).where(User.username == username)
query_user = db_session.execute(stmt1).first()
if not query_user:
flash('Some error message')
# Some password verifications and other things
session['username'] = query_user[0].username
flash('Successfully logged in')
return True
Until that point, I don't have any problem, the problem comes when I try to do unit testing with unittest, I can't set the test environment and I don't know how can I use the Session object defined in flaskapp/db.py for testing in a separate database. Everything I've tried until now gets me an error, this is my tests/__init__.py:
import unittest
from flaskapp import create_app
from flaskapp.db import Session
from flaskapp.models import User
class BaseTestClass(unittest.TestCase):
def setUp(self):
self.app = create_app(test_config={
'TESTING': True,
'DEBUG': True,
'APP_ENV': 'testing',
# I pass the test database URI expecting the engine to use it
'SQLALCHEMY_DATABASE_URI': 'sqlite:///testdb.sqlite',
})
self.client = self.app.test_client()
# Context
with self.app.app_context():
self.populate_db()
def tearDown(self):
pass
def populate_db(self):
with Session() as db_session:
db_session.add(User(
username='Harry',
email='harry#yahoo.es',
password = 'Harry123.'
))
db_session.commit()
When I try to use the Session object inside populate_db() I get this error:
=====================================================================
ERROR: tests.test_auth (unittest.loader._FailedTest.tests.test_auth)
----------------------------------------------------------------------
ImportError: Failed to import test module: tests.test_auth
Traceback (most recent call last):
File "/usr/local/lib/python3.11/unittest/loader.py", line 407, in _find_test_path
module = self._get_module_from_name(name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/unittest/loader.py", line 350, in _get_module_from_name
__import__(name)
File "/home/chus/usb/soft/proy/SAGC/tests/test_auth.py", line 1, in <module>
from flaskapp.db import Session
File "/home/chus/usb/soft/proy/SAGC/flaskapp/db.py", line 5, in <module>
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 316, in __get__
obj = instance._get_current_object() # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 513, in _get_current_object
raise RuntimeError(unbound_message) from None
RuntimeError: Working outside of application context.
This typically means that you attempted to use functionality that needed
the current application. To solve this, set up an application context
with app.app_context(). See the documentation for more information.
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (errors=1)
Can someone help me out? I've tried everything to make the tests use the app context in order to populate the test database and make queries but it appears to be conflicting with it being created in create_app() or something.

Firsly, the error you posted is about tests/test_auth.py which you did not include.
However, your code is tightly coupling the flask app and the db.py module with the use of flask.current_app to obtain the engine url.
So the engine creation will only work if there is an app context, which is probably absent when you do a top level import (line 1) of from flaskapp.db import Session in tests/test_auth.py.
A solution could be creating the engine and sessionmaker (or scoped_session) with your create_app and injecting the engine or Session in your functions (look up dependency injection principle):
def init_db(engine):
models.Base.metadata.create_all(engine)
def drop_db(engine):
models.Base.metadata.drop_all(engine)
def log_user(session, username: str, password: str) -> bool:
stmt1 = select(User).where(User.username == username)
query_user = session.execute(stmt1).first()
# skipping the rest
Or look into Flask-SQLAlchemy.

Assigning global variable that is pickable

I aplogize if this is an easy problem to fix, I've tried searching but couldn't come up with a solution. I come from PHP, so maybe what I am trying to achieve is not possible, or it needs to be done differently in python.
Okay, so I have a class called database.py which parses a config file and depending on what 'database' type I am using returns an object of either sqlite or mysql
database.py
import mysql.connector
from mysql.connector import Error
from mysql.connector import pooling
class Database:
# Connect to the database
#staticmethod
def connect():
if Config().type() == 'mysql':
return mysql.connector.pooling.MySQLConnectionPool(**Config().mysql())
elif Config().type() == 'sqlite':
return sqlite3.connect("%s.sqlite" % Config().sqlite())
So, as you can see, I am returning an object of the database I am using, for this example, let's say Config().type() == 'mysql' is true, and we are using mysql, and the Config().mysql() is just a dictionary like so:
{
'pool_name': "indexit",
'pool_size': config['DATABASE']['mysql']['pool_size'],
'pool_reset_session': True,
'user': config['DATABASE']['mysql']['user'],
'password': config['DATABASE']['mysql']['password'],
'host': config['DATABASE']['mysql']['host'],
'database': config['DATABASE']['mysql']['database'],
'auth_plugin': 'mysql_native_password',
'charset': 'utf8',
'use_unicode': True
}
In my main python file, I am trying to assign the Database.connect() to a variable, and then use it in my threaded function and get the mysql pool from the object:
from multiprocessing import Pool
from core.database import Database
class Indexit:
# MyProgram constructor
def __init__(self):
# 1 connection for threads
self.database = Database.connect()
# Thread to run
def run(self, id):
# return the id
print(id)
# Get the database pool
print(self.database.get_connection())
# Do the threads
def main(self):
# Pool connections
with Pool(processes=5) as pool:
pool.map(self.run, range(10000000))
Indexit().main()
But my code returns an error: (you can view the full code here: https://github.com/filtration/indexit)
Traceback (most recent call last):
File "indexit.py", line 75, in <module>
Indexit().main()
File "indexit.py", line 68, in main
pool.map(self.run, range(10000000))
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/pool.py", line 268, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/pool.py", line 657, in get
raise self._value
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/pool.py", line 431, in _handle_tasks
put(task)
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/connection.py", line 206, in send
self._send_bytes(_ForkingPickler.dumps(obj))
File "/usr/local/Cellar/python/3.7.6_1/Frameworks/Python.framework/Versions/3.7/lib/python3.7/multiprocessing/reduction.py", line 51, in dumps
cls(buf, protocol).dump(obj)
TypeError: can't pickle _thread.lock objects
So, my question is, how can I use the global connection and then use my threaded function to get their own pool worker?

Apparently, MySQLConnectionPool uses threads internally. When you invoke the map method of a multiprocessing.Pool, it tries to send everything that is needed to run the given function. In this case including the database connection pool.
When a new process is created, it only has one thread. So it doesn't make sense to try and send threading related stuff like locks to a child process. Whatever data that lock has would be meaningless in the child process. Hence the error.
You should probably create the database connection inside the pool worker or inside the initializer for the Pool.

Best practices for cleaning up SQLAlchemy engine and sessionmaker instances?

According to SQLAlchemy documentation, engine and sessionmaker instances should be created in the application's global scope:
When do I make a sessionmaker? Just one time, somewhere in your application’s global scope. It should be looked upon as part of your application’s configuration. If your application has three .py files in a package, you could, for example, place the sessionmaker line in your __init__.py file; from that point on your other modules say “from mypackage import Session”. That way, everyone else just uses Session(), and the configuration of that session is controlled by that central point.
Questions:
What is the best practice for cleaning up SQLAlchemy engine and sessionmaker instances? Please refer to my example below, while I could call engine.dispose() in main.py, it does not seem a good practice to clean up a global object from a different module (database.py) in __main__ (main.py), is there a better way to do it?
Do we need to clean up sessionmaker instances? It seems there is no method for closing the sessionmaker instance (sessionmaker.close_all() is deprecated, and session.close_all_sessions() is a session instance method and not sessionmaker method.)
Example:
I created the engine and sessionmaker object in a module called database.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from contextlib import contextmanager
DB_ENGINE = create_engine(DB_CONNECTION_STRING, pool_size=5, max_overflow=10)
DB_SESSION = sessionmaker(bind=DB_ENGINE, autocommit=False, autoflush=True, expire_on_commit=False)
#contextmanager
def db_session(db_session_factory):
"""Provide a transactional scope around a series of operations."""
session = db_session_factory()
try:
yield session
session.commit()
except:
session.rollback()
raise
finally:
session.close()
In my main application main.py, I import the module and use the engine and sessionmaker instances as follows. I cleaned up the engine instance at the end of __main__.
from multiprocessing.pool import ThreadPool
from database import DB_ENGINE, DB_SESSION, db_session
def worker_func(data):
with db_session(DB_SESSION) as session:
[...database operations using session object...]
if __name__ == '__main__':
try:
data = [1,2,3,4,5]
with ThreadPool(processes=5) as thread_pool:
results = thread_pool.map(worker_func, data)
except:
raise
finally:
# Cleanup
DB_ENGINE.dispose()

Create set of Python functions

I would like to create a number of functions which start by calling one particular function, and end by calling another.
Each function would take a different number of arguments, but they would share the first and last line. Is this possible?
To give you an example, I am trying to use this to create a set of functions which can connect to my database via sqlalchemy, add an entry to it, and exit nicely:
from sqlalchemy import create_engine
from os import path
from common_classes import *
from sqlalchemy.orm import sessionmaker
def loadSession():
db_path = "sqlite:///" + path.expanduser("~/animal_data.db")
engine = create_engine(db_path, echo=False)
Session = sessionmaker(bind=engine)
session = Session()
Base.metadata.create_all(engine)
return session, engine
def add_animal(id_eth, cage_eth, sex, ear_punches, id_uzh="", cage_uzh=""):
session, engine = loadSession()
new_animal = Animal(id_eth=id_eth, cage_eth=cage_eth, sex=sex, ear_punches=ear_punches, id_uzh=id_uzh, cage_uzh=cage_uzh)
session.add(new_animal)
commit_and_close(session, engine)
def add_genotype(name, zygosity):
session, engine = loadSession()
new_genotype = Genotype(name=name, zygosity=zygosity)
session.add(new_animal)
commit_and_close(session, engine)
def commit_and_close(session, engine):
session.commit()
session.close()
engine.dispose()
Again, what I am trying to do is collapse add_animal() and add_genotype() (and prospectively many more functions) into a single constructor.
I have thought maybe I can use a class for this, and while I believe loadSession() could be called from __init__ I have no idea how to call the commit_and_close() function at the end - nor how to manage the variable number of arguments of every subclass...

Instead of having add_X functions for every type X, just create a single add function that adds an object which you create on the “outside” of the funcition:
So add_animal(params…) becomes add(Animal(params…)), and add_genotype(params…) becomes add(Genotype(params…)).
That way, your add function would just look like this:
def add (obj):
session, engine = loadSession()
session.add(obj)
commit_and_close(session, engine)
Then it’s up to the caller of that function to create the object, which opens up the interface and allows you to get objects from elsewhere too. E.g. something like this would be possible too then:
for animal in zoo.getAnimals():
add(animal)

Issue with multiple sessions

In my application I'm using SQLAlchemy for storing most persistent data across app restarts. For this I have a db package containing my mapper classes (like Tag, Group etc.) and a support class creating a single engine instance using create_engine and a single, global, Session factory using sessionmaker.
Now my understanding of how to use SQLAlchemys sessions is, that I don't pass them around in my app but rather create instances using the global factory whenever I need database access.
This leads to situations were a record is queried in one session and then passed on to another part of the app, which uses a different session instance. This gives me exceptions like this one:
Traceback (most recent call last):
File "…", line 29, in delete
session.delete(self.record)
File "/usr/lib/python3.3/site-packages/sqlalchemy/orm/session.py", line 1444, in delete
self._attach(state, include_before=True)
File "/usr/lib/python3.3/site-packages/sqlalchemy/orm/session.py", line 1748, in _attach
state.session_id, self.hash_key))
sqlalchemy.exc.InvalidRequestError: Object '<Group at 0x7fb64c7b3f90>' is already attached to session '1' (this is '3')
Now my question is: did I get the usage of Session completly wrong (so I should use one session only at a time and pass that session around to other components together with records from the database) or could this result from actual code issue?
Some example code demonstrating my exact problem:
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base, declared_attr
Base = declarative_base()
class Record(Base):
__tablename__ = "record"
id = Column(Integer, primary_key=True)
name = Column(String)
def __init__(self, name):
self.name = name
def __repr__(self):
return "<%s('%s')>" % (type(self).__name__, self.name)
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
Session = sessionmaker(bind=engine)
s1 = Session()
record = Record("foobar")
s1.add(record)
s1.commit()
# This would be a completly different part of app
s2 = Session()
record = s2.query(Record).filter(Record.name == "foobar").first()
def delete_record(record):
session = Session()
session.delete(record)
session.commit()
delete_record(record)

For now I switched over to using a single, global session instance. That's neither nice nor clean in my opinion, but including lots and lots of boiler plate code to expunge objects from one session just to add them back to their original session after handing it over to some other application part was no realistic option, either.
I suppose this will completely blow up if I start using multiple threads to access the database via the very same session…