We Keep Coding

Cannot use SQLAlchemy Session() outside Flask application context for unit testing

This is my inside flaskapp/__init__.py for creating my Flask app, with it I can access Session inside any module in the package, by just importing Session from flaskapp/db.py:
import os
from flask import Flask
def create_app(test_config=None):
app = Flask(__name__, instance_relative_config=True)
app.config.from_mapping(
SECRET_KEY=b'some_secret_key',
DEBUG=True,
SQLALCHEMY_DATABASE_URI=f'sqlite:///{os.path.join(app.instance_path, "flaskapp.sqlite")}',
)
if test_config is None:
app.config.from_pyfile('config.py', silent=True)
else:
app.config.from_mapping(test_config)
try:
os.makedirs(app.instance_path)
except OSError:
pass
with app.app_context():
from flaskapp.routes import home, auth
from flaskapp.db import init_db
init_db()
return app
This is my flaskapp/db.py:
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from flask import current_app
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
Session = sessionmaker(bind=engine)
def init_db():
import flaskapp.models as models
models.Base.metadata.create_all(engine)
def drop_db():
import flaskapp.models as models
models.Base.metadata.drop_all(engine)
With that, I can access the database in any other module, for example, flaskapp/auth.py:
from flask import flash, session
from sqlalchemy import select
from flaskapp.db import Session
from flaskapp.models import User
def log_user(username: str, password: str) -> bool:
with Session() as db_session:
stmt1 = select(User).where(User.username == username)
query_user = db_session.execute(stmt1).first()
if not query_user:
flash('Some error message')
# Some password verifications and other things
session['username'] = query_user[0].username
flash('Successfully logged in')
return True
Until that point, I don't have any problem, the problem comes when I try to do unit testing with unittest, I can't set the test environment and I don't know how can I use the Session object defined in flaskapp/db.py for testing in a separate database. Everything I've tried until now gets me an error, this is my tests/__init__.py:
import unittest
from flaskapp import create_app
from flaskapp.db import Session
from flaskapp.models import User
class BaseTestClass(unittest.TestCase):
def setUp(self):
self.app = create_app(test_config={
'TESTING': True,
'DEBUG': True,
'APP_ENV': 'testing',
# I pass the test database URI expecting the engine to use it
'SQLALCHEMY_DATABASE_URI': 'sqlite:///testdb.sqlite',
})
self.client = self.app.test_client()
# Context
with self.app.app_context():
self.populate_db()
def tearDown(self):
pass
def populate_db(self):
with Session() as db_session:
db_session.add(User(
username='Harry',
email='harry#yahoo.es',
password = 'Harry123.'
))
db_session.commit()
When I try to use the Session object inside populate_db() I get this error:
=====================================================================
ERROR: tests.test_auth (unittest.loader._FailedTest.tests.test_auth)
----------------------------------------------------------------------
ImportError: Failed to import test module: tests.test_auth
Traceback (most recent call last):
File "/usr/local/lib/python3.11/unittest/loader.py", line 407, in _find_test_path
module = self._get_module_from_name(name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/local/lib/python3.11/unittest/loader.py", line 350, in _get_module_from_name
__import__(name)
File "/home/chus/usb/soft/proy/SAGC/tests/test_auth.py", line 1, in <module>
from flaskapp.db import Session
File "/home/chus/usb/soft/proy/SAGC/flaskapp/db.py", line 5, in <module>
engine = create_engine(current_app.config['SQLALCHEMY_DATABASE_URI'], echo=True)
^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 316, in __get__
obj = instance._get_current_object() # type: ignore[misc]
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/chus/usb/soft/proy/SAGC/venv/lib/python3.11/site-packages/werkzeug/local.py", line 513, in _get_current_object
raise RuntimeError(unbound_message) from None
RuntimeError: Working outside of application context.
This typically means that you attempted to use functionality that needed
the current application. To solve this, set up an application context
with app.app_context(). See the documentation for more information.
----------------------------------------------------------------------
Ran 1 test in 0.000s
FAILED (errors=1)
Can someone help me out? I've tried everything to make the tests use the app context in order to populate the test database and make queries but it appears to be conflicting with it being created in create_app() or something.

Firsly, the error you posted is about tests/test_auth.py which you did not include.
However, your code is tightly coupling the flask app and the db.py module with the use of flask.current_app to obtain the engine url.
So the engine creation will only work if there is an app context, which is probably absent when you do a top level import (line 1) of from flaskapp.db import Session in tests/test_auth.py.
A solution could be creating the engine and sessionmaker (or scoped_session) with your create_app and injecting the engine or Session in your functions (look up dependency injection principle):
def init_db(engine):
models.Base.metadata.create_all(engine)
def drop_db(engine):
models.Base.metadata.drop_all(engine)
def log_user(session, username: str, password: str) -> bool:
stmt1 = select(User).where(User.username == username)
query_user = session.execute(stmt1).first()
# skipping the rest
Or look into Flask-SQLAlchemy.

Python circular import error with flask and celery

How can I make #celery.task available on user resources package?
I'm new to python and confused about the "circular imports error" can somebody explain how it works? and how to handle this kind of errors in flask application. This is the part where I always get stuck.
This is my project current folder structure
code
|_resources
| |_user.py
|
|_utils
| |_flask_celery.py
|
|_flask_app.py
--- SOURCE CODE ---
flask_app.py
from flask import Flask
from flask_cors import CORS
from flask_restful import Api
from util.flask_celery import make_celery
from routes.endpoint import urls
from resources.user import Users
app = Flask(__name__)
CORS(app, resources={r"/*": {"origins": "*"}})
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql:///username:StrongPassword#localhost:3306/db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['CELERY_BROKER_URL'] = 'amqp//admin:StrongPassword#localhost:5672'
app.config['CELERY_RESULT_BACKEND'] = 'db+mysql:///username:StrongPassword#localhost:3306/db'
app.config['PROPAGATE_EXCEPTIONS'] = True
api = Api(app)
celery = make_celery(app)
url = urls()
api.add_resource(Users, url.get('users'))
if __name__ == "__main__":
# sqlalchemy
from db import db
db.__init__(app)
app.run(host='0.0.0.0', debug=True, port=5000)
flask_celery.py
from celery import Celery
def make_celery(celery_app):
celery = Celery(
celery_app.import_name,
backend=celery_app.config['CELERY_RESULT_BACKEND'],
broker=celery_app.config['CELERY_BROKER_URL']
)
celery.conf.update(celery_app.config)
class ContextTask(celery.Task):
def __call__(self, *args, **kwargs):
with celery_app.app_context():
return self.run(*args, **kwargs)
celery.Task = ContextTask
return celery
user.py
from flask_restful import Resource, reqparse
from app import celery # <-- this cause the error
from util.zk_connector import zk_connect
from util.zk_error import error
class Users(Resource):
def post(self, ip, comkey):
zk = zk_connect(ip=ip, password=int(comkey))
try:
session = zk.connect()
session.disable_device()
users = session.get_users()
print(users)
session.enable_device()
session.disconnect()
return {'message': 'success'}, 200
except Exception as e:
return error(e)
#celery.task(name='user.reverse')
def reverse(string):
return string[::-1]
Error:
Traceback (most recent call last):
File ".\flask_app.py", line 6, in <module>
from resources.user import User
File "C:\Users\Gelo\Documents\Brand new clean arch pyzk\code\resources\user.py", line 2, in <module>
from flask_app import celery
File "C:\Users\Gelo\Documents\Brand new clean arch pyzk\code\flask_app.py", line 6, in <module>
from resources.user import User
ImportError: cannot import name 'User' from partially initialized module 'resources.user' (most likely due to a circular import) (C:\Users\Gelo\Documents\Brand new clean arch pyzk\code\resources\user.py)

Using flask track usage with blueprint and config from object

I have a question regarding the flask_track_usage module.
All my blueprints should have the Trackusage function included.
Unfortunately i didn't find a way to solve my problem.
Why isn't it possible to simply use the flask.current_app?
route.py
import datetime
from uuid import uuid4
from flask import Blueprint, session, render_template, url_for, flash, redirect, current_app, request, jsonify
from flask_template.main.utils import my_func
import os
import json
from flask_track_usage import TrackUsage
from flask_track_usage.storage.printer import PrintWriter
from flask_track_usage.storage.output import OutputWriter
main = Blueprint('main', __name__)
t = TrackUsage(current_app, [
PrintWriter(),
OutputWriter(transform=lambda s: "OUTPUT: " + str(s))
])
#t.include
#main.before_request
def session_management():
now = datetime.datetime.now()
session_lifetime = current_app.config['SESSION_DURATION']
# Create session, if not already existing
if session.get('session_ID') is None:
# Initiate session, set a random UUID as Session ID
session['session_ID'] = str(uuid4())
session.permanent = True # will expire after 30 minutes of inactivity
session['timeout'] = False
print(f'Initated session with ID:', session.get('session_ID'))
return redirect(url_for('main.index'))
else:
try:
last_active = session.get('last_active')
delta = now - last_active
if delta.seconds > 1740:
print(f'Note: Session lifetime less than one minute. Expires in:',
session_lifetime - delta.seconds, 'sec')
if delta.seconds > session_lifetime:
session['last_active'] = now
session['timeout'] = True
print(f'Your session has expired after 30 minutes, you have been logged out (files are deleted).')
return redirect(url_for('main.logout'))
except:
pass
try:
session['last_active'] = now
except:
pass
#main.route('/')
def index():
return redirect(url_for('main.home'))
#main.route('/home')
def home():
return render_template('home.html', title='home', subheader='Template Main Page')
__init__.py
from flask import Flask
from flask_template.config import Config
def create_app(config_class=Config):
app = Flask(__name__)
app.config.from_object(Config)
from flask_template.sample_extension.routes import sample_extension
from flask_template.main.routes import main
from flask_template.errors.handlers import errors
app.register_blueprint(sample_extension)
app.register_blueprint(main)
app.register_blueprint(errors)
return app
Config
import os
import json
from datetime import timedelta
with open(os.path.join(os.getcwd(), 'etc', 'app_config.json')) as config_file:
config = json.load(config_file)
class Config:
SECRET_KEY = config.get('SECRET_KEY')
SESSION_DURATION = 1800 # 30 minutes for delete function
PERMANENT_SESSION_LIFETIME = timedelta(minutes=30) # session lifetime
root_dir = os.path.join(os.path.realpath('.'), 'flask_template')
# Path to template to illustrate download functionality
TEMPLATE_FOLDER = os.path.join(root_dir, 'sample_extension', 'assets', 'template')
TRACK_USAGE_USE_FREEGEOIP = False
TRACK_USAGE_INCLUDE_OR_EXCLUDE_VIEWS = 'include'
run.py
from flask_template import create_app
from flask_track_usage import TrackUsage
from flask_track_usage.storage.printer import PrintWriter
from flask_track_usage.storage.output import OutputWriter
app = create_app()
if __name__ == '__main__':
app.run(ssl_context=('******'),
host='0.0.0.0',
port=5000,
debug=True)
error
RuntimeError: Working outside of application context.

There are several global Flask variables such as current_app and g that can be accessed only while the application is running (more about them in the Flask Application Context documentation entry). Using them outside the application context raises RuntimeError.
You can instantiate TrackUsage without parameters in your routes.py module:
track_usage = TrackUsage()
#track_usage.include
#main.before_request
def session_management():
...
And then you can import it in your __init__.py module and apply to your application instance:
from flask import Flask
from flask_template.config import Config
from flask_track_usage.storage.printer import PrintWriter
from flask_track_usage.storage.output import OutputWriter
def create_app(config_class=Config):
app = Flask(__name__)
app.config.from_object(Config)
from flask_template.sample_extension.routes import sample_extension, track_usage
from flask_template.main.routes import main
from flask_template.errors.handlers import errors
track_usage.init_app(app, [
PrintWriter(),
OutputWriter(transform=lambda s: "OUTPUT: " + str(s)),
])
app.register_blueprint(sample_extension)
app.register_blueprint(main)
app.register_blueprint(errors)
return app
I have not found this solution in the Flask-Track-Usage documentation but it is a common interface for Flask extentions. It allows to instantiate an extension in one module and connect it to Flask application in main module.

RuntimeError: assets instance not bound to an application, and no application in current context

I'm working to modify a cookiecutter Flask app. I'm currently trying to add a datepicker to a page. I've found https://eonasdan.github.io/bootstrap-datetimepicker/. This cookiecutter uses flask-assets to manage the project assets.
Following https://adambard.com/blog/fresh-flask-setup/ I've modified my assets.py file :
from flask_assets import Bundle, Environment
import os
css = Bundle(
"libs/bootstrap/dist/css/spacelab/bootstrap.css",
"bower_components/eonasdan-bootstrap-datetimepicker/build/css/bootstrap-datetimepicker.css",
"css/style.css",
"css/home.css",
# "css/style.css",
filters="cssmin",
output="public/css/common.css"
)
js = Bundle(
"libs/jQuery/dist/jquery.js",
"libs/bootstrap/dist/js/bootstrap.js",
"bower_components/moment/moment.js",
"bower_components/eonasdan-bootstrap-datetimepicker/build/js/bootstrap-datetimepicker.min.js",
"js/plugins.js",
filters='jsmin',
output="public/js/common.js"
)
# Tell flask-assets where to look for our coffeescript and sass files.
assets = Environment()
assets.load_path = [os.path.join(os.path.dirname(__file__), 'myflaskapp/static/bower_components')]
assets.register("js_all", js)
assets.register("css_all", css)
When I do this I get:
Traceback (most recent call last):
File "C:/envs/r2/myproject/manage.py", line 8, in <module>
from myflaskapp.app import create_app
File "C:\envs\r2\myproject\myflaskapp\app.py", line 8, in <module>
from myflaskapp.assets import assets
File "C:\envs\r2\myproject\myflaskapp\assets.py", line 41, in <module>
assets.load_path = [os.path.join(os.path.dirname(__file__), 'myflaskapp/static/bower_components')]
File "C:\envs\virtalenvs\flask_myproject\lib\site-packages\webassets\env.py", line 639, in _set_load_path
self._storage['load_path'] = load_path
File "C:\envs\virtalenvs\flask_myproject\lib\site-packages\flask_assets.py", line 104, in __setitem__
self.env._app.config[self._transform_key(key)] = value
File "C:\envs\virtalenvs\flask_myproject\lib\site-packages\flask_assets.py", line 292, in _app
'and no application in current context')
RuntimeError: assets instance not bound to an application, and no application in current context
What am I doing wrong?
edit: app.py initializes the assets:
from flask import Flask, render_template
from myflaskapp.assets import assets
:param config_object: The configuration object to use.
"""
app = Flask(__name__)
app.config.from_object(config_object)
register_extensions(app)
register_blueprints(app)
register_errorhandlers(app)
return app
def register_extensions(app):
assets.init_app(app)
def register_blueprints(app):
app.register_blueprint(public.blueprint)
app.register_blueprint(user.blueprint)

As I noted in my comment, you have to a Flask app bound to the object, as it notes in the traceback:
>>> RuntimeError: assets instance not bound to an application, and no
application in current context
This will fix your problem, whether or not it's relevant for you use case...:
def register_extensions(app):
assets.init_app(app)
with app.app_context():
assets.load_path = ['static']
or you could re-write your create app to have
def create_assets(app):
assets = Environment(app)
....
assets.load_path ['static']
return assets
def create_app():
app = Flask()
....
assets = create_assets(app)
return app
The whole reason for your errors is your call to load_path. This tries to set the attribute in webassets, which you can see here: https://github.com/miracle2k/webassets/blob/95dff0ad6dcc25b81790a1585c67f5393e7d32af/src/webassets/env.py#L656
def _set_load_path(self, load_path):
self._storage['load_path'] = load_path
In turn, this now activates the attribute of _storage on your flask-asset object, which results in it trying to do this: https://github.com/miracle2k/flask-assets/blob/eb7f1905410828689086b80eb19be9331041ac52/src/flask_assets.py#L102
def __setitem__(self, key, value):
if not self._set_deprecated(key, value):
self.env._app.config[self._transform_key(key)] = value
As you see, it needs access to an app, and as you haven't given one when you used load_path it will complain to you, as it explains so nicely in the Traceback. I found a discussion about this on the flask-asset page: https://github.com/miracle2k/flask-assets/issues/35
You may, quite rightly, think that as you called init_app() that everything should be fine, but that's not the case, init_app() does not give any reference of the app to Environment: https://github.com/miracle2k/flask-assets/blob/eb7f1905410828689086b80eb19be9331041ac52/src/flask_assets.py#L338
def init_app(self, app):
app.jinja_env.add_extension('webassets.ext.jinja2.AssetsExtension')
app.jinja_env.assets_environment = self
I don't use flask-assets at all, and so I'm not particularly sure why they haven't referenced the app with the Environment().init_app, so if someone else knows, shout out.

Gettings settings and config from INI file for Pyramid functional testing

In a real Pyramid app it does not work per docs http://docs.pylonsproject.org/projects/pyramid//en/latest/narr/testing.html :
class FunctionalTests(unittest.TestCase):
def setUp(self):
from myapp import main
app = main({})
Exception:
Traceback (most recent call last):
File "C:\projects\myapp\tests\model\task_dispatcher_integration_test.py", line 35, in setUp
app = main({})
File "C:\projects\myapp\myapp\__init__.py", line 207, in main
engine = engine_from_config(settings, 'sqlalchemy.')
File "C:\projects\myapp\ve\lib\site-packages\sqlalchemy\engine\__init__.py", line 407, in engine_from_config
url = options.pop('url')
KeyError: 'url'
The reason is trivial: an empty dictionary is passed to main, while it seems that while running real app (from __init__.py) it gets settings pre-filled with values from [app:main] section of development.ini / production.ini:
settings {'ldap_port': '4032', 'sqlalchemy.url': 'postgresql://.....}
Is there some way of reconstructing settings easily from an .ini file for functional testing?

pyramid.paster.get_appsettings is the only thing you need:
from pyramid.paster import get_appsettings
settings = get_appsettings('test.ini', name='main')
app = main(settings)
That test.ini can include all the settings of another .ini file easily like this:
[app:main]
use = config:development.ini#main
and then you only need to override those keys that change (I guess you'd want to rather test against a separate DB):
[app:main]
use = config:development.ini#main
sqlalchemy.uri = postgresql://....

In case anyone else doesn't get #antti-haapala's answer right away:
Create a test.ini filled with:
[app:main]
use = config:development.ini#main
(Actually this step is not necessary. You could also keep your development.ini and use it instead of test.ini in the following code. A separate test.ini might however be useful if you want separate settings for testing)
In your tests.py add:
from pyramid.paster import get_appsettings
settings = get_appsettings('test.ini', name='main')
and replace
app = TestApp(main({}))
with
app = TestApp(main(global_config = None, **settings))
Relevant for this answer was the following comment: Pyramid fails to start when webtest and sqlalchemy are used together

Actually, you don't need import get_appsettings, just add the
parameters like this:
class FunctionalTests(unittest.TestCase):
def setUp(self):
from myapp import main
settings = {'sqlalchemy.url': 'sqlite://'}
app = main({}, **settings)
here is the source: functional test, it is in the second block code, line 31.

Yes there is, though the easy is a subject to debate.
I am using the following py.test test fixture to make --ini option passed to the tests. However this approach is limited to py.test test runner, as other test runner do not have such flexibility.
Also my test.ini has special settings like disabling outgoing mail and instead printing it out to terminal and test accessible backlog.
#pytest.fixture(scope='session')
def ini_settings(request):
"""Load INI settings for test run from py.test command line.
Example:
py.test yourpackage -s --ini=test.ini
:return: Adictionary representing the key/value pairs in an ``app`` section within the file represented by ``config_uri``
"""
if not getattr(request.config.option, "ini", None):
raise RuntimeError("You need to give --ini test.ini command line option to py.test to find our test settings")
# Unrelated, but if you need to poke standard Python ConfigParser do it here
# from websauna.utils.configincluder import monkey_patch_paster_config_parser
# monkey_patch_paster_config_parser()
config_uri = os.path.abspath(request.config.option.ini)
setup_logging(config_uri)
config = get_appsettings(config_uri)
# To pass the config filename itself forward
config["_ini_file"] = config_uri
return config
Then I can set up app (note that here pyramid.paster.bootstrap parses the config file again:
#pytest.fixture(scope='session')
def app(request, ini_settings, **settings_overrides):
"""Initialize WSGI application from INI file given on the command line.
TODO: This can be run only once per testing session, as SQLAlchemy does some stupid shit on import, leaks globals and if you run it again it doesn't work. E.g. trying to manually call ``app()`` twice::
Class <class 'websauna.referral.models.ReferralProgram'> already has been instrumented declaratively
:param settings_overrides: Override specific settings for the test case.
:return: WSGI application instance as created by ``Initializer.make_wsgi_app()``.
"""
if not getattr(request.config.option, "ini", None):
raise RuntimeError("You need to give --ini test.ini command line option to py.test to find our test settings")
data = bootstrap(ini_settings["_ini_file"])
return data["app"]
Furthermore setting up a functional test server:
import threading
import time
from wsgiref.simple_server import make_server
from urllib.parse import urlparse
from pyramid.paster import bootstrap
import pytest
from webtest import TestApp
from backports import typing
#: The URL where WSGI server is run from where Selenium browser loads the pages
HOST_BASE = "http://localhost:8521"
class ServerThread(threading.Thread):
""" Run WSGI server on a background thread.
Pass in WSGI app object and serve pages from it for Selenium browser.
"""
def __init__(self, app, hostbase=HOST_BASE):
threading.Thread.__init__(self)
self.app = app
self.srv = None
self.daemon = True
self.hostbase = hostbase
def run(self):
"""Open WSGI server to listen to HOST_BASE address
"""
parts = urlparse(self.hostbase)
domain, port = parts.netloc.split(":")
self.srv = make_server(domain, int(port), self.app)
try:
self.srv.serve_forever()
except Exception as e:
# We are a background thread so we have problems to interrupt tests in the case of error
import traceback
traceback.print_exc()
# Failed to start
self.srv = None
def quit(self):
"""Stop test webserver."""
if self.srv:
self.srv.shutdown()
#pytest.fixture(scope='session')
def web_server(request, app) -> str:
"""py.test fixture to create a WSGI web server for functional tests.
:param app: py.test fixture for constructing a WSGI application
:return: localhost URL where the web server is running.
"""
server = ServerThread(app)
server.start()
# Wait randomish time to allows SocketServer to initialize itself.
# TODO: Replace this with proper event telling the server is up.
time.sleep(0.1)
assert server.srv is not None, "Could not start the test web server"
host_base = HOST_BASE
def teardown():
server.quit()
request.addfinalizer(teardown)
return host_base