We Keep Coding

How does one get Flask-Migrate to read the current app config?

I am having trouble running flask db migrate. I have run flask db init without issue and have modified the resulting migrations/env.py file to focus only on a specific schema in an existing MS SQL Server database. It is acting like the migrations/env.py: run_migrations_online() method is not finding the correct app configuration. It wants to create a default sqlite in memory db instead.
I am defining SQLALCHEMY_DATABASE_URI in create_app() where that reads in a local config file:
app.config.from_object('app.local_settings')
The config for the mssql db should be:
SQLALCHEMY_DATABASE_URI = \
"mssql+pyodbc://db-dev.my.company.com/devdb?driver=SQL+Server"
My app entry point looks like:
from app import create_app
try:
app = create_app()
except Exception as e:
print(repr(e))
raise
if __name__ == "__main__":
try:
app.run(debug=True)
except Exception as e:
app.logger.error(repr(e))
print(repr(e))
The definition of create_app() is in the __init__.py of the app module:
import pyodbc
from flask import Flask
from flask_mail import Mail
from flask_migrate import Migrate
from flask_sqlalchemy import SQLAlchemy
from flask_wtf.csrf import CSRFProtect
from flask_security import Security, SQLAlchemyUserDatastore, auth_required, \
hash_password
from flask_security.models import fsqla_v2 as fsqla
# Instantiate Flask
app = Flask(__name__)
# Instantiate Flask extensions
csrf_protect = CSRFProtect(app=app)
db = SQLAlchemy(app=app)
mail = Mail(app=app)
migrate = Migrate(app=app, db=db)
# Initialize Flask Application
def create_app(extra_config_settings={}):
# Load common settings
app.config.from_object('app.settings')
# Load environment specific settings
app.config.from_object('app.local_settings')
# Load extra settings from extra_config_settings param
app.config.update(extra_config_settings)
# print(pyodbc.drivers())
# Setup Flask-SQLAlchemy
db.init_app(app)
# Setup Flask-Migrate
migrate.init_app(app=app, db=db)
# Setup Flask-Mail
mail.init_app(app)
# Setup WTForms CSRFProtect
csrf_protect.init_app(app)
# Register blueprints
from .views import register_blueprints
register_blueprints(app)
# Setup an error-logger to send emails to app.config.ADMINS
init_email_error_handler(app)
# Setup Flask-User to handle user account related forms
from .models.user import User, UserRegisterForm, UserProfileForm, \
UserLoginForm
from .views.main import user_profile_page
from .models.roles import Role
# APIs
from .views import register_api, SchoolAPI
register_api(app, SchoolAPI, 'school_api', '/school/', pk='school_id')
# #app.context_processor
# def context_processor():
# return dict(user_manager=user_manager)
fsqla.FsModels.set_db_info(db)
user_datastore = SQLAlchemyUserDatastore(db, User, Role)
security = Security(app, user_datastore)
return app
I have modified the migrations/env.py file as:
from __future__ import with_statement
import logging
from logging.config import fileConfig
from flask import current_app
from alembic import context
config = context.config
fileConfig(config.config_file_name)
logger = logging.getLogger('alembic.env')
config.set_main_option(
'sqlalchemy.url',
str(current_app.extensions['migrate'].db.get_engine().url).replace(
'%', '%%'))
target_metadata = current_app.extensions['migrate'].db.metadata
def include_name(name, type_, parent_names):
result = False
if type_ == "schema":
return name in ["MySchema"]
else:
return True
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
url = config.get_main_option("sqlalchemy.url")
print(f'run_migrations_offline: {url}')
context.configure(
url=url,
target_metadata=target_metadata,
literal_binds=True,
version_table_schema='MySchema',
include_schemas=True,
include_name=include_name
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
# this callback is used to prevent an auto-migration from being generated
# when there are no changes to the schema
# reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html
def process_revision_directives(context, revision, directives):
if getattr(config.cmd_opts, 'autogenerate', False):
script = directives[0]
if script.upgrade_ops.is_empty():
directives[:] = []
logger.info('No changes in schema detected.')
connectable = current_app.extensions['migrate'].db.get_engine()
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
process_revision_directives=process_revision_directives,
version_table_schema='MySchema',
include_schemas=True,
include_name=include_name,
**current_app.extensions['migrate'].configure_args
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()
When I run flask db migrate I get:
H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask_sqlalchemy\__init__.py:851: UserWarning: Neither SQLALCHEMY_DATABASE_URI nor SQLALCHEMY_BINDS is set. Defaulting SQLALCHEMY_DATABASE_URI to "sqlite:///:memory:".
warnings.warn(
H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask_sqlalchemy\__init__.py:872: FSADeprecationWarning: SQLALCHEMY_TRACK_MODIFICATIONS adds signi
ficant overhead and will be disabled by default in the future. Set it to True or False to suppress this warning.
warnings.warn(FSADeprecationWarning(
INFO [alembic.runtime.migration] Context impl SQLiteImpl.
INFO [alembic.runtime.migration] Will assume non-transactional DDL.
Traceback (most recent call last):
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1819, in _execute_context
self.dialect.do_execute(
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\engine\default.py", line 732, in do_execute
cursor.execute(statement, parameters)
sqlite3.OperationalError: unknown database "MyProjectDB"
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Python310\lib\runpy.py", line 196, in _run_module_as_main
return _run_code(code, main_globals, None,
File "C:\Python310\lib\runpy.py", line 86, in _run_code
exec(code, run_globals)
File "H:\path\to\my\project\UserInterface\venv\Scripts\flask.exe\__main__.py", line 7, in <module>
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask\cli.py", line 988, in main
cli.main()
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask\cli.py", line 579, in main
return super().main(*args, **kwargs)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\click\core.py", line 1055, in main
rv = self.invoke(ctx)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\click\core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\click\core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\click\core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\click\core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\click\decorators.py", line 26, in new_func
return f(get_current_context(), *args, **kwargs)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask\cli.py", line 427, in decorator
return __ctx.invoke(f, *args, **kwargs)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\click\core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask_migrate\cli.py", line 104, in migrate
_migrate(directory, message, sql, head, splice, branch_label, version_path,
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask_migrate\__init__.py", line 98, in wrapped
f(*args, **kwargs)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\flask_migrate\__init__.py", line 155, in migrate
command.revision(config, message, autogenerate=True, sql=sql,
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\command.py", line 229, in revision
script_directory.run_env()
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\script\base.py", line 569, in run_env
util.load_python_file(self.dir, "env.py")
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\util\pyfiles.py", line 94, in load_python_file
module = load_module_py(module_id, path)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\util\pyfiles.py", line 110, in load_module_py
spec.loader.exec_module(module) # type: ignore
File "<frozen importlib._bootstrap_external>", line 883, in exec_module
File "<frozen importlib._bootstrap>", line 241, in _call_with_frames_removed
File "H:\path\to\my\project\UserInterface\MyProjectDB\migrations\env.py", line 108, in <module>
run_migrations_online()
File "H:\path\to\my\project\UserInterface\MyProjectDB\migrations\env.py", line 102, in run_migrations_online
context.run_migrations()
File "<string>", line 8, in run_migrations
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\runtime\environment.py", line 853, in run_migrations
self.get_context().run_migrations(**kw)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\runtime\migration.py", line 601, in run_migrations
heads = self.get_current_heads()
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\runtime\migration.py", line 533, in get_current_heads
if not self._has_version_table():
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\runtime\migration.py", line 549, in _has_version_table
return sqla_compat._connectable_has_table(
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\alembic\util\sqla_compat.py", line 195, in _connectable_has_table
return inspect(connectable).has_table(tablename, schemaname)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\engine\reflection.py", line 283, in has_table
return self.dialect.has_table(conn, table_name, schema)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\dialects\sqlite\base.py", line 2018, in has_table
info = self._get_table_pragma(
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\dialects\sqlite\base.py", line 2545, in _get_table_pragma
cursor = connection.exec_driver_sql(statement)
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1686, in exec_driver_sql
return self._exec_driver_sql(
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1595, in _exec_driver_sql
ret = self._execute_context(
File "H:\path\to\my\project\UserInterface\venv\lib\site-packages\sqlalchemy\engine\base.py", line 1862, in _execute_context
Thoughts?

Turns out that I was trying to follow Flask-Security-Too's "Basic SQLAlchemy Application with session" quick start guide. I am not claiming it is incorrect but my understanding is probably flawed.
I decided to replace FST's "session" idea with the simpler "SQLAlchemy Application" sample and everything is rolling along just fine.
My best guess at the general issue is that maybe Alembic was not at the correct version or there are changes to the env.py file that I could not figure out.

AttributeError: 'NoneType' object has no attribute '_instantiate_plugins'

I'm trying to get started using Flask with PostgreSQL to create a simple project. For the sake of completeness I will include all terminal commands along with my code. Working in my SQL_Example folder and a local PostgreSQL database "lecture3" with table "flights":
$ python3 -m venv venv
$ . venv/bin/activate
$ pip install Flask
Now I continued with the following code:
import os
from flask import Flask, render_template, request
from sqlalchemy import create_engine
from sqlalchemy.orm import scoped_session, sessionmaker
app = Flask(__name__)
engine = create_engine(os.getenv("DATABASE_URL"))
db = scoped_session(sessionmaker(bind=engine))
#app.route("/")
def index():
flights = db.execute("SELECT * FROM flights").fetchall()
return render_template("index.html", flights=flights)
#app.route("/book", methods=["POST"])
def book():
name = request.form.get("name")
try:
flight_id = int(request.form.get("flight_id"))
except ValueError:
return render_template("error.html", message="Invalid flight number.")
if db.execute("SELECT * FROM flights WHERE id = :id", {"id": flight_id}).rowcount == 0:
return render_template("error.html", message="No such flight with that id.")
db.execute("INSERT INTO passengers (name, flight_id) VALUES (:name, :flight_id)", {"name": name, "flight_id": flight_id})
db.commit()
return render_template("success.html")
Executing the following commands:
$ export FLASK_APP="application.py"
$ flask run
Encountered the following error:
Traceback (most recent call last):
File "/Users/spencermiller/Desktop/SQL_Example/venv/bin/flask", line 11, in <module>
sys.exit(main())
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/flask/cli.py", line 967, in main
cli.main(args=sys.argv[1:], prog_name="python -m flask" if as_module else None)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/flask/cli.py", line 586, in main
return super(FlaskGroup, self).main(*args, **kwargs)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/click/core.py", line 782, in main
rv = self.invoke(ctx)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/click/core.py", line 1259, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/click/core.py", line 1066, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/click/decorators.py", line 73, in new_func
return ctx.invoke(f, obj, *args, **kwargs)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/click/core.py", line 610, in invoke
return callback(*args, **kwargs)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/flask/cli.py", line 848, in run_command
app = DispatchingApp(info.load_app, use_eager_loading=eager_loading)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/flask/cli.py", line 305, in __init__
self._load_unlocked()
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/flask/cli.py", line 330, in _load_unlocked
self._app = rv = self.loader()
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/flask/cli.py", line 388, in load_app
app = locate_app(self, import_name, name)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/flask/cli.py", line 240, in locate_app
__import__(module_name)
File "/Users/spencermiller/Desktop/SQL_Example/application.py", line 9, in <module>
engine = create_engine(os.getenv("DATABASE_URL"))
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/sqlalchemy/engine/__init__.py", line 479, in create_engine
return strategy.create(*args, **kwargs)
File "/Users/spencermiller/Desktop/SQL_Example/venv/lib/python3.7/site-packages/sqlalchemy/engine/strategies.py", line 56, in create
plugins = u._instantiate_plugins(kwargs)
AttributeError: 'NoneType' object has no attribute '_instantiate_plugins'
I have seen some similar issues but none of those solutions have worked for me.
Environment variable:
TERM_PROGRAM=Apple_Terminal
SHELL=/bin/bash
TERM=xterm-256color
TMPDIR=/var/folders/pf/2_4xxn2s0jl7m8r0xv9wg9fr0000gp/T/
Apple_PubSub_Socket_Render=/private/tmp/com.apple.launchd.CyouqBAyRx/Render
TERM_PROGRAM_VERSION=421.1.1
OLDPWD=/Users/spencermiller
TERM_SESSION_ID=3CA9A778-D58A-405E-8EC3-D5F86D87FFFA
USER=spencermiller
SSH_AUTH_SOCK=/private/tmp/com.apple.launchd.aXcWzrdMYX/Listeners
PATH=/Library/Frameworks/Python.framework/Versions/3.7/bin:/Library/Frameworks/Python.framework/Versions/3.7/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin
PWD=/Users/spencermiller/Desktop/SQL_Example
LANG=en_CA.UTF-8
XPC_FLAGS=0x0
XPC_SERVICE_NAME=0
SHLVL=1
HOME=/Users/spencermiller
LOGNAME=spencermiller
DISPLAY=/private/tmp/com.apple.launchd.WkWoFkNc6W/org.macosforge.xquartz:0
_=/usr/bin/printenv

in this piece of code, you are trying to create your flask's engine with a database URL that is on you environment variables.
engine = create_engine(os.getenv("DATABASE_URL"))
My guess is that you didn't set up this environment variable, so you're trying to create an engine with a None url.
You need to set your env variable, or hard code your DATABASE_URL into python.
db_url = 'postgresql+psycopg2://{user}:{password}#{url}/{db}'
engine = create_engine(db_url)
Hope it helps.

Follow the other answers, if still you're getting following error:
AttributeError: 'NoneType' object has no attribute '_instantiate_plugins'
do (line 5)
engine = create_engine("DATABASE_URL")
instead of
engine = create_engine(os.getenv("DATABASE_URL"))

Flask + SQLAlchemy OperationalError

I am trying to play around with deploying Flask application. When I run the code using docker swarm on Amazon's EC2 I started to get following errors:
Error message:
sqlalchemy.exc:OperationalError: (psycopg2.OperationalError) server closed the connection unexpectedly This probably means the server terminated abnormally before or while processing the request.[SQL: SELECT visitor."ID" AS "visitor_ID", visitor.username AS visitor_username, visitor.visits AS visitor_visits FROM visitor](Background on this error at: http://sqlalche.me/e/e3q8)
Stack trace
Traceback (most recent call last):
File "/usr/local/bin/gunicorn", line 8, in <module>
File "/usr/local/lib/python3.7/site-packages/gunicorn/app/wsgiapp.py", line 58, in run
File "/usr/local/lib/python3.7/site-packages/gunicorn/app/base.py", line 228, in run
File "/usr/local/lib/python3.7/site-packages/gunicorn/app/base.py", line 72, in run
File "/usr/local/lib/python3.7/site-packages/gunicorn/arbiter.py", line 202, in run
File "/usr/local/lib/python3.7/site-packages/gunicorn/arbiter.py", line 545, in manage_workers
File "/usr/local/lib/python3.7/site-packages/gunicorn/arbiter.py", line 616, in spawn_workers
File "/usr/local/lib/python3.7/site-packages/gunicorn/arbiter.py", line 583, in spawn_worker
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/base.py", line 140, in init_process
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 123, in run
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 67, in run_for_one
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 29, in accept
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 134, in handle
File "/usr/local/lib/python3.7/site-packages/gunicorn/workers/sync.py", line 175, in handle_request
File "/usr/local/lib/python3.7/site-packages/newrelic/api/wsgi_application.py", line 665, in _nr_wsgi_application_wrapper_
File "/usr/local/lib/python3.7/site-packages/newrelic/api/wsgi_application.py", line 193, in __init__
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2463, in __call__
File "/usr/local/lib/python3.7/site-packages/newrelic/api/wsgi_application.py", line 555, in _nr_wsgi_application_wrapper_
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 2446, in wsgi_app
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1949, in full_dispatch_request
File "/usr/local/lib/python3.7/site-packages/flask/app.py", line 1935, in dispatch_request
File "/usr/local/lib/python3.7/site-packages/newrelic/hooks/framework_flask.py", line 45, in _nr_wrapper_handler_
File "/app/src/views.py", line 5, in index
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3233, in all
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3389, in __iter__
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/orm/query.py", line 3414, in _execute_and_instances
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 982, in execute
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/sql/elements.py", line 293, in _execute_on_connection
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1101, in _execute_clauseelement
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1250, in _execute_context
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1476, in _handle_dbapi_exception
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 398, in raise_from_cause
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/util/compat.py", line 152, in reraise
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/base.py", line 1246, in _execute_context
File "/usr/local/lib/python3.7/site-packages/sqlalchemy/engine/default.py", line 588, in do_execute
File "/usr/local/lib/python3.7/site-packages/newrelic/hooks/database_psycopg2.py", line 51, in execute
File "/usr/local/lib/python3.7/site-packages/newrelic/hooks/database_dbapi2.py", line 25, in execute
Initially I thought that this might be a result of not closing db connections after using them, but when I added db.session.close() the error still occurs.
Full code can be found here, the most important parts are:
views.py file:
from models import Visitor, db
def index():
visitors = Visitor.query.all()
response = [
{"id": visitor.ID, "username": visitor.username, "visits": visitor.visits}
for visitor in visitors
]
db.session.close()
return {"results": response}
def increment_visits(username: str) -> dict:
if username == "favicon.ico":
return {}
visitor = Visitor.query.filter_by(username=username).first()
if visitor is None:
visitor = Visitor(username=username, visits=1)
db.session.add(visitor)
else:
visitor.visits += 1
db.session.commit()
return {"id": visitor.ID, "username": visitor.username, "visits": visitor.visits}
main.py file:
import sys
from flask import Flask
from config import SQLALCHEMY_DATABASE_URI, HOST, PORT, DEBUG, DB_POOL_SIZE
from models import db
def make_app() -> Flask:
flask_app = Flask(__name__)
flask_app.config['SQLALCHEMY_DATABASE_URI'] = SQLALCHEMY_DATABASE_URI
flask_app.config['SQLALCHEMY_POOL_SIZE'] = DB_POOL_SIZE
add_urls(flask_app)
db.init_app(app=flask_app)
return flask_app
def add_urls(flask_app: Flask):
from views import index, increment_visits
flask_app.add_url_rule('/', view_func=index)
flask_app.add_url_rule('/<username>/', view_func=increment_visits)
app = make_app()
if __name__ == '__main__':
if 'createdb' in sys.argv:
app.app_context().push()
db.create_all()
else:
app.run(host=HOST, port=PORT, debug=DEBUG)
models.py:
from flask_sqlalchemy import SQLAlchemy
db = SQLAlchemy()
class Visitor(db.Model):
ID = db.Column(db.Integer, primary_key=True)
username = db.Column(db.String(80), unique=True)
visits = db.Column(db.Integer(), default=1)
Docker services:
ubuntu#srv1:~$ docker stack ps --filter "desired-state=running" demo
ID NAME IMAGE NODE DESIRED STATE CURRENT STATE ERROR PORTS
frbdra27cdcb demo_web.1 gonczor/aws-simple-app:prod srv1 Running Running 7 hours ago
0by6yhvr9n4a demo_nginx.1 gonczor/aws-simple-app-nginx:prod srv1 Running Running 3 days ago
ym2t3we6r5b1 demo_db.1 postgres:11 srv1 Running Running 4 days ago
luwgpr3jnsj8 demo_web.2 gonczor/aws-simple-app:prod srv1 Running Running 7 hours ago
I have no clue where this error might come from, so I am happy to provide any further details.
EDIT
Output of requested command:
simple_app=# SELECT sum(numbackends) FROM pg_stat_database;
sum
-----
2
(1 row)

I think I found a solution, though I have very vague why it works. I found an answer in this reddit post, so here we go.
The error mainly occurred after a long idleness. When I was flooding the app with requests during benchmarking nothing happened. It seems that some connections were kept by SQLAlchemy for too long. I lowered the POOL_RECYCLE_TIME to 10 minutes and updated the settings format to avoid deprecation that will be introduced in version 3.0 of Flask-SQLAlchemy.
DB_POOL_SIZE = int(os.environ.get('DB_POOL_SIZE', '10'))
DB_POOL_RECYCLE = int(os.environ.get('DB_POOL_RECYCLE', '60'))
SQLALCHEMY_ENGINE_OPTIONS = {
'pool_size': DB_POOL_SIZE,
'pool_recycle': DB_POOL_RECYCLE,
}
As I say, I don't fully understand why this was a problem, but after introducing those changes no errors occurred. Anyway comments and links to materials explaining this issue are welcome.

Django test parallel AppRegistryNotReady

I am trying to understand how to run django tests in parallel with in memory sqlite3.
I have django app with that structure:
gbook
order
...
tests
__init__.py
test_a1.py
test_b1.py
utils.py
test_a1.py and test_b1.py contains same code:
import time
from order import models
from .utils import BackendTestCase
class ATestCase(BackendTestCase):
def test_a(self):
time.sleep(1)
a = models.City.objects.count()
self.assertEqual(a, a)
class BTestCase(BackendTestCase):
def test_b(self):
time.sleep(1)
a = models.City.objects.count()
self.assertEqual(a, a)
utils.py is:
from django.test import TestCase, Client
from order import models
from django.conf import settings
from order.utils import to_hash
class BackendTestCase(TestCase):
fixtures = ['City.json', 'Agency.json']
def setUp(self):
self.client = Client()
self.lang_codes = (i[0] for i in settings.LANGUAGES)
...
settings_test.py:
from .settings import *
DEBUG = False
TEMPLATE_DEBUG = False
STATICFILES_STORAGE = 'django.contrib.staticfiles.storage.StaticFilesStorage'
PASSWORD_HASHERS = ['django.contrib.auth.hashers.MD5PasswordHasher',] # faster
DATABASES['default'] = {
'ENGINE': 'django.db.backends.sqlite3',
}
When I run test in single process, all goes well (about 4 sec):
python.exe manage.py test order --settings=gbook.settings_test
Then I trying to run tests in parallel:
python.exe manage.py test order --settings=gbook.settings_test --parallel=2
I get this trace (console):
Creating test database for alias 'default'...
Cloning test database for alias 'default'...
Cloning test database for alias 'default'...
System check identified no issues (0 silenced).
Process SpawnPoolWorker-2:
Process SpawnPoolWorker-1:
Traceback (most recent call last):
Traceback (most recent call last):
File "C:\python\Python36-32\lib\multiprocessing\process.py", line 258, in _bootstrap
self.run()
File "C:\python\Python36-32\lib\multiprocessing\process.py", line 258, in _bootstrap
self.run()
File "C:\python\Python36-32\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\python\Python36-32\lib\multiprocessing\process.py", line 93, in run
self._target(*self._args, **self._kwargs)
File "C:\python\Python36-32\lib\multiprocessing\pool.py", line 108, in worker
task = get()
File "C:\python\Python36-32\lib\multiprocessing\pool.py", line 108, in worker
task = get()
File "C:\python\Python36-32\lib\multiprocessing\queues.py", line 337, in get
return _ForkingPickler.loads(res)
File "C:\python\Python36-32\lib\multiprocessing\queues.py", line 337, in get
return _ForkingPickler.loads(res)
File "C:\kvk\develop\Python\gbook\order\tests\test_a1.py", line 2, in <module>
from order import models
File "C:\kvk\develop\Python\gbook\order\tests\test_a1.py", line 2, in <module>
from order import models
File "C:\kvk\develop\Python\gbook\order\models.py", line 79, in <module>
class Agency(models.Model):
File "C:\kvk\develop\Python\gbook\order\models.py", line 79, in <module>
class Agency(models.Model):
File "C:\python\venv\gbook\lib\site-packages\django\db\models\base.py", line 110, in __new__
app_config = apps.get_containing_app_config(module)
File "C:\python\venv\gbook\lib\site-packages\django\db\models\base.py", line 110, in __new__
app_config = apps.get_containing_app_config(module)
File "C:\python\venv\gbook\lib\site-packages\django\apps\registry.py", line 247, in get_containing_app_config
self.check_apps_ready()
File "C:\python\venv\gbook\lib\site-packages\django\apps\registry.py", line 247, in get_containing_app_config
self.check_apps_ready()
File "C:\python\venv\gbook\lib\site-packages\django\apps\registry.py", line 125, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
File "C:\python\venv\gbook\lib\site-packages\django\apps\registry.py", line 125, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
From Pycharm trace is different:
...
Traceback (most recent call last):
File "C:\python\Python36-32\lib\unittest\suite.py", line 163, in _handleClassSetUp
setUpClass()
File "C:\python\venv\gbook\lib\site-packages\django\test\testcases.py", line 1036, in setUpClass
'database': db_name,
File "C:\python\venv\gbook\lib\site-packages\django\core\management\__init__.py", line 131, in call_command
return command.execute(*args, **defaults)
File "C:\python\venv\gbook\lib\site-packages\django\core\management\base.py", line 330, in execute
output = self.handle(*args, **options)
File "C:\python\venv\gbook\lib\site-packages\modeltranslation\management\commands\loaddata.py", line 61, in handle
return super(Command, self).handle(*fixture_labels, **options)
File "C:\python\venv\gbook\lib\site-packages\django\core\management\commands\loaddata.py", line 69, in handle
self.loaddata(fixture_labels)
File "C:\python\venv\gbook\lib\site-packages\django\core\management\commands\loaddata.py", line 109, in loaddata
self.load_label(fixture_label)
File "C:\python\venv\gbook\lib\site-packages\django\core\management\commands\loaddata.py", line 175, in load_label
obj.save(using=self.using)
File "C:\python\venv\gbook\lib\site-packages\django\core\serializers\base.py", line 205, in save
models.Model.save_base(self.object, using=using, raw=True, **kwargs)
File "C:\python\venv\gbook\lib\site-packages\django\db\models\base.py", line 838, in save_base
updated = self._save_table(raw, cls, force_insert, force_update, using, update_fields)
File "C:\python\venv\gbook\lib\site-packages\django\db\models\base.py", line 905, in _save_table
forced_update)
File "C:\python\venv\gbook\lib\site-packages\django\db\models\base.py", line 955, in _do_update
return filtered._update(values) > 0
File "C:\python\venv\gbook\lib\site-packages\django\db\models\query.py", line 664, in _update
return query.get_compiler(self.db).execute_sql(CURSOR)
File "C:\python\venv\gbook\lib\site-packages\django\db\models\sql\compiler.py", line 1204, in execute_sql
cursor = super(SQLUpdateCompiler, self).execute_sql(result_type)
File "C:\python\venv\gbook\lib\site-packages\django\db\models\sql\compiler.py", line 899, in execute_sql
raise original_exception
File "C:\python\venv\gbook\lib\site-packages\django\db\models\sql\compiler.py", line 889, in execute_sql
cursor.execute(sql, params)
File "C:\python\venv\gbook\lib\site-packages\django\db\backends\utils.py", line 64, in execute
return self.cursor.execute(sql, params)
File "C:\python\venv\gbook\lib\site-packages\django\db\utils.py", line 94, in __exit__
six.reraise(dj_exc_type, dj_exc_value, traceback)
File "C:\python\venv\gbook\lib\site-packages\django\utils\six.py", line 685, in reraise
raise value.with_traceback(tb)
File "C:\python\venv\gbook\lib\site-packages\django\db\backends\utils.py", line 64, in execute
return self.cursor.execute(sql, params)
File "C:\python\venv\gbook\lib\site-packages\django\db\backends\sqlite3\base.py", line 328, in execute
return Database.Cursor.execute(self, query, params)
django.db.utils.OperationalError: Problem installing fixture 'C:\kvk\develop\Python\gbook\order\fixtures\AirportInfo.json': Could not load order.AirportInfo(pk=2411): no such table: GB_AIRPORT_INFO
It seem like migrations not works for parallel, but why?
Docs says: "--parallel" Runs tests in separate parallel processes. Each process gets its own database. And I do not need to change my code for use it.
Please, help me to understand, what am i doing wrong.
multiprocessing.cpu_count() = 4
Django version 1.11.10
Python 3.6.5

Same issue as above with MacOS and Python 3.8+. You have to explicitly set import multiprocessing; multiprocessing.set_start_method('fork') at the top of your settings.py file. But be sure to understand the side effects before you do!

I ran into a similar issue trying to use the --parallel feature on Windows.
Django's documentation states
This feature isn’t available on Windows. It doesn’t work with the Oracle database backend either.
Running the same command on Linux completed with no issues.

Parallel running is still disabled on Windows as of today. You can track the ticket that keeps progress of this feature here: https://code.djangoproject.com/ticket/31169.
And here's the code block that disables this option on Windows:
def default_test_processes():
"""Default number of test processes when using the --parallel option."""
# The current implementation of the parallel test runner requires
# multiprocessing to start subprocesses with fork().
if multiprocessing.get_start_method() != 'fork':
return 1
try:
return int(os.environ['DJANGO_TEST_PROCESSES'])
except KeyError:
return multiprocessing.cpu_count()
Source: https://github.com/django/django/blob/59b4e99dd00b9c36d56055b889f96885995e4240/django/test/runner.py#L286-L295

In reply to #Menth, this is how I enable for testing only:
# near the top of settings.py
if "test" in sys.argv[1:]:
import multiprocessing
logging.info("Using multiproc for testing.")
multiprocessing.set_start_method("fork")

ImportError: No module named google.appengine.api

when I migrate my database, i see that error.
i did init. (python manager.py db init)
and. migrate (python manager.py db migrate)
i use google app engine, flask, mac, python
what should i do?
$ python manager.py db migrate
/Library/Python/2.7/site-packages/flask_sqlalchemy/__init__.py:839: FSADeprecationWarning: SQLALCHEMY_TRACK_MODIFICATIONS adds significant overhead and will be disabled by default in the future. Set it to True or False to suppress this warning.
'SQLALCHEMY_TRACK_MODIFICATIONS adds significant overhead and '
Traceback (most recent call last):
File "manager.py", line 3, in <module>
manager.run()
File "/Library/Python/2.7/site-packages/flask_script/__init__.py", line 412, in run
result = self.handle(sys.argv[0], sys.argv[1:])
File "/Library/Python/2.7/site-packages/flask_script/__init__.py", line 383, in handle
res = handle(*args, **config)
File "/Library/Python/2.7/site-packages/flask_script/commands.py", line 216, in __call__
return self.run(*args, **kwargs)
File "/Library/Python/2.7/site-packages/flask_migrate/__init__.py", line 182, in migrate
version_path=version_path, rev_id=rev_id)
File "/Library/Python/2.7/site-packages/alembic/command.py", line 176, in revision
script_directory.run_env()
File "/Library/Python/2.7/site-packages/alembic/script/base.py", line 421, in run_env
util.load_python_file(self.dir, 'env.py')
File "/Library/Python/2.7/site-packages/alembic/util/pyfiles.py", line 93, in load_python_file
module = load_module_py(module_id, path)
File "/Library/Python/2.7/site-packages/alembic/util/compat.py", line 75, in load_module_py
mod = imp.load_source(module_id, path, fp)
File "migrations/env.py", line 87, in <module>
run_migrations_online()
File "migrations/env.py", line 70, in run_migrations_online
poolclass=pool.NullPool)
File "/Library/Python/2.7/site-packages/sqlalchemy/engine/__init__.py", line 428, in engine_from_config
return create_engine(url, **options)
File "/Library/Python/2.7/site-packages/sqlalchemy/engine/__init__.py", line 387, in create_engine
return strategy.create(*args, **kwargs)
File "/Library/Python/2.7/site-packages/sqlalchemy/engine/strategies.py", line 80, in create
dbapi = dialect_cls.dbapi(**dbapi_args)
File "/Library/Python/2.7/site-packages/sqlalchemy/dialects/mysql/gaerdbms.py", line 68, in dbapi
from google.appengine.api import apiproxy_stub_map
ImportError: No module named google.appengine.api

As the Error was suggesting, your runtime environment couldn't find the module. Either install it in your virtualenvironment or globally by following the instruction:
https://cloud.google.com/appengine/downloads#Google_App_Engine_SDK_for_Python