I use flask app factory:
app.py
from flask import Flask, g
from . import commands
from .database import DatabaseManager
from .extensions import db
from .settings import ProdConfig
def create_app(config_object=ProdConfig):
'''Flask app factory'''
app = Flask(__name__)
app.url_map.strict_slashes = False
app.config.from_object(config_object)
register_extensions(app)
register_blueprints(app)
register_commands(app)
#app.before_request
def create_context():
g.db = db.database
g.db_manager = DatabaseManager()
return app
# Another functions
# ...
def register_commands(app):
app.cli.add_command(commands.root_group)
Then i wrote command in:
commands.py
from flask import g
from flask.cli import AppGroup
root_group = AppGroup('myapp')
#root_group.command('test')
def test():
print(g.db)
Then when i run command flask myapp test i get AttributeError on g.db. So somehow flask.g dont receive attrs g.db and g.db_manager initialized in before_request. As i understand it have to receive, because app_context must be pushed when cli command based on AppGroup runs. How to fix this?
From testing i understand that when CLI command executes, Flask not push Request context, so #before_request hook not executes. I dont find good way to push Request context in my cli command and make this hack:
#root_group.command()
def test():
exec_before_request_hooks()
print(g.db_manager)
# HACK: run #app.before_request for CLI command
def exec_before_request_hooks():
current_app.test_client().get()
If anybody know better solution, i will be very appreciate to hear it)
Related
I am trying to add some monitoring to a simple REST web service with flask and mongoengine and have come across what I think is a lack of understanding on my part of how imports and mongoengine is working in flask applications.
I'm following pymongo's documentation on monitoring : https://pymongo.readthedocs.io/en/3.7.2/api/pymongo/monitoring.html
I defined the following CommandListener in a separate file:
import logging
from pymongo import monitoring
log = logging.getLogger('my_logger')
class CommandLogger(monitoring.CommandListener):
def started(self, event):
log.debug("Command {0.command_name} with request id "
"{0.request_id} started on server "
"{0.connection_id}".format(event))
monitoring.register(CommandLogger())
I made an application_builder.py file to create my flask App, code looks something like this:
from flask_restful import Api
from flask import Flask
from command_logger import CommandLogger # <----
from db import initialize_db
from routes import initialize_routes
def create_app():
app = Flask(__name__)
api = Api(app)
initialize_db(app)
initialize_routes(api)
return app
The monitoring only seems to works if I import : CommandLogger in application_builder.py. I'd like to understand what is going on here, how does the import affect the monitoring registration?
Also I'd like to extract monitoring.register(CommandLogger()) as a function and call it at a latter stage in my code something like def register(): monitoring.register(CommandLogger())
But this doesn't seem to work, "registration' only works when it is in the same file as the CommandLogger class...
From the MongoEngine's doc, it seems important that the listener gets registered before connecting mongoengine
To use pymongo.monitoring with MongoEngine, you need to make sure that
you are registering the listeners before establishing the database
connection (i.e calling connect)
This worked for me. I'm just initializing/registering it the same way as I did other modules to avoid circular imports.
# admin/logger.py
import logging
from pymongo import monitoring
log = logging.getLogger()
log.setLevel(logging.DEBUG)
logging.basicConfig(level=logging.DEBUG)
class CommandLogger(monitoring.CommandListener):
# def methods...
class ServerLogger(monitoring.ServerListener):
# def methods
class HeartbeatLogger(monitoring.ServerHeartbeatListener):
# def methods
def initialize_logger():
monitoring.register(CommandLogger())
monitoring.register(ServerLogger())
monitoring.register(HeartbeatLogger())
monitoring.register(TopologyLogger())
# /app.py
from flask import Flask
from admin.toolbar import initialize_debugtoolbar
from admin.admin import initialize_admin
from admin.views import initialize_views
from admin.logger import initialize_logger
from database.db import initialize_db
from flask_restful import Api
from resources.errors import errors
app = Flask(__name__)
# imports requiring app
from resources.routes import initialize_routes
api = Api(app, errors=errors)
# Logger before db
initialize_logger()
# Database and Routes
initialize_db(app)
initialize_routes(api)
# Admin and Development
initialize_admin(app)
initialize_views()
initialize_debugtoolbar(app)
# /run.py
from app import app
app.run(debug=True)
then in any module...
from admin.logger import log
from db.models import User
# inside some class/view/queryset or however your objects are written...
log.info('Saving an item through MongoEngine...')
User(name='Foo').save()
What I'm trying to figure out now is how to integrate Flask DebuggerToolbar's Logging panel with the monitoring messages from these listeners...
I'm using a application factory pattern, and when I tried to run my test, I get "Attempted to generate a URL without the application context being". I created a fixture to create the application:
#pytest.fixture
def app():
yield create_app()
but when I run my test
def test_get_activation_link(self, app):
user = User()
user.set_password(self.VALID_PASS)
generated_link = user.get_activation_link()
I get the above error (from the line of code url = url_for("auth.activate")). I'm also trying to figure out to have the app creation run for every test, without having to import it into every test, but I can't seem to find if that's possible.
This works for my app
import pytest
from xxx import create_app
#pytest.fixture
def client():
app = create_app()
app.config['TESTING'] = True
with app.app_context():
with app.test_client() as client:
yield client
def smoke_test_homepage(client):
"""basic tests to make sure test setup works"""
rv = client.get("/")
assert b"Login" in rv.data
So, you missed the application context.
At this year's Flaskcon there was an excellent talk about the Flask context - I highly recommend this video.
https://www.youtube.com/watch?v=fq8y-9UHjyk
The Case:
We have some time-consuming functional/integration tests that utilize Flask's current_app for configuration (global variables etc.) and some logging.
We are trying to distribute and parallelize those tests on a cluster (for the moment a local "cluster" created from Dask's Docker image.).
The Issue(s?):
Let's assume the following example:
A time-consuming function:
def will_take_my_time(n)
# Add the 'TAKE_YOUR_TIME' in the config in how many seconds you want
time.sleep(current_app.config['TAKE_YOUR_TIME'])
return n
A time-consuming test:
def need_my_time_test(counter=None):
print(f"Test No. {will_take_my_time(counter)}")
A Flask CLI command that creates a Dask Client to connect to the cluster and execute 10 tests of need_my_time_test:
#app.cli.command()
def itests(extended):
with Client(processes=False) as dask_client:
futures = dask_client.map(need_my_time_test, range(10))
print(f"Futures: {futures}")
print(f"Gathered: {dask_client.gather(futures)}")
EDIT: For convenience let's add an application factory for an easier reproducible example:
def create_app():
app = Flask(__name__)
app.config.from_mapping(
SECRET_KEY='dev',
DEBUG=True,
)
#app.route('/hello')
def hello():
return 'Hello, World!'
#app.cli.command()
def itests(extended):
with Client(processes=False) as dask_client:
futures = dask_client.map(need_my_time_test, range(10))
print(f"Futures: {futures}")
print(f"Gathered: {dask_client.gather(futures)}")
Using the above with flask itests, we are running into the following error (described here):
RuntimeError: Working outside of application context.
This typically means that you attempted to use functionality that
needed to interface with the current application object in some way.
To solve this, set up an application context with app.app_context().
We have tried:
Pushing the app_context (app.app_context().push()) on the app singleton creation.
Using with current_app.app_context(): on the CLI command and some of the functions that use the current_app.
Sending the app_context through a Dask Variable but it cannot serialize the context.
With no avail.
The questions:
Are there any suggestions on what should we try (containing the "where" will be highly appreciated)?
Is there something that we tried correct but misused and we should retry it differently?
When using current_app proxy, it is assumed that the Flask app is created in the same process that the proxy is used.
This is not the situation when tasks submitted to the workers are run.
The tasks are executed isolated away from the Flask app created in the process that submitted the tasks.
In the task, define the flask app and provide the application context there.
import time
from flask import Flask
from dask.distributed import Client
def _create_app():
app = Flask(__name__)
app.config.from_mapping(
SECRET_KEY='dev',
DEBUG=True,
TAKE_YOUR_TIME=0.2
)
return app
def will_take_my_time(n):
# Add the 'TAKE_YOUR_TIME' in the config in how many seconds you want
app = _create_app()
with app.app_context():
time.sleep(app.config['TAKE_YOUR_TIME'])
return n
def need_my_time_test(counter=None):
print(f"Test No. {will_take_my_time(counter)}")
def create_app():
app = _create_app()
#app.route('/hello')
def hello():
return 'Hello, World!'
#app.cli.command()
def itests():
with Client(processes=False) as dask_client:
futures = dask_client.map(need_my_time_test, range(10))
print(f"Futures: {futures}")
print(f"Gathered: {dask_client.gather(futures)}")
return app
app = create_app()
I've found similar questions, but they seem to only cover mocking MongoDB and don't mention Flask.
I have a Flask app and I'm trying to unit test it with PyTest (including PyTest-Mongo and PyTest-Flask). However, before I can even get to the point of writing any tests, my test script crashes. The crash happens when importing the script with my Flash app: It's trying to create the PyMongo object without a url.
My question is: How can I ensure that PyMongo is mocked correctly at this point? According to the PyTest-Mongo documentation, the MongoDB test fixture should be passed to each of the test functions, but that doesn't help me if it's crashing on import.
test_app.py:
import pytest
import pytest_mongodb
from app import app
#pytest.fixture
def client():
app.config['TESTING'] = True
return client
app.py:
import ...
app = Flask(__name__)
app.config["MONGO_DBNAME"] = os.environ.get('DB_NAME')
app.config["MONGO_URI"] = os.environ.get('MONGO_URI')
app.secret_key = os.environ.get('SECRET')
mongo = PyMongo(app)
...
if __name__ == '__main__':
app.run(host=os.environ.get('IP'),
port=int(os.environ.get('PORT')),
debug=False)
we can wrap app and mongo in a function
This works because mongo is used as a local variable.
app.py
from flask import Flask
from flask_pymongo import PyMongo
def get_app_with_config(config):
app = Flask(__name__)
app.config.from_object(config)
mongo = PyMongo(app)
#app.route("/")
def index():
pass
.
.
return app, mongo
then we can create a test file and an application execution file with different databases:
test_app.py
from app import get_app_with_config
from config import TestConfig
app, mongo = get_app_with_config(TestConfig)
run.py
from app import get_app_with_config
from config import RunConfig
app, mongo = get_app_with_config(RunConfig)
if __name__ == '__main__':
app.run(port=8000)
Sample of config.py file:
class RunConfig:
MONGO_HOST = '192.168.1.37'
MONGO_PORT = 27017
MONGO_DBNAME = 'my_database'
MONGO_URI = f"mongodb://{MONGO_HOST}:{MONGO_PORT}/{MONGO_DBNAME}"
class TestConfig:
MONGO_HOST = '192.168.1.37'
MONGO_PORT = 27017
MONGO_DBNAME = 'my_database_test'
MONGO_URI = f"mongodb://{MONGO_HOST}:{MONGO_PORT}/{MONGO_DBNAME}"
TESTING = True
Needed a quick fix so I edited app.py so that it only hard-fails if PyMongo doesn't initialise when the file is executed (i.e. it ignores PyMongo's failed initialisation when running unit-tests.)
app = Flask(__name__)
app.config["MONGO_DBNAME"] = os.environ.get('DB_NAME')
app.config["MONGO_URI"] = os.environ.get('MONGO_URI')
app.secret_key = os.environ.get('SECRET')
try:
mongodb = PyMongo(app).db
except ValueError:
"""We don't provide a URI when running unit tests, so PyMongo will fail to initialize.
This is okay because we replace it with a version for testing anyway. """
print('PyMongo not initialized!')
mongodb = None
.
.
.
if __name__ == '__main__':
if not mongodb:
print('Cannot run. PyMongo failed to initialize. Double check environment variables.')
exit(1)
app.run(host=os.environ.get('IP'),
port=int(os.environ.get('PORT')),
debug=False)
In my tests file, I just assign the mocked mongoDB client to the app in the tests that need it. Definitely not the ideal solution.
def test_redacted(client, mongodb):
app.mongodb = mongodb
...
I have been trying to follow the tutorials to get flask apps to run on Heroku, like this one: https://dev.to/emcain/how-to-set-up-a-twitter-bot-with-python-and-heroku-1n39.
They all tell you to put this in your code in a file server.py:
from flask import Flask
app = Flask(__name__)
app.run(host='0.0.0.0')
And then run the app via the following command:
python3 server.py
But the tutorials don't explain how to connect the actual function you want to run using the app. In my case, I have a File testbot.py that has the function test(arg1) that contains the code I want to execute:
def test(arg1):
while(1):
#do stuff with arg1 on twitter
I want to do something like this:
from flask import Flask
from testbot import test
from threading import Thread
app = Flask(__name__)
app.addfunction(test(arg1='hardcodedparameter'))
app.run(host='0.0.0.0')
So that when the app runs my test() function executes with the argument. Right now my server is starting, but nothing is happening.
Am I thinking about this correctly?
*Edit: I got it working with the solution, so my server.py now looks like this:
from flask import Flask
from testbot import test
def main_process():
test("hardcodeparam")
app = Flask(__name__)
Thread(target=main_process).start()
app.run(debug=True,host='0.0.0.0')
And now test runs as expected.
Before app.run, register the function with a path, e.g.
#app.route('/')
def test(): # no argument
... do one iteration
return 'ok'
Then visiting the URL will trigger the function. Sites such as https://cron-job.org/ can automate that visiting on a regular basis for free, as suggested here.
If the regular intervals aren't good enough, then you could try:
#app.route('/')
def index(): # no argument
return 'ok'
def test():
while True:
# do stuff
from threading import Thread
Thread(target=test).start()
app.run(...)
You will probably still need to have a job regularly visiting the URL so that Heroku sees that the server is alive and in use.