When integrating celery with a Flask app does celery need to be aware of the Flask application context?
Can I just do something like:
import celery from Celery
celery = Celery()
#task
def mytask():
Or do I have to do this:
def make_celery(app=None):
app = app or create_app(os.getenv('FLASK_CONFIG') or 'default')
celery = Celery(__name__, broker=app.config.CELERY_BROKER_URL)
celery.conf.update(app.conf)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
And then run celery = make_celery(app)?
Celery tasks only need to be aware of the application context if you're doing things that requires it (database queries, etc). Otherwise you can just use Celery as is.
Related
Very simple question, I hope. I have a flask service that needs to listen to a subscription. I have all the code written to listen to the subscription and run some code when triggered.
The flask app code is:
from flask import Flask, jsonify
import logging
from celery.bin.worker import worker
from celery import shared_task
from proj.celery_config import make_celery, get_options
from proj.config import Config
app = Flask(__name__)
app.config.from_object(Config)
celery = make_celery(app)
options = get_options(app)
worker = worker(app)
#shared_task()
def run_listener():
listen()
#app.route('/actuator/health')
def health_check():
return jsonify({'status': 'UP'})
#app.route('/')
def hello_world():
"""
A simple test route for verifying flask is working.
:return: string containing "Hello, World!"
"""
logger.info("Hello world called!")
return 'Hello, World!'
def main():
return app
The code for make_celery and get_options is:
from celery import Celery
def make_celery(app):
celery = Celery(
app.import_name,
backend=app.config['CELERY_BROKER_URL'],
broker=app.config['CELERY_BROKER_URL']
)
celery.conf.update(app.config)
class ContextTask(celery.Task):
def __call__(self, *args, **kwargs):
with app.app_context():
return self.run(*args, **kwargs)
celery.Task = ContextTask
return celery
def get_options(app):
return {
'broker': app.config['CELERY_BROKER_URL'],
'traceback': True,
'loglevel': 'info',
'queues': 'q1'
}
I have not set up all the endpoints yet, but here is something very important here: the task run_listener needs to work on startup without running the worker outside the app. Meaning I cannot do a celery -A tasks .... I just want it to run whenever the flask app is run. Is there a way to do this?
I have tried running worker.run(**options) but I get an error, no matter if I use the flask app or the celery app when initializing the worker object. Am I missing something?
i have a simple flask restful API where I want to execute the request as Celery task since some endpoints need a lot of execution time.
main.py:
from flask import Flask
from flask_restful import Api
from flask_celery import make_celery
app = Flask(__name__)
app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0',
app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0'
celery = make_celery(app)
api = Api(app)
api.add_resource(someResource, '/someendpoint/')
if __name__ == '__main__':
app.run(debug=True)
with make_celery.py:
from celery import Celery
def make_celery(app):
celery = Celery(
app.import_name,
backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL']
)
celery.conf.update(app.config)
class ContextTask(celery.Task):
def __call__(self, *args, **kwargs):
with app.app_context():
return self.run(*args, **kwargs)
celery.Task = ContextTask
return celery
I now want to define that the Resource I defined in resource.py is a celery task:
class Cost(Resource):
def get(self):
result = some_code
return result
what is the most convenient way to make the get method a celery task here?
Thanks a lot!
The closest working answer is that:
How to use Flask-SQLAlchemy in a Celery task
I aim this question at someone who is actually using python, flask, factory pattern and celery. Python is 2.7, others are latest version today.
I am trying to avoid circular dependencies and do it flasky way,
I have gone through 10 pages of google and all possible solutions and I could not solve this.
~/git/project celery -A app worker --loglevel=info
Celery is still connecting to:
[2017-11-10 16:08:12,208: ERROR/MainProcess] consumer: Cannot connect to amqp://guest:**#127.0.0.1:5672//: [Errno 111] Connection refused.
Trying again in 32.00 seconds...
Despite various attempts to start the app
app/extensions.py
from flask.ext.marshmallow import Marshmallow
from flask.ext.sqlalchemy import SQLAlchemy
from flask_mail import Mail
import flask
from celery import Celery
class FlaskCelery(Celery):
def __init__(self, *args, **kwargs):
super(FlaskCelery, self).__init__(*args, **kwargs)
self.patch_task()
if 'app' in kwargs:
self.init_app(kwargs['app'])
def patch_task(self):
TaskBase = self.Task
_celery = self
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
if flask.has_app_context():
return TaskBase.__call__(self, *args, **kwargs)
else:
with _celery.app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
self.Task = ContextTask
def init_app(self, app):
self.app = app
self.config_from_object(app.config)
print self._conf['broker_url']
celery = FlaskCelery()
db = SQLAlchemy()
ma = Marshmallow()
mail = Mail()
!!!!! print self._conf['broker_url']: redis://localhost:6379/0
app/init.py
from flask import Flask, render_template
from app.extensions import db, ma, mail, celery
from celerytasks import save_mailbox_items, sumf
from config import config
from utils import encoding_utils
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
# SQLAlchemy configuration
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql://...'
# Celery configuration
app.config['BROKER_URL'] = 'redis://localhost:6379/0'
app.config['broker_url'] = 'redis://localhost:6379/0'
app.config['celery_broker_url'] = 'redis://localhost:6379/0'
app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0'
register_extensions(app)
return app
def register_extensions(app):
db.init_app(app)
with app.app_context():
db.create_all()
ma.init_app(app)
mail.init_app(app)
celery.init_app(app)
from .api_v1 import api as api_v1_blueprint
app.register_blueprint(api_v1_blueprint, url_prefix='/api/v1')
#app.route('/', methods=['GET'])
def index():
return render_template('index.html')
./manager.py
import os
from flask.ext.script import Manager
from app import create_app
app = create_app(os.getenv('APP_CONFIG', 'default'))
manager = Manager(app)
#manager.shell
def make_shell_context():
return dict(app=app)
if __name__ == '__main__':
manager.run()
When you run your celery worker, it will use the one created with
celery = FlaskCelery()
But because it does not receive a Flask app as an argument, you never go through self.init_app(kwargs['app']) and thus it will use the default configuration.
Several options are possible to fix this here:
instantiate a FlaskCelery object and passing a Flask instance when doing so
in your FlaskCelery class, instantiate a flask app in your init function if no argument is passed in the constructor.
For the latest point, this would give something like
class FlaskCelery(Celery):
def __init__(self, *args, **kwargs):
super(FlaskCelery, self).__init__(*args, **kwargs)
self.patch_task()
if 'app' in kwargs:
self.init_app(kwargs['app'])
else:
self.init_app(create_app(os.getenv('APP_CONFIG', 'default')))
def patch_task(self):
TaskBase = self.Task
_celery = self
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
if flask.has_app_context():
return TaskBase.__call__(self, *args, **kwargs)
else:
with _celery.app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
self.Task = ContextTask
def init_app(self, app):
self.app = app
self.config_from_object(app.config)
print self._conf['broker_url']
I'm using Flask-SQLAlchemy, Celery and uWSGI.
I know that Flask-SQLAlchemy automatically manages the session for you. I'm not sure how this works with Celery workers, but it seems that when I run a task a second time, I get the following error: DatabaseError: (psycopg2.DatabaseError) server closed the connection unexpectedly.
Here's how I create the app context and celery tasks:
def make_celery(app):
celery = Celery(
app.import_name,
backend=app.config['CELERY_BACKEND'],
broker=app.config['CELERY_BROKER_URL'],
)
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
It seems that maybe the workers are using the same database connection and after a task completes that connection is not replenished?
It may be related to the following question?
I'm not sure how to correctly setup the workers or celery so that they're using new connections to the database..
Okay. I figured it out, for every process that's using an application context, you must use a new application context. Before, in my app/__init__.py I was simply creating the application globally like so:
from flask import Flask
app = Flask(__name__)
I then changed my app to use create_app like in this pattern
Now, my tasks.py looks like this:
from myapp import create_app
from celery import Celery
def make_celery(app=None):
app = app or create_app()
celery = Celery(
app.import_name,
backend=app.config['CELERY_BACKEND'],
broker=app.config['CELERY_BROKER_URL'],
)
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
celery = make_celery()
Make sure in your create_app you are calling db.init_app(app).
I'm writing web application with Flask and SQLAlchemy. My program needs to process some stuff in the background and then mark this stuff as processed in the database. Using standard Flask/Celery example, I have something like this:
from flask import Flask
from celery import Celery
def make_celery(app):
celery = Celery(app.import_name, broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
app = Flask(__name__)
celery = make_celery(app)
class Stuff(db.Model):
id = db.Column(db.Integer, primary_key=True)
processed = db.Column(db.Boolean)
#celery.task()
def process_stuff(stuff):
# process stuff here
stuff.processed = True
db.session.commit()
#app.route("/process_stuff/<id>")
def do_process_stuff(id):
stuff = Stuff.query.get_or_404(id)
process_stuff.delay(stuff)
return redirect(url_for("now_wait"))
I can access my database from process_stuff (e.g. submit queries like Stuff.query.get(some_id) work), but db.session.commit() do nothing: my stuff record is not updated. According to Celery worker log, commit occures but nothing changes in the database. Is there something wrong with my db.session.commit()? Is it possible to make such commit somehow?
Okay, I got it. stuff passed to process_stuff() is not attached to db.session. I have to make explicit request in process_stuff() to get the right stuff object like this:
#celery.task()
def process_stuff(stuff):
# process stuff here
my_stuff = Stuff.query.get(stuff.id)
my_stuff.processed = True
db.session.commit()
Now it works.