I am trying to use celery==3.1.25 (Python 2.7) to run some expensive tasks separately to my main flask app. However, when I start the celery worker using celery -A run.celery worker --loglevel=info the process fails with:
[ERROR/MainProcess] Process 'Worker' exited with 'exitcode 1'
Here is my file structure:
app/
__init__.py
celery_functions.py
routes.py
...
run.py
run.py:
from app import create_app
from app.config import Config
from celery import Celery
app = create_app('default')
app.app_context().push()
from app.routes import *
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000)
__init__.py:
from flask import Flask
from flask_bootstrap import Bootstrap
from config import Config
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
bootstrap = Bootstrap(app)
return app
celery_functions.py:
import celery
#celery.task(name='celery_functions.archive_repo')
def archive_repo():
# do something
routes.py:
from celery_functions import *
#app.route('/archive', methods=['GET', 'POST'])
#login_required
def archive():
archive_repo.delay()
return ''
Related
I am trying to find a way to structure my code such that I can view logs in the production server hosted on Heroku.
Every tutorial I found seems to do logging like this:
How to show stdout logs in Heroku using Flask?
However, I am using the application factory pattern which utilizes Blueprints. As such here are some sample:
main.py
from app import create_app
app = create_app()
if __name__ == "__main__":
app.run()
app/_ _ init _ _.py
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
...
db = SQLAlchemy()
...
def create_app():
app = Flask(__name__)
...
db.init_app(app)
from .routes.demo_blueprint import demo_blueprint
...
# Register the routes to the Flask object
app.register_blueprint(demo_blueprint)
...
app/routes/demo_blueprint.py
from app import db
...
demo_blueprint = Blueprint('demo_blueprint', __name__)
#demo_blueprint.route('/demo', methods=['GET'])
...
In order to perform logging at the blueprint level, I would need to import app from main.py. However, this would cause an import error since __init__.py imports the blueprint before app is created. I was wondering if there were any work arounds for this.
Turns out it was a simple fix. To access the application context in the Blueprint, just use current_app. Following the example:
How to show stdout logs in Heroku using Flask?
main.py
from app import create_app
app = create_app()
if __name__ == "__main__":
app.run()
app/_ _ init _ _.py
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
...
db = SQLAlchemy()
...
def create_app():
app = Flask(__name__)
if __name__ != '__main__':
gunicorn_logger = logging.getLogger('gunicorn.error')
app.logger.handlers = gunicorn_logger.handlers
app.logger.setLevel(gunicorn_logger.level)
...
db.init_app(app)
from .routes.demo_blueprint import demo_blueprint
...
# Register the routes to the Flask object
app.register_blueprint(demo_blueprint)
...
app/routes/demo_blueprint.py
from flask import ***current_user***
from app import db
...
demo_blueprint = Blueprint('demo_blueprint', __name__)
#demo_blueprint.route('/demo', methods=['GET'])
def demo():
current_app.logger.debug('debug message: %s', 'test')
...
I am trying to organize project structure for Flask+Celery app.
When i worked in single file all works fine.
But when i distributed code in modules, calling test_task.apply_async() is locking flask.
My project structure:
web_spider/
app/
__init__.py
rest/
__init__.py
views/
__init__.py
test_view.py
flask_app.py
task_runner/
__init__.py
celery_app.py
tasks.py
requirements.txt
test_view.py
import flask
from app.task_runner.tasks import test_task
api_test_view = flask.Blueprint('api_test_view', __name__)
#api_test_view.route('/')
def test_view():
test_task.apply_async() #lock there
return 'Hello, World!'
flask_app.py
import flask
from app.rest.views.api_test_view import test_view
flask_app = flask.Flask(__name__)
flask_app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'
flask_app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0'
flask_app.register_blueprint(test_view)
if __name__ == '__main__':
flask_app.run(debug=True)
celery_app.py
from app.rest.flask_app import flask_app
import celery
celery_app = celery.Celery(flask_app.name, broker=flask_app.config['CELERY_BROKER_URL'])
celery_app.conf.update(flask_app.config)
tasks.py
from celery import shared_task
#shared_task
def test_task():
return 1 + 1
Probably, you don't load celery_app and shared_task can't find the Celery app to work with. Add to your web_spider/app/__init__.py or to web_spider/app/task_runner/__init__.py:
from app.task_runner.celery_app import celery_app
__all__ = ('celery_app',)
It's documented at https://docs.celeryproject.org/en/stable/django/first-steps-with-django.html#django-first-steps search for shared_task.
I tried to add celery to my existing flask project. After adding, I got an "working outside of application context" error while running. It seems that the celery worker lacks of my application context. But I am not sure where to pass the applicaiton context to celery worker in this case.
Here is my current structure (I tried to follow a factory pattern with blueprints and api documentions):
-run.py
-app
-module1
-controller.py
-model.py
-service.py
-__init__.py
-config.py
For the init.py
# __init__.py
import os
from flask import Flask
from flask_sqlalchemy import SQLAlchemy
from flask_cors import CORS
from app.config import Config
from flask_restplus import Api
from celery import Celery
cors = CORS()
db = SQLAlchemy()
api = Api()
celery = Celery(__name__, broker=Config.CELERY_BROKER_URL, include=["app.module1.service"])
def create_app(config_class = Config):
app = Flask(__name__, static_url_path='')
app.config.from_object(Config)
cors.init_app(app)
db.init_app(app)
api.init_app(app=app)
celery.conf.update(app.config)
from app.module1.controller import blueprint
from app.module1.controller import ns
app.register_blueprint(blueprint)
api.add_namespace(ns)
return app
For the run.py
from app import create_app
app = create_app()
if __name__ == '__main__':
app.run(threaded=True, debug=True)
For the service.py
from app import db, celery
#celery.task(bind=True)
def service1(self):
# do somethigng & return
For the controller.py
from flask import Blueprint
from flask_restplus import Api, Resouce
blueprint = Blueprint('service', __name__)
apis = Api(app = blueprint)
ns = apis.namespace('service', 'service description')
#ns.route("/")
class SomeList(Resource):
def get(self):
service1.apply_async()
# return
I think the confusion is based on the fact that you are trying to "pass" an application context to the Celery worker. In reality the Flask process cannot pass a context to the worker because they are different processes. The Celery worker process needs to create its own Flask application instance by calling create_app() so that it can push its own app contexts when needed.
So for example, in your service1 task:
from app import db, celery, create_app
#celery.task(bind=True)
def service1(self):
app = create_app()
with app.app_context():
# do somethigng & return
To make this a bit more efficient, you can create a single global app that is shared by all your tasks:
from app import db, celery, create_app
app = create_app()
#celery.task(bind=True)
def service1(self):
with app.app_context():
# do somethigng & return
I'm making an app in FLASK and I've incorporated Celery into it. However, I have to run the app via the terminal if I want the Celery worker to work as well. (celery -A app.celery worker). I tried running it from the main run.py file as follows
init.py
from flask import Flask
from flask.ext.sqlalchemy import SQLAlchemy
from flask.ext.mail import Mail
from celery import Celery
from kombu import serialization
app = Flask(__name__)
app.config.from_object('config')
db = SQLAlchemy(app)
mail = Mail(app)
app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0'
app.config['CELERY_ACCEPT_CONTENT'] = ['json']
app.config['CELERY_TASK_SERIALIZER'] = 'json'
app.config['CELERY_RESULT_SERIALIZER'] = 'json'
celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
serialization.registry._decoders.pop("application/x-python-serialize")
from app import views
and run.py
#!flask/bin/python
from __future__ import absolute_import, unicode_literals
from app import app
# app.run(debug=True, port=9001)
from celery import current_app
from celery.bin import worker
app = current_app._get_current_object()
worker = worker.worker(app=app)
options = {
'broker': app.config['CELERY_BROKER_URL'],
'loglevel': 'INFO',
'traceback': True,
}
worker.run(**options)
But this gives the error AttributeError: 'Celery' object has no attribute 'config'
Any pointers as to what Im doing wrong would be much appreciated.
Your run.py should be:
#!flask/bin/python
from __future__ import absolute_import, unicode_literals
from app import app
# app.run(debug=True, port=9001)
from celery import current_app
from celery.bin import worker
application = current_app._get_current_object()
worker = worker.worker(app=application)
options = {
'broker': app.config['CELERY_BROKER_URL'],
'loglevel': 'INFO',
'traceback': True,
}
worker.run(**options)
I'm using Flask-Script and I've got a manage.py that looks like this.
from mypackage import create_app
from flask.ext.script import Manager
app = create_app()
manager = Manager(app)
if __name__ == '__main__':
manager.run()
I'll start my app with python manage.py runserver --debug
From within manage.py, how can I find out that runserver was called with --debug?
I've tried app.debug but that returns False and manager.get_options() returns an empty list.
The code you've provided is fine-- here's a mypackage.py file to demonstrate that:
from flask import Flask, Blueprint, current_app
test = Blueprint('user', __name__, url_prefix='/')
#test.route('/')
def home():
return str(current_app.debug)
def create_app():
app = Flask(__name__)
app.register_blueprint(test)
return app
Adding --debug reflects accurately (True) when you access the index page.