I am trying to organize project structure for Flask+Celery app.
When i worked in single file all works fine.
But when i distributed code in modules, calling test_task.apply_async() is locking flask.
My project structure:
web_spider/
app/
__init__.py
rest/
__init__.py
views/
__init__.py
test_view.py
flask_app.py
task_runner/
__init__.py
celery_app.py
tasks.py
requirements.txt
test_view.py
import flask
from app.task_runner.tasks import test_task
api_test_view = flask.Blueprint('api_test_view', __name__)
#api_test_view.route('/')
def test_view():
test_task.apply_async() #lock there
return 'Hello, World!'
flask_app.py
import flask
from app.rest.views.api_test_view import test_view
flask_app = flask.Flask(__name__)
flask_app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'
flask_app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0'
flask_app.register_blueprint(test_view)
if __name__ == '__main__':
flask_app.run(debug=True)
celery_app.py
from app.rest.flask_app import flask_app
import celery
celery_app = celery.Celery(flask_app.name, broker=flask_app.config['CELERY_BROKER_URL'])
celery_app.conf.update(flask_app.config)
tasks.py
from celery import shared_task
#shared_task
def test_task():
return 1 + 1
Probably, you don't load celery_app and shared_task can't find the Celery app to work with. Add to your web_spider/app/__init__.py or to web_spider/app/task_runner/__init__.py:
from app.task_runner.celery_app import celery_app
__all__ = ('celery_app',)
It's documented at https://docs.celeryproject.org/en/stable/django/first-steps-with-django.html#django-first-steps search for shared_task.
Related
I have a Flask Application with an MVC structure:
my_app
├── server.py
├── requirements.txt
├── models
│ ├── __init__.py
└── model.py
├── controllers
├── __init__.py
├── client_controllers
└──controller.py
└── another_controller.py
└── templates
I use blueprints to split the server code in "controllers" so I have something like this:
server.py:
from flask import Flask
from celery import Celery
from controllers.client_controllers.controller import controller
app = Flask(__name__)
app.secret_key = 'SECRET'
app.register_blueprint(controller)
# Celery Configuration
def make_celery(app):
celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
app.config.update(
CELERY_BROKER_URL='redis://localhost:6379',
CELERY_RESULT_BACKEND='redis://localhost:6379'
)
celery = make_celery(app)
if __name__ == "__main__":
app.run(host='0.0.0.0', debug=True)
controller.py:
from flask import Blueprint, render_template, json, request, redirect, url_for, abort, session
controller = Blueprint('controller', __name__,
template_folder='templates/')
#celery.task()
def add_together(a, b):
return a + b
#controller.route('/add', methods=['GET'])
def add():
result = add_together.delay(23, 42)
result.wait()
return 'Processing'
As you may notice, celery is not imported into the controller, because I don't know how to import the celery instance from server.py into my controller.py without getting an error, I've been trying with:
from ...server import celery
from ..server import celery
...etc
but still failing with errors.
The flask Error RuntimeError: Working outside of application context. happens because you are not in a Flask application_context().
You should use celery shared_task which is what you need given your MVC structure.
celery_flask/
├── celery_tasks
│ ├── app_tasks.py
│ ├── __init__.py
├── celery_worker.py
├── controllers
│ ├── __init__.py
│ ├── some_controller.py
├── __init__.py
└── server.py
Script app_tasks.py
#=====================
# app_tasks.py
#=====================
from __future__ import absolute_import, unicode_literals
from celery import shared_task
#shared_task(name='celery_tasks.add_together')
def add_together(x, y):
return x + y
The #shared_task decorator returns a proxy that always points to the active Celery instances:
>>> from celery import Celery, shared_task
>>> #shared_task
... def add_together(x, y):
... return x + y
...
>>> app1 = Celery(broker='amqp://')
>>> add_together.app is app1
True
>>> app2 = Celery(broker='redis://')
>>> add_together.app is app2
True
After you define you task you can call them using a reference to a Celery app. This celery app could be part of the flask application_context(). Example:
Script server.py
from __future__ import absolute_import
from flask import Flask
from celery import Celery
from controllers.some_controller import controller
flask_app = Flask(__name__)
flask_app.secret_key = 'SECRET'
flask_app.register_blueprint(controller)
# Celery Configuration
def make_celery( app ):
celery = Celery('flask-celery-app', backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL'],
include=['celery_tasks.app_tasks'])
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
def list_celery_task( ):
from celery.task.control import inspect
i = inspect()
i.registered_tasks()
from itertools import chain
t = set(chain.from_iterable( i.registered_tasks().values() ))
print "registered_tasks={}".format( t )
#======================================
# MAIN
#======================================
flask_app.config.update(
CELERY_BROKER_URL='redis://localhost:6379',
CELERY_RESULT_BACKEND='redis://localhost:6379'
)
celery = make_celery(flask_app)
flask_app.celery = celery
list_celery_task( )
if __name__ == "__main__":
flask_app.run(host='0.0.0.0', debug=True)
Script some_controller.py
#============================
# some_controller.py
#============================
from __future__ import absolute_import
from flask import Blueprint
from flask import current_app
controller = Blueprint('controller', __name__,
template_folder='templates/')
#controller.route('/add', methods=['GET'])
def add():
print "calling add"
result = current_app.celery.send_task('celery_tasks.add_together',args=[12,6])
r = result.get()
print 'Processing is {}'.format( r )
return 'Processing is {}'.format( r )
Finally, start the worker to consume the tasks:
celery -A celery_worker worker --loglevel=DEBUG
Script celery_worker.py
#============================
# celery_worker.py
#============================
from __future__ import absolute_import
from celery import Celery
# Celery Configuration
def make_celery():
celery = Celery('flask-celery-app', backend='redis://localhost:6379',
broker='redis://localhost:6379',
include=['celery_tasks.app_tasks'])
return celery
celery = make_celery()
print "tasks={}".format( celery.tasks.keys() )
One option is to assign celery instance to the app instance and then access it through flask's current_app.
In you server.py, just add:
celery = make_celery(app)
app.celery = celery
Then you can access this in your controller.py:
from flask import current_app
#current_app.celery.task()
def add_together(a, b):
return a + b
I am trying to use celery==3.1.25 (Python 2.7) to run some expensive tasks separately to my main flask app. However, when I start the celery worker using celery -A run.celery worker --loglevel=info the process fails with:
[ERROR/MainProcess] Process 'Worker' exited with 'exitcode 1'
Here is my file structure:
app/
__init__.py
celery_functions.py
routes.py
...
run.py
run.py:
from app import create_app
from app.config import Config
from celery import Celery
app = create_app('default')
app.app_context().push()
from app.routes import *
if __name__ == '__main__':
app.run(host='0.0.0.0', port=5000)
__init__.py:
from flask import Flask
from flask_bootstrap import Bootstrap
from config import Config
def create_app(config_name):
app = Flask(__name__)
app.config.from_object(config[config_name])
bootstrap = Bootstrap(app)
return app
celery_functions.py:
import celery
#celery.task(name='celery_functions.archive_repo')
def archive_repo():
# do something
routes.py:
from celery_functions import *
#app.route('/archive', methods=['GET', 'POST'])
#login_required
def archive():
archive_repo.delay()
return ''
I want to create a background task to update a record on a specific date. I'm using Django and Celery with RabbitMQ.
I've managed to get the task called when the model is saved with this dummy task function:
tasks.py
from __future__ import absolute_import
from celery import Celery
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
app = Celery('tasks', broker='amqp://localhost//')
#app.task(name='news.tasks.update_news_status')
def update_news_status(news_id):
# (I pass the news id and return it, nothing complicated about it)
return news_id
this task is called from my save() method in my models.py
from django.db import models
from celery import current_app
class News(models.model):
(...)
def save(self, *args, **kwargs):
current_app.send_task('news.tasks.update_news_status', args=(self.id,))
super(News, self).save(*args, **kwargs)
Thing is I want to import my News model in tasks.py but if I try to like this:
from .models import News
I get this error :
django.core.exceptions.ImproperlyConfigured: Requested setting
DEFAULT_INDEX_TABLESPACE, but settings are not configured. You must
either define the environment variable DJANGO_SETTINGS_MODULE or call
settings.configure() before accessing settings.
This is how mi celery.py looks like
from __future__ import absolute_import, unicode_literals
from celery import Celery
import os
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myapp.settings')
app = Celery('myapp')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
I have already tried this:
can't import django model into celery task
I have tried to make the import inside the task method Django and Celery, AppRegisteredNotReady exception
I have also tried this Celery - importing models in tasks.py
I also tried to create a utils.py and import it and was not possible.
and ran into different errors but in the end I'm not able to import any module in tasks.py
There might be something wrong with my config but I can't see the error, I followed the steps in The Celery Docs: First steps with Django
Also, my project structure looks like this:
├── myapp
│ ├── __init__.py
├── ├── celery.py
│ ├── settings.py
│ ├── urls.py
│ └── wsgi.py
├── news
│ ├── __init__.py
│ ├── admin.py
│ ├── apps.py
│ ├── tasks.py
│ ├── urls.py
│ ├── models.py
│ ├── views.py
├── manage.py
I'm executing the worker from myapp directory like this:
celery -A news.tasks worker --loglevel=info
What am I missing here? Thanks in advance for your help!
lambda: settings.INSTALLED_APPS
EDIT
After making the changes suggested in comments:
Add this to celery.py
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
and import inside method: tasks.py
from __future__ import absolute_import
from celery import Celery
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
app = Celery('tasks', broker='amqp://localhost//')
#app.task(name='news.tasks.update_news_status')
def update_news_status(news_id):
from .models import News
return news_id
I get the following error:
[2018-07-20 12:24:29,337: ERROR/ForkPoolWorker-1] Task news.tasks.update_news_status[87f9ec92-c260-4ee9-a3bc-5f684c819f79] raised unexpected: ValueError('Attempted relative import in non-package',)
Traceback (most recent call last):
File "/Users/carla/Develop/App/backend/myapp-venv/lib/python2.7/site-packages/celery/app/trace.py", line 382, in trace_task
R = retval = fun(*args, **kwargs)
File "/Users/carla/Develop/App/backend/myapp-venv/lib/python2.7/site-packages/celery/app/trace.py", line 641, in __protected_call__
return self.run(*args, **kwargs)
File "/Users/carla/Develop/App/backend/news/tasks.py", line 12, in update_news_status
from .models import News
ValueError: Attempted relative import in non-package
Ok so for anyone struggling with this... turns out my celery.py wasn't reading env variables from the settings.
After a week and lots of research I realised that Celery is not a process of Django but a process running outside of it (duh), so when I tried to load the settings they were loaded but then I wasn't able to access the env variables I have defined in my .env ( I use the dotenv library). Celery was trying to look up for the env variables in my .bash_profile (of course)
So in the end my solution was to create a helper module in the same directory where my celery.py is defined, called load_env.py with the following
from os.path import dirname, join
import dotenv
def load_env():
"Get the path to the .env file and load it."
project_dir = dirname(dirname(__file__))
dotenv.read_dotenv(join(project_dir, '.env'))
and then on my celery.py (note the last import and first instruction)
from __future__ import absolute_import, unicode_literals
from celery import Celery
from django.conf import settings
import os
from .load_env import load_env
load_env()
# set the default Django settings module for the 'celery' program.
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "myapp.settings")
app = Celery('myapp')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('myapp.settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
after the call to load_env() env variables are loaded and the celery worker has access to them. By doing this I am now able to access other modules from my tasks.py, which was my main problem.
Credits to this guys (Caktus Consulting Group) and their django-project-template because if it wasn't for them I wouldn't find the answer. Thanks.
try something like this. its working in 3.1 celery, import should happen inside save method and after super()
from django.db import models
class News(models.model):
(...)
def save(self, *args, **kwargs):
(...)
super(News, self).save(*args, **kwargs)
from task import update_news_status
update_news_status.apply_async((self.id,)) #apply_async or delay
Here what i would do (Django 1.11 and celery 4.2), you have a problem in your celery config and you try to re-declare the Celery instance :
tasks.py
from myapp.celery import app # would contain what you need :)
from celery.utils.log import get_task_logger
logger = get_task_logger(__name__)
#app.task(name='news.tasks.update_news_status')
def update_news_status(news_id):
# (I pass the news id and return it, nothing complicated about it)
return news_id
celery.py
from __future__ import absolute_import, unicode_literals
from celery import Celery
from django.conf import settings
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "myapp.settings")
app = Celery('myapp', backend='rpc://', broker=BROKER_URL) # your config here
app.config_from_object('django.myapp:settings', namespace='CELERY') # change here
app.autodiscover_tasks()
models.py
from django.db import models
class News(models.model):
(...)
def save(self, *args, **kwargs):
super(News, self).save(*args, **kwargs)
from news.tasks import update_news_status
update_news_status.delay(self.id) # change here
And launch it with celery -A myapp worker --loglevel=info because your app is defined in myapp.celery so -A parameter need to be the app where the conf is declared
I am trying to setup a task to run every ten seconds.Using Celery Beat.
I am using:
Django==1.11.3
celery==4.1.0
django-celery-beat==1.1.1
django-celery-results==1.0.1
It is giving me the following error:
Received unregistered task of type 'operations.tasks.message'
I am new to Celery, I have tried numerous solutions and cannot seem to find a solution,would appreciate the help
settings.py
CELERY_BROKER_URL = 'pyamqp://guest#localhost//'
CELERY_RESULT_BACKEND = 'django-db'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TASK_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Africa/Johannesburg'
CELERY_BEAT_SCHEDULE = {
'message': {
'task': 'operations.tasks.message',
'schedule': 10.0
}
}
celery.py
from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'nodiso.settings')
app = Celery('nodiso')
# Using a string here means the worker don't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
__init__.py
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ['celery_app']
task.py
from __future__ import absolute_import, unicode_literals
from celery import shared_task
from operations import models
from .celery import periodic_task
#task
def message():
t = models.Celerytest.objects.create(Message='Hello World')
t.save()
files structure
proj-
proj-
__init__.py
settings.py-
celery.py-
app-
tasks.py-
Within my celery.py file I define app like this:
app = Celery(
'your_celery_app_name',
include=[
'your_celery_app_name.module.task1',
'your_celery_app_name.module.task2',
]
)
app.config_from_object('your_celery_app_name.celeryconfig')
My celeryconfig.py is where I define my beats and other settings (I think this would be same as your settings.py).
Below is probably not relevant - I'm not an expert with Python and how package should be put together - but from my limited understanding your tasks should be a submodule of your celery app module. Take this with pinch of salt though.
My project structure looks more like this:
your_celery_app_name (dir)
setup.py (file)
your_celery_app_name (dir)
__init__.py (file)
celery.py (file)
celeryconfig.py (file)
module (dir)
__init__.py (importing task1 and task2 from tasks)
tasks.py (implementing task1 and task2)
I'm trying to learn Flask from Instant Flask Web Development book following it step-by-step but I get the error presented below.
(It seems to be a problem with circular imports.)
The questions are:
Can someone explain what would be the fix and detail the circular imports?
Directory structure:
├── manage.py
├── requirements.txt
└── sched
├── __init__.py
├── __init__.pyc
├── app.py
├── app.pyc
├── static
└── templates
manager.py:
from flask.ext.script import Manager
from sched import app
manager = Manager(app)
app.config['DEBUG'] = True
if __name__ == '__main__':
manager.run()
app.py:
from flask import Flask
app = Flask(__name__)
#app.route('/')
def hello_world():
return 'Hello World!'
if __name__ == '__main__':
app.run()
__init__.py is ampty.
Command line generated error:
$ python manage.py runserver
Traceback (most recent call last):
File "manage.py", line 8, in <module>
app.config['DEBUG'] = True
AttributeError: 'module' object has no attribute 'config'
When you do
from sched import app
you are importing the app module. The app object is inside the module. The import you want to use is
from sched.app import app
Alternative architecture :
# app.py:
from flask import Flask
app = Flask(__name__)
app.debug = True
from flask.ext.script import Manager
manager = Manager(app)
# __init__.py:
from .sched import app, manager
import sched.views # for your views
import sched.commands # for additional commands
# manager.py:
from sched import manager
if __name__ == '__main__':
manager.run()
#views.py:
from .app import app
#app.route('/')
def hello_world():
return 'Hello World!'
I suggest following an app factory pattern for your app.
manage.py
sched/
__init__.py
in __init__.py you can do the following:
from flask import Flask
def create_app():
app = Flask(__name__)
app.config['DEBUG'] = True
return app
in manage.py
from app import create_app
from flask.ext.script import Manager
app = create_app()
manager = Manager(app)
if __name__ == '__main__':
manager.run()
This tends to be an extensible solution for larger application structures.