My Celery task isn't executing in the background in my Django 1.7/Python3 project.
# settings.py
BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULTBACKEND = BROKER_URL
CELERYBEAT_SCHEDULER = 'djcelery.schedulers.DatabaseScheduler'
CELERY_ALWAYS_EAGER = False
I have celery.py in my root app module as such:
from __future__ import absolute_import
import os
import django
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'my_app.settings')
django.setup()
app = Celery('my_app')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
and load the app in __init__.py in the root module:
from __future__ import absolute_import
from .celery import app as celery_app
My task is set up as a shared task in a tasks.py file in my app module:
from __future__ import absolute_import
from celery import shared_task
#shared_task
def update_statistics(profile, category):
# more code
and I call the task as a group:
. . .
job = group([update_statistics(f.profile, category)
for f in forecasts])
job.apply_async()
However, I'm not seeing any status updates in my task queue, which I am starting via:
$ celery -A my_app worker -l info
The task is being executed, just not in the background. If I add a print statement to the task code, I will see the output in my Django development server console instead of the Celery queue.
After the task runs in the foreground, I'm greeted with this exception:
'NoneType' object has no attribute 'app'
Here's the full traceback if you're interested: https://gist.github.com/alsoicode/0263d251e3744227ba46
You're calling the tasks directly in your list comprehension when you create the group, so they're executed then and there. You need to use the .subtask() method (or its shortcut, .s()) to create the subtasks without calling them:
job = group([update_statistics.s(f.profile, category) for f in forecasts])
Related
this function i was using under management/commands and it was working but i'd like to use it to update feeds periodically. like every 15 mins or so.
celery.py
from celery import Celery
import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mysite.settings")
app = Celery('mysite',
broker='redis://',
backend='rpc://',
include=['mysite.tasks'],
timezone='UTC')
if __name__ == '__main__':
app.start()
tasks.py
from .celery import app
from news.models import Feed, Article
import feedparser
from datetime import datetime, timezone
#app.on_after_configure.connect
def setup_periodic_tasks(sender, **kwargs):
sender.add_periodic_task(60.0, handle.s(), name='update feeds every 60 seconds')
#app.task
def handle(self, *args, **kwargs):
feeds = Feed.objects.all()
for feed in feeds:
new_list = feedparser.parse(feed.url)
for entry in new_list.entries:
start = datetime.now(timezone.utc)
end = datetime(*(entry.published_parsed[0:6])).replace(tzinfo=timezone.utc)
if (start - end).days < 2 and not Article.objects.filter(url=entry.link).exists():
article = Article()
article.title = entry.title
article.url = entry.link
article.description = entry.description
dateString = end.strftime('%Y-%m-%d %H:%M:%S %z')
article.publication_date = dateString
article.save()
else:
pass
i run celery -A mysite worker -l INFO and there is
[tasks]
. mysite.tasks.handle
i tried also celery -A mysite beat
no errors but i don't see any effect on my site
i also tried to run both commands in two terminals.
in settings directory create file for celery
celery_conf.py
import os
from celery import Celery
from datetime import timedelta
os.environ.setdefault('DJANGO_SETTINGS_MODULE', '<path to settings like=> config.settings>')
celery_app = Celery('<settings direcory name like=>config>')
celery_app.autodiscover_tasks()
celery_app.conf.broker_url = 'amqp://'
celery_app.conf.result_backend = 'rpc://'
in your apps,Wherever you need celery,create tasks.py file
Create your own function(in tasks.py file) and set shared_task decorator to function
tasks.py
from celery import shared_task
#shared_task
def send_email():
.....
Celery using shared_task to identify tasks
*** Don't forget to use delay in task calling***
I need a minimum example to do periodic task (run some function after every 5 minutes, or run something at 12:00:00 etc.).
In my myapp/tasks.py, I have,
from celery.task.schedules import crontab
from celery.decorators import periodic_task
from celery import task
#periodic_task(run_every=(crontab(hour="*", minute=1)), name="run_every_1_minutes", ignore_result=True)
def return_5():
return 5
#task
def test():
return "test"
When I run celery workers it does show the tasks (given below) but does not return any values (in either terminal or flower).
[tasks]
. mathematica.core.tasks.test
. run_every_1_minutes
Please provide a minimum example or hints to achieve the desired results.
Background:
I have a config/celery.py which contains the following:
import os
from celery import Celery
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
app = Celery('config')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
And in my config/__init__.py, I have
from .celery import app as celery_app
__all__ = ['celery_app']
I added a function something like below in myapp/tasks.py
from celery import task
#task
def test():
return "test"
When I run test.delay() from shell, it runs successfully and also shows the task information in flower
To run periodic task you should run celery beat also. You can run it with this command:
celery -A proj beat
Or if you are using one worker:
celery -A proj worker -B
I can't import my celery app to run tasks from my main Python application. I want to be able to run celery tasks from the myprogram.py file.
My celery_app.py file is as follows:
import celery
app = celery.Celery('MyApp', broker='redis://localhost:6379/0')
app.conf.broker_url = 'redis://localhost:6379/0'
app.conf.result_backend = 'redis://localhost:6379/0'
app.autodiscover_tasks()
#app.task(ignore_result=True)
def task_to_run():
print("Task Running")
# The following call runs a worker in celery
task_to_run.delay()
if __name__ == '__main__':
app.start()
Application structure
projectfolder/core/celery_app.py # Celery app
projectfolder/core/myprogram.py # My Python application
projectfolder/core/other python files...
The file myprogram.py contains the following:
from .celery_app import task_to_run
task_to_run.delay()
Error:
Received unregistered task of type 'projectfolder.core.celery_app.task_to_run'.
The message has been ignored and discarded.
Did you remember to import the module containing this task?
Or maybe you're using relative imports?
strategy = strategies[type_]
KeyError: 'projectfolder.core.celery_app.task_to_run'
Thanks
interesting, I didn't know about autodiscover_tasks, I guess it's new in 4.1
As I see in the documentation, this function takes list of packages to search. You might want to call it with:
app.autodiscover_tasks(['core.celery_app'])
or it might be better to extract the task to a seperate file called tasks.py and then it would be just:
app.autodiscover_tasks(['core']).
Alternatively, you can use the inculde parameter when creating the Celery instance:
app = celery.Celery('MyApp', broker='redis://localhost:6379/0', include=['core.celery_app']) or wherever your tasks are.
Good luck
I have an issue with Celery queue routing when using current_app.send_task
I have two workers (each one for each queue)
python manage.py celery worker -E -Q priority --concurrency=8 --loglevel=DEBUG
python manage.py celery worker -Q low --concurrency=8 -E -B --loglevel=DEBUG
I have two queues defined in celeryconfig.py file:
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.core.exceptions import ImproperlyConfigured
from celery import Celery
from django.conf import settings
try:
app = Celery('proj', broker=getattr(settings, 'BROKER_URL', 'redis://'))
except ImproperlyConfigured:
app = Celery('proj', broker='redis://')
app.conf.update(
CELERY_TASK_SERIALIZER='json',
CELERY_ACCEPT_CONTENT=['json'],
CELERY_RESULT_SERIALIZER='json',
CELERY_RESULT_BACKEND='djcelery.backends.database:DatabaseBackend',
CELERY_DEFAULT_EXCHANGE='tasks',
CELERY_DEFAULT_EXCHANGE_TYPE='topic',
CELERY_DEFAULT_ROUTING_KEY='task.priority',
CELERY_QUEUES=(
Queue('priority',routing_key='priority.#'),
Queue('low', routing_key='low.#'),
),
CELERY_DEFAULT_EXCHANGE='priority',
CELERY_IMPORTS=('mymodule.tasks',)
CELERY_ENABLE_UTC = True
CELERY_TIMEZONE = 'UTC'
if __name__ == '__main__':
app.start()
In the definition of tasks, we use decorator to explicit the queue:
#task(name='mymodule.mytask', routing_key='low.mytask', queue='low')
def mytask():
# does something
pass
This task is run indeed in the low queue when this task is run using:
from mymodule.tasks import mytask
mytask.delay()
But it's not the case when it's run using: (it's run in the default queue: "priority")
from celery import current_app
current_app.send_task('mymodule.mytask')
I wonder why this later way doesn't route the task to the "low" queue!
p.s: I use redis.
send_task is a low-level method. It sends directly to the broker the task signature without going through your task decorator.
With this method, you can even send a task without loading the task code/module.
To solve your problem, you can fetch the routing_key/queue from configuration directly:
route = celery.amqp.routes[0].route_for_task("mymodule.mytask")
Out[10]: {'queue': 'low', 'routing_key': 'low.mytask'}
celery.send_task("myodule.mytask", queue=route['queue'], routing_key=route['routing_key']`
I have a Deadlock when using a Celery task to save new customers from a CSV. This is what I have working so far.
for line in csv.reader(instance.data_file.read().splitlines()):
for index, item in enumerate(line):
number = int(item)
# TODO: Turn into task
Customer.objects.create_customer(
mobile=number,
campaign=instance.campaign,
reward_group=instance.reward_group,
company=instance.company,
)
No errors.
However, when add this same code to a Celery task I get the following error...
Deadlock found when trying to get lock; try restarting transaction'
So, this leads me to believe that I have done something wrong with my celery setup here. Can anyone spot what?
Here is the new Celery task that gives the deadlock error. I'm using shared_task as these task will at some point run on a different machine without Django, but that should not matter for now.
The first row in the CSV import ok, then I get a deadlock error...
for line in csv.reader(instance.data_file.read().splitlines()):
for index, item in enumerate(line):
number = int(item)
celery_app.send_task('test.tasks.create_customer_from_import', args=[number, instance.id], kwargs={})
tasks.py
# Python imports
from __future__ import absolute_import
# Core Django imports
from celery import shared_task
from mgm.core.celery import app as celery_app
#shared_task
def create_customer_from_import(number, customer_upload_id):
customer_upload = CustomerUpload.objects.get(pk=customer_upload_id)
new_customer = Customer.objects.create_customer(
mobile=number,
campaign=customer_upload.campaign,
reward_group=customer_upload.reward_group,
company=customer_upload.company,
)
return new_customer
celery.py
from __future__ import absolute_import
import os
from celery import Celery
from django.conf import settings
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'test.settings')
app = Celery('test-tasks')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
This is the CustomerManager:
class CustomerManager(models.Manager):
def create_customer(self, mobile, campaign, reward_group, company, password=None):.
user = AppUser.objects.create_user(mobile=mobile)
# Creates a new customer for a company and campaign
customer = self.model(
user=user,
campaign=campaign,
reward_group=reward_group,
company=company
)
customer.save(using=self._db)
Your code doesn't look wrong, but you're probably getting the deadlock because of the concurrency of multiple celery workers. From http://celery.readthedocs.org/en/latest/faq.html#mysql-is-throwing-deadlock-errors-what-can-i-do:
MySQL has default isolation level set to REPEATABLE-READ, if you don’t
really need that, set it to READ-COMMITTED. You can do that by adding
the following to your my.cnf:
[mysqld]
transaction-isolation = READ-COMMITTED