I've been reading through the answers to some previous questions as well as the celery docs and I just can't quite fix this issue. My task.py, celery.py, and settings.py are all contained within the same app RBWebfiles which is contained within the project called recruiting board. I'm attempting to creating a periodic-task that will run using celery beat's scheduler.
This is the portion of my settings.py that deals with celery:
from __future__ import absolute_import
import os
from datetime import timedelta
CELERY_IMPORTS = ('RBWebfiles.tasks')
CELERYBEAT_SCHEDULE = 'djcelery.schedulers.DatabaseScheduler'
CELERYBEAT_SCHEDULE = {
'schedule-name':{
'task': 'RBWebfiles.tasks.requestRefresher',
'schedule': timedelta(seconds = 30),
},
}
BROKER_URL = 'django://'
I also added 'djcelery' to my installed apps within that file
This is my tasks.py file:
from __future__ import absolute_import
import datetime
from celery.task.base import periodic_task
from student.models import StudentAccount
from celery import Celery
from celery.utils.log import get_task_logger
import os
celery = Celery('tasks', broker='django://')
logger = get_task_logger(__name__)
os.environ['DJANGO_SETTINGS_MODULE'] = 'RBWebfiles.settings'
#periodic_task(run_every=datetime.timedelta(seconds=30))
def requestRefresher(self):
logger.info("start task")
for s in StudentAccount.objects.all():
s.requestLimit = 0
s.save()
return None
and lastly this is my celery.py file:
from __future__ import absolute_import
import os
from celery import Celery
from django.conf import settings
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'conf.settings')
app = Celery('RBWebfiles.celery')
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda : settings.INSTALLED_APPS)
app.conf.update(
CELERY_RESULT_BACKEND ='djcelery.backends.database:DatabaseBackend',
)
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
I've tried uninstalling and reinstalling both celery and django-celery, I'm not sure if I just don't understand something or I'm making a massive error. I attempt to run it using the command : celery beat -A RBWebfiles
this is the traceback:
C:\Users\Lexie Infantino\PycharmProjects\recruitingboard>celery beat -A RBWebfiles
Traceback (most recent call last):
File "C:\Python34\lib\site-packages\celery\app\utils.py", line 235, in find_app
found = sym.app
AttributeError: 'module' object has no attribute 'app'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Python34\lib\runpy.py", line 170, in _run_module_as_main
"__main__", mod_spec)
File "C:\Python34\lib\runpy.py", line 85, in _run_code
exec(code, run_globals)
File "C:\Python34\Scripts\celery.exe\__main__.py", line 9, in <module>
File "C:\Python34\lib\site-packages\celery\__main__.py", line 30, in main
main()
File "C:\Python34\lib\site-packages\celery\bin\celery.py", line 81, in main
cmd.execute_from_commandline(argv)
File "C:\Python34\lib\site-packages\celery\bin\celery.py", line 769, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "C:\Python34\lib\site-packages\celery\bin\base.py", line 309, in execute_from_commandline
argv = self.setup_app_from_commandline(argv)
File "C:\Python34\lib\site-packages\celery\bin\base.py", line 469, in setup_app_from_commandline
self.app = self.find_app(app)
File "C:\Python34\lib\site-packages\celery\bin\base.py", line 489, in find_app
return find_app(app, symbol_by_name=self.symbol_by_name)
File "C:\Python34\lib\site-packages\celery\app\utils.py", line 240, in find_app
found = sym.celery
AttributeError: 'module' object has no attribute 'celery'
Make sure djcelery is in your INSTALLED_APPS and try starting celery beat with manage.py celerybeat instead.
Related
i am extremely new to django-celery, its doc have been confusing to me and i have been following tutorial, here is just a basic setup and i have encountered a untrackable error for me, the error is:
AttributeError: 'EntryPoint' object has no attribute 'module_name'
full traceback:
Traceback (most recent call last):
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/app/base.py", line 1250, in backend
return self._local.backend
AttributeError: '_thread._local' object has no attribute 'backend'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/worker/worker.py", line 203, in start
self.blueprint.start(self)
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/bootsteps.py", line 112, in start
self.on_start()
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/apps/worker.py", line 136, in on_start
self.emit_banner()
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/apps/worker.py", line 170, in emit_banner
' \n', self.startup_info(artlines=not use_image))),
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/apps/worker.py", line 232, in startup_info
results=self.app.backend.as_uri(),
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/app/base.py", line 1252, in backend
self._local.backend = new_backend = self._get_backend()
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/app/base.py", line 955, in _get_backend
backend, url = backends.by_url(
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/app/backends.py", line 69, in by_url
return by_name(backend, loader), url
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/app/backends.py", line 47, in by_name
aliases.update(load_extension_class_names(extension_namespace))
File "/home/muhammad/Desktop/celery/env/lib/python3.8/site-packages/celery/utils/imports.py", line 146, in load_extension_class_names
yield ep.name, ':'.join([ep.module_name, ep.attrs[0]])
AttributeError: 'EntryPoint' object has no attribute 'module_name'
the celery.py, init.py and and task are just the basic:
init.py:
from __future__ import absolute_import, unicode_literals
from .celery import app as celery_app
__all__ = ('celery_app',)
celery.py:
from __future__ import absolute_import, unicode_literals
from celery import Celery
import os
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'core.settings')
app = Celery('core')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()
task.py:
from celery import shared_task
#shared_task(bind=True)
def add(x, y):
return x + y
settings conf for celery is:
CELERY_BROKER_URL = 'localhost'
CELERY_ACCEPT_CONTENT = ['json',]
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_BACKEND = 'django-db'
also i have installed and added django_celery_result in INSTALLED APPS
INSTALLED_APPS = [
...
'celery_practice',
'django_celery_results'
]
For your information, rabbitmq-server is running on localhost thats why i have set BROKER_URL TO 'localhost':
rabbitmq-server.service - RabbitMQ Messaging Server
Loaded: loaded (/lib/systemd/system/rabbitmq-server.service; enabled; vendor preset: enabled)
Active: active (running) since Sun 2022-04-03 16:14:04 PKT; 24h ago
Main PID: 1005 (beam.smp)
Status: "Initialized"
Tasks: 91 (limit: 9090)
any help would be appreciated thanks!
My celery task is to update the mongodb database. I've created a handler for it which i'm importing to celery.py. Celery seems to work fine but when i try to import the handler module it throws an error.
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet
celery.py:
from __future__ import absolute_import, unicode_literals
from django.conf import settings
import os
from celery import Celery
# from handlers.ordercount import OrderCount
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'di_idealsteel.settings')
app = Celery('di_idealsteel')
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
app.config_from_object('django.conf:settings')
# Load task modules from all registered Django app configs.
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
#app.task(bind=True)
def debug_task(self):
print('Request: {0!r}'.format(self.request))
#app.task(bind=True)
def add(self,number,customer_id):
ordercount = OrderCount()
opencount = ordercount.open_enquiries(number,customer_id)
When i try to import 5th line it throws this error
Traceback (most recent call last):
File "c:\users\ashish\envs\idealvenv\lib\site-packages\celery\app\trace.py", line 240, in trace_task
R = retval = fun(*args, **kwargs)
File "c:\users\ashish\envs\idealvenv\lib\site-packages\celery\app\trace.py", line 437, in __protected_call__
return self.run(*args, **kwargs)
File "C:\Users\Ashish\Desktop\Internship\di_idealsteel\di_idealsteel\celery.py", line 32, in add
from handlers.ordercount import OrderCount
File "C:\Users\Ashish\Desktop\Internship\di_idealsteel\handlers\ordercount.py", line 4, in <module>
from handlers.user_handler import UserHandler
File "C:\Users\Ashish\Desktop\Internship\di_idealsteel\handlers\user_handler.py", line 12, in <module>
from app.models import (AuthUser,Customers,CustomerUsers,UserHistory,CustomerUsersHistory)
File "C:\Users\Ashish\Desktop\Internship\di_idealsteel\app\models.py", line 10, in <module>
from django.contrib.auth.models import User
File "c:\users\ashish\envs\idealvenv\lib\site-packages\django\contrib\auth\models.py", line 4, in <module>
from django.contrib.auth.base_user import AbstractBaseUser, BaseUserManager
File "c:\users\ashish\envs\idealvenv\lib\site-packages\django\contrib\auth\base_user.py", line 49, in <module>
class AbstractBaseUser(models.Model):
File "c:\users\ashish\envs\idealvenv\lib\site-packages\django\db\models\base.py", line 94, in __new__
app_config = apps.get_containing_app_config(module)
File "c:\users\ashish\envs\idealvenv\lib\site-packages\django\apps\registry.py", line 239, in get_containing_app_config
self.check_apps_ready()
File "c:\users\ashish\envs\idealvenv\lib\site-packages\django\apps\registry.py", line 124, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
Other files:
__init__.py:
from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
__all__ = ('celery_app',)
settings.py:
# CELERY STUFF
BROKER_URL = 'redis://localhost:6379'
CELERY_RESULT_BACKEND = 'redis://localhost:6379'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Africa/Nairobi'
This line in celery.py is causing the error -->
from handlers.ordercount import OrderCount
If you look at the error stacktrace, internally the OrderCount module/class is calling AbstractBaseUser model, from django.contrib.auth.models import User / AbstractBaseUser. And till now django hasn't had the chance to load all the apps inside its registry and you are trying to find the User model from the registry.
You can create a tasks.py file in your django app and register your tasks there.
You have to just remove the add celery task from celery.py file
Alternatively, you could also import OrderCount locally in the add function, to avoid the error.
I am trying out celery with Python Flask.
celery_example.py
from __future__ import absolute_import, unicode_literals
from flask import Flask
from flask_celery import make_celery
flask_app = Flask(__name__)
flask_app.config.update(
CELERY_BROKER_URL='amqp://rsrc:rsrc#localhost:5672/localhost',
CELERY_RESULT_BACKEND='db+postgresql://postgres:postgres#localhost/rsrc_celery'
)
celery = make_celery(flask_app)
#flask_app.route('/process/<name>')
def process(name):
reverse.delay(name)
return "Async !"
#celery.task(name="celery_example.reverse")
def reverse(nm):
return nm[::-1]
if __name__ == "__main__":
flask_app.run(debug=True)
flask_celery.py
from __future__ import absolute_import, unicode_literals
from celery import Celery
def make_celery(app):
celery = Celery(app.import_name, backend=app.config['CELERY_RESULT_BACKEND'], broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
FLask server is up and running fine, I'm able to call the route that is defined.
But I'm unable to start the Celery worker (celery worker -A celery_example.celery --loglevel=info)
This is the error I get always.
Traceback (most recent call last):
File "/usr/local/bin/celery", line 10, in <module>
sys.exit(main())
File "/Library/Python/2.7/site-packages/celery/__main__.py", line 16, in main
_main()
File "/Library/Python/2.7/site-packages/celery/bin/celery.py", line 322, in main
cmd.execute_from_commandline(argv)
File "/Library/Python/2.7/site-packages/celery/bin/celery.py", line 496, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/Library/Python/2.7/site-packages/celery/bin/base.py", line 290, in execute_from_commandline
self.on_error(UNABLE_TO_LOAD_APP_MODULE_NOT_FOUND.format(e.name))
AttributeError: 'exceptions.ImportError' object has no attribute 'name'
My directory structure is
--Flask_Celery
-- celery_example.py
-- flask_celery.py
-- __init__.py
Ok so what the problem was the environment.
I was creating the environment by "virtualenv env" which was creating the environment with python, whereas I needed the python3 environment.
So, I created virtual environment using "python3 -m venv env" and then loaded all the dependencies which resolved the error.
I'm trying to use celery for background process in my Django application. Django version is 1.4.8 and latest suitable celery version is 3.1.25.
I use Redis (3.1.0) as broker and backend, json as serializer.
When I start the worker
celery -A celery_app worker -l info I'm getting Attribute error 'unicode' object has no attribute 'iteritems'
My settings.py file:
BROKER_URL = 'redis://localhost'
CELERY_RESULT_BACKEND = 'redis://localhost/'
CELERY_ACCEPT_CONTENT = ['json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_STORE_ERRORS_EVEN_IF_IGNORED = True
celery_app.py:
import sys
from django.conf import settings
from celery import Celery
project_root = os.path.dirname(__file__)
sys.path.insert(0, os.path.join(project_root, '../env'))
sys.path.insert(0, os.path.join(project_root, '../'))
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'project.settings')
app = Celery('project')
app.config_from_object('project.settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS, force=True)
tasks.py:
#celery_app.task
def sample_task(x):
return 'Test response'
and that's how I run this task:
sample_task.delay({'key': 'test'})
And I get the following error:
File "/Users/user/project/venv/lib/python2.7/site-packages/redis/_compat.py", line 94, in iteritems
return x.iteritems()
AttributeError: 'unicode' object has no attribute 'iteritems'
full traceback:
[2019-01-31 16:43:08,909: ERROR/MainProcess] Unrecoverable error: AttributeError("'unicode' object has no attribute 'iteritems'",)
Traceback (most recent call last):
File "/Users/user/project/venv/lib/python2.7/site-packages/celery/worker/__init__.py", line 206, in start
self.blueprint.start(self)
File "/Users/user/project/venv/lib/python2.7/site-packages/celery/bootsteps.py", line 123, in start
step.start(parent)
File "/Users/user/project/venv/lib/python2.7/site-packages/celery/bootsteps.py", line 374, in start
return self.obj.start()
File "/Users/user/project/venv/lib/python2.7/site-packages/celery/worker/consumer.py", line 280, in start
blueprint.start(self)
File "/Users/user/project/venv/lib/python2.7/site-packages/celery/bootsteps.py", line 123, in start
step.start(parent)
File "/Users/user/project/venv/lib/python2.7/site-packages/celery/worker/consumer.py", line 884, in start
c.loop(*c.loop_args())
File "/Users/user/project/venv/lib/python2.7/site-packages/celery/worker/loops.py", line 76, in asynloop
next(loop)
File "/Users/user/project/venv/lib/python2.7/site-packages/kombu/async/hub.py", line 340, in create_loop
cb(*cbargs)
File "/Users/user/project/venv/lib/python2.7/site-packages/kombu/transport/redis.py", line 1019, in on_readable
self._callbacks[queue](message)
File "/Users/user/project/venv/lib/python2.7/site-packages/kombu/transport/virtual/__init__.py", line 534, in _callback
self.qos.append(message, message.delivery_tag)
File "/Users/user/project/venv/lib/python2.7/site-packages/kombu/transport/redis.py", line 146, in append
pipe.zadd(self.unacked_index_key, delivery_tag, time()) \
File "/Users/user/project/venv/lib/python2.7/site-packages/redis/client.py", line 2320, in zadd
for pair in iteritems(mapping):
File "/Users/user/project/venv/lib/python2.7/site-packages/redis/_compat.py", line 94, in iteritems
return x.iteritems()
AttributeError: 'unicode' object has no attribute 'iteritems'
I tried to find the issue on the internet, tried to pass another params to task. I don't know how to debug celery process and could not find the solution by myself. Please help me
Seems that this Celery version doesn't support Redis 3. Try to install Redis 2.10.6.
I try to start a Celery worker server from a command line:
celery -A server application worker --loglevel=info
The code and folder path:
server.py
application/controllers/routes.py
server.py
app = Flask(__name__)
from application.controllers import routes
app.run(host='127.0.0.1',port=5051,debug=True)
route.py
from flask import Flask,
from celery import Celery
from server import app
app.config['CELERY_BROKER_URL'] = 'redis://localhost:6379/0'
app.config['CELERY_RESULT_BACKEND'] = 'redis://localhost:6379/0'
celery = Celery(app.name, broker=app.config['CELERY_BROKER_URL'])
celery.conf.update(app.config)
#celery.task()
def add_together(self, count):
return "First success"
#app.route("/queing")
def testsfunction():
count = 1
add_together.delay(count)
return "cool"
Trace back:
Traceback (most recent call last):
File "/Library/Frameworks/Python.framework/Versions/2.7/bin/celery", line 11, in <module>
sys.exit(main())
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/celery/__main__.py", line 30, in main
main()
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/celery/bin/celery.py", line 81, in main
cmd.execute_from_commandline(argv)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/celery/bin/celery.py", line 770, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/celery/bin/base.py", line 309, in execute_from_commandline
argv = self.setup_app_from_commandline(argv)
File "/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/site-packages/celery/bin/base.py", line 477, in setup_app_from_commandline
user_preload = tuple(self.app.user_options['preload'] or ())
AttributeError: 'Flask' object has no attribute 'user_options'
I got this error when I'm running a celery worker in terminal.
just run the celery with this command instead of yours:
celery -A application.controllers.routes:celery worker --loglevel=info
this will solve your current problem however your codes have a plenty of mistakes for example if you want to have a self argument inside your add_together function you you should declare a task like this:
#celery.task(bind=True)
It seems like you have typo mistake:
def add_together(self, count):
to
def add_together(count):