diff --git a/cms/__init__.py b/cms/__init__.py index e69de29bb2..7ca1196bb0 100644 --- a/cms/__init__.py +++ b/cms/__init__.py @@ -0,0 +1,9 @@ +""" +Celery needs to be loaded when the cms modules are so that task +registration and discovery can work correctly. +""" +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import APP as CELERY_APP diff --git a/cms/celery.py b/cms/celery.py new file mode 100644 index 0000000000..d3a44fc6d8 --- /dev/null +++ b/cms/celery.py @@ -0,0 +1,23 @@ +""" +Import celery, load its settings from the django settings +and auto discover tasks in all installed django apps. + +Taken from: http://celery.readthedocs.org/en/latest/django/first-steps-with-django.html +""" +from __future__ import absolute_import + +import os + +from celery import Celery + +from django.conf import settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') + +APP = Celery('proj') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +APP.config_from_object('django.conf:settings') +APP.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/cms/envs/aws.py b/cms/envs/aws.py index ea9b758003..44b6ea2ffb 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -60,8 +60,7 @@ BROKER_POOL_LIMIT = 0 BROKER_CONNECTION_TIMEOUT = 1 # For the Result Store, use the django cache named 'celery' -CELERY_RESULT_BACKEND = 'cache' -CELERY_CACHE_BACKEND = 'celery' +CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend' # When the broker is behind an ELB, use a heartbeat to refresh the # connection and to detect if it has been dropped. diff --git a/cms/envs/test.py b/cms/envs/test.py index 2d10aaad49..63da126ae4 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -187,9 +187,7 @@ simplefilter('ignore') ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True -CELERY_RESULT_BACKEND = 'cache' -BROKER_TRANSPORT = 'memory' - +CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend' ########################### Server Ports ################################### diff --git a/cms/envs/yaml_config.py b/cms/envs/yaml_config.py index 4fa64674bc..bb7d532ee5 100644 --- a/cms/envs/yaml_config.py +++ b/cms/envs/yaml_config.py @@ -75,8 +75,7 @@ BROKER_POOL_LIMIT = 0 BROKER_CONNECTION_TIMEOUT = 1 # For the Result Store, use the django cache named 'celery' -CELERY_RESULT_BACKEND = 'cache' -CELERY_CACHE_BACKEND = 'celery' +CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend' # When the broker is behind an ELB, use a heartbeat to refresh the # connection and to detect if it has been dropped. diff --git a/lms/__init__.py b/lms/__init__.py index e69de29bb2..7ca1196bb0 100644 --- a/lms/__init__.py +++ b/lms/__init__.py @@ -0,0 +1,9 @@ +""" +Celery needs to be loaded when the cms modules are so that task +registration and discovery can work correctly. +""" +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import APP as CELERY_APP diff --git a/lms/celery.py b/lms/celery.py new file mode 100644 index 0000000000..d3a44fc6d8 --- /dev/null +++ b/lms/celery.py @@ -0,0 +1,23 @@ +""" +Import celery, load its settings from the django settings +and auto discover tasks in all installed django apps. + +Taken from: http://celery.readthedocs.org/en/latest/django/first-steps-with-django.html +""" +from __future__ import absolute_import + +import os + +from celery import Celery + +from django.conf import settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings') + +APP = Celery('proj') + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +APP.config_from_object('django.conf:settings') +APP.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/lms/djangoapps/bulk_email/tasks.py b/lms/djangoapps/bulk_email/tasks.py index 45b57d3111..9b12740413 100644 --- a/lms/djangoapps/bulk_email/tasks.py +++ b/lms/djangoapps/bulk_email/tasks.py @@ -773,7 +773,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current # needs to be returned back to Celery. If it fails, we return the existing # exception. try: - send_course_email.retry( + retry_task = send_course_email.retry( args=[ entry_id, email_id, @@ -786,6 +786,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current max_retries=max_retries, throw=True, ) + raise retry_task except RetryTaskError as retry_error: # If the retry call is successful, update with the current progress: log.exception(u'Task %s: email with id %d caused send_course_email task to retry.', @@ -800,7 +801,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current log.exception(u'Task %s: email with id %d caused send_course_email task to fail to retry. To list: %s', task_id, email_id, [i['email'] for i in to_list]) num_failed = len(to_list) - subtask_status.increment(subtask_status, failed=num_failed, state=FAILURE) + subtask_status.increment(failed=num_failed, state=FAILURE) return subtask_status, retry_exc diff --git a/lms/djangoapps/bulk_email/tests/test_tasks.py b/lms/djangoapps/bulk_email/tests/test_tasks.py index 759025335a..e76867f816 100644 --- a/lms/djangoapps/bulk_email/tests/test_tasks.py +++ b/lms/djangoapps/bulk_email/tests/test_tasks.py @@ -104,15 +104,9 @@ class TestBulkEmailInstructorTask(InstructorTaskCourseTestCase): return instructor_task def _run_task_with_mock_celery(self, task_class, entry_id, task_id): - """Submit a task and mock how celery provides a current_task.""" - mock_current_task = Mock() - mock_current_task.max_retries = settings.BULK_EMAIL_MAX_RETRIES - mock_current_task.default_retry_delay = settings.BULK_EMAIL_DEFAULT_RETRY_DELAY + """Mock was not needed for some tests, testing to see if it's needed at all.""" task_args = [entry_id, {}] - - with patch('bulk_email.tasks._get_current_task') as mock_get_task: - mock_get_task.return_value = mock_current_task - return task_class.apply(task_args, task_id=task_id).get() + return task_class.apply(task_args, task_id=task_id).get() def test_email_missing_current_task(self): task_entry = self._create_input_entry() diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 45104a17c4..de19b68f3b 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -64,8 +64,7 @@ BROKER_POOL_LIMIT = 0 BROKER_CONNECTION_TIMEOUT = 1 # For the Result Store, use the django cache named 'celery' -CELERY_RESULT_BACKEND = 'cache' -CELERY_CACHE_BACKEND = 'celery' +CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend' # When the broker is behind an ELB, use a heartbeat to refresh the # connection and to detect if it has been dropped. diff --git a/lms/envs/bok_choy.py b/lms/envs/bok_choy.py index 75e66f4215..e7efd6ca73 100644 --- a/lms/envs/bok_choy.py +++ b/lms/envs/bok_choy.py @@ -55,8 +55,7 @@ MEDIA_URL = "/static/uploads/" ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True -CELERY_RESULT_BACKEND = 'cache' -BROKER_TRANSPORT = 'memory' +CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend' ###################### Grade Downloads ###################### GRADES_DOWNLOAD = { diff --git a/lms/envs/test.py b/lms/envs/test.py index ff0946b288..b545a562df 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -299,8 +299,7 @@ GIT_REPO_DIR = TEST_ROOT / "course_repos" ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True -CELERY_RESULT_BACKEND = 'cache' -BROKER_TRANSPORT = 'memory' +CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend' ######################### MARKETING SITE ############################### diff --git a/lms/envs/yaml_config.py b/lms/envs/yaml_config.py index 65a3a8090d..328d5c07a5 100644 --- a/lms/envs/yaml_config.py +++ b/lms/envs/yaml_config.py @@ -112,8 +112,7 @@ BROKER_POOL_LIMIT = 0 BROKER_CONNECTION_TIMEOUT = 1 # For the Result Store, use the django cache named 'celery' -CELERY_RESULT_BACKEND = 'cache' -CELERY_CACHE_BACKEND = 'celery' +CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend' # When the broker is behind an ELB, use a heartbeat to refresh the # connection and to detect if it has been dropped. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 0d892a5464..882510006a 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -9,13 +9,13 @@ beautifulsoup==3.2.1 bleach==1.4 html5lib==0.999 boto==2.32.1 -celery==3.0.19 +celery==3.1.18 cssselect==0.9.1 dealer==2.0.4 defusedxml==0.4.1 distribute>=0.6.28, <0.7 django-babel-underscore==0.1.0 -django-celery==3.0.17 +django-celery==3.1.16 django-countries==2.1.2 django-extensions==1.2.5 django-filter==0.6.0 diff --git a/requirements/edx/github.txt b/requirements/edx/github.txt index 0000a4d8d6..c61b66d63d 100644 --- a/requirements/edx/github.txt +++ b/requirements/edx/github.txt @@ -36,7 +36,7 @@ git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c -e git+https://github.com/edx/bok-choy.git@82d2f4b72e807b10d112179c0a4abd810a001b82#egg=bok_choy -e git+https://github.com/edx-solutions/django-splash.git@7579d052afcf474ece1239153cffe1c89935bc4f#egg=django-splash -e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock --e git+https://github.com/edx/edx-ora2.git@release-2015-05-07T14.06#egg=edx-ora2 +-e git+https://github.com/edx/edx-ora2.git@release-2015-05-08T16.15#egg=edx-ora2 -e git+https://github.com/edx/edx-submissions.git@e2361932b9bce061a018a31bb3929e9cade80f49#egg=edx-submissions -e git+https://github.com/edx/opaque-keys.git@1254ed4d615a428591850656f39f26509b86d30a#egg=opaque-keys -e git+https://github.com/edx/ease.git@c6dee053eae6b3ac4fdf6be11fa7a9f8265540aa#egg=ease