Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #7842 from edx/feanil/update_celery

Browse Source 
Update to the latest version of celery, and django-celery.
This commit is contained in:
Feanil Patel
2015-05-08 13:41:18 -04:00
parent fa2b84d95d e0acd7151e
commit 219b9a0cc6
15 changed files with 79 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
cms/__init__.py
View File

@@ -0,0 +1,9 @@
"""
Celery needs to be loaded when the cms modules are so that task
registration and discovery can work correctly.
"""
from __future__ import absolute_import
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import APP as CELERY_APP

23
cms/celery.py Normal file
View File

@@ -0,0 +1,23 @@
"""
Import celery, load its settings from the django settings
and auto discover tasks in all installed django apps.
Taken from: http://celery.readthedocs.org/en/latest/django/first-steps-with-django.html
"""
from __future__ import absolute_import
import os
from celery import Celery
from django.conf import settings
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings')
APP = Celery('proj')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
APP.config_from_object('django.conf:settings')
APP.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

3
cms/envs/aws.py
View File

@@ -60,8 +60,7 @@ BROKER_POOL_LIMIT = 0
BROKER_CONNECTION_TIMEOUT = 1
# For the Result Store, use the django cache named 'celery'
CELERY_RESULT_BACKEND = 'cache'
CELERY_CACHE_BACKEND = 'celery'
CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
# When the broker is behind an ELB, use a heartbeat to refresh the
# connection and to detect if it has been dropped.

4
cms/envs/test.py
View File

@@ -187,9 +187,7 @@ simplefilter('ignore')
################################# CELERY ######################################
CELERY_ALWAYS_EAGER = True
CELERY_RESULT_BACKEND = 'cache'
BROKER_TRANSPORT = 'memory'
CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
########################### Server Ports ###################################

3
cms/envs/yaml_config.py
View File

@@ -75,8 +75,7 @@ BROKER_POOL_LIMIT = 0
BROKER_CONNECTION_TIMEOUT = 1
# For the Result Store, use the django cache named 'celery'
CELERY_RESULT_BACKEND = 'cache'
CELERY_CACHE_BACKEND = 'celery'
CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
# When the broker is behind an ELB, use a heartbeat to refresh the
# connection and to detect if it has been dropped.

9
lms/__init__.py
View File

@@ -0,0 +1,9 @@
"""
Celery needs to be loaded when the cms modules are so that task
registration and discovery can work correctly.
"""
from __future__ import absolute_import
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import APP as CELERY_APP

23
lms/celery.py Normal file
View File

@@ -0,0 +1,23 @@
"""
Import celery, load its settings from the django settings
and auto discover tasks in all installed django apps.
Taken from: http://celery.readthedocs.org/en/latest/django/first-steps-with-django.html
"""
from __future__ import absolute_import
import os
from celery import Celery
from django.conf import settings
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'proj.settings')
APP = Celery('proj')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
APP.config_from_object('django.conf:settings')
APP.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

5
lms/djangoapps/bulk_email/tasks.py
View File

@@ -773,7 +773,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
# needs to be returned back to Celery. If it fails, we return the existing
# exception.
try:
send_course_email.retry(
retry_task = send_course_email.retry(
args=[
entry_id,
email_id,
@@ -786,6 +786,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
max_retries=max_retries,
throw=True,
)
raise retry_task
except RetryTaskError as retry_error:
# If the retry call is successful, update with the current progress:
log.exception(u'Task %s: email with id %d caused send_course_email task to retry.',
@@ -800,7 +801,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
log.exception(u'Task %s: email with id %d caused send_course_email task to fail to retry. To list: %s',
task_id, email_id, [i['email'] for i in to_list])
num_failed = len(to_list)
subtask_status.increment(subtask_status, failed=num_failed, state=FAILURE)
subtask_status.increment(failed=num_failed, state=FAILURE)
return subtask_status, retry_exc

10
lms/djangoapps/bulk_email/tests/test_tasks.py
View File

@@ -104,15 +104,9 @@ class TestBulkEmailInstructorTask(InstructorTaskCourseTestCase):
return instructor_task
def _run_task_with_mock_celery(self, task_class, entry_id, task_id):
"""Submit a task and mock how celery provides a current_task."""
mock_current_task = Mock()
mock_current_task.max_retries = settings.BULK_EMAIL_MAX_RETRIES
mock_current_task.default_retry_delay = settings.BULK_EMAIL_DEFAULT_RETRY_DELAY
"""Mock was not needed for some tests, testing to see if it's needed at all."""
task_args = [entry_id, {}]
with patch('bulk_email.tasks._get_current_task') as mock_get_task:
mock_get_task.return_value = mock_current_task
return task_class.apply(task_args, task_id=task_id).get()
return task_class.apply(task_args, task_id=task_id).get()
def test_email_missing_current_task(self):
task_entry = self._create_input_entry()

3
lms/envs/aws.py
View File

@@ -64,8 +64,7 @@ BROKER_POOL_LIMIT = 0
BROKER_CONNECTION_TIMEOUT = 1
# For the Result Store, use the django cache named 'celery'
CELERY_RESULT_BACKEND = 'cache'
CELERY_CACHE_BACKEND = 'celery'
CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
# When the broker is behind an ELB, use a heartbeat to refresh the
# connection and to detect if it has been dropped.

3
lms/envs/bok_choy.py
View File

@@ -55,8 +55,7 @@ MEDIA_URL = "/static/uploads/"
################################# CELERY ######################################
CELERY_ALWAYS_EAGER = True
CELERY_RESULT_BACKEND = 'cache'
BROKER_TRANSPORT = 'memory'
CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
###################### Grade Downloads ######################
GRADES_DOWNLOAD = {

3
lms/envs/test.py
View File

@@ -299,8 +299,7 @@ GIT_REPO_DIR = TEST_ROOT / "course_repos"
################################# CELERY ######################################
CELERY_ALWAYS_EAGER = True
CELERY_RESULT_BACKEND = 'cache'
BROKER_TRANSPORT = 'memory'
CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
######################### MARKETING SITE ###############################

3
lms/envs/yaml_config.py
View File

@@ -112,8 +112,7 @@ BROKER_POOL_LIMIT = 0
BROKER_CONNECTION_TIMEOUT = 1
# For the Result Store, use the django cache named 'celery'
CELERY_RESULT_BACKEND = 'cache'
CELERY_CACHE_BACKEND = 'celery'
CELERY_RESULT_BACKEND = 'djcelery.backends.cache:CacheBackend'
# When the broker is behind an ELB, use a heartbeat to refresh the
# connection and to detect if it has been dropped.

4
requirements/edx/base.txt
View File

@@ -9,13 +9,13 @@ beautifulsoup==3.2.1
bleach==1.4
html5lib==0.999
boto==2.32.1
celery==3.0.19
celery==3.1.18
cssselect==0.9.1
dealer==2.0.4
defusedxml==0.4.1
distribute>=0.6.28, <0.7
django-babel-underscore==0.1.0
django-celery==3.0.17
django-celery==3.1.16
django-countries==2.1.2
django-extensions==1.2.5
django-filter==0.6.0

2
requirements/edx/github.txt
View File

@@ -36,7 +36,7 @@ git+https://github.com/hmarr/django-debug-toolbar-mongo.git@b0686a76f1ce3532088c
-e git+https://github.com/edx/bok-choy.git@82d2f4b72e807b10d112179c0a4abd810a001b82#egg=bok_choy
-e git+https://github.com/edx-solutions/django-splash.git@7579d052afcf474ece1239153cffe1c89935bc4f#egg=django-splash
-e git+https://github.com/edx/acid-block.git@e46f9cda8a03e121a00c7e347084d142d22ebfb7#egg=acid-xblock
-e git+https://github.com/edx/edx-ora2.git@release-2015-05-07T14.06#egg=edx-ora2
-e git+https://github.com/edx/edx-ora2.git@release-2015-05-08T16.15#egg=edx-ora2
-e git+https://github.com/edx/edx-submissions.git@e2361932b9bce061a018a31bb3929e9cade80f49#egg=edx-submissions
-e git+https://github.com/edx/opaque-keys.git@1254ed4d615a428591850656f39f26509b86d30a#egg=opaque-keys
-e git+https://github.com/edx/ease.git@c6dee053eae6b3ac4fdf6be11fa7a9f8265540aa#egg=ease