From b37d2d739af36213fe09ca05d4953f0c5cf7c9c5 Mon Sep 17 00:00:00 2001
From: Nimisha Asthagiri <nasthagiri@edx.org>
Date: Thu, 1 Sep 2016 15:27:57 -0400
Subject: [PATCH] Make BlockStructure tasks more resilient against failures.
 https://openedx.atlassian.net/browse/TNL-5041

---
 cms/envs/common.py                              |  1 +
 lms/envs/common.py                              | 17 +++++++++++++++++
 .../content/block_structure/signals.py          |  6 +++++-
 .../djangoapps/content/block_structure/tasks.py |  6 +++++-
 4 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/cms/envs/common.py b/cms/envs/common.py
index b27a9dcab0..373bf3104d 100644
--- a/cms/envs/common.py
+++ b/cms/envs/common.py
@@ -80,6 +80,7 @@ from lms.envs.common import (
 
     # django-debug-toolbar
     DEBUG_TOOLBAR_PATCH_SETTINGS,
+    BLOCK_STRUCTURES_SETTINGS,
 )
 from path import Path as path
 from warnings import simplefilter
diff --git a/lms/envs/common.py b/lms/envs/common.py
index f8de19911a..94c01207ac 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -1771,6 +1771,23 @@ CELERY_QUEUES = {
 # let logging work as configured:
 CELERYD_HIJACK_ROOT_LOGGER = False
 
+################################ Block Structures ###################################
+
+BLOCK_STRUCTURES_SETTINGS = dict(
+    # Delay, in seconds, after a new edit of a course is published
+    # before updating the block structures cache.  This is needed
+    # for a better chance at getting the latest changes when there
+    # are secondary reads in sharded mongoDB clusters. See TNL-5041
+    # for more info.
+    BLOCK_STRUCTURES_COURSE_PUBLISH_TASK_DELAY=30,
+
+    # Delay, in seconds, between retry attempts if a task fails.
+    BLOCK_STRUCTURES_TASK_DEFAULT_RETRY_DELAY=30,
+
+    # Maximum number of retries per task.
+    BLOCK_STRUCTURES_TASK_MAX_RETRIES=5,
+)
+
 ################################ Bulk Email ###################################
 
 # Suffix used to construct 'from' email address for bulk emails.
diff --git a/openedx/core/djangoapps/content/block_structure/signals.py b/openedx/core/djangoapps/content/block_structure/signals.py
index b8e65a723c..adc12c5c04 100644
--- a/openedx/core/djangoapps/content/block_structure/signals.py
+++ b/openedx/core/djangoapps/content/block_structure/signals.py
@@ -1,6 +1,7 @@
 """
 Signal handlers for invalidating cached data.
 """
+from django.conf import settings
 from django.dispatch.dispatcher import receiver
 
 from xmodule.modulestore.django import SignalHandler
@@ -19,7 +20,10 @@ def _listen_for_course_publish(sender, course_key, **kwargs):  # pylint: disable
 
     # The countdown=0 kwarg ensures the call occurs after the signal emitter
     # has finished all operations.
-    update_course_in_cache.apply_async([unicode(course_key)], countdown=0)
+    update_course_in_cache.apply_async(
+        [unicode(course_key)],
+        countdown=settings.BLOCK_STRUCTURES_SETTINGS['BLOCK_STRUCTURES_COURSE_PUBLISH_TASK_DELAY'],
+    )
 
 
 @receiver(SignalHandler.course_deleted)
diff --git a/openedx/core/djangoapps/content/block_structure/tasks.py b/openedx/core/djangoapps/content/block_structure/tasks.py
index 1ec5ec23ed..d4f01ffc03 100644
--- a/openedx/core/djangoapps/content/block_structure/tasks.py
+++ b/openedx/core/djangoapps/content/block_structure/tasks.py
@@ -3,6 +3,7 @@ Asynchronous tasks related to the Course Blocks sub-application.
 """
 import logging
 from celery.task import task
+from django.conf import settings
 from opaque_keys.edx.keys import CourseKey
 
 from openedx.core.djangoapps.content.block_structure import api
@@ -10,7 +11,10 @@ from openedx.core.djangoapps.content.block_structure import api
 log = logging.getLogger('edx.celery.task')
 
 
-@task
+@task(
+    default_retry_delay=settings.BLOCK_STRUCTURES_SETTINGS['BLOCK_STRUCTURES_TASK_DEFAULT_RETRY_DELAY'],
+    max_retries=settings.BLOCK_STRUCTURES_SETTINGS['BLOCK_STRUCTURES_TASK_MAX_RETRIES'],
+)
 def update_course_in_cache(course_key):
     """
     Updates the course blocks (in the database) for the specified course.