From b9559fc934310182e1b5903b4872105bee90c189 Mon Sep 17 00:00:00 2001
From: Alan Evangelista <alan@opencraft.com>
Date: Fri, 20 Mar 2020 09:39:46 -0300
Subject: [PATCH] Move Celery check task to the high priority queue

Currently, LMS uses 3 Celery workers: lms_default_1, lms_high_1 and
lms_high_mem_1. Each Celery worker sends messages to a single queue:
edx.core.default, edx.core.high and edx.core.high_mem, respectively.
The number of child processes per Celery worker is set to 1. Due to
this configuration, any task in a queue blocks all other tasks.

Currently, the Celery check task submitted by the /heartbeat?extended
LMS HTTP API endpoint runs in the default queue. When some slow task
(eg course grades creation) is sent to the default queue, it will
block the Celery check task, which will expire and the heartbeat endpoint
will fail. This patch moves the task to another queue which has
only shorter tasks and in which this problem will not occur.

Use a Django setting for the Celery check task routing key so that
it can be overriden by individual OpenEDX instances via JSON
env files.
---
 cms/envs/common.py                         |  1 +
 cms/envs/production.py                     |  3 +++
 lms/envs/common.py                         | 31 +++++++++++-----------
 lms/envs/production.py                     |  3 +++
 openedx/core/djangoapps/heartbeat/tasks.py |  3 ++-
 5 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/cms/envs/common.py b/cms/envs/common.py
index 31b53faebb..e61a23ed0e 100644
--- a/cms/envs/common.py
+++ b/cms/envs/common.py
@@ -75,6 +75,7 @@ from lms.envs.common import (
     HEARTBEAT_CHECKS,
     HEARTBEAT_EXTENDED_CHECKS,
     HEARTBEAT_CELERY_TIMEOUT,
+    HEARTBEAT_CELERY_ROUTING_KEY,
 
     # Default site to use if no site exists matching request headers
     SITE_ID,
diff --git a/cms/envs/production.py b/cms/envs/production.py
index 25b8ebef1d..2ef4a7dd6f 100644
--- a/cms/envs/production.py
+++ b/cms/envs/production.py
@@ -294,6 +294,9 @@ STUDIO_SHORT_NAME = ENV_TOKENS.get('STUDIO_SHORT_NAME') or STUDIO_SHORT_NAME
 if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS:
     TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS")
 
+# Heartbeat
+HEARTBEAT_CELERY_ROUTING_KEY = ENV_TOKENS.get('HEARTBEAT_CELERY_ROUTING_KEY', HIGH_PRIORITY_QUEUE)
+
 LOGIN_REDIRECT_WHITELIST = [reverse_lazy('home')]
 
 
diff --git a/lms/envs/common.py b/lms/envs/common.py
index 6f6981ae6d..621bf684da 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -1062,21 +1062,6 @@ USERNAME_REGEX_PARTIAL = r'[\w .@_+-]+'
 USERNAME_PATTERN = r'(?P<username>{regex})'.format(regex=USERNAME_REGEX_PARTIAL)
 
 
-############################## HEARTBEAT ######################################
-
-# Checks run in normal mode by the heartbeat djangoapp
-HEARTBEAT_CHECKS = [
-    'openedx.core.djangoapps.heartbeat.default_checks.check_modulestore',
-    'openedx.core.djangoapps.heartbeat.default_checks.check_database',
-]
-
-# Other checks to run by default in "extended"/heavy mode
-HEARTBEAT_EXTENDED_CHECKS = (
-    'openedx.core.djangoapps.heartbeat.default_checks.check_celery',
-)
-
-HEARTBEAT_CELERY_TIMEOUT = 5
-
 ############################## EVENT TRACKING #################################
 LMS_SEGMENT_KEY = None
 
@@ -2326,6 +2311,22 @@ CELERY_BROKER_HOSTNAME = 'localhost'
 CELERY_BROKER_USER = 'celery'
 CELERY_BROKER_PASSWORD = 'celery'
 
+############################## HEARTBEAT ######################################
+
+# Checks run in normal mode by the heartbeat djangoapp
+HEARTBEAT_CHECKS = [
+    'openedx.core.djangoapps.heartbeat.default_checks.check_modulestore',
+    'openedx.core.djangoapps.heartbeat.default_checks.check_database',
+]
+
+# Other checks to run by default in "extended"/heavy mode
+HEARTBEAT_EXTENDED_CHECKS = (
+    'openedx.core.djangoapps.heartbeat.default_checks.check_celery',
+)
+
+HEARTBEAT_CELERY_TIMEOUT = 5
+HEARTBEAT_CELERY_ROUTING_KEY = HIGH_PRIORITY_QUEUE
+
 ################################ Block Structures ###################################
 
 BLOCK_STRUCTURES_SETTINGS = dict(
diff --git a/lms/envs/production.py b/lms/envs/production.py
index b238a3f2fb..24e317bb61 100644
--- a/lms/envs/production.py
+++ b/lms/envs/production.py
@@ -568,6 +568,9 @@ TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES = ENV_TOKENS.get(
 )
 TRACKING_SEGMENTIO_SOURCE_MAP = ENV_TOKENS.get("TRACKING_SEGMENTIO_SOURCE_MAP", TRACKING_SEGMENTIO_SOURCE_MAP)
 
+# Heartbeat
+HEARTBEAT_CELERY_ROUTING_KEY = ENV_TOKENS.get('HEARTBEAT_CELERY_ROUTING_KEY', HIGH_PRIORITY_QUEUE)
+
 # Student identity verification settings
 VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT)
 DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH = ENV_TOKENS.get(
diff --git a/openedx/core/djangoapps/heartbeat/tasks.py b/openedx/core/djangoapps/heartbeat/tasks.py
index b7dbfc6007..dca2849b51 100644
--- a/openedx/core/djangoapps/heartbeat/tasks.py
+++ b/openedx/core/djangoapps/heartbeat/tasks.py
@@ -4,8 +4,9 @@ A trivial task for health checks
 
 
 from celery.task import task
+from django.conf import settings
 
 
-@task()
+@task(routing_key=settings.HEARTBEAT_CELERY_ROUTING_KEY)
 def sample_task():
     return True