Move Celery check task to the high priority queue

Currently, LMS uses 3 Celery workers: lms_default_1, lms_high_1 and
lms_high_mem_1. Each Celery worker sends messages to a single queue:
edx.core.default, edx.core.high and edx.core.high_mem, respectively.
The number of child processes per Celery worker is set to 1. Due to
this configuration, any task in a queue blocks all other tasks.

Currently, the Celery check task submitted by the /heartbeat?extended
LMS HTTP API endpoint runs in the default queue. When some slow task
(eg course grades creation) is sent to the default queue, it will
block the Celery check task, which will expire and the heartbeat endpoint
will fail. This patch moves the task to another queue which has
only shorter tasks and in which this problem will not occur.

Use a Django setting for the Celery check task routing key so that
it can be overriden by individual OpenEDX instances via JSON
env files.

cms/envs/common.py

@@ -75,6 +75,7 @@ from lms.envs.common import (
     HEARTBEAT_CHECKS,
     HEARTBEAT_EXTENDED_CHECKS,
     HEARTBEAT_CELERY_TIMEOUT,
+    HEARTBEAT_CELERY_ROUTING_KEY,
 
     # Default site to use if no site exists matching request headers
     SITE_ID,

cms/envs/production.py

@@ -294,6 +294,9 @@ STUDIO_SHORT_NAME = ENV_TOKENS.get('STUDIO_SHORT_NAME') or STUDIO_SHORT_NAME
 if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS:
     TRACKING_IGNORE_URL_PATTERNS = ENV_TOKENS.get("TRACKING_IGNORE_URL_PATTERNS")
 
+# Heartbeat
+HEARTBEAT_CELERY_ROUTING_KEY = ENV_TOKENS.get('HEARTBEAT_CELERY_ROUTING_KEY', HIGH_PRIORITY_QUEUE)
+
 LOGIN_REDIRECT_WHITELIST = [reverse_lazy('home')]
 
 

lms/envs/common.py

@@ -1062,21 +1062,6 @@ USERNAME_REGEX_PARTIAL = r'[\w .@_+-]+'
 USERNAME_PATTERN = r'(?P<username>{regex})'.format(regex=USERNAME_REGEX_PARTIAL)
 
 
-############################## HEARTBEAT ######################################
-
-# Checks run in normal mode by the heartbeat djangoapp
-HEARTBEAT_CHECKS = [
-    'openedx.core.djangoapps.heartbeat.default_checks.check_modulestore',
-    'openedx.core.djangoapps.heartbeat.default_checks.check_database',
-]
-
-# Other checks to run by default in "extended"/heavy mode
-HEARTBEAT_EXTENDED_CHECKS = (
-    'openedx.core.djangoapps.heartbeat.default_checks.check_celery',
-)
-
-HEARTBEAT_CELERY_TIMEOUT = 5
-
 ############################## EVENT TRACKING #################################
 LMS_SEGMENT_KEY = None
 
@@ -2326,6 +2311,22 @@ CELERY_BROKER_HOSTNAME = 'localhost'
 CELERY_BROKER_USER = 'celery'
 CELERY_BROKER_PASSWORD = 'celery'
 
+############################## HEARTBEAT ######################################
+
+# Checks run in normal mode by the heartbeat djangoapp
+HEARTBEAT_CHECKS = [
+    'openedx.core.djangoapps.heartbeat.default_checks.check_modulestore',
+    'openedx.core.djangoapps.heartbeat.default_checks.check_database',
+]
+
+# Other checks to run by default in "extended"/heavy mode
+HEARTBEAT_EXTENDED_CHECKS = (
+    'openedx.core.djangoapps.heartbeat.default_checks.check_celery',
+)
+
+HEARTBEAT_CELERY_TIMEOUT = 5
+HEARTBEAT_CELERY_ROUTING_KEY = HIGH_PRIORITY_QUEUE
+
 ################################ Block Structures ###################################
 
 BLOCK_STRUCTURES_SETTINGS = dict(

lms/envs/production.py

@@ -568,6 +568,9 @@ TRACKING_SEGMENTIO_DISALLOWED_SUBSTRING_NAMES = ENV_TOKENS.get(
 )
 TRACKING_SEGMENTIO_SOURCE_MAP = ENV_TOKENS.get("TRACKING_SEGMENTIO_SOURCE_MAP", TRACKING_SEGMENTIO_SOURCE_MAP)
 
+# Heartbeat
+HEARTBEAT_CELERY_ROUTING_KEY = ENV_TOKENS.get('HEARTBEAT_CELERY_ROUTING_KEY', HIGH_PRIORITY_QUEUE)
+
 # Student identity verification settings
 VERIFY_STUDENT = AUTH_TOKENS.get("VERIFY_STUDENT", VERIFY_STUDENT)
 DISABLE_ACCOUNT_ACTIVATION_REQUIREMENT_SWITCH = ENV_TOKENS.get(

openedx/core/djangoapps/heartbeat/tasks.py

@@ -4,8 +4,9 @@ A trivial task for health checks
 
 
 from celery.task import task
+from django.conf import settings
 
 
-@task()
+@task(routing_key=settings.HEARTBEAT_CELERY_ROUTING_KEY)
 def sample_task():
     return True