From c783684a099ab9d490e33fd34694ad79bfd96382 Mon Sep 17 00:00:00 2001
From: Ken Clary <kclary@edx.org>
Date: Fri, 29 Oct 2021 16:00:35 -0400
Subject: [PATCH 1/2] fix: move coursegraph celery tasks to cms (from lms), and
 on a non-default, non-high-priority queue.

This is so that the lms default celery queue does not get backed up
when coursegraph is hosed (which is likely when coursegraph has been
redeployed and needs to get the full set of courses).

TNL-8386
---
 cms/envs/common.py     | 6 ++++--
 cms/envs/production.py | 4 +++-
 lms/envs/common.py     | 3 ---
 lms/envs/production.py | 5 -----
 4 files changed, 7 insertions(+), 11 deletions(-)

diff --git a/cms/envs/common.py b/cms/envs/common.py
index ae2b65709e..9ad011750c 100644
--- a/cms/envs/common.py
+++ b/cms/envs/common.py
@@ -1381,13 +1381,15 @@ CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core'
 
 HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high'
 DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default'
+LOW_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.low'
 
 CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
 CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
 
 CELERY_QUEUES = {
     HIGH_PRIORITY_QUEUE: {},
-    DEFAULT_PRIORITY_QUEUE: {}
+    DEFAULT_PRIORITY_QUEUE: {},
+    LOW_PRIORITY_QUEUE: {}
 }
 
 # Queues configuration
@@ -2175,7 +2177,7 @@ SOFTWARE_SECURE_VERIFICATION_ROUTING_KEY = 'edx.lms.core.default'
 POLICY_CHANGE_TASK_RATE_LIMIT = '300/h'
 
 ############## Settings for CourseGraph ############################
-COURSEGRAPH_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE
+COURSEGRAPH_JOB_QUEUE = LOW_PRIORITY_QUEUE
 
 ########## Settings for video transcript migration tasks ############
 VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE
diff --git a/cms/envs/production.py b/cms/envs/production.py
index 0f46c094f0..3aebf6a6ac 100644
--- a/cms/envs/production.py
+++ b/cms/envs/production.py
@@ -492,7 +492,7 @@ if FEATURES.get('CUSTOM_COURSES_EDX'):
     INSTALLED_APPS.append('openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig')
 
 ############## Settings for CourseGraph ############################
-COURSEGRAPH_JOB_QUEUE = ENV_TOKENS.get('COURSEGRAPH_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE)
+COURSEGRAPH_JOB_QUEUE = ENV_TOKENS.get('COURSEGRAPH_JOB_QUEUE', LOW_PRIORITY_QUEUE)
 
 ########## Settings for video transcript migration tasks ############
 VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE = ENV_TOKENS.get('VIDEO_TRANSCRIPT_MIGRATIONS_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE)
@@ -587,6 +587,8 @@ EXPLICIT_QUEUES = {
         'queue': POLICY_CHANGE_GRADES_ROUTING_KEY},
     'cms.djangoapps.contentstore.tasks.update_search_index': {
         'queue': UPDATE_SEARCH_INDEX_JOB_QUEUE},
+    'openedx.core.djangoapps.coursegraph.dump_course_to_neo4j': {
+        'queue': COURSEGRAPH_JOB_QUEUE},
 }
 
 LOGO_IMAGE_EXTRA_TEXT = ENV_TOKENS.get('LOGO_IMAGE_EXTRA_TEXT', '')
diff --git a/lms/envs/common.py b/lms/envs/common.py
index 6d8bbe2bce..0b2926d68a 100644
--- a/lms/envs/common.py
+++ b/lms/envs/common.py
@@ -4527,9 +4527,6 @@ CONTENT_TYPE_GATE_GROUP_IDS = {
 
 COURSES_API_CACHE_TIMEOUT = 3600  # Value is in seconds
 
-############## Settings for CourseGraph ############################
-COURSEGRAPH_JOB_QUEUE = DEFAULT_PRIORITY_QUEUE
-
 
 # Initialize to 'unknown', but read from JSON in production.py
 EDX_PLATFORM_REVISION = 'release'
diff --git a/lms/envs/production.py b/lms/envs/production.py
index a92af271ec..5e2737d925 100644
--- a/lms/envs/production.py
+++ b/lms/envs/production.py
@@ -927,9 +927,6 @@ SYSTEM_TO_FEATURE_ROLE_MAPPING = ENV_TOKENS.get(
 ICP_LICENSE = ENV_TOKENS.get('ICP_LICENSE', None)
 ICP_LICENSE_INFO = ENV_TOKENS.get('ICP_LICENSE_INFO', {})
 
-############## Settings for CourseGraph ############################
-COURSEGRAPH_JOB_QUEUE = ENV_TOKENS.get('COURSEGRAPH_JOB_QUEUE', DEFAULT_PRIORITY_QUEUE)
-
 # How long to cache OpenAPI schemas and UI, in seconds.
 OPENAPI_CACHE_TIMEOUT = ENV_TOKENS.get('OPENAPI_CACHE_TIMEOUT', 60 * 60)
 
@@ -1029,8 +1026,6 @@ EXPLICIT_QUEUES = {
         'queue': PROGRAM_CERTIFICATES_ROUTING_KEY},
     'openedx.core.djangoapps.programs.tasks.award_course_certificate': {
         'queue': PROGRAM_CERTIFICATES_ROUTING_KEY},
-    'openedx.core.djangoapps.coursegraph.dump_course_to_neo4j': {
-        'queue': COURSEGRAPH_JOB_QUEUE},
 }
 
 LOGO_IMAGE_EXTRA_TEXT = ENV_TOKENS.get('LOGO_IMAGE_EXTRA_TEXT', '')

From 109fbb9f7cf88f790b4c6b8370d13ed6775af3a3 Mon Sep 17 00:00:00 2001
From: Ken Clary <kclary@edx.org>
Date: Fri, 29 Oct 2021 16:00:35 -0400
Subject: [PATCH 2/2] fix: move coursegraph celery tasks to cms (from lms), and
 on a non-default, non-high-priority queue.

This is so that the lms default celery queue does not get backed up
when coursegraph is hosed (which is likely when coursegraph has been
redeployed and needs to get the full set of courses).

TNL-8386
---
 cms/envs/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cms/envs/common.py b/cms/envs/common.py
index 9ad011750c..7c5753a5bf 100644
--- a/cms/envs/common.py
+++ b/cms/envs/common.py
@@ -1389,7 +1389,7 @@ CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
 CELERY_QUEUES = {
     HIGH_PRIORITY_QUEUE: {},
     DEFAULT_PRIORITY_QUEUE: {},
-    LOW_PRIORITY_QUEUE: {}
+    LOW_PRIORITY_QUEUE: {},
 }
 
 # Queues configuration