Revert "Update routing config" (#25536)" (#25549)" (#25553)" (#25561)

This reverts commit db4c3b1210.

openedx/core/djangoapps/coursegraph/tasks.py

@@ -246,7 +246,7 @@ def should_dump_course(course_key, graph):
     return last_this_command_was_run < course_last_published_date
 
 
-@task
+@task(routing_key=settings.COURSEGRAPH_JOB_QUEUE)
 def dump_course_to_neo4j(course_key_string, credentials):
     """
     Serializes a course and writes it to neo4j.

openedx/core/djangoapps/credentials/tasks/v1/tasks.py

@@ -13,6 +13,11 @@ from openedx.core.djangoapps.credentials.utils import get_credentials_api_client
 
 logger = get_task_logger(__name__)
 
+# Under cms the following setting is not defined, leading to errors during tests.
+# These tasks aren't strictly credentials generation, but are similar in the sense
+# that they generate records on the credentials side. And have a similar SLA.
+ROUTING_KEY = getattr(settings, 'CREDENTIALS_GENERATION_ROUTING_KEY', None)
+
 # Maximum number of retries before giving up.
 # For reference, 11 retries with exponential backoff yields a maximum waiting
 # time of 2047 seconds (about 30 minutes). Setting this to None could yield
@@ -20,7 +25,7 @@ logger = get_task_logger(__name__)
 MAX_RETRIES = 11
 
 
-@task(bind=True, ignore_result=True)
+@task(bind=True, ignore_result=True, routing_key=ROUTING_KEY)
 def send_grade_to_credentials(self, username, course_run_key, verified, letter_grade, percent_grade):
     """ Celery task to notify the Credentials IDA of a grade change via POST. """
     logger.info(u'Running task send_grade_to_credentials for username %s and course %s', username, course_run_key)

openedx/core/djangoapps/heartbeat/tasks.py

@@ -4,8 +4,9 @@ A trivial task for health checks
 
 
 from celery.task import task
+from django.conf import settings
 
 
-@task
+@task(routing_key=settings.HEARTBEAT_CELERY_ROUTING_KEY)
 def sample_task():
     return True

openedx/core/djangoapps/programs/tasks.py

@@ -22,6 +22,9 @@ from openedx.core.djangoapps.programs.utils import ProgramProgressMeter
 from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
 
 LOGGER = get_task_logger(__name__)
+# Under cms the following setting is not defined, leading to errors during tests.
+ROUTING_KEY = getattr(settings, 'CREDENTIALS_GENERATION_ROUTING_KEY', None)
+PROGRAM_CERTIFICATES_ROUTING_KEY = getattr(settings, 'PROGRAM_CERTIFICATES_ROUTING_KEY', None)
 # Maximum number of retries before giving up on awarding credentials.
 # For reference, 11 retries with exponential backoff yields a maximum waiting
 # time of 2047 seconds (about 30 minutes). Setting this to None could yield
@@ -120,7 +123,7 @@ def award_program_certificate(client, username, program_uuid, visible_date):
     })
 
 
-@task(bind=True, ignore_result=True)
+@task(bind=True, ignore_result=True, routing_key=PROGRAM_CERTIFICATES_ROUTING_KEY)
 def award_program_certificates(self, username):
     """
     This task is designed to be called whenever a student's completion status
@@ -282,7 +285,7 @@ def post_course_certificate(client, username, certificate, visible_date):
     })
 
 
-@task(bind=True, ignore_result=True)
+@task(bind=True, ignore_result=True, routing_key=ROUTING_KEY)
 def award_course_certificate(self, username, course_run_key):
     """
     This task is designed to be called whenever a student GeneratedCertificate is updated.
@@ -396,7 +399,7 @@ def revoke_program_certificate(client, username, program_uuid):
     })
 
 
-@task(bind=True, ignore_result=True)
+@task(bind=True, ignore_result=True, routing_key=PROGRAM_CERTIFICATES_ROUTING_KEY)
 def revoke_program_certificates(self, username, course_key):
     """
     This task is designed to be called whenever a student's course certificate is
@@ -519,7 +522,7 @@ def revoke_program_certificates(self, username, course_key):
     LOGGER.info(u'Successfully completed the task revoke_program_certificates for username %s', username)
 
 
-@task(bind=True, ignore_result=True)
+@task(bind=True, ignore_result=True, routing_key=PROGRAM_CERTIFICATES_ROUTING_KEY)
 def update_certificate_visible_date_on_course_update(self, course_key):
     """
     This task is designed to be called whenever a course is updated with

openedx/core/djangoapps/schedules/tasks.py

@@ -143,7 +143,7 @@ class BinnedScheduleMessageBaseTask(ScheduleMessageBaseTask):
         raise NotImplementedError
 
 
-@task(base=LoggedTask, ignore_result=True)
+@task(base=LoggedTask, ignore_result=True, routing_key=ROUTING_KEY)
 def _recurring_nudge_schedule_send(site_id, msg_str):
     _schedule_send(
         msg_str,
@@ -153,7 +153,7 @@ def _recurring_nudge_schedule_send(site_id, msg_str):
     )
 
 
-@task(base=LoggedTask, ignore_result=True)
+@task(base=LoggedTask, ignore_result=True, routing_key=ROUTING_KEY)
 def _upgrade_reminder_schedule_send(site_id, msg_str):
     _schedule_send(
         msg_str,
@@ -163,7 +163,7 @@ def _upgrade_reminder_schedule_send(site_id, msg_str):
     )
 
 
-@task(base=LoggedTask, ignore_result=True)
+@task(base=LoggedTask, ignore_result=True, routing_key=ROUTING_KEY)
 def _course_update_schedule_send(site_id, msg_str):
     _schedule_send(
         msg_str,

openedx/core/lib/celery/routers.py

@@ -5,41 +5,62 @@ For more, see https://celery.readthedocs.io/en/latest/userguide/routing.html#rou
 """
 
 import logging
+from abc import ABCMeta, abstractproperty
 
+from django.conf import settings
+import six
 
 log = logging.getLogger(__name__)
 
 
-def route_task_queue(name):
+class AlternateEnvironmentRouter(six.with_metaclass(ABCMeta, object)):
     """
-    Helper method allowing for custom routing logic.
-
-    If None is returned from this method, default routing logic is used.
+    A custom Router class for use in routing celery tasks to non-default queues.
     """
-    from django.conf import settings  # pylint: disable=import-outside-toplevel
 
-    if name in settings.EXPLICIT_QUEUES:
-        return settings.EXPLICIT_QUEUES[name]
+    @abstractproperty
+    def alternate_env_tasks(self):
+        """
+        Defines the task -> alternate worker environment to be used when routing.
 
-    alternate_env = settings.ALTERNATE_ENV_TASKS.get(name, None)
-    if alternate_env:
-        return ensure_queue_env(alternate_env)
+        Subclasses must override this property with their own specific mappings.
+        """
+        return {}
 
+    @property
+    def explicit_queues(self):
+        """
+        Defines the task -> alternate worker queue to be used when routing.
+        """
+        return {}
 
-def ensure_queue_env(desired_env):
-    """
-    Helper method to get the desired type of queue.
+    def route_for_task(self, task, args=None, kwargs=None):  # pylint: disable=unused-argument
+        """
+        Celery-defined method allowing for custom routing logic.
 
-    If no such queue is defined, default routing logic is used.
-    """
-    from django.conf import settings  # pylint: disable=import-outside-toplevel
+        If None is returned from this method, default routing logic is used.
+        """
+        if task in self.explicit_queues:
+            return self.explicit_queues[task]
 
-    queues = getattr(settings, 'CELERY_QUEUES', None)
-    return next(
-        (
-            queue
-            for queue in queues
-            if '.{}.'.format(desired_env) in queue
-        ),
-        None
-    )
+        alternate_env = self.alternate_env_tasks.get(task, None)
+        if alternate_env:
+            return self.ensure_queue_env(alternate_env)
+
+        return None
+
+    def ensure_queue_env(self, desired_env):
+        """
+        Helper method to get the desired type of queue.
+
+        If no such queue is defined, default routing logic is used.
+        """
+        queues = getattr(settings, 'CELERY_QUEUES', None)
+        return next(
+            (
+                queue
+                for queue in queues
+                if '.{}.'.format(desired_env) in queue
+            ),
+            None
+        )