From c2d848860d5b86ef717932eba4053fe70c72c96b Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 30 Jun 2021 00:27:40 -0400 Subject: [PATCH] fix!: Make default queue names service variant specific. common.py has queue names that always get overridden by production.py and lead to confusion. Set a default SERVICE_VANIANT in common.py and then set the queue names based on that in common.py so that production.py doesn't make it more complicated. This should prevent the issue where if you copy a queue name in common.py it ends up being incorrect in the production system. This is what happened with the sample_task change. https://github.com/edx/edx-platform/pull/23731 made it so that the queue name for that queue is independently configurable but the default was set to the value of HIGH_PRIORITY_QUEUE in common.py which is not the same as the value set in production.py leading to stale tasks that never get picked up in production. BREAKING_CHANGE: If anyone was building a different settings file on top of common, the default names in common.py are now change to be service variant specific. eg 'edx.cms.core.high' instead of 'edx.core.high' --- cms/envs/common.py | 37 ++++++++++++++++++++++++++++--------- cms/envs/production.py | 26 -------------------------- lms/envs/common.py | 38 ++++++++++++++++++++++++++++---------- lms/envs/production.py | 31 ------------------------------- 4 files changed, 56 insertions(+), 76 deletions(-) diff --git a/cms/envs/common.py b/cms/envs/common.py index 2529107744..5c5db8a594 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1266,6 +1266,19 @@ WEBPACK_LOADER = { } WEBPACK_CONFIG_PATH = 'webpack.prod.config.js' + +############################ SERVICE_VARIANT ################################## + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', 'cms') + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + + ################################# CELERY ###################################### # Message configuration @@ -1292,22 +1305,28 @@ CELERY_SEND_TASK_SENT_EVENT = True CELERY_DEFAULT_EXCHANGE = 'edx.core' CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' -# Queues configuration +# Name the exchange and queues for each variant -HIGH_PRIORITY_QUEUE = 'edx.core.high' -DEFAULT_PRIORITY_QUEUE = 'edx.core.default' +QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_QUEUE_HA_POLICY = 'all' +CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' -CELERY_CREATE_MISSING_QUEUES = True +HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' +DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE -CELERY_QUEUES = [ - 'edx.cms.core.default', - 'edx.cms.core.high', -] +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {} +} + +# Queues configuration + +CELERY_QUEUE_HA_POLICY = 'all' + +CELERY_CREATE_MISSING_QUEUES = True CELERY_BROKER_TRANSPORT = 'amqp' CELERY_BROKER_HOSTNAME = 'localhost' diff --git a/cms/envs/production.py b/cms/envs/production.py index 1ce1a223ae..908997ab31 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -93,15 +93,6 @@ except Exception: # pylint: disable=broad-except # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) -# SERVICE_VARIANT specifies name of the variant used, which decides what JSON -# configuration files are read during startup. -SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) - -# CONFIG_PREFIX specifies the prefix of the JSON configuration files, -# based on the service variant. If no variant is use, don't use a -# prefix. -CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" - ###################################### CELERY ################################ # Don't use a connection pool, since connections are dropped by ELB. @@ -119,23 +110,6 @@ BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2) # Each worker should only fetch one message at a time CELERYD_PREFETCH_MULTIPLIER = 1 -# Rename the exchange and queues for each variant - -QUEUE_VARIANT = CONFIG_PREFIX.lower() - -CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' - -HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' -DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' - -CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE -CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE - -CELERY_QUEUES = { - HIGH_PRIORITY_QUEUE: {}, - DEFAULT_PRIORITY_QUEUE: {} -} - CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task" # STATIC_URL_BASE specifies the base url to use for static files diff --git a/lms/envs/common.py b/lms/envs/common.py index f0051b8f39..45e8e9aa95 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2617,24 +2617,42 @@ CELERY_SEND_TASK_SENT_EVENT = True CELERY_DEFAULT_EXCHANGE = 'edx.core' CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', "lms") + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + # Queues configuration -HIGH_PRIORITY_QUEUE = 'edx.core.high' -DEFAULT_PRIORITY_QUEUE = 'edx.core.default' -HIGH_MEM_QUEUE = 'edx.core.high_mem' +# Name the exchange and queues w.r.t the SERVICE_VARIANT +QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_QUEUE_HA_POLICY = 'all' +CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' -CELERY_CREATE_MISSING_QUEUES = True +HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' +DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' +HIGH_MEM_QUEUE = f'edx.{QUEUE_VARIANT}core.high_mem' CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE -CELERY_QUEUES = [ - 'edx.lms.core.default', - 'edx.lms.core.high', - 'edx.lms.core.high_mem' -] +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {}, + HIGH_MEM_QUEUE: {}, +} + +CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task" +CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict + +CELERY_QUEUE_HA_POLICY = 'all' + +CELERY_CREATE_MISSING_QUEUES = True # let logging work as configured: CELERYD_HIJACK_ROOT_LOGGER = False diff --git a/lms/envs/production.py b/lms/envs/production.py index 70bfa038ad..adcfce2af7 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -103,15 +103,6 @@ except Exception: # pylint: disable=broad-except # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) -# SERVICE_VARIANT specifies name of the variant used, which decides what JSON -# configuration files are read during startup. -SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) - -# CONFIG_PREFIX specifies the prefix of the JSON configuration files, -# based on the service variant. If no variant is use, don't use a -# prefix. -CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" - ###################################### CELERY ################################ # Don't use a connection pool, since connections are dropped by ELB. @@ -129,28 +120,6 @@ BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2) # Each worker should only fetch one message at a time CELERYD_PREFETCH_MULTIPLIER = 1 -# Rename the exchange and queues for each variant - -QUEUE_VARIANT = CONFIG_PREFIX.lower() - -CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' - -HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' -DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' -HIGH_MEM_QUEUE = f'edx.{QUEUE_VARIANT}core.high_mem' - -CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE -CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE - -CELERY_QUEUES = { - HIGH_PRIORITY_QUEUE: {}, - DEFAULT_PRIORITY_QUEUE: {}, - HIGH_MEM_QUEUE: {}, -} - -CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task" -CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict - # STATIC_ROOT specifies the directory where static files are # collected STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None)