diff --git a/cms/envs/common.py b/cms/envs/common.py index 2529107744..5c5db8a594 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1266,6 +1266,19 @@ WEBPACK_LOADER = { } WEBPACK_CONFIG_PATH = 'webpack.prod.config.js' + +############################ SERVICE_VARIANT ################################## + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', 'cms') + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + + ################################# CELERY ###################################### # Message configuration @@ -1292,22 +1305,28 @@ CELERY_SEND_TASK_SENT_EVENT = True CELERY_DEFAULT_EXCHANGE = 'edx.core' CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' -# Queues configuration +# Name the exchange and queues for each variant -HIGH_PRIORITY_QUEUE = 'edx.core.high' -DEFAULT_PRIORITY_QUEUE = 'edx.core.default' +QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_QUEUE_HA_POLICY = 'all' +CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' -CELERY_CREATE_MISSING_QUEUES = True +HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' +DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE -CELERY_QUEUES = [ - 'edx.cms.core.default', - 'edx.cms.core.high', -] +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {} +} + +# Queues configuration + +CELERY_QUEUE_HA_POLICY = 'all' + +CELERY_CREATE_MISSING_QUEUES = True CELERY_BROKER_TRANSPORT = 'amqp' CELERY_BROKER_HOSTNAME = 'localhost' diff --git a/cms/envs/production.py b/cms/envs/production.py index 1ce1a223ae..908997ab31 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -93,15 +93,6 @@ except Exception: # pylint: disable=broad-except # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) -# SERVICE_VARIANT specifies name of the variant used, which decides what JSON -# configuration files are read during startup. -SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) - -# CONFIG_PREFIX specifies the prefix of the JSON configuration files, -# based on the service variant. If no variant is use, don't use a -# prefix. -CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" - ###################################### CELERY ################################ # Don't use a connection pool, since connections are dropped by ELB. @@ -119,23 +110,6 @@ BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2) # Each worker should only fetch one message at a time CELERYD_PREFETCH_MULTIPLIER = 1 -# Rename the exchange and queues for each variant - -QUEUE_VARIANT = CONFIG_PREFIX.lower() - -CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' - -HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' -DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' - -CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE -CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE - -CELERY_QUEUES = { - HIGH_PRIORITY_QUEUE: {}, - DEFAULT_PRIORITY_QUEUE: {} -} - CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task" # STATIC_URL_BASE specifies the base url to use for static files diff --git a/lms/envs/common.py b/lms/envs/common.py index f0051b8f39..45e8e9aa95 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2617,24 +2617,42 @@ CELERY_SEND_TASK_SENT_EVENT = True CELERY_DEFAULT_EXCHANGE = 'edx.core' CELERY_DEFAULT_EXCHANGE_TYPE = 'direct' + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', "lms") + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + # Queues configuration -HIGH_PRIORITY_QUEUE = 'edx.core.high' -DEFAULT_PRIORITY_QUEUE = 'edx.core.default' -HIGH_MEM_QUEUE = 'edx.core.high_mem' +# Name the exchange and queues w.r.t the SERVICE_VARIANT +QUEUE_VARIANT = CONFIG_PREFIX.lower() -CELERY_QUEUE_HA_POLICY = 'all' +CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' -CELERY_CREATE_MISSING_QUEUES = True +HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' +DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' +HIGH_MEM_QUEUE = f'edx.{QUEUE_VARIANT}core.high_mem' CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE -CELERY_QUEUES = [ - 'edx.lms.core.default', - 'edx.lms.core.high', - 'edx.lms.core.high_mem' -] +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {}, + HIGH_MEM_QUEUE: {}, +} + +CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task" +CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict + +CELERY_QUEUE_HA_POLICY = 'all' + +CELERY_CREATE_MISSING_QUEUES = True # let logging work as configured: CELERYD_HIJACK_ROOT_LOGGER = False diff --git a/lms/envs/production.py b/lms/envs/production.py index 70bfa038ad..adcfce2af7 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -103,15 +103,6 @@ except Exception: # pylint: disable=broad-except # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) -# SERVICE_VARIANT specifies name of the variant used, which decides what JSON -# configuration files are read during startup. -SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) - -# CONFIG_PREFIX specifies the prefix of the JSON configuration files, -# based on the service variant. If no variant is use, don't use a -# prefix. -CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" - ###################################### CELERY ################################ # Don't use a connection pool, since connections are dropped by ELB. @@ -129,28 +120,6 @@ BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2) # Each worker should only fetch one message at a time CELERYD_PREFETCH_MULTIPLIER = 1 -# Rename the exchange and queues for each variant - -QUEUE_VARIANT = CONFIG_PREFIX.lower() - -CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core' - -HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high' -DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default' -HIGH_MEM_QUEUE = f'edx.{QUEUE_VARIANT}core.high_mem' - -CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE -CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE - -CELERY_QUEUES = { - HIGH_PRIORITY_QUEUE: {}, - DEFAULT_PRIORITY_QUEUE: {}, - HIGH_MEM_QUEUE: {}, -} - -CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task" -CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict - # STATIC_ROOT specifies the directory where static files are # collected STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None)