Merge pull request #28066 from edx/feanil/arch-1821-alternative
fix!: Make default queue names service variant specific.
This commit is contained in:
Feanil Patel
@@ -1266,6 +1266,19 @@ WEBPACK_LOADER = {
|
||||
}
|
||||
WEBPACK_CONFIG_PATH = 'webpack.prod.config.js'
|
||||
|
||||
|
||||
############################ SERVICE_VARIANT ##################################
|
||||
|
||||
# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
|
||||
# configuration files are read during startup.
|
||||
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', 'cms')
|
||||
|
||||
# CONFIG_PREFIX specifies the prefix of the JSON configuration files,
|
||||
# based on the service variant. If no variant is use, don't use a
|
||||
# prefix.
|
||||
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
|
||||
|
||||
|
||||
################################# CELERY ######################################
|
||||
|
||||
# Message configuration
|
||||
@@ -1292,22 +1305,28 @@ CELERY_SEND_TASK_SENT_EVENT = True
|
||||
CELERY_DEFAULT_EXCHANGE = 'edx.core'
|
||||
CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
|
||||
|
||||
# Queues configuration
|
||||
# Name the exchange and queues for each variant
|
||||
|
||||
HIGH_PRIORITY_QUEUE = 'edx.core.high'
|
||||
DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
|
||||
QUEUE_VARIANT = CONFIG_PREFIX.lower()
|
||||
|
||||
CELERY_QUEUE_HA_POLICY = 'all'
|
||||
CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core'
|
||||
|
||||
CELERY_CREATE_MISSING_QUEUES = True
|
||||
HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high'
|
||||
DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default'
|
||||
|
||||
CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
|
||||
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
|
||||
|
||||
CELERY_QUEUES = [
|
||||
'edx.cms.core.default',
|
||||
'edx.cms.core.high',
|
||||
]
|
||||
CELERY_QUEUES = {
|
||||
HIGH_PRIORITY_QUEUE: {},
|
||||
DEFAULT_PRIORITY_QUEUE: {}
|
||||
}
|
||||
|
||||
# Queues configuration
|
||||
|
||||
CELERY_QUEUE_HA_POLICY = 'all'
|
||||
|
||||
CELERY_CREATE_MISSING_QUEUES = True
|
||||
|
||||
CELERY_BROKER_TRANSPORT = 'amqp'
|
||||
CELERY_BROKER_HOSTNAME = 'localhost'
|
||||
|
||||
@@ -93,15 +93,6 @@ except Exception: # pylint: disable=broad-except
|
||||
# Do NOT calculate this dynamically at startup with git because it's *slow*.
|
||||
EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION)
|
||||
|
||||
# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
|
||||
# configuration files are read during startup.
|
||||
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
|
||||
|
||||
# CONFIG_PREFIX specifies the prefix of the JSON configuration files,
|
||||
# based on the service variant. If no variant is use, don't use a
|
||||
# prefix.
|
||||
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
|
||||
|
||||
###################################### CELERY ################################
|
||||
|
||||
# Don't use a connection pool, since connections are dropped by ELB.
|
||||
@@ -119,23 +110,6 @@ BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2)
|
||||
# Each worker should only fetch one message at a time
|
||||
CELERYD_PREFETCH_MULTIPLIER = 1
|
||||
|
||||
# Rename the exchange and queues for each variant
|
||||
|
||||
QUEUE_VARIANT = CONFIG_PREFIX.lower()
|
||||
|
||||
CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core'
|
||||
|
||||
HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high'
|
||||
DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default'
|
||||
|
||||
CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
|
||||
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
|
||||
|
||||
CELERY_QUEUES = {
|
||||
HIGH_PRIORITY_QUEUE: {},
|
||||
DEFAULT_PRIORITY_QUEUE: {}
|
||||
}
|
||||
|
||||
CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task"
|
||||
|
||||
# STATIC_URL_BASE specifies the base url to use for static files
|
||||
|
||||
@@ -2623,24 +2623,42 @@ CELERY_SEND_TASK_SENT_EVENT = True
|
||||
CELERY_DEFAULT_EXCHANGE = 'edx.core'
|
||||
CELERY_DEFAULT_EXCHANGE_TYPE = 'direct'
|
||||
|
||||
|
||||
# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
|
||||
# configuration files are read during startup.
|
||||
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', "lms")
|
||||
|
||||
# CONFIG_PREFIX specifies the prefix of the JSON configuration files,
|
||||
# based on the service variant. If no variant is use, don't use a
|
||||
# prefix.
|
||||
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
|
||||
|
||||
# Queues configuration
|
||||
|
||||
HIGH_PRIORITY_QUEUE = 'edx.core.high'
|
||||
DEFAULT_PRIORITY_QUEUE = 'edx.core.default'
|
||||
HIGH_MEM_QUEUE = 'edx.core.high_mem'
|
||||
# Name the exchange and queues w.r.t the SERVICE_VARIANT
|
||||
QUEUE_VARIANT = CONFIG_PREFIX.lower()
|
||||
|
||||
CELERY_QUEUE_HA_POLICY = 'all'
|
||||
CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core'
|
||||
|
||||
CELERY_CREATE_MISSING_QUEUES = True
|
||||
HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high'
|
||||
DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default'
|
||||
HIGH_MEM_QUEUE = f'edx.{QUEUE_VARIANT}core.high_mem'
|
||||
|
||||
CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
|
||||
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
|
||||
|
||||
CELERY_QUEUES = [
|
||||
'edx.lms.core.default',
|
||||
'edx.lms.core.high',
|
||||
'edx.lms.core.high_mem'
|
||||
]
|
||||
CELERY_QUEUES = {
|
||||
HIGH_PRIORITY_QUEUE: {},
|
||||
DEFAULT_PRIORITY_QUEUE: {},
|
||||
HIGH_MEM_QUEUE: {},
|
||||
}
|
||||
|
||||
CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task"
|
||||
CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict
|
||||
|
||||
CELERY_QUEUE_HA_POLICY = 'all'
|
||||
|
||||
CELERY_CREATE_MISSING_QUEUES = True
|
||||
|
||||
# let logging work as configured:
|
||||
CELERYD_HIJACK_ROOT_LOGGER = False
|
||||
|
||||
@@ -103,15 +103,6 @@ except Exception: # pylint: disable=broad-except
|
||||
# Do NOT calculate this dynamically at startup with git because it's *slow*.
|
||||
EDX_PLATFORM_REVISION = REVISION_CONFIG.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION)
|
||||
|
||||
# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
|
||||
# configuration files are read during startup.
|
||||
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
|
||||
|
||||
# CONFIG_PREFIX specifies the prefix of the JSON configuration files,
|
||||
# based on the service variant. If no variant is use, don't use a
|
||||
# prefix.
|
||||
CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else ""
|
||||
|
||||
###################################### CELERY ################################
|
||||
|
||||
# Don't use a connection pool, since connections are dropped by ELB.
|
||||
@@ -129,28 +120,6 @@ BROKER_HEARTBEAT_CHECKRATE = ENV_TOKENS.get('BROKER_HEARTBEAT_CHECKRATE', 2)
|
||||
# Each worker should only fetch one message at a time
|
||||
CELERYD_PREFETCH_MULTIPLIER = 1
|
||||
|
||||
# Rename the exchange and queues for each variant
|
||||
|
||||
QUEUE_VARIANT = CONFIG_PREFIX.lower()
|
||||
|
||||
CELERY_DEFAULT_EXCHANGE = f'edx.{QUEUE_VARIANT}core'
|
||||
|
||||
HIGH_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.high'
|
||||
DEFAULT_PRIORITY_QUEUE = f'edx.{QUEUE_VARIANT}core.default'
|
||||
HIGH_MEM_QUEUE = f'edx.{QUEUE_VARIANT}core.high_mem'
|
||||
|
||||
CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE
|
||||
CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE
|
||||
|
||||
CELERY_QUEUES = {
|
||||
HIGH_PRIORITY_QUEUE: {},
|
||||
DEFAULT_PRIORITY_QUEUE: {},
|
||||
HIGH_MEM_QUEUE: {},
|
||||
}
|
||||
|
||||
CELERY_ROUTES = "openedx.core.lib.celery.routers.route_task"
|
||||
CELERYBEAT_SCHEDULE = {} # For scheduling tasks, entries can be added to this dict
|
||||
|
||||
# STATIC_ROOT specifies the directory where static files are
|
||||
# collected
|
||||
STATIC_ROOT_BASE = ENV_TOKENS.get('STATIC_ROOT_BASE', None)
|
||||
|
||||
Reference in New Issue
Block a user