From 2f0f011a211b74deeadb7149a69dc9633ea68a2f Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 26 Nov 2014 21:14:39 +0000 Subject: [PATCH 1/5] config refactor wip --- cms/envs/common.py | 14 +++ cms/envs/prod.py | 206 +++++++++++++++++++++++++++++++++++++++ lms/envs/common.py | 27 ++++++ lms/envs/prod.py | 235 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 482 insertions(+) create mode 100644 cms/envs/prod.py create mode 100644 lms/envs/prod.py diff --git a/cms/envs/common.py b/cms/envs/common.py index 93c0fb4a74..e72b5693f2 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -752,3 +752,17 @@ ADVANCED_PROBLEM_TYPES = [ 'boilerplate_name': None, } ] + +CELERY_ALWAYS_EAGER = False +GIT_REPO_EXPORT_DIR = '/edx/var/edxapp/export_course_repos' +SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = None +EMAIL_FILE_PATH = None +STATIC_URL_BASE = None +STATIC_ROOT_BASE = None +SESSION_COOKIE_NAME = None +ADDL_INSTALLED_APPS = [] +AUTH_USE_CAS = False +CAS_ATTRIBUTE_CALLBACK = None +MICROSITE_ROOT_DIR = '' +SEGMENT_IO = False +DATADOG = {} diff --git a/cms/envs/prod.py b/cms/envs/prod.py new file mode 100644 index 0000000000..df6bbf2e61 --- /dev/null +++ b/cms/envs/prod.py @@ -0,0 +1,206 @@ +""" +This is the default template for our main set of AWS servers. +""" + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + +import yaml + +from .common import * + +from logsettings import get_logger_config +import os + +from path import path +from dealer.git import git +from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed + +# https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects +from yaml import Loader, SafeLoader +def construct_yaml_str(self, node): + # Override the default string handling function + # to always return unicode objects + return self.construct_scalar(node) +Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) +SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) + +def convert_tokens(tokens): + + for k, v in tokens.iteritems(): + if v == 'None': + tokens[k] = None + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) + +# CONFIG_ROOT specifies the directory where the JSON configuration +# files are expected to be found. If not specified, use the project +# directory. +CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + + +############### ALWAYS THE SAME ################################ + +DEBUG = False +TEMPLATE_DEBUG = False + +EMAIL_BACKEND = 'django_ses.SESBackend' +SESSION_ENGINE = 'django.contrib.sessions.backends.cache' +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + +###################################### CELERY ################################ + +# Don't use a connection pool, since connections are dropped by ELB. +BROKER_POOL_LIMIT = 0 +BROKER_CONNECTION_TIMEOUT = 1 + +# For the Result Store, use the django cache named 'celery' +CELERY_RESULT_BACKEND = 'cache' +CELERY_CACHE_BACKEND = 'celery' + +# When the broker is behind an ELB, use a heartbeat to refresh the +# connection and to detect if it has been dropped. +BROKER_HEARTBEAT = 10.0 +BROKER_HEARTBEAT_CHECKRATE = 2 + +# Each worker should only fetch one message at a time +CELERYD_PREFETCH_MULTIPLIER = 1 + +# Skip djcelery migrations, since we don't use the database as the broker +SOUTH_MIGRATION_MODULES = { + 'djcelery': 'ignore', +} + +# Rename the exchange and queues for each variant + +QUEUE_VARIANT = CONFIG_PREFIX.lower() + +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) +LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT) + +CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {} +} + +####################### START ENV_TOKENS ####################### + +with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file: + ENV_TOKENS = yaml.load(env_file) + +convert_tokens(ENV_TOKENS) + +ENV_FEATURES = ENV_TOKENS.get('FEATURES', ENV_TOKENS.get('MITX_FEATURES', {})) +for feature, value in ENV_FEATURES.items(): + FEATURES[feature] = value + +if 'FEATURES' in ENV_TOKENS: + del ENV_TOKENS['FEATURES'] + +vars().update(ENV_TOKENS) + + +################## ENV_TOKENS Modifications ####################### +# +# Some Django settings need to be modified after they are +# read in from yaml and imported into settings. +# + +if STATIC_URL_BASE: + # collectstatic will fail if STATIC_URL is a unicode string + STATIC_URL = STATIC_URL_BASE.encode('ascii') + if not STATIC_URL.endswith("/"): + STATIC_URL += "/" + STATIC_URL += git.revision + "/" + +if STATIC_ROOT_BASE: + STATIC_ROOT = path(STATIC_ROOT_BASE) / git.revision + + +# Cache used for location mapping -- called many times with the same key/value +# in a given request. +if 'loc_cache' not in CACHES: + CACHES['loc_cache'] = { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'edx_location_mem_cache', + } + + +# allow for environments to specify what cookie name our login subsystem should use +# this is to fix a bug regarding simultaneous logins between edx.org and edge.edx.org which can +# happen with some browsers (e.g. Firefox) +if SESSION_COOKIE_NAME: + # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str() + SESSION_COOKIE_NAME = str(SESSION_COOKIE_NAME) + + +# Additional installed apps +for app in ADDL_INSTALLED_APPS: + INSTALLED_APPS += (app,) + + +LOGGING = get_logger_config(LOG_DIR, + logging_env=LOGGING_ENV, + debug=False, + service_variant=SERVICE_VARIANT) + +if AUTH_USE_CAS: + AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'django_cas.backends.CASBackend', + ) + INSTALLED_APPS += ('django_cas',) + MIDDLEWARE_CLASSES += ('django_cas.middleware.CASMiddleware',) + if CAS_ATTRIBUTE_CALLBACK: + import importlib + CAS_USER_DETAILS_RESOLVER = getattr( + importlib.import_module(CAS_ATTRIBUTE_CALLBACK['module']), + CAS_ATTRIBUTE_CALLBACK['function'] + ) + +MICROSITE_ROOT_DIR = path(MICROSITE_ROOT_DIR) + +################ SECURE AUTH ITEMS ############################### +# Secret things: passwords, access keys, etc. +with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file: + AUTH_TOKENS = yaml.load(auth_file) + +convert_tokens(AUTH_TOKENS) + +vars().update(AUTH_TOKENS) + +if SEGMENT_IO_KEY: + FEATURES['SEGMENT_IO'] = SEGMENT_IO + +if AWS_ACCESS_KEY_ID == "": + AWS_ACCESS_KEY_ID = None + +if AWS_SECRET_ACCESS_KEY == "": + AWS_SECRET_ACCESS_KEY = None + +MODULESTORE = convert_module_store_setting_if_needed(MODULESTORE) + +# TODO: deprecated (compatibility with previous settings) +if 'DATADOG_API' in AUTH_TOKENS: + DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API'] + +BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT, + CELERY_BROKER_USER, + CELERY_BROKER_PASSWORD, + CELERY_BROKER_HOSTNAME, + CELERY_BROKER_VHOST) diff --git a/lms/envs/common.py b/lms/envs/common.py index 97900cd92a..55b46243c0 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1844,3 +1844,30 @@ INVOICE_PAYMENT_INSTRUCTIONS = "This is where you can\nput directions on how peo COUNTRIES_OVERRIDE = { "TW": _("Taiwan"), } + +SESSION_COOKIE_NAME = None +GIT_REPO_DIR = '/edx/var/edxapp/course_repos' +MICROSITE_ROOT_DIR = '' +CAS_SERVER_URL = None +CAS_ATTRIBUTE_CALLBACK = None + +##### Defaults for OAUTH2 Provider ############## +OAUTH_OIDC_ISSUER = None +OAUTH_ENFORCE_SECURE = True +OAUTH_ENFORCE_CLIENT_SECURE = True + +#### Course Registration Code length #### +REGISTRATION_CODE_LENGTH = 8 + +# SSL external authentication settings +SSL_AUTH_EMAIL_DOMAIN = "MIT.EDU" +SSL_AUTH_DN_FORMAT_STRING = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN={0}/emailAddress={1}" + +GIT_IMPORT_STATIC = True +META_UNIVERSITIES = {} +DATADOG = {} +EMAIL_FILE_PATH = None +SEGMENT_IO_LMS = False + +MONGODB_LOG = {} +SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = None diff --git a/lms/envs/prod.py b/lms/envs/prod.py new file mode 100644 index 0000000000..0a87b934ba --- /dev/null +++ b/lms/envs/prod.py @@ -0,0 +1,235 @@ +""" +This is the default template for our main set of AWS servers. This does NOT +cover the content machines, which use content.py + +Common traits: +* Use memcached, and cache-backed sessions +* Use a MySQL 5.1 database +""" + +# We intentionally define lots of variables that aren't used, and +# want to import all variables from base settings files +# pylint: disable=W0401, W0614 + +import json +import yaml +from pprint import pprint + +from .common import * +from logsettings import get_logger_config +import os + +from path import path +from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed + +# https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects +from yaml import Loader, SafeLoader +def construct_yaml_str(self, node): + # Override the default string handling function + # to always return unicode objects + return self.construct_scalar(node) +Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) +SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) + +def convert_tokens(tokens): + + for k, v in tokens.iteritems(): + if v == 'None': + tokens[k] = None + +# SERVICE_VARIANT specifies name of the variant used, which decides what JSON +# configuration files are read during startup. +SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) + +# CONFIG_ROOT specifies the directory where the JSON configuration +# files are expected to be found. If not specified, use the project +# directory. +CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) + +# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# based on the service variant. If no variant is use, don't use a +# prefix. +CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" + + +################################ ALWAYS THE SAME ############################## + +DEBUG = False +TEMPLATE_DEBUG = False + +EMAIL_BACKEND = 'django_ses.SESBackend' +SESSION_ENGINE = 'django.contrib.sessions.backends.cache' +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' + +# IMPORTANT: With this enabled, the server must always be behind a proxy that +# strips the header HTTP_X_FORWARDED_PROTO from client requests. Otherwise, +# a user can fool our server into thinking it was an https connection. +# See +# https://docs.djangoproject.com/en/dev/ref/settings/#secure-proxy-ssl-header +# for other warnings. +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + +###################################### CELERY ################################ + +# Don't use a connection pool, since connections are dropped by ELB. +BROKER_POOL_LIMIT = 0 +BROKER_CONNECTION_TIMEOUT = 1 + +# For the Result Store, use the django cache named 'celery' +CELERY_RESULT_BACKEND = 'cache' +CELERY_CACHE_BACKEND = 'celery' + +# When the broker is behind an ELB, use a heartbeat to refresh the +# connection and to detect if it has been dropped. +BROKER_HEARTBEAT = 10.0 +BROKER_HEARTBEAT_CHECKRATE = 2 + +# Each worker should only fetch one message at a time +CELERYD_PREFETCH_MULTIPLIER = 1 + +# Skip djcelery migrations, since we don't use the database as the broker +SOUTH_MIGRATION_MODULES = { + 'djcelery': 'ignore', +} + +# Rename the exchange and queues for each variant + +QUEUE_VARIANT = CONFIG_PREFIX.lower() + +CELERY_DEFAULT_EXCHANGE = 'edx.{0}core'.format(QUEUE_VARIANT) + +HIGH_PRIORITY_QUEUE = 'edx.{0}core.high'.format(QUEUE_VARIANT) +DEFAULT_PRIORITY_QUEUE = 'edx.{0}core.default'.format(QUEUE_VARIANT) +LOW_PRIORITY_QUEUE = 'edx.{0}core.low'.format(QUEUE_VARIANT) +HIGH_MEM_QUEUE = 'edx.{0}core.high_mem'.format(QUEUE_VARIANT) + +CELERY_DEFAULT_QUEUE = DEFAULT_PRIORITY_QUEUE +CELERY_DEFAULT_ROUTING_KEY = DEFAULT_PRIORITY_QUEUE + +CELERY_QUEUES = { + HIGH_PRIORITY_QUEUE: {}, + LOW_PRIORITY_QUEUE: {}, + DEFAULT_PRIORITY_QUEUE: {}, + HIGH_MEM_QUEUE: {}, +} + +# If we're a worker on the high_mem queue, set ourselves to die after processing +# one request to avoid having memory leaks take down the worker server. This env +# var is set in /etc/init/edx-workers.conf -- this should probably be replaced +# with some celery API call to see what queue we started listening to, but I +# don't know what that call is or if it's active at this point in the code. +if os.environ.get('QUEUE') == 'high_mem': + CELERYD_MAX_TASKS_PER_CHILD = 1 + + +####################### START ENV_TOKENS ####################### + +with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file: + ENV_TOKENS = yaml.load(env_file) + +convert_tokens(ENV_TOKENS) + +# These settings are merged from common.py +ENV_FEATURES = ENV_TOKENS.get('FEATURES', ENV_TOKENS.get('MITX_FEATURES', {})) +for feature, value in ENV_FEATURES.items(): + FEATURES[feature] = value + +MKTG_URL_LINK_MAP.update(ENV_TOKENS.get('MKTG_URL_LINK_MAP', {})) + +if 'FEATURES' in ENV_TOKENS: + del ENV_TOKENS['FEATURES'] + +if 'MKTG_URL_LINK_MAP' in ENV_TOKENS: + del ENV_TOKENS['MKTG_URL_LINK_MAP'] + +vars().update(ENV_TOKENS) + +if SESSION_COOKIE_NAME: + # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str() + SESSION_COOKIE_NAME = str(SESSION_COOKIE_NAME) + +MICROSITE_ROOT_DIR = path(MICROSITE_ROOT_DIR) + +# Cache used for location mapping -- called many times with the same key/value +# in a given request. +if 'loc_cache' not in CACHES: + CACHES['loc_cache'] = { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'edx_location_mem_cache', + } + +# We want Bulk Email running on the high-priority queue, so we define the +# routing key that points to it. At the moment, the name is the same. +# We have to reset the value here, since we have changed the value of the queue name. +BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE + +LANGUAGE_DICT = dict(LANGUAGES) + +# Additional installed apps +for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): + INSTALLED_APPS += (app,) + +local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') + +LOGGING = get_logger_config(LOG_DIR, + logging_env=LOGGING_ENV, + local_loglevel=local_loglevel, + debug=False, + service_variant=SERVICE_VARIANT) + +for name, value in ENV_TOKENS.get("CODE_JAIL", {}).items(): + oldvalue = CODE_JAIL.get(name) + if isinstance(oldvalue, dict): + for subname, subvalue in value.items(): + oldvalue[subname] = subvalue + else: + CODE_JAIL[name] = value + + +if FEATURES.get('AUTH_USE_CAS'): + AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'django_cas.backends.CASBackend', + ) + INSTALLED_APPS += ('django_cas',) + MIDDLEWARE_CLASSES += ('django_cas.middleware.CASMiddleware',) + if CAS_ATTRIBUTE_CALLBACK: + import importlib + CAS_USER_DETAILS_RESOLVER = getattr( + importlib.import_module(CAS_ATTRIBUTE_CALLBACK['module']), + CAS_ATTRIBUTE_CALLBACK['function'] + ) + + +STATIC_ROOT = path(STATIC_ROOT_BASE) + +####################### END ENV_TOKENS ####################### + + +with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file: + AUTH_TOKENS = yaml.load(auth_file) + +convert_tokens(AUTH_TOKENS) + +vars().update(AUTH_TOKENS) + +FEATURES['SEGMENT_IO_LMS'] = SEGMENT_IO_LMS + +if AWS_ACCESS_KEY_ID == "": + AWS_ACCESS_KEY_ID = None + +if AWS_SECRET_ACCESS_KEY == "": + AWS_SECRET_ACCESS_KEY = None + +# TODO: deprecated (compatibility with previous settings) +if 'DATADOG_API' in AUTH_TOKENS: + DATADOG['api_key'] = AUTH_TOKENS['DATADOG_API'] + +BROKER_URL = "{0}://{1}:{2}@{3}/{4}".format(CELERY_BROKER_TRANSPORT, + CELERY_BROKER_USER, + CELERY_BROKER_PASSWORD, + CELERY_BROKER_HOSTNAME, + CELERY_BROKER_VHOST) + +# Grades download +GRADES_DOWNLOAD_ROUTING_KEY = HIGH_MEM_QUEUE From 73c9b4c93bd989d6d19d9ce4a54af08816d97829 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Mon, 1 Dec 2014 10:31:30 -0500 Subject: [PATCH 2/5] pep8 and comments --- cms/envs/prod.py | 88 +++++++++++++++++++++++++++++++---------- lms/envs/prod.py | 100 +++++++++++++++++++++++++++++++++++------------ 2 files changed, 142 insertions(+), 46 deletions(-) diff --git a/cms/envs/prod.py b/cms/envs/prod.py index df6bbf2e61..fa8b2215b3 100644 --- a/cms/envs/prod.py +++ b/cms/envs/prod.py @@ -1,5 +1,13 @@ """ This is the default template for our main set of AWS servers. + +Before importing this settings file the following MUST be +defined in the environment: + + * SERVICE_VARIANT - can be either "lms" or "cms" + * CONFIG_ROOT - the directory where the application + yaml config files are located + """ # We intentionally define lots of variables that aren't used, and @@ -9,7 +17,6 @@ This is the default template for our main set of AWS servers. import yaml from .common import * - from logsettings import get_logger_config import os @@ -19,35 +26,49 @@ from xmodule.modulestore.modulestore_settings import convert_module_store_settin # https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects from yaml import Loader, SafeLoader + + def construct_yaml_str(self, node): - # Override the default string handling function + # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) + def convert_tokens(tokens): - + """ + This function is called on the token + dictionary, at the top level it converts + all strings containing 'None' to a literal + None due to a bug in Ansible which creates + the yaml files + """ + for k, v in tokens.iteritems(): if v == 'None': tokens[k] = None - -# SERVICE_VARIANT specifies name of the variant used, which decides what JSON + +# SERVICE_VARIANT specifies name of the variant used, which decides what YAML # configuration files are read during startup. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) -# CONFIG_ROOT specifies the directory where the JSON configuration +# CONFIG_ROOT specifies the directory where the YAML configuration # files are expected to be found. If not specified, use the project # directory. CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) -# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# CONFIG_PREFIX specifies the prefix of the YAML configuration files, # based on the service variant. If no variant is use, don't use a # prefix. CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" - -############### ALWAYS THE SAME ################################ +############################################################## +# +# DEFAULT SETTINGS FOR PRODUCTION +# +# These are defaults common for all production deployments +# DEBUG = False TEMPLATE_DEBUG = False @@ -56,7 +77,10 @@ EMAIL_BACKEND = 'django_ses.SESBackend' SESSION_ENGINE = 'django.contrib.sessions.backends.cache' DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' -###################################### CELERY ################################ +############################################################## +# +# DEFAULT SETTINGS FOR CELERY +# # Don't use a connection pool, since connections are dropped by ELB. BROKER_POOL_LIMIT = 0 @@ -98,28 +122,45 @@ CELERY_QUEUES = { DEFAULT_PRIORITY_QUEUE: {} } -####################### START ENV_TOKENS ####################### +############################################################## +# +# ENV TOKEN IMPORT +# +# Currently non-secure and secure settings are managed +# in two yaml files. This section imports the non-secure +# settings and modifies them in code if necessary. +# with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file: ENV_TOKENS = yaml.load(env_file) convert_tokens(ENV_TOKENS) +########################################## +# Merge settings from common.py +# +# Before the tokens are imported directly +# into settings some dictionary settings +# need to be merged from common.py + ENV_FEATURES = ENV_TOKENS.get('FEATURES', ENV_TOKENS.get('MITX_FEATURES', {})) for feature, value in ENV_FEATURES.items(): FEATURES[feature] = value +# Delete keys from ENV_TOKENS so that when it's imported +# into settings it doesn't override what was set above if 'FEATURES' in ENV_TOKENS: - del ENV_TOKENS['FEATURES'] + del ENV_TOKENS['FEATURES'] vars().update(ENV_TOKENS) - -################## ENV_TOKENS Modifications ####################### -# -# Some Django settings need to be modified after they are -# read in from yaml and imported into settings. -# +########################################## +# Manipulate imported settings with code +# +# For historical reasons some settings need +# to be modified in code. For example +# conversions to other data structures that +# cannot be represented in YAML. if STATIC_URL_BASE: # collectstatic will fail if STATIC_URL is a unicode string @@ -175,8 +216,11 @@ if AUTH_USE_CAS: MICROSITE_ROOT_DIR = path(MICROSITE_ROOT_DIR) -################ SECURE AUTH ITEMS ############################### -# Secret things: passwords, access keys, etc. +############################################################## +# +# AUTH TOKEN IMPORT +# + with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file: AUTH_TOKENS = yaml.load(auth_file) @@ -184,6 +228,10 @@ convert_tokens(AUTH_TOKENS) vars().update(AUTH_TOKENS) +########################################## +# Manipulate imported settings with code +# + if SEGMENT_IO_KEY: FEATURES['SEGMENT_IO'] = SEGMENT_IO diff --git a/lms/envs/prod.py b/lms/envs/prod.py index 0a87b934ba..7e1a4be2a8 100644 --- a/lms/envs/prod.py +++ b/lms/envs/prod.py @@ -1,58 +1,69 @@ """ -This is the default template for our main set of AWS servers. This does NOT -cover the content machines, which use content.py +This is the default settings files for all +production servers. -Common traits: -* Use memcached, and cache-backed sessions -* Use a MySQL 5.1 database +Before importing this settings file the following MUST be +defined in the environment: + + * SERVICE_VARIANT - can be either "lms" or "cms" + * CONFIG_ROOT - the directory where the application + yaml config files are located """ -# We intentionally define lots of variables that aren't used, and -# want to import all variables from base settings files -# pylint: disable=W0401, W0614 - -import json import yaml -from pprint import pprint from .common import * from logsettings import get_logger_config import os from path import path -from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed # https://stackoverflow.com/questions/2890146/how-to-force-pyyaml-to-load-strings-as-unicode-objects from yaml import Loader, SafeLoader + + def construct_yaml_str(self, node): - # Override the default string handling function + # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) + def convert_tokens(tokens): - + """ + This function is called on the token + dictionary, at the top level it converts + all strings containing 'None' to a literal + None due to a bug in Ansible which creates + the yaml files + """ + for k, v in tokens.iteritems(): if v == 'None': tokens[k] = None - -# SERVICE_VARIANT specifies name of the variant used, which decides what JSON + +# SERVICE_VARIANT specifies name of the variant used, which decides what YAML # configuration files are read during startup. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) -# CONFIG_ROOT specifies the directory where the JSON configuration +# CONFIG_ROOT specifies the directory where the YAML configuration # files are expected to be found. If not specified, use the project # directory. CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) -# CONFIG_PREFIX specifies the prefix of the JSON configuration files, +# CONFIG_PREFIX specifies the prefix of the YAML configuration files, # based on the service variant. If no variant is use, don't use a # prefix. CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" -################################ ALWAYS THE SAME ############################## +############################################################## +# +# DEFAULT SETTINGS FOR PRODUCTION +# +# These are defaults common for all production deployments +# DEBUG = False TEMPLATE_DEBUG = False @@ -69,7 +80,11 @@ DEFAULT_FILE_STORAGE = 'storages.backends.s3boto.S3BotoStorage' # for other warnings. SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') -###################################### CELERY ################################ +############################################################## +# +# DEFAULT SETTINGS FOR CELERY +# + # Don't use a connection pool, since connections are dropped by ELB. BROKER_POOL_LIMIT = 0 @@ -122,28 +137,54 @@ if os.environ.get('QUEUE') == 'high_mem': CELERYD_MAX_TASKS_PER_CHILD = 1 -####################### START ENV_TOKENS ####################### +############################################################## +# +# ENV TOKEN IMPORT +# +# Currently non-secure and secure settings are managed +# in two yaml files. This section imports the non-secure +# settings and modifies them in code if necessary. +# with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file: ENV_TOKENS = yaml.load(env_file) +# Works around an Ansible bug convert_tokens(ENV_TOKENS) -# These settings are merged from common.py +########################################## +# Merge settings from common.py +# +# Before the tokens are imported directly +# into settings some dictionary settings +# need to be merged from common.py + ENV_FEATURES = ENV_TOKENS.get('FEATURES', ENV_TOKENS.get('MITX_FEATURES', {})) for feature, value in ENV_FEATURES.items(): FEATURES[feature] = value MKTG_URL_LINK_MAP.update(ENV_TOKENS.get('MKTG_URL_LINK_MAP', {})) +# Delete keys from ENV_TOKENS so that when it's imported +# into settings it doesn't override what was set above if 'FEATURES' in ENV_TOKENS: - del ENV_TOKENS['FEATURES'] + del ENV_TOKENS['FEATURES'] if 'MKTG_URL_LINK_MAP' in ENV_TOKENS: - del ENV_TOKENS['MKTG_URL_LINK_MAP'] + del ENV_TOKENS['MKTG_URL_LINK_MAP'] +# Update the token dictionary directly into settings vars().update(ENV_TOKENS) +########################################## +# Manipulate imported settings with code +# +# For historical reasons some settings need +# to be modified in code. For example +# conversions to other data structures that +# cannot be represented in YAML. + + if SESSION_COOKIE_NAME: # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str() SESSION_COOKIE_NAME = str(SESSION_COOKIE_NAME) @@ -203,16 +244,23 @@ if FEATURES.get('AUTH_USE_CAS'): STATIC_ROOT = path(STATIC_ROOT_BASE) -####################### END ENV_TOKENS ####################### - +############################################################## +# +# AUTH TOKEN IMPORT +# with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file: AUTH_TOKENS = yaml.load(auth_file) +# Works around an Ansible bug convert_tokens(AUTH_TOKENS) vars().update(AUTH_TOKENS) +########################################## +# Manipulate imported settings with code +# + FEATURES['SEGMENT_IO_LMS'] = SEGMENT_IO_LMS if AWS_ACCESS_KEY_ID == "": From a32f3373264dc4273e35300e2090f2ef4aadaf23 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 4 Dec 2014 16:43:17 -0500 Subject: [PATCH 3/5] renmaing prod.py and setting more defaults --- cms/envs/common.py | 2 ++ cms/envs/{prod.py => yaml-config.py} | 1 + lms/envs/common.py | 2 ++ lms/envs/{prod.py => yaml-config.py} | 6 ++---- 4 files changed, 7 insertions(+), 4 deletions(-) rename cms/envs/{prod.py => yaml-config.py} (99%) rename lms/envs/{prod.py => yaml-config.py} (98%) diff --git a/cms/envs/common.py b/cms/envs/common.py index 84a43de7ef..d336a4939b 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -769,3 +769,5 @@ CAS_ATTRIBUTE_CALLBACK = None MICROSITE_ROOT_DIR = '' SEGMENT_IO = False DATADOG = {} +ADDL_INSTALLED_APPS = [] +LOCAL_LOGLEVEL = 'INFO' diff --git a/cms/envs/prod.py b/cms/envs/yaml-config.py similarity index 99% rename from cms/envs/prod.py rename to cms/envs/yaml-config.py index fa8b2215b3..a4580654d6 100644 --- a/cms/envs/prod.py +++ b/cms/envs/yaml-config.py @@ -196,6 +196,7 @@ for app in ADDL_INSTALLED_APPS: LOGGING = get_logger_config(LOG_DIR, + local_loglevel=LOCAL_LOGLEVEL, logging_env=LOGGING_ENV, debug=False, service_variant=SERVICE_VARIANT) diff --git a/lms/envs/common.py b/lms/envs/common.py index 8f184fff7d..28423dc8c8 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1921,3 +1921,5 @@ SEGMENT_IO_LMS = False MONGODB_LOG = {} SESSION_INACTIVITY_TIMEOUT_IN_SECONDS = None +ADDL_INSTALLED_APPS = [] +LOCAL_LOGLEVEL = 'INFO' diff --git a/lms/envs/prod.py b/lms/envs/yaml-config.py similarity index 98% rename from lms/envs/prod.py rename to lms/envs/yaml-config.py index 7e1a4be2a8..1a06702029 100644 --- a/lms/envs/prod.py +++ b/lms/envs/yaml-config.py @@ -207,14 +207,12 @@ BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE LANGUAGE_DICT = dict(LANGUAGES) # Additional installed apps -for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): +for app in ADDL_INSTALLED_APPS: INSTALLED_APPS += (app,) -local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') - LOGGING = get_logger_config(LOG_DIR, logging_env=LOGGING_ENV, - local_loglevel=local_loglevel, + local_loglevel=LOCAL_LOGLEVEL, debug=False, service_variant=SERVICE_VARIANT) From de98bf8fac064cbf79d6f69c9105ece44ad35954 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Thu, 4 Dec 2014 18:01:34 -0500 Subject: [PATCH 4/5] renaming to use underscores --- cms/envs/{yaml-config.py => yaml_config.py} | 0 lms/envs/{yaml-config.py => yaml_config.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename cms/envs/{yaml-config.py => yaml_config.py} (100%) rename lms/envs/{yaml-config.py => yaml_config.py} (100%) diff --git a/cms/envs/yaml-config.py b/cms/envs/yaml_config.py similarity index 100% rename from cms/envs/yaml-config.py rename to cms/envs/yaml_config.py diff --git a/lms/envs/yaml-config.py b/lms/envs/yaml_config.py similarity index 100% rename from lms/envs/yaml-config.py rename to lms/envs/yaml_config.py From c711ed50915d4b60ba76277d03514960205f48fb Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Fri, 5 Dec 2014 14:00:25 -0500 Subject: [PATCH 5/5] moving the convert_tokens function to a common file --- cms/envs/yaml_config.py | 19 +++---------------- common/djangoapps/util/config_parse.py | 26 ++++++++++++++++++++++++++ lms/envs/yaml_config.py | 19 +++---------------- 3 files changed, 32 insertions(+), 32 deletions(-) create mode 100644 common/djangoapps/util/config_parse.py diff --git a/cms/envs/yaml_config.py b/cms/envs/yaml_config.py index a4580654d6..1a8fa255de 100644 --- a/cms/envs/yaml_config.py +++ b/cms/envs/yaml_config.py @@ -18,6 +18,7 @@ import yaml from .common import * from logsettings import get_logger_config +from util.config_parse import convert_tokens import os from path import path @@ -35,20 +36,6 @@ def construct_yaml_str(self, node): Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) - -def convert_tokens(tokens): - """ - This function is called on the token - dictionary, at the top level it converts - all strings containing 'None' to a literal - None due to a bug in Ansible which creates - the yaml files - """ - - for k, v in tokens.iteritems(): - if v == 'None': - tokens[k] = None - # SERVICE_VARIANT specifies name of the variant used, which decides what YAML # configuration files are read during startup. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) @@ -134,7 +121,7 @@ CELERY_QUEUES = { with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file: ENV_TOKENS = yaml.load(env_file) -convert_tokens(ENV_TOKENS) +ENV_TOKENS = convert_tokens(ENV_TOKENS) ########################################## # Merge settings from common.py @@ -225,7 +212,7 @@ MICROSITE_ROOT_DIR = path(MICROSITE_ROOT_DIR) with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file: AUTH_TOKENS = yaml.load(auth_file) -convert_tokens(AUTH_TOKENS) +AUTH_TOKENS = convert_tokens(AUTH_TOKENS) vars().update(AUTH_TOKENS) diff --git a/common/djangoapps/util/config_parse.py b/common/djangoapps/util/config_parse.py new file mode 100644 index 0000000000..3d76aed308 --- /dev/null +++ b/common/djangoapps/util/config_parse.py @@ -0,0 +1,26 @@ +""" +Helper functions for configuration parsing +""" +import collections + + +def convert_tokens(tokens): + """ + This function is called on the token + dictionary that is imported from a yaml file. + It returns a new dictionary where + all strings containing 'None' are converted + to a literal None due to a bug in Ansible + """ + + if tokens == 'None': + return None + elif isinstance(tokens, basestring) or (not isinstance(tokens, collections.Iterable)): + return tokens + elif isinstance(tokens, dict): + return { + convert_tokens(k): convert_tokens(v) + for k, v in tokens.items() + } + else: + return [convert_tokens(v) for v in tokens] diff --git a/lms/envs/yaml_config.py b/lms/envs/yaml_config.py index 1a06702029..282a257f6e 100644 --- a/lms/envs/yaml_config.py +++ b/lms/envs/yaml_config.py @@ -14,6 +14,7 @@ import yaml from .common import * from logsettings import get_logger_config +from util.config_parse import convert_tokens import os from path import path @@ -29,20 +30,6 @@ def construct_yaml_str(self, node): Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) - -def convert_tokens(tokens): - """ - This function is called on the token - dictionary, at the top level it converts - all strings containing 'None' to a literal - None due to a bug in Ansible which creates - the yaml files - """ - - for k, v in tokens.iteritems(): - if v == 'None': - tokens[k] = None - # SERVICE_VARIANT specifies name of the variant used, which decides what YAML # configuration files are read during startup. SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None) @@ -150,7 +137,7 @@ with open(CONFIG_ROOT / CONFIG_PREFIX + "env.yaml") as env_file: ENV_TOKENS = yaml.load(env_file) # Works around an Ansible bug -convert_tokens(ENV_TOKENS) +ENV_TOKENS = convert_tokens(ENV_TOKENS) ########################################## # Merge settings from common.py @@ -251,7 +238,7 @@ with open(CONFIG_ROOT / CONFIG_PREFIX + "auth.yaml") as auth_file: AUTH_TOKENS = yaml.load(auth_file) # Works around an Ansible bug -convert_tokens(AUTH_TOKENS) +AUTH_TOKENS = convert_tokens(AUTH_TOKENS) vars().update(AUTH_TOKENS)