From de80ab751a3193adb7d12c15ea2b2e48704f9a93 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 10:47:31 -0500 Subject: [PATCH 01/12] Remove the debug override. --- cms/envs/aws.py | 1 - lms/envs/aws.py | 1 - lms/envs/static.py | 3 +-- openedx/core/lib/logsettings.py | 3 +-- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cms/envs/aws.py b/cms/envs/aws.py index 3fe992d1ab..5e96be8785 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -256,7 +256,6 @@ WIKI_ENABLED = ENV_TOKENS.get('WIKI_ENABLED', WIKI_ENABLED) LOGGING = get_logger_config(LOG_DIR, logging_env=ENV_TOKENS['LOGGING_ENV'], - debug=False, service_variant=SERVICE_VARIANT) #theming start: diff --git a/lms/envs/aws.py b/lms/envs/aws.py index b8dcbdb500..8e60fe02bd 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -345,7 +345,6 @@ local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') LOGGING = get_logger_config(LOG_DIR, logging_env=ENV_TOKENS['LOGGING_ENV'], local_loglevel=local_loglevel, - debug=False, service_variant=SERVICE_VARIANT) COURSE_LISTINGS = ENV_TOKENS.get('COURSE_LISTINGS', {}) diff --git a/lms/envs/static.py b/lms/envs/static.py index 3a02a8140d..3c015e1c84 100644 --- a/lms/envs/static.py +++ b/lms/envs/static.py @@ -20,8 +20,7 @@ STATIC_GRAB = True LOGGING = get_logger_config(ENV_ROOT / "log", logging_env="dev", - tracking_filename="tracking.log", - debug=False) + tracking_filename="tracking.log") DATABASES = { 'default': { diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index a600449e5c..11f6c3938a 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -16,7 +16,6 @@ def get_logger_config(log_dir, edx_filename="edx.log", dev_env=False, syslog_addr=None, - debug=False, local_loglevel='INFO', console_loglevel=None, service_variant=None): @@ -42,7 +41,7 @@ def get_logger_config(log_dir, local_loglevel = 'INFO' if console_loglevel is None or console_loglevel not in LOG_LEVELS: - console_loglevel = 'DEBUG' if debug else 'INFO' + console_loglevel = 'INFO' if service_variant is None: # default to a blank string so that if SERVICE_VARIANT is not From 37ae5e5218f4536b5a83e6fa5cafe444822398db Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 10:48:02 -0500 Subject: [PATCH 02/12] Move LOG_DIR setting next to LOGGING setting. --- lms/envs/aws.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 8e60fe02bd..972e7d47ed 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -213,8 +213,6 @@ if ENV_TOKENS.get('SESSION_COOKIE_NAME', None): # NOTE, there's a bug in Django (http://bugs.python.org/issue18012) which necessitates this being a str() SESSION_COOKIE_NAME = str(ENV_TOKENS.get('SESSION_COOKIE_NAME')) -LOG_DIR = ENV_TOKENS['LOG_DIR'] - CACHES = ENV_TOKENS['CACHES'] # Cache used for location mapping -- called many times with the same key/value # in a given request. @@ -340,7 +338,9 @@ for app in ENV_TOKENS.get('ADDL_INSTALLED_APPS', []): INSTALLED_APPS.append(app) WIKI_ENABLED = ENV_TOKENS.get('WIKI_ENABLED', WIKI_ENABLED) + local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') +LOG_DIR = ENV_TOKENS['LOG_DIR'] LOGGING = get_logger_config(LOG_DIR, logging_env=ENV_TOKENS['LOGGING_ENV'], From 147ea0c12c3d311b1ce7201a8a53c609789dc5a4 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 10:51:01 -0500 Subject: [PATCH 03/12] Remove un-necessary conditional. --- openedx/core/lib/logsettings.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index 11f6c3938a..c24b30e5f0 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -18,7 +18,7 @@ def get_logger_config(log_dir, syslog_addr=None, local_loglevel='INFO', console_loglevel=None, - service_variant=None): + service_variant=""): """ @@ -43,11 +43,6 @@ def get_logger_config(log_dir, if console_loglevel is None or console_loglevel not in LOG_LEVELS: console_loglevel = 'INFO' - if service_variant is None: - # default to a blank string so that if SERVICE_VARIANT is not - # set we will not log to a sub directory - service_variant = '' - hostname = platform.node().split(".")[0] syslog_format = ("[service_variant={service_variant}]" "[%(name)s][env:{logging_env}] %(levelname)s " From 779bc1d7724f305c974b5bdfbb082aec7f4634d3 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 11:04:21 -0500 Subject: [PATCH 04/12] The dev_env parameter is never used in any of our configs. --- openedx/core/lib/logsettings.py | 68 ++++++++++----------------------- 1 file changed, 21 insertions(+), 47 deletions(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index c24b30e5f0..2a77adb70c 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -14,7 +14,6 @@ def get_logger_config(log_dir, logging_env="no_env", tracking_filename="tracking.log", edx_filename="edx.log", - dev_env=False, syslog_addr=None, local_loglevel='INFO', console_loglevel=None, @@ -28,9 +27,6 @@ def get_logger_config(log_dir, about resetting the logging state if this is called multiple times when settings are extended. - If dev_env is set to true logging will not be done via local rsyslogd, - instead, tracking and application logs will be dropped in log_dir. - "tracking_filename" and "edx_filename" are ignored unless dev_env is set to true since otherwise logging is handled by rsyslogd. @@ -112,49 +108,27 @@ def get_logger_config(log_dir, }, }) - if dev_env: - tracking_file_loc = os.path.join(log_dir, tracking_filename) - edx_file_loc = os.path.join(log_dir, edx_filename) - logger_config['handlers'].update({ - 'local': { - 'class': 'logging.handlers.RotatingFileHandler', - 'level': local_loglevel, - 'formatter': 'standard', - 'filename': edx_file_loc, - 'maxBytes': 1024 * 1024 * 2, - 'backupCount': 5, - }, - 'tracking': { - 'level': 'DEBUG', - 'class': 'logging.handlers.RotatingFileHandler', - 'filename': tracking_file_loc, - 'formatter': 'raw', - 'maxBytes': 1024 * 1024 * 2, - 'backupCount': 5, - }, - }) - else: - # for production environments we will only - # log INFO and up - logger_config['loggers']['']['level'] = 'INFO' - # requests is so loud at INFO (logs every connection) that we force it to warn in production environments - logger_config['loggers']['requests.packages.urllib3'] = {'level': 'WARN'} - logger_config['handlers'].update({ - 'local': { - 'level': local_loglevel, - 'class': 'logging.handlers.SysLogHandler', - 'address': '/dev/log', - 'formatter': 'syslog_format', - 'facility': SysLogHandler.LOG_LOCAL0, - }, - 'tracking': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'address': '/dev/log', - 'facility': SysLogHandler.LOG_LOCAL1, - 'formatter': 'raw', - }, - }) + # for production environments we will only + # log INFO and up + logger_config['loggers']['']['level'] = 'INFO' + # requests is so loud at INFO (logs every connection) that we force it to warn in production environments + logger_config['loggers']['requests.packages.urllib3'] = {'level': 'WARN'} + logger_config['handlers'].update({ + 'local': { + 'level': local_loglevel, + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'formatter': 'syslog_format', + 'facility': SysLogHandler.LOG_LOCAL0, + }, + 'tracking': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'facility': SysLogHandler.LOG_LOCAL1, + 'formatter': 'raw', + }, + }) return logger_config From 49cbd581e9a6e6c0b27b4fae3c0bb2167aa0f9da Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 11:06:41 -0500 Subject: [PATCH 05/12] Remove tracking_filename override. The override value is the same as the default value. --- lms/envs/static.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/envs/static.py b/lms/envs/static.py index 3c015e1c84..0d9f080bed 100644 --- a/lms/envs/static.py +++ b/lms/envs/static.py @@ -20,7 +20,7 @@ STATIC_GRAB = True LOGGING = get_logger_config(ENV_ROOT / "log", logging_env="dev", - tracking_filename="tracking.log") + ) DATABASES = { 'default': { From bc0869859299125229def66e3a67a58b4099596e Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 11:08:36 -0500 Subject: [PATCH 06/12] Remove dev_env specific settings. Since we never used dev_env and have removed it. --- openedx/core/lib/logsettings.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index 2a77adb70c..5aaa6260f1 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -12,8 +12,6 @@ LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] def get_logger_config(log_dir, logging_env="no_env", - tracking_filename="tracking.log", - edx_filename="edx.log", syslog_addr=None, local_loglevel='INFO', console_loglevel=None, @@ -26,10 +24,6 @@ def get_logger_config(log_dir, this way instead of registering directly is because I didn't want to worry about resetting the logging state if this is called multiple times when settings are extended. - - "tracking_filename" and "edx_filename" are ignored unless dev_env - is set to true since otherwise logging is handled by rsyslogd. - """ # Revert to INFO if an invalid string is passed in From 3d6fe857038c5550cd0cb0b3ae9a0dc237a557d3 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 11:12:27 -0500 Subject: [PATCH 07/12] syslog_addr config never used. --- openedx/core/lib/logsettings.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index 5aaa6260f1..d6f4450f35 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -12,7 +12,6 @@ LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] def get_logger_config(log_dir, logging_env="no_env", - syslog_addr=None, local_loglevel='INFO', console_loglevel=None, service_variant=""): @@ -42,9 +41,6 @@ def get_logger_config(log_dir, hostname=hostname) handlers = ['console', 'local'] - if syslog_addr: - handlers.append('syslogger-remote') - logger_config = { 'version': 1, 'disable_existing_loggers': False, @@ -92,15 +88,6 @@ def get_logger_config(log_dir, }, } } - if syslog_addr: - logger_config['handlers'].update({ - 'syslogger-remote': { - 'level': 'INFO', - 'class': 'logging.handlers.SysLogHandler', - 'address': syslog_addr, - 'formatter': 'syslog_format', - }, - }) # for production environments we will only # log INFO and up From caff4b572049b754024c7652ec023234422118f5 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 11:14:06 -0500 Subject: [PATCH 08/12] console_loglevel never used, remove it. --- openedx/core/lib/logsettings.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index d6f4450f35..3385a67d9b 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -13,7 +13,6 @@ LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'] def get_logger_config(log_dir, logging_env="no_env", local_loglevel='INFO', - console_loglevel=None, service_variant=""): """ @@ -29,9 +28,6 @@ def get_logger_config(log_dir, if local_loglevel not in LOG_LEVELS: local_loglevel = 'INFO' - if console_loglevel is None or console_loglevel not in LOG_LEVELS: - console_loglevel = 'INFO' - hostname = platform.node().split(".")[0] syslog_format = ("[service_variant={service_variant}]" "[%(name)s][env:{logging_env}] %(levelname)s " @@ -59,7 +55,7 @@ def get_logger_config(log_dir, }, 'handlers': { 'console': { - 'level': console_loglevel, + 'level': 'INFO', 'class': 'logging.StreamHandler', 'formatter': 'standard', 'stream': sys.stderr, From 8d84e8b4a25d493b704912df22dc080639f5201a Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 11:51:47 -0500 Subject: [PATCH 09/12] Consolidate overrides into base config. They used to be in a conditional but aren't anymore so we don't need to have them be separate. --- openedx/core/lib/logsettings.py | 43 +++++++++++++++------------------ 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index 3385a67d9b..4800ff3f7f 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -65,6 +65,20 @@ def get_logger_config(log_dir, 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' }, + 'local': { + 'level': local_loglevel, + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'formatter': 'syslog_format', + 'facility': SysLogHandler.LOG_LOCAL0, + }, + 'tracking': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'facility': SysLogHandler.LOG_LOCAL1, + 'formatter': 'raw', + }, }, 'loggers': { 'tracking': { @@ -74,7 +88,7 @@ def get_logger_config(log_dir, }, '': { 'handlers': handlers, - 'level': 'DEBUG', + 'level': 'INFO', 'propagate': False }, 'django.request': { @@ -82,31 +96,14 @@ def get_logger_config(log_dir, 'level': 'ERROR', 'propagate': True, }, + # requests is so loud at INFO (logs every connection) that we + # force it to warn in production environments + 'requests.packages.urllib3': { + 'level': 'WARN' + } } } - # for production environments we will only - # log INFO and up - logger_config['loggers']['']['level'] = 'INFO' - # requests is so loud at INFO (logs every connection) that we force it to warn in production environments - logger_config['loggers']['requests.packages.urllib3'] = {'level': 'WARN'} - logger_config['handlers'].update({ - 'local': { - 'level': local_loglevel, - 'class': 'logging.handlers.SysLogHandler', - 'address': '/dev/log', - 'formatter': 'syslog_format', - 'facility': SysLogHandler.LOG_LOCAL0, - }, - 'tracking': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'address': '/dev/log', - 'facility': SysLogHandler.LOG_LOCAL1, - 'formatter': 'raw', - }, - }) - return logger_config From 14b57f14ff7428ccf86048ae1d6fd18d4487f047 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 12:00:30 -0500 Subject: [PATCH 10/12] Remove unnecessary var. Previously it was useful because we updated it conditionally but now it's just reducing information locality. --- openedx/core/lib/logsettings.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index 4800ff3f7f..d0f9b6d2cc 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -36,7 +36,6 @@ def get_logger_config(log_dir, logging_env=logging_env, hostname=hostname) - handlers = ['console', 'local'] logger_config = { 'version': 1, 'disable_existing_loggers': False, @@ -87,7 +86,7 @@ def get_logger_config(log_dir, 'propagate': False, }, '': { - 'handlers': handlers, + 'handlers': ['console', 'local'], 'level': 'INFO', 'propagate': False }, From 41879e0087daf7a718795c64739b2927f3274230 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 14:43:27 -0500 Subject: [PATCH 11/12] Update comment. --- openedx/core/lib/logsettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/lib/logsettings.py b/openedx/core/lib/logsettings.py index d0f9b6d2cc..ce1cae7ddb 100644 --- a/openedx/core/lib/logsettings.py +++ b/openedx/core/lib/logsettings.py @@ -96,7 +96,7 @@ def get_logger_config(log_dir, 'propagate': True, }, # requests is so loud at INFO (logs every connection) that we - # force it to warn in production environments + # force it to warn by default. 'requests.packages.urllib3': { 'level': 'WARN' } From 0d8c30e58d8b53f90f9318cdf3db26ed1e272602 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Wed, 31 Jan 2018 14:48:30 -0500 Subject: [PATCH 12/12] Fix pep8 violation. --- lms/envs/static.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lms/envs/static.py b/lms/envs/static.py index 0d9f080bed..1954f73504 100644 --- a/lms/envs/static.py +++ b/lms/envs/static.py @@ -19,8 +19,7 @@ from openedx.core.lib.logsettings import get_logger_config STATIC_GRAB = True LOGGING = get_logger_config(ENV_ROOT / "log", - logging_env="dev", - ) + logging_env="dev") DATABASES = { 'default': {