From 1e2406fc0c79b893981944c3fc3de9d30d014c88 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 29 Aug 2012 10:05:08 -0400 Subject: [PATCH 1/8] updating logging config to allow for local logging --- lms/envs/logsettings.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 3683314d02..72b4a0b251 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -29,7 +29,7 @@ def get_logger_config(log_dir, " %(process)d] [%(filename)s:%(lineno)d] - %(message)s").format( logging_env=logging_env, hostname=hostname) - handlers = ['console'] if debug else ['console', 'syslogger', 'newrelic'] + handlers = ['console'] if debug else ['console', 'syslogger-remote', 'syslogger-local', 'newrelic'] return { 'version': 1, @@ -48,16 +48,24 @@ def get_logger_config(log_dir, 'formatter' : 'standard', 'stream' : sys.stdout, }, - 'syslogger' : { + 'syslogger-remote' : { 'level' : 'INFO', 'class' : 'logging.handlers.SysLogHandler', 'address' : syslog_addr, 'formatter' : 'syslog_format', }, + 'syslogger-local' : { + 'level' : 'DEBUG', + 'class' : 'logging.handlers.SysLogHandler', + 'address' : '/dev/log', + 'formatter' : 'syslog_format', + 'facility': SysLogHandler.LOG_LOCAL0, + }, 'tracking' : { 'level' : 'DEBUG', - 'class' : 'logging.handlers.WatchedFileHandler', - 'filename' : tracking_file_loc, + 'class' : 'logging.handlers.SysLogHandler', + 'address' : '/dev/log', + 'facility' : SysLogHandler.LOG_LOCAL1, 'formatter' : 'raw', }, 'newrelic' : { From 2e1c8666e1186be8adf897056e1470994bafdce6 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 29 Aug 2012 10:59:33 -0400 Subject: [PATCH 2/8] adding SysLogHandler --- lms/envs/logsettings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 72b4a0b251..0351ee9cc9 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -2,6 +2,7 @@ import os import os.path import platform import sys +from logging.handlers import SysLogHandler def get_logger_config(log_dir, logging_env="no_env", From 27725ab686559c4e125e34fcc876e17786bd8bc7 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 29 Aug 2012 13:09:00 -0400 Subject: [PATCH 3/8] making loglevel configurable for local logging --- lms/envs/aws.py | 2 ++ lms/envs/logsettings.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lms/envs/aws.py b/lms/envs/aws.py index fbcb9b8e4c..136f311a5d 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -48,10 +48,12 @@ for feature, value in ENV_TOKENS.get('MITX_FEATURES', {}).items(): MITX_FEATURES[feature] = value WIKI_ENABLED = ENV_TOKENS.get('WIKI_ENABLED', WIKI_ENABLED) +local_loglevel = ENV_TOKENS.get('LOCAL_LOGLEVEL', 'INFO') LOGGING = get_logger_config(LOG_DIR, logging_env=ENV_TOKENS['LOGGING_ENV'], syslog_addr=(ENV_TOKENS['SYSLOG_SERVER'], 514), + local_loglevel=local_loglevel, debug=False) COURSE_LISTINGS = ENV_TOKENS.get('COURSE_LISTINGS', {}) diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 0351ee9cc9..0c8a9a681c 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -8,13 +8,19 @@ def get_logger_config(log_dir, logging_env="no_env", tracking_filename=None, syslog_addr=None, - debug=False): + debug=False, + local_loglevel='INFO'): + """Return the appropriate logging config dictionary. You should assign the result of this to the LOGGING var in your settings. The reason it's done this way instead of registering directly is because I didn't want to worry about resetting the logging state if this is called multiple times when settings are extended.""" + # Revert to INFO if an invalid string is passed in + if local_loglevel not in ['DEBUG','INFO','WARNING','ERROR','CRITICAL']: + local_loglevel = 'INFO' + # If we're given an explicit place to put tracking logs, we do that (say for # debugging). However, logging is not safe for multiple processes hitting # the same file. So if it's left blank, we dynamically create the filename @@ -56,7 +62,7 @@ def get_logger_config(log_dir, 'formatter' : 'syslog_format', }, 'syslogger-local' : { - 'level' : 'DEBUG', + 'level' : local_loglevel, 'class' : 'logging.handlers.SysLogHandler', 'address' : '/dev/log', 'formatter' : 'syslog_format', From cef089f722e2e540c4b0b38fcee8958542ec4e81 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Fri, 31 Aug 2012 09:38:13 -0400 Subject: [PATCH 4/8] tracking log filename now determined by puppet, formatting cleanup --- lms/envs/dev.py | 7 +++ lms/envs/logsettings.py | 116 ++++++++++++++++++++-------------------- 2 files changed, 64 insertions(+), 59 deletions(-) diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 5da84f59f0..d122b167f2 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -13,6 +13,7 @@ from .logsettings import get_logger_config DEBUG = True TEMPLATE_DEBUG = True + MITX_FEATURES['DISABLE_START_DATES'] = True MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True MITX_FEATURES['SUBDOMAIN_COURSE_LISTINGS'] = False # Enable to test subdomains--otherwise, want all courses to show up @@ -55,6 +56,7 @@ CACHES = { } } + XQUEUE_INTERFACE = { "url": "https://sandbox-xqueue.edx.org", "django_auth": { @@ -70,6 +72,7 @@ CACHE_TIMEOUT = 0 # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' + COURSE_LISTINGS = { 'default': ['BerkeleyX/CS169.1x/2012_Fall', 'BerkeleyX/CS188.1x/2012_Fall', @@ -85,6 +88,7 @@ COURSE_LISTINGS = { 'sjsu': ['MITx/6.002x-EE98/2012_Fall_SJSU'], } + SUBDOMAIN_BRANDING = { 'sjsu': 'MITx', 'mit': 'MITx', @@ -94,6 +98,8 @@ SUBDOMAIN_BRANDING = { COMMENTS_SERVICE_KEY = "PUT_YOUR_API_KEY_HERE" + + ################################ LMS Migration ################################# MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False # require that user be in the staff_* group to be able to enroll @@ -116,6 +122,7 @@ OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' # TODO: accept m OPENID_USE_AS_ADMIN_LOGIN = False ################################ MIT Certificates SSL Auth ################################# + MITX_FEATURES['AUTH_USE_MIT_CERTIFICATES'] = True ################################ DEBUG TOOLBAR ################################# diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 0c8a9a681c..749ad6ba71 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -4,6 +4,7 @@ import platform import sys from logging.handlers import SysLogHandler + def get_logger_config(log_dir, logging_env="no_env", tracking_filename=None, @@ -18,94 +19,91 @@ def get_logger_config(log_dir, settings are extended.""" # Revert to INFO if an invalid string is passed in - if local_loglevel not in ['DEBUG','INFO','WARNING','ERROR','CRITICAL']: + if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: local_loglevel = 'INFO' - # If we're given an explicit place to put tracking logs, we do that (say for - # debugging). However, logging is not safe for multiple processes hitting - # the same file. So if it's left blank, we dynamically create the filename + # If we're given an explicit place to put tracking logs, + # we do that (say for debugging). However, logging is not + # safe for multiple processes hitting the same file. + # So if it's left blank, we dynamically create the filename # based on the PID of this worker process. - if tracking_filename: - tracking_file_loc = os.path.join(log_dir, tracking_filename) - else: - pid = os.getpid() # So we can log which process is creating the log - tracking_file_loc = os.path.join(log_dir, "tracking_{0}.log".format(pid)) hostname = platform.node().split(".")[0] syslog_format = ("[%(name)s][env:{logging_env}] %(levelname)s [{hostname} " + " %(process)d] [%(filename)s:%(lineno)d] - %(message)s").format( logging_env=logging_env, hostname=hostname) - handlers = ['console'] if debug else ['console', 'syslogger-remote', 'syslogger-local', 'newrelic'] + handlers = ['console'] if debug else ['console', 'syslogger-remote', + 'syslogger-local', 'newrelic'] return { 'version': 1, 'disable_existing_loggers': False, - 'formatters' : { - 'standard' : { - 'format' : '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', + 'formatters': { + 'standard': { + 'format': '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', }, - 'syslog_format' : { 'format' : syslog_format }, - 'raw' : { 'format' : '%(message)s' }, + 'syslog_format': {'format': syslog_format , + 'raw': {'format': '%(message)s'}, }, - 'handlers' : { - 'console' : { - 'level' : 'DEBUG' if debug else 'INFO', - 'class' : 'logging.StreamHandler', - 'formatter' : 'standard', - 'stream' : sys.stdout, + 'handlers': { + 'console': { + 'level': 'DEBUG' if debug else 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'standard', + 'stream': sys.stdout, }, - 'syslogger-remote' : { - 'level' : 'INFO', - 'class' : 'logging.handlers.SysLogHandler', - 'address' : syslog_addr, - 'formatter' : 'syslog_format', + 'syslogger-remote': { + 'level': 'INFO', + 'class': 'logging.handlers.SysLogHandler', + 'address': syslog_addr, + 'formatter': 'syslog_format', }, - 'syslogger-local' : { - 'level' : local_loglevel, - 'class' : 'logging.handlers.SysLogHandler', - 'address' : '/dev/log', - 'formatter' : 'syslog_format', + 'syslogger-local': { + 'level': local_loglevel, + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'formatter': 'syslog_format', 'facility': SysLogHandler.LOG_LOCAL0, }, - 'tracking' : { - 'level' : 'DEBUG', - 'class' : 'logging.handlers.SysLogHandler', - 'address' : '/dev/log', - 'facility' : SysLogHandler.LOG_LOCAL1, - 'formatter' : 'raw', + 'tracking': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'facility': SysLogHandler.LOG_LOCAL1, + 'formatter': 'raw', }, - 'newrelic' : { + 'newrelic': { 'level': 'ERROR', 'class': 'newrelic_logging.NewRelicHandler', 'formatter': 'raw', } }, - 'loggers' : { - 'django' : { - 'handlers' : handlers, - 'propagate' : True, - 'level' : 'INFO' + 'loggers': { + 'django': { + 'handlers': handlers, + 'propagate': True, + 'level': 'INFO' }, - 'tracking' : { - 'handlers' : ['tracking'], - 'level' : 'DEBUG', - 'propagate' : False, + 'tracking': { + 'handlers': ['tracking'], + 'level': 'DEBUG', + 'propagate': False, }, - '' : { - 'handlers' : handlers, - 'level' : 'DEBUG', - 'propagate' : False + '': { + 'handlers': handlers, + 'level': 'DEBUG', + 'propagate': False }, - 'mitx' : { - 'handlers' : handlers, - 'level' : 'DEBUG', - 'propagate' : False + 'mitx': { + 'handlers': handlers, + 'level': 'DEBUG', + 'propagate': False }, - 'keyedcache' : { - 'handlers' : handlers, - 'level' : 'DEBUG', - 'propagate' : False + 'keyedcache': { + 'handlers': handlers, + 'level': 'DEBUG', + 'propagate': False }, } } From 66c5894e91575fac121d0a27e00f5ea12c02e7ea Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Fri, 31 Aug 2012 10:03:14 -0400 Subject: [PATCH 5/8] syntax error --- lms/envs/logsettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 749ad6ba71..1cb5171891 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -43,7 +43,7 @@ def get_logger_config(log_dir, 'standard': { 'format': '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', }, - 'syslog_format': {'format': syslog_format , + 'syslog_format': {'format': syslog_format ,} 'raw': {'format': '%(message)s'}, }, 'handlers': { From 164dae71fb077be2a4b67fe7b53079179d998cd1 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Fri, 31 Aug 2012 10:09:15 -0400 Subject: [PATCH 6/8] Fixing syntax error --- lms/envs/logsettings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 1cb5171891..60cd1f8675 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -43,8 +43,8 @@ def get_logger_config(log_dir, 'standard': { 'format': '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', }, - 'syslog_format': {'format': syslog_format ,} - 'raw': {'format': '%(message)s'}, + 'syslog_format': {'format': syslog_format }, + 'raw': {'format': '%(message)s' }, }, 'handlers': { 'console': { From 54f5e30992c7e0219ceff3f2bfdb34e2382fd83a Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Tue, 4 Sep 2012 09:39:54 -0400 Subject: [PATCH 7/8] * Removed tracking log comment * Additional PEP8 format cleanup * Removed imports that are no longer necessary. --- lms/envs/logsettings.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 60cd1f8675..847992359d 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -1,5 +1,3 @@ -import os -import os.path import platform import sys from logging.handlers import SysLogHandler @@ -22,12 +20,6 @@ def get_logger_config(log_dir, if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: local_loglevel = 'INFO' - # If we're given an explicit place to put tracking logs, - # we do that (say for debugging). However, logging is not - # safe for multiple processes hitting the same file. - # So if it's left blank, we dynamically create the filename - # based on the PID of this worker process. - hostname = platform.node().split(".")[0] syslog_format = ("[%(name)s][env:{logging_env}] %(levelname)s [{hostname} " + " %(process)d] [%(filename)s:%(lineno)d] - %(message)s").format( @@ -43,8 +35,8 @@ def get_logger_config(log_dir, 'standard': { 'format': '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', }, - 'syslog_format': {'format': syslog_format }, - 'raw': {'format': '%(message)s' }, + 'syslog_format': {'format': syslog_format}, + 'raw': {'format': '%(message)s'}, }, 'handlers': { 'console': { From f7baa0d8f078588b9c4640feeb7e8335f07b19d1 Mon Sep 17 00:00:00 2001 From: John Jarvis Date: Wed, 5 Sep 2012 11:25:33 -0400 Subject: [PATCH 8/8] * By default local dev environments will be setup to send tracking logs and application logs (DEBUG) to ENV_ROOT/log. Logs are rotated using the RotatingFileHandler * For non dev environments logging will continue to be handled by rsyslogd --- lms/envs/dev.py | 3 +- lms/envs/logsettings.py | 88 ++++++++++++++++++++++++++++++----------- 2 files changed, 67 insertions(+), 24 deletions(-) diff --git a/lms/envs/dev.py b/lms/envs/dev.py index d122b167f2..0867d0c897 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -23,7 +23,8 @@ WIKI_ENABLED = True LOGGING = get_logger_config(ENV_ROOT / "log", logging_env="dev", - tracking_filename="tracking.log", + local_loglevel="DEBUG", + dev_env=True, debug=True) DATABASES = { diff --git a/lms/envs/logsettings.py b/lms/envs/logsettings.py index 847992359d..2b001b0517 100644 --- a/lms/envs/logsettings.py +++ b/lms/envs/logsettings.py @@ -1,3 +1,4 @@ +import os import platform import sys from logging.handlers import SysLogHandler @@ -5,35 +6,49 @@ from logging.handlers import SysLogHandler def get_logger_config(log_dir, logging_env="no_env", - tracking_filename=None, + tracking_filename="tracking.log", + edx_filename="edx.log", + dev_env=False, syslog_addr=None, debug=False, local_loglevel='INFO'): - """Return the appropriate logging config dictionary. You should assign the + """ + + Return the appropriate logging config dictionary. You should assign the result of this to the LOGGING var in your settings. The reason it's done this way instead of registering directly is because I didn't want to worry about resetting the logging state if this is called multiple times when - settings are extended.""" + settings are extended. + + If dev_env is set to true logging will not be done via local rsyslogd, + instead, tracking and application logs will be dropped in log_dir. + + "tracking_filename" and "edx_filename" are ignored unless dev_env + is set to true since otherwise logging is handled by rsyslogd. + + """ # Revert to INFO if an invalid string is passed in if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: local_loglevel = 'INFO' hostname = platform.node().split(".")[0] - syslog_format = ("[%(name)s][env:{logging_env}] %(levelname)s [{hostname} " + - " %(process)d] [%(filename)s:%(lineno)d] - %(message)s").format( + syslog_format = ("[%(name)s][env:{logging_env}] %(levelname)s " + "[{hostname} %(process)d] [%(filename)s:%(lineno)d] " + "- %(message)s").format( logging_env=logging_env, hostname=hostname) - handlers = ['console'] if debug else ['console', 'syslogger-remote', - 'syslogger-local', 'newrelic'] + handlers = ['console', 'local'] if debug else ['console', + 'syslogger-remote', 'local', 'newrelic'] - return { + logger_config = { 'version': 1, 'disable_existing_loggers': False, 'formatters': { 'standard': { - 'format': '%(asctime)s %(levelname)s %(process)d [%(name)s] %(filename)s:%(lineno)d - %(message)s', + 'format': '%(asctime)s %(levelname)s %(process)d ' + '[%(name)s] %(filename)s:%(lineno)d - %(message)s', }, 'syslog_format': {'format': syslog_format}, 'raw': {'format': '%(message)s'}, @@ -51,20 +66,6 @@ def get_logger_config(log_dir, 'address': syslog_addr, 'formatter': 'syslog_format', }, - 'syslogger-local': { - 'level': local_loglevel, - 'class': 'logging.handlers.SysLogHandler', - 'address': '/dev/log', - 'formatter': 'syslog_format', - 'facility': SysLogHandler.LOG_LOCAL0, - }, - 'tracking': { - 'level': 'DEBUG', - 'class': 'logging.handlers.SysLogHandler', - 'address': '/dev/log', - 'facility': SysLogHandler.LOG_LOCAL1, - 'formatter': 'raw', - }, 'newrelic': { 'level': 'ERROR', 'class': 'newrelic_logging.NewRelicHandler', @@ -99,3 +100,44 @@ def get_logger_config(log_dir, }, } } + + if dev_env: + tracking_file_loc = os.path.join(log_dir, tracking_filename) + edx_file_loc = os.path.join(log_dir, edx_filename) + logger_config['handlers'].update({ + 'local': { + 'class': 'logging.handlers.RotatingFileHandler', + 'level': local_loglevel, + 'formatter': 'standard', + 'filename': edx_file_loc, + 'maxBytes': 1024 * 1024 * 2, + 'backupCount': 5, + }, + 'tracking': { + 'level': 'DEBUG', + 'class': 'logging.handlers.RotatingFileHandler', + 'filename': tracking_file_loc, + 'formatter': 'raw', + 'maxBytes': 1024 * 1024 * 2, + 'backupCount': 5, + }, + }) + else: + logger_config['handlers'].update({ + 'local': { + 'level': local_loglevel, + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'formatter': 'syslog_format', + 'facility': SysLogHandler.LOG_LOCAL0, + }, + 'tracking': { + 'level': 'DEBUG', + 'class': 'logging.handlers.SysLogHandler', + 'address': '/dev/log', + 'facility': SysLogHandler.LOG_LOCAL1, + 'formatter': 'raw', + }, + }) + + return logger_config