diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 1630f53ace..ff74b9873b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -9,6 +9,8 @@ LMS: Fix issue with CourseMode expiration dates LMS: Ported bulk emailing to the beta instructor dashboard. +LMS: Add monitoring of bulk email subtasks to display progress on instructor dash. + LMS: Add PaidCourseRegistration mode, where payment is required before course registration. diff --git a/lms/djangoapps/bulk_email/tasks.py b/lms/djangoapps/bulk_email/tasks.py index 1a0e77b61b..1fa4fa3530 100644 --- a/lms/djangoapps/bulk_email/tasks.py +++ b/lms/djangoapps/bulk_email/tasks.py @@ -469,8 +469,10 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas else: dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)]) - - log.debug('Email with id %s sent to %s', email_id, email) + if settings.BULK_EMAIL_LOG_SENT_EMAILS: + log.info('Email with id %s sent to %s', email_id, email) + else: + log.debug('Email with id %s sent to %s', email_id, email) num_sent += 1 # Pop the user that was emailed off the end of the list: @@ -611,21 +613,23 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current task_id, subtask_status['succeeded'], subtask_status['failed'], subtask_status['skipped']) # Calculate time until we retry this task (in seconds): + # The value for max_retries is increased by the number of times an "infinite-retry" exception + # has been retried. We want the regular retries to trigger max-retry checking, but not these + # special retries. So we count them separately. max_retries = _get_current_task().max_retries + subtask_status['retried_nomax'] base_delay = _get_current_task().default_retry_delay if skip_retry_max: - retry_index = subtask_status['retried_nomax'] - exp = min(retry_index, 5) - countdown = ((2 ** exp) * base_delay) * random.uniform(.5, 1.25) + # once we reach five retries, don't increase the countdown further. + retry_index = min(subtask_status['retried_nomax'], 5) exception_type = 'sending-rate' else: retry_index = subtask_status['retried_withmax'] - countdown = ((2 ** retry_index) * base_delay) * random.uniform(.75, 1.5) exception_type = 'transient' - # max_retries is increased by the number of times an "infinite-retry" exception - # has been retried. We want the regular retries to trigger max-retry checking, but not these - # special retries. So we count them separately. + # Skew the new countdown value by a random factor, so that not all + # retries are deferred by the same amount. + countdown = ((2 ** retry_index) * base_delay) * random.uniform(.75, 1.25) + log.warning('Task %s: email with id %d not delivered due to %s error %s, retrying send to %d recipients in %s seconds (with max_retry=%s)', task_id, email_id, exception_type, current_exception, len(to_list), countdown, max_retries) @@ -644,7 +648,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current throw=True, ) except RetryTaskError as retry_error: - # If retry call is successful, update with the current progress: + # If the retry call is successful, update with the current progress: log.exception('Task %s: email with id %d caused send_course_email task to retry.', task_id, email_id) return subtask_status, retry_error diff --git a/lms/envs/aws.py b/lms/envs/aws.py index 4dc928a208..f77691d92e 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -93,10 +93,6 @@ CELERY_QUEUES = { DEFAULT_PRIORITY_QUEUE: {} } -# We want Bulk Email running on the high-priority queue, so we define the -# routing key that points to it. At the moment, the name is the same. -BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE - ########################## NON-SECURE ENV CONFIG ############################## # Things like server locations, ports, etc. @@ -111,8 +107,6 @@ EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', None) EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', 'localhost') # django default is localhost EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', 25) # django default is 25 EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', False) # django default is False -EMAILS_PER_TASK = ENV_TOKENS.get('EMAILS_PER_TASK', 100) -EMAILS_PER_QUERY = ENV_TOKENS.get('EMAILS_PER_QUERY', 1000) SITE_NAME = ENV_TOKENS['SITE_NAME'] SESSION_ENGINE = ENV_TOKENS.get('SESSION_ENGINE', SESSION_ENGINE) SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN') @@ -135,7 +129,6 @@ CACHES = ENV_TOKENS['CACHES'] # Email overrides DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL) DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL) -DEFAULT_BULK_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_BULK_FROM_EMAIL', DEFAULT_BULK_FROM_EMAIL) ADMINS = ENV_TOKENS.get('ADMINS', ADMINS) SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL) TECH_SUPPORT_EMAIL = ENV_TOKENS.get('TECH_SUPPORT_EMAIL', TECH_SUPPORT_EMAIL) @@ -144,8 +137,18 @@ BUGS_EMAIL = ENV_TOKENS.get('BUGS_EMAIL', BUGS_EMAIL) PAYMENT_SUPPORT_EMAIL = ENV_TOKENS.get('PAYMENT_SUPPORT_EMAIL', PAYMENT_SUPPORT_EMAIL) PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CURRENCY', PAID_COURSE_REGISTRATION_CURRENCY) + +# Bulk Email overrides +DEFAULT_BULK_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_BULK_FROM_EMAIL', DEFAULT_BULK_FROM_EMAIL) +EMAILS_PER_TASK = ENV_TOKENS.get('EMAILS_PER_TASK', 100) +EMAILS_PER_QUERY = ENV_TOKENS.get('EMAILS_PER_QUERY', 1000) BULK_EMAIL_DEFAULT_RETRY_DELAY = ENV_TOKENS.get('BULK_EMAIL_DEFAULT_RETRY_DELAY', BULK_EMAIL_DEFAULT_RETRY_DELAY) BULK_EMAIL_MAX_RETRIES = ENV_TOKENS.get('BULK_EMAIL_MAX_RETRIES', BULK_EMAIL_MAX_RETRIES) +BULK_EMAIL_INFINITE_RETRY_CAP = ENV_TOKENS.get('BULK_EMAIL_INFINITE_RETRY_CAP', BULK_EMAIL_INFINITE_RETRY_CAP) +BULK_EMAIL_LOG_SENT_EMAILS = ENV_TOKENS.get('BULK_EMAIL_LOG_SENT_EMAILS', BULK_EMAIL_LOG_SENT_EMAILS) +# We want Bulk Email running on the high-priority queue, so we define the +# routing key that points to it. At the moment, the name is the same. +BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE # Theme overrides THEME_NAME = ENV_TOKENS.get('THEME_NAME', None) diff --git a/lms/envs/common.py b/lms/envs/common.py index 0c12d088c9..dbfb28e86f 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -114,6 +114,7 @@ MITX_FEATURES = { # analytics experiments 'ENABLE_INSTRUCTOR_ANALYTICS': False, + # bulk email available to instructors: 'ENABLE_INSTRUCTOR_EMAIL': False, # enable analytics server. @@ -333,7 +334,7 @@ TRACKING_BACKENDS = { } } -# Backawrds compatibility with ENABLE_SQL_TRACKING_LOGS feature flag. +# Backwards compatibility with ENABLE_SQL_TRACKING_LOGS feature flag. # In the future, adding the backend to TRACKING_BACKENDS enough. if MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): TRACKING_BACKENDS.update({ @@ -414,12 +415,9 @@ HTTPS = 'on' ROOT_URLCONF = 'lms.urls' IGNORABLE_404_ENDS = ('favicon.ico') -# Email +# Platform Email EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' DEFAULT_FROM_EMAIL = 'registration@edx.org' -DEFAULT_BULK_FROM_EMAIL = 'course-updates@edx.org' -EMAILS_PER_TASK = 100 -EMAILS_PER_QUERY = 1000 DEFAULT_FEEDBACK_EMAIL = 'feedback@edx.org' SERVER_EMAIL = 'devops@edx.org' TECH_SUPPORT_EMAIL = 'technical@edx.org' @@ -800,11 +798,29 @@ CELERYD_HIJACK_ROOT_LOGGER = False ################################ Bulk Email ################################### +DEFAULT_BULK_FROM_EMAIL = 'no-reply@courseupdates.edx.org' +EMAILS_PER_TASK = 100 +EMAILS_PER_QUERY = 1000 + +# Initial delay used for retrying tasks. Additional retries use +# longer delays. Value is in seconds. +BULK_EMAIL_DEFAULT_RETRY_DELAY = 30 + +# Maximum number of retries per task for errors that are not related +# to throttling. +BULK_EMAIL_MAX_RETRIES = 5 + +# Maximum number of retries per task for errors that are related to +# throttling. If this is not set, then there is no cap on such retries. +BULK_EMAIL_INFINITE_RETRY_CAP = 1000 + # We want Bulk Email running on the high-priority queue, so we define the # routing key that points to it. At the moment, the name is the same. BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE -BULK_EMAIL_DEFAULT_RETRY_DELAY = 15 -BULK_EMAIL_MAX_RETRIES = 5 + +# Flag to indicate if individual email addresses should be logged as they are sent +# a bulk email message. +BULK_EMAIL_LOG_SENT_EMAILS = False ################################### APPS ###################################### INSTALLED_APPS = (