Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Add settings to cap infinite retries.

Browse Source
This commit is contained in:
Brian Wilson
2013-10-08 13:58:00 -04:00
parent 41fcd9627b
commit c5debc22e7
4 changed files with 49 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CHANGELOG.rst
View File

@@ -9,6 +9,8 @@ LMS: Fix issue with CourseMode expiration dates
LMS: Ported bulk emailing to the beta instructor dashboard.
LMS: Add monitoring of bulk email subtasks to display progress on instructor dash.
LMS: Add PaidCourseRegistration mode, where payment is required before course
registration.

24
lms/djangoapps/bulk_email/tasks.py
View File

@@ -469,8 +469,10 @@ def _send_course_email(entry_id, email_id, to_list, global_email_context, subtas
else:
dog_stats_api.increment('course_email.sent', tags=[_statsd_tag(course_title)])
log.debug('Email with id %s sent to %s', email_id, email)
if settings.BULK_EMAIL_LOG_SENT_EMAILS:
log.info('Email with id %s sent to %s', email_id, email)
else:
log.debug('Email with id %s sent to %s', email_id, email)
num_sent += 1
# Pop the user that was emailed off the end of the list:
@@ -611,21 +613,23 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
task_id, subtask_status['succeeded'], subtask_status['failed'], subtask_status['skipped'])
# Calculate time until we retry this task (in seconds):
# The value for max_retries is increased by the number of times an "infinite-retry" exception
# has been retried. We want the regular retries to trigger max-retry checking, but not these
# special retries. So we count them separately.
max_retries = _get_current_task().max_retries + subtask_status['retried_nomax']
base_delay = _get_current_task().default_retry_delay
if skip_retry_max:
retry_index = subtask_status['retried_nomax']
exp = min(retry_index, 5)
countdown = ((2 ** exp) * base_delay) * random.uniform(.5, 1.25)
# once we reach five retries, don't increase the countdown further.
retry_index = min(subtask_status['retried_nomax'], 5)
exception_type = 'sending-rate'
else:
retry_index = subtask_status['retried_withmax']
countdown = ((2 ** retry_index) * base_delay) * random.uniform(.75, 1.5)
exception_type = 'transient'
# max_retries is increased by the number of times an "infinite-retry" exception
# has been retried. We want the regular retries to trigger max-retry checking, but not these
# special retries. So we count them separately.
# Skew the new countdown value by a random factor, so that not all
# retries are deferred by the same amount.
countdown = ((2 ** retry_index) * base_delay) * random.uniform(.75, 1.25)
log.warning('Task %s: email with id %d not delivered due to %s error %s, retrying send to %d recipients in %s seconds (with max_retry=%s)',
task_id, email_id, exception_type, current_exception, len(to_list), countdown, max_retries)
@@ -644,7 +648,7 @@ def _submit_for_retry(entry_id, email_id, to_list, global_email_context, current
throw=True,
)
except RetryTaskError as retry_error:
# If retry call is successful, update with the current progress:
# If the retry call is successful, update with the current progress:
log.exception('Task %s: email with id %d caused send_course_email task to retry.',
task_id, email_id)
return subtask_status, retry_error

17
lms/envs/aws.py
View File

@@ -93,10 +93,6 @@ CELERY_QUEUES = {
DEFAULT_PRIORITY_QUEUE: {}
}
# We want Bulk Email running on the high-priority queue, so we define the
# routing key that points to it. At the moment, the name is the same.
BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE
########################## NON-SECURE ENV CONFIG ##############################
# Things like server locations, ports, etc.
@@ -111,8 +107,6 @@ EMAIL_FILE_PATH = ENV_TOKENS.get('EMAIL_FILE_PATH', None)
EMAIL_HOST = ENV_TOKENS.get('EMAIL_HOST', 'localhost') # django default is localhost
EMAIL_PORT = ENV_TOKENS.get('EMAIL_PORT', 25) # django default is 25
EMAIL_USE_TLS = ENV_TOKENS.get('EMAIL_USE_TLS', False) # django default is False
EMAILS_PER_TASK = ENV_TOKENS.get('EMAILS_PER_TASK', 100)
EMAILS_PER_QUERY = ENV_TOKENS.get('EMAILS_PER_QUERY', 1000)
SITE_NAME = ENV_TOKENS['SITE_NAME']
SESSION_ENGINE = ENV_TOKENS.get('SESSION_ENGINE', SESSION_ENGINE)
SESSION_COOKIE_DOMAIN = ENV_TOKENS.get('SESSION_COOKIE_DOMAIN')
@@ -135,7 +129,6 @@ CACHES = ENV_TOKENS['CACHES']
# Email overrides
DEFAULT_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_FROM_EMAIL', DEFAULT_FROM_EMAIL)
DEFAULT_FEEDBACK_EMAIL = ENV_TOKENS.get('DEFAULT_FEEDBACK_EMAIL', DEFAULT_FEEDBACK_EMAIL)
DEFAULT_BULK_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_BULK_FROM_EMAIL', DEFAULT_BULK_FROM_EMAIL)
ADMINS = ENV_TOKENS.get('ADMINS', ADMINS)
SERVER_EMAIL = ENV_TOKENS.get('SERVER_EMAIL', SERVER_EMAIL)
TECH_SUPPORT_EMAIL = ENV_TOKENS.get('TECH_SUPPORT_EMAIL', TECH_SUPPORT_EMAIL)
@@ -144,8 +137,18 @@ BUGS_EMAIL = ENV_TOKENS.get('BUGS_EMAIL', BUGS_EMAIL)
PAYMENT_SUPPORT_EMAIL = ENV_TOKENS.get('PAYMENT_SUPPORT_EMAIL', PAYMENT_SUPPORT_EMAIL)
PAID_COURSE_REGISTRATION_CURRENCY = ENV_TOKENS.get('PAID_COURSE_REGISTRATION_CURRENCY',
PAID_COURSE_REGISTRATION_CURRENCY)
# Bulk Email overrides
DEFAULT_BULK_FROM_EMAIL = ENV_TOKENS.get('DEFAULT_BULK_FROM_EMAIL', DEFAULT_BULK_FROM_EMAIL)
EMAILS_PER_TASK = ENV_TOKENS.get('EMAILS_PER_TASK', 100)
EMAILS_PER_QUERY = ENV_TOKENS.get('EMAILS_PER_QUERY', 1000)
BULK_EMAIL_DEFAULT_RETRY_DELAY = ENV_TOKENS.get('BULK_EMAIL_DEFAULT_RETRY_DELAY', BULK_EMAIL_DEFAULT_RETRY_DELAY)
BULK_EMAIL_MAX_RETRIES = ENV_TOKENS.get('BULK_EMAIL_MAX_RETRIES', BULK_EMAIL_MAX_RETRIES)
BULK_EMAIL_INFINITE_RETRY_CAP = ENV_TOKENS.get('BULK_EMAIL_INFINITE_RETRY_CAP', BULK_EMAIL_INFINITE_RETRY_CAP)
BULK_EMAIL_LOG_SENT_EMAILS = ENV_TOKENS.get('BULK_EMAIL_LOG_SENT_EMAILS', BULK_EMAIL_LOG_SENT_EMAILS)
# We want Bulk Email running on the high-priority queue, so we define the
# routing key that points to it. At the moment, the name is the same.
BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE
# Theme overrides
THEME_NAME = ENV_TOKENS.get('THEME_NAME', None)

30
lms/envs/common.py
View File

@@ -114,6 +114,7 @@ MITX_FEATURES = {
# analytics experiments
'ENABLE_INSTRUCTOR_ANALYTICS': False,
# bulk email available to instructors:
'ENABLE_INSTRUCTOR_EMAIL': False,
# enable analytics server.
@@ -333,7 +334,7 @@ TRACKING_BACKENDS = {
}
}
# Backawrds compatibility with ENABLE_SQL_TRACKING_LOGS feature flag.
# Backwards compatibility with ENABLE_SQL_TRACKING_LOGS feature flag.
# In the future, adding the backend to TRACKING_BACKENDS enough.
if MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
TRACKING_BACKENDS.update({
@@ -414,12 +415,9 @@ HTTPS = 'on'
ROOT_URLCONF = 'lms.urls'
IGNORABLE_404_ENDS = ('favicon.ico')
# Email
# Platform Email
EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend'
DEFAULT_FROM_EMAIL = 'registration@edx.org'
DEFAULT_BULK_FROM_EMAIL = 'course-updates@edx.org'
EMAILS_PER_TASK = 100
EMAILS_PER_QUERY = 1000
DEFAULT_FEEDBACK_EMAIL = 'feedback@edx.org'
SERVER_EMAIL = 'devops@edx.org'
TECH_SUPPORT_EMAIL = 'technical@edx.org'
@@ -800,11 +798,29 @@ CELERYD_HIJACK_ROOT_LOGGER = False
################################ Bulk Email ###################################
DEFAULT_BULK_FROM_EMAIL = 'no-reply@courseupdates.edx.org'
EMAILS_PER_TASK = 100
EMAILS_PER_QUERY = 1000
# Initial delay used for retrying tasks. Additional retries use
# longer delays. Value is in seconds.
BULK_EMAIL_DEFAULT_RETRY_DELAY = 30
# Maximum number of retries per task for errors that are not related
# to throttling.
BULK_EMAIL_MAX_RETRIES = 5
# Maximum number of retries per task for errors that are related to
# throttling. If this is not set, then there is no cap on such retries.
BULK_EMAIL_INFINITE_RETRY_CAP = 1000
# We want Bulk Email running on the high-priority queue, so we define the
# routing key that points to it. At the moment, the name is the same.
BULK_EMAIL_ROUTING_KEY = HIGH_PRIORITY_QUEUE
BULK_EMAIL_DEFAULT_RETRY_DELAY = 15
BULK_EMAIL_MAX_RETRIES = 5
# Flag to indicate if individual email addresses should be logged as they are sent
# a bulk email message.
BULK_EMAIL_LOG_SENT_EMAILS = False
################################### APPS ######################################
INSTALLED_APPS = (