From 36b0c593ccd13e5b690e90b3034588b8939de091 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Mon, 8 Feb 2016 13:32:05 -0500 Subject: [PATCH] Retry failed Programs tasks using exponential backoff and a configurable retry limit --- openedx/core/djangoapps/programs/tasks/v1/tasks.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/programs/tasks/v1/tasks.py b/openedx/core/djangoapps/programs/tasks/v1/tasks.py index 42ec5c4b37..98a35a9286 100644 --- a/openedx/core/djangoapps/programs/tasks/v1/tasks.py +++ b/openedx/core/djangoapps/programs/tasks/v1/tasks.py @@ -136,11 +136,14 @@ def award_program_certificates(self, username): """ LOGGER.info('Running task award_program_certificates for username %s', username) + config = ProgramsApiConfig.current() + countdown = 2 ** self.request.retries + # If either programs or credentials config models are disabled for this # feature, this task should not have been invoked in the first place, and # an error somewhere is likely (though a race condition is also possible). # In either case, the task should not be executed nor should it be retried. - if not ProgramsApiConfig.current().is_certification_enabled: + if not config.is_certification_enabled: LOGGER.warning( 'Task award_program_certificates cannot be executed when program certification is disabled in API config', ) @@ -179,7 +182,7 @@ def award_program_certificates(self, username): # Invoke the Programs API completion check endpoint to identify any # programs that are satisfied by these course completions. - programs_client = get_api_client(ProgramsApiConfig.current(), student) + programs_client = get_api_client(config, student) program_ids = get_completed_programs(programs_client, course_certs) if not program_ids: # Again, no reason to continue beyond this point unless/until this @@ -192,7 +195,7 @@ def award_program_certificates(self, username): except Exception as exc: # pylint: disable=broad-except LOGGER.exception('Failed to determine program certificates to be awarded for user %s', username) - raise self.retry(exc=exc) + raise self.retry(exc=exc, countdown=countdown, max_retries=config.max_retries) # For each completed program for which the student doesn't already have a # certificate, award one now. @@ -208,7 +211,7 @@ def award_program_certificates(self, username): except Exception as exc: # pylint: disable=broad-except LOGGER.exception('Failed to create a credentials API client to award program certificates') # Retry because a misconfiguration could be fixed - raise self.retry(exc=exc) + raise self.retry(exc=exc, countdown=countdown, max_retries=config.max_retries) for program_id in new_program_ids: try: