From c931d4d14b6ee3cfbe0babb2b8dbc6617b4292c6 Mon Sep 17 00:00:00 2001 From: Renzo Lucioni Date: Thu, 5 May 2016 14:27:58 -0400 Subject: [PATCH] Add management command for backpopulating missing program credentials This command triggers program certification tasks for any users in the system who may qualify for a program credential. ECOM-3924. --- .../programs/management/__init__.py | 0 .../programs/management/commands/__init__.py | 0 .../backpopulate_program_credentials.py | 119 +++++++ .../test_backpopulate_program_credentials.py | 294 ++++++++++++++++++ 4 files changed, 413 insertions(+) create mode 100644 openedx/core/djangoapps/programs/management/__init__.py create mode 100644 openedx/core/djangoapps/programs/management/commands/__init__.py create mode 100644 openedx/core/djangoapps/programs/management/commands/backpopulate_program_credentials.py create mode 100644 openedx/core/djangoapps/programs/tests/test_backpopulate_program_credentials.py diff --git a/openedx/core/djangoapps/programs/management/__init__.py b/openedx/core/djangoapps/programs/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/djangoapps/programs/management/commands/__init__.py b/openedx/core/djangoapps/programs/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/djangoapps/programs/management/commands/backpopulate_program_credentials.py b/openedx/core/djangoapps/programs/management/commands/backpopulate_program_credentials.py new file mode 100644 index 0000000000..101133d0ee --- /dev/null +++ b/openedx/core/djangoapps/programs/management/commands/backpopulate_program_credentials.py @@ -0,0 +1,119 @@ +"""Management command for backpopulating missing program credentials.""" +from collections import namedtuple +import logging + +from django.core.management import BaseCommand, CommandError +from django.db.models import Q +from opaque_keys.edx.keys import CourseKey +from provider.oauth2.models import Client + +from certificates.models import GeneratedCertificate # pylint: disable=import-error +from openedx.core.djangoapps.programs.models import ProgramsApiConfig +from openedx.core.djangoapps.programs.tasks.v1.tasks import award_program_certificates +from openedx.core.djangoapps.programs.utils import get_programs + + +# TODO: Log to console, even with debug mode disabled? +logger = logging.getLogger(__name__) # pylint: disable=invalid-name +RunMode = namedtuple('RunMode', ['course_key', 'mode_slug']) + + +class Command(BaseCommand): + """Management command for backpopulating missing program credentials. + + The command's goal is to pass a narrow subset of usernames to an idempotent + Celery task for further (parallelized) processing. + """ + help = 'Backpopulate missing program credentials.' + client = None + run_modes = None + usernames = None + + def add_arguments(self, parser): + parser.add_argument( + '-c', '--commit', + action='store_true', + dest='commit', + default=False, + help='Submit tasks for processing.' + ) + + def handle(self, *args, **options): + programs_config = ProgramsApiConfig.current() + self.client = Client.objects.get(name=programs_config.OAUTH2_CLIENT_NAME) + + if self.client.user is None: + msg = ( + 'No user is associated with the {} OAuth2 client. ' + 'A service user is necessary to make requests to the Programs API. ' + 'No tasks have been enqueued. ' + 'Associate a user with the client and try again.' + ).format(programs_config.OAUTH2_CLIENT_NAME) + + raise CommandError(msg) + + self._load_run_modes() + + logger.info('Looking for users who may be eligible for a program certificate.') + + self._load_usernames() + + if options.get('commit'): + logger.info('Enqueuing program certification tasks for %d candidates.', len(self.usernames)) + else: + logger.info( + 'Found %d candidates. To enqueue program certification tasks, pass the -c or --commit flags.', + len(self.usernames) + ) + return + + succeeded, failed = 0, 0 + for username in self.usernames: + try: + award_program_certificates.delay(username) + except: # pylint: disable=bare-except + failed += 1 + logger.exception('Failed to enqueue task for user [%s]', username) + else: + succeeded += 1 + logger.debug('Successfully enqueued task for user [%s]', username) + + logger.info( + 'Done. Successfully enqueued tasks for %d candidates. ' + 'Failed to enqueue tasks for %d candidates.', + succeeded, + failed + ) + + def _load_run_modes(self): + """Find all run modes which are part of a program.""" + programs = get_programs(self.client.user) + self.run_modes = self._flatten(programs) + + def _flatten(self, programs): + """Flatten program dicts into a set of run modes.""" + run_modes = set() + for program in programs: + for course_code in program['course_codes']: + for run in course_code['run_modes']: + course_key = CourseKey.from_string(run['course_key']) + run_modes.add( + RunMode(course_key, run['mode_slug']) + ) + + return run_modes + + def _load_usernames(self): + """Identify a subset of users who may be eligible for a program certificate. + + This is done by finding users who have earned a certificate in at least one + program course code's run mode. + """ + query = reduce( + lambda x, y: x | y, + [Q(course_id=r.course_key, mode=r.mode_slug) for r in self.run_modes] + ) + + # TODO: Filter further, by passing status? + username_dicts = GeneratedCertificate.eligible_certificates.filter(query).values('user__username').distinct() + self.usernames = [d['user__username'] for d in username_dicts] diff --git a/openedx/core/djangoapps/programs/tests/test_backpopulate_program_credentials.py b/openedx/core/djangoapps/programs/tests/test_backpopulate_program_credentials.py new file mode 100644 index 0000000000..5143cf1cdc --- /dev/null +++ b/openedx/core/djangoapps/programs/tests/test_backpopulate_program_credentials.py @@ -0,0 +1,294 @@ +"""Tests for the backpopulate_program_credentials management command.""" +import json +from unittest import skipUnless + +import ddt +from django.conf import settings +from django.core.management import call_command, CommandError +from django.test import TestCase +from edx_oauth2_provider.tests.factories import ClientFactory +import httpretty +import mock +from provider.constants import CONFIDENTIAL + +from lms.djangoapps.certificates.api import MODES +from lms.djangoapps.certificates.tests.factories import GeneratedCertificateFactory +from openedx.core.djangoapps.programs.models import ProgramsApiConfig +from openedx.core.djangoapps.programs.tests import factories +from openedx.core.djangoapps.programs.tests.mixins import ProgramsApiConfigMixin +from student.tests.factories import UserFactory + + +COMMAND_MODULE = 'openedx.core.djangoapps.programs.management.commands.backpopulate_program_credentials' + + +@ddt.ddt +@httpretty.activate +@mock.patch(COMMAND_MODULE + '.award_program_certificates.delay') +@skipUnless(settings.ROOT_URLCONF == 'lms.urls', 'Test only valid in lms') +class BackpopulateProgramCredentialsTests(ProgramsApiConfigMixin, TestCase): + """Tests for the backpopulate_program_credentials management command.""" + course_id, alternate_course_id = 'org/course/run', 'org/alternate/run' + + def setUp(self): + super(BackpopulateProgramCredentialsTests, self).setUp() + + self.alice = UserFactory() + self.bob = UserFactory() + self.oauth2_user = UserFactory() + self.oauth2_client = ClientFactory(name=ProgramsApiConfig.OAUTH2_CLIENT_NAME, client_type=CONFIDENTIAL) + + self.create_programs_config() + + def _link_oauth2_user(self): + """Helper to link user and OAuth2 client.""" + self.oauth2_client.user = self.oauth2_user + self.oauth2_client.save() # pylint: disable=no-member + + def _mock_programs_api(self, data): + """Helper for mocking out Programs API URLs.""" + self.assertTrue(httpretty.is_enabled(), msg='httpretty must be enabled to mock Programs API calls.') + + url = ProgramsApiConfig.current().internal_api_url.strip('/') + '/programs/' + body = json.dumps({'results': data}) + + httpretty.register_uri(httpretty.GET, url, body=body, content_type='application/json') + + @ddt.data(True, False) + def test_handle(self, commit, mock_task): + """Verify that relevant tasks are only enqueued when the commit option is passed.""" + data = [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=self.course_id), + ]), + ] + ), + ] + self._mock_programs_api(data) + self._link_oauth2_user() + + GeneratedCertificateFactory( + user=self.alice, + course_id=self.course_id, + mode=MODES.verified, + ) + + GeneratedCertificateFactory( + user=self.bob, + course_id=self.alternate_course_id, + mode=MODES.verified, + ) + + call_command('backpopulate_program_credentials', commit=commit) + + if commit: + mock_task.assert_called_once_with(self.alice.username) + else: + mock_task.assert_not_called() + + @ddt.data( + [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=course_id), + ]), + ] + ), + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=alternate_course_id), + ]), + ] + ), + ], + [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=course_id), + ]), + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=alternate_course_id), + ]), + ] + ), + ], + [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=course_id), + factories.RunMode(course_key=alternate_course_id), + ]), + ] + ), + ], + ) + def test_handle_flatten(self, data, mock_task): + """Verify that program structures are flattened correctly.""" + self._mock_programs_api(data) + self._link_oauth2_user() + + GeneratedCertificateFactory( + user=self.alice, + course_id=self.course_id, + mode=MODES.verified, + ) + + GeneratedCertificateFactory( + user=self.bob, + course_id=self.alternate_course_id, + mode=MODES.verified, + ) + + call_command('backpopulate_program_credentials', commit=True) + + calls = [ + mock.call(self.alice.username), + mock.call(self.bob.username) + ] + mock_task.assert_has_calls(calls, any_order=True) + + def test_handle_username_dedup(self, mock_task): + """Verify that only one task is enqueued for a user with multiple eligible certs.""" + data = [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=self.course_id), + factories.RunMode(course_key=self.alternate_course_id), + ]), + ] + ), + ] + self._mock_programs_api(data) + self._link_oauth2_user() + + GeneratedCertificateFactory( + user=self.alice, + course_id=self.course_id, + mode=MODES.verified, + ) + + GeneratedCertificateFactory( + user=self.alice, + course_id=self.alternate_course_id, + mode=MODES.verified, + ) + + call_command('backpopulate_program_credentials', commit=True) + + mock_task.assert_called_once_with(self.alice.username) + + def test_handle_mode_slugs(self, mock_task): + """Verify that mode slugs are taken into account.""" + data = [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode( + course_key=self.course_id, + mode_slug=MODES.honor + ), + ]), + ] + ), + ] + self._mock_programs_api(data) + self._link_oauth2_user() + + GeneratedCertificateFactory( + user=self.alice, + course_id=self.course_id, + ) + + GeneratedCertificateFactory( + user=self.bob, + course_id=self.course_id, + mode=MODES.verified, + ) + + call_command('backpopulate_program_credentials', commit=True) + + mock_task.assert_called_once_with(self.alice.username) + + def test_handle_unlinked_oauth2_user(self, mock_task): + """Verify that the command fails when no user is associated with the OAuth2 client.""" + data = [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=self.course_id), + ]), + ] + ), + ] + self._mock_programs_api(data) + + GeneratedCertificateFactory( + user=self.alice, + course_id=self.course_id, + mode=MODES.verified, + ) + + with self.assertRaises(CommandError): + call_command('backpopulate_program_credentials') + + mock_task.assert_not_called() + + @mock.patch(COMMAND_MODULE + '.logger.exception') + def test_handle_enqueue_failure(self, mock_log, mock_task): + """Verify that failure to enqueue a task doesn't halt execution.""" + def side_effect(username): + """Simulate failure to enqueue a task.""" + if username == self.alice.username: + raise Exception + + mock_task.side_effect = side_effect + + data = [ + factories.Program( + organizations=[factories.Organization()], + course_codes=[ + factories.CourseCode(run_modes=[ + factories.RunMode(course_key=self.course_id), + ]), + ] + ), + ] + self._mock_programs_api(data) + self._link_oauth2_user() + + GeneratedCertificateFactory( + user=self.alice, + course_id=self.course_id, + mode=MODES.verified, + ) + + GeneratedCertificateFactory( + user=self.bob, + course_id=self.course_id, + mode=MODES.verified, + ) + + call_command('backpopulate_program_credentials', commit=True) + + self.assertTrue(mock_log.called) + + calls = [ + mock.call(self.alice.username), + mock.call(self.bob.username) + ] + mock_task.assert_has_calls(calls, any_order=True)