From 65977a9e02a1ca4979d8b1be2022db8f929be19c Mon Sep 17 00:00:00 2001 From: "Albert (AJ) St. Aubin" Date: Wed, 5 May 2021 11:48:11 -0400 Subject: [PATCH] [feat] Added program_uuids to notify_credentials mgmt cmd (#27503) * [feat] Added program_uuids to notify_credentials mgmt cmd [MICROBA-951] To support updating a users credentials in the Credentials services for all users enrolled in a program we have added a command line argument to the notify_credentials command called program_uuids. This supports a list of program uuids. It will retrieve all course runs in the listed programs and update the related credentials data. * updated comments --- openedx/core/djangoapps/catalog/api.py | 28 ++++++ .../management/commands/notify_credentials.py | 87 +++++++++++++------ .../commands/tests/test_notify_credentials.py | 63 ++++++++++++++ 3 files changed, 153 insertions(+), 25 deletions(-) diff --git a/openedx/core/djangoapps/catalog/api.py b/openedx/core/djangoapps/catalog/api.py index 74d6b54804..baf3cd878c 100644 --- a/openedx/core/djangoapps/catalog/api.py +++ b/openedx/core/djangoapps/catalog/api.py @@ -3,6 +3,8 @@ Python APIs exposed by the catalog app to other in-process apps. """ from .utils import get_programs_by_type_slug as _get_programs_by_type_slug +from .utils import get_programs as _get_programs +from .utils import course_run_keys_for_program as _course_run_keys_for_program def get_programs_by_type(site, program_type_slug): @@ -19,3 +21,29 @@ def get_programs_by_type(site, program_type_slug): A list of programs (dicts) for the given site with the given type slug """ return _get_programs_by_type_slug(site, program_type_slug) + + +def get_programs_from_cache_by_uuid(uuids): + """ + Retrieves the programs for the provided UUIDS. Relies on + the Program cache, if it is not updated or data is missing the result + will be missing data or empty. + + Params: + uuids (list): A list of Program UUIDs to get Program data for from the cache. + Returns: + (list): list of dictionaries representing programs. + """ + return _get_programs(uuids=uuids) + + +def get_course_run_key_for_program_from_cache(program): + """ + Retrieves a list of Course Run Keys from the Program. + + Params: + program (dict): A dictionary from the program cache containing the data for a program. + Returns: + (set): A set of Course Run Keys. + """ + return _course_run_keys_for_program(parent_program=program) diff --git a/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py index 078588400b..840403420d 100644 --- a/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py +++ b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py @@ -18,13 +18,16 @@ import sys from datetime import datetime, timedelta import dateutil.parser from django.core.management.base import BaseCommand, CommandError -from MySQLdb import OperationalError from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey from pytz import UTC from openedx.core.djangoapps.credentials.models import NotifyCredentialsConfig from openedx.core.djangoapps.credentials.tasks.v1.tasks import handle_notify_credentials +from openedx.core.djangoapps.catalog.api import ( + get_programs_from_cache_by_uuid, + get_course_run_key_for_program_from_cache, +) log = logging.getLogger(__name__) @@ -79,7 +82,12 @@ class Command(BaseCommand): parser.add_argument( '--courses', nargs='+', - help='Send information only for specific courses.', + help='Send information only for specific course runs.', + ) + parser.add_argument( + '--program_uuids', + nargs='+', + help='Send user data for course runs for courses within a program based on program uuids provided.', ) parser.add_argument( '--start-date', @@ -164,35 +172,64 @@ class Command(BaseCommand): 'auto' if options['auto'] else 'manual', ) - course_keys = self.get_course_keys(options['courses']) - if not (course_keys or options['start_date'] or options['end_date'] or options['user_ids']): - raise CommandError('You must specify a filter (e.g. --courses= or --start-date or --user_ids)') + program_course_run_keys = self._get_course_run_keys_for_programs(options["program_uuids"]) - handle_notify_credentials.delay(options, course_keys) + course_runs = options["courses"] + if not course_runs: + course_runs = [] + if program_course_run_keys: + course_runs.extend(program_course_run_keys) - def get_course_keys(self, courses=None): + course_run_keys = self._get_validated_course_run_keys(course_runs) + if not ( + course_run_keys or + options['start_date'] or + options['end_date'] or + options['user_ids'] + ): + raise CommandError( + 'You must specify a filter (e.g. --courses, --program_uuids, --start-date, or --user_ids)' + ) + + handle_notify_credentials.delay(options, course_run_keys) + + def _get_course_run_keys_for_programs(self, uuids): """ - Return a list of CourseKeys that we will emit signals to. + Retrieve all course runs for all of the given program UUIDs. + + Params: + uuids (list): List of programs UUIDs. + + Returns: + (list): List of Course Run Keys as Strings. - `courses` is an optional list of strings that can be parsed into - CourseKeys. If `courses` is empty or None, we will default to returning - all courses in the modulestore (which can be very expensive). If one of - the strings passed in the list for `courses` does not parse correctly, - it is a fatal error and will cause us to exit the entire process. """ - # Use specific courses if specified, but fall back to all courses. - if not courses: - courses = [] - course_keys = [] + program_course_run_keys = [] + if uuids: + programs = get_programs_from_cache_by_uuid(uuids=uuids) + for program in programs: + program_course_run_keys.extend(get_course_run_key_for_program_from_cache(program)) + return program_course_run_keys - log.info("%d courses specified: %s", len(courses), ", ".join(courses)) - for course_id in courses: + def _get_validated_course_run_keys(self, course_run_keys): + """ + Validates a list of course run keys and returns the validated keys. + + Params: + courses (list): list of strings that can be parsed by CourseKey to verify the keys. + Returns: + (list): Containing a series of validated course keys as strings. + """ + if not course_run_keys: + course_run_keys = [] + validated_course_run_keys = [] + + log.info("%d courses specified: %s", len(course_run_keys), ", ".join(course_run_keys)) + for course_run_key in course_run_keys: try: # Use CourseKey to check if the course_id is parsable, but just # keep the string; the celery task needs JSON serializable data. - course_keys.append(str(CourseKey.from_string(course_id))) - except InvalidKeyError: - log.fatal("%s is not a parseable CourseKey", course_id) - sys.exit(1) - - return course_keys + validated_course_run_keys.append(str(CourseKey.from_string(course_run_key))) + except InvalidKeyError as exc: + raise CommandError("{} is not a parsable CourseKey".format(course_run_key)) from exc + return validated_course_run_keys diff --git a/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py b/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py index 1abc35a661..be70f413f8 100644 --- a/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py +++ b/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py @@ -11,6 +11,7 @@ from django.core.management.base import CommandError from django.test import TestCase, override_settings from freezegun import freeze_time +from openedx.core.djangoapps.catalog.tests.factories import ProgramFactory, CourseFactory, CourseRunFactory from openedx.core.djangoapps.credentials.models import NotifyCredentialsConfig from openedx.core.djangoapps.site_configuration.tests.factories import SiteConfigurationFactory from openedx.core.djangolib.testing.utils import skip_unless_lms @@ -41,6 +42,7 @@ class TestNotifyCredentials(TestCase): 'no_color': False, 'notify_programs': False, 'page_size': 100, + 'program_uuids': None, 'pythonpath': None, 'settings': None, 'site': None, @@ -62,6 +64,67 @@ class TestNotifyCredentials(TestCase): assert mock_task.called assert mock_task.call_args[0][0] == self.expected_options + @mock.patch(NOTIFY_CREDENTIALS_TASK) + @mock.patch( + 'openedx.core.djangoapps.credentials.management.commands.notify_credentials.get_programs_from_cache_by_uuid' + ) + def test_program_uuid_args(self, mock_get_programs, mock_task): + course_1_id = 'course-v1:edX+Test+1' + course_2_id = 'course-v1:edX+Test+2' + program = ProgramFactory( + courses=[ + CourseFactory( + course_runs=[ + CourseRunFactory(key=course_1_id), + CourseRunFactory(key=course_2_id) + ] + ) + ], + curricula=[], + ) + self.expected_options['program_uuids'] = [program['uuid']] + mock_get_programs.return_value = [program] + call_command(Command(), '--program_uuids', program['uuid']) + assert mock_task.called + assert mock_task.call_args[0][0] == self.expected_options + assert mock_task.call_args[0][1].sort() == [course_1_id, course_2_id].sort() + + @mock.patch(NOTIFY_CREDENTIALS_TASK) + @mock.patch( + 'openedx.core.djangoapps.credentials.management.commands.notify_credentials.get_programs_from_cache_by_uuid' + ) + def test_multiple_programs_uuid_args(self, mock_get_programs, mock_task): + course_1_id = 'course-v1:edX+Test+1' + course_2_id = 'course-v1:edX+Test+2' + program = ProgramFactory( + courses=[ + CourseFactory( + course_runs=[ + CourseRunFactory(key=course_1_id), + ] + ) + ], + curricula=[], + ) + + program2 = ProgramFactory( + courses=[ + CourseFactory( + course_runs=[ + CourseRunFactory(key=course_2_id) + ] + ) + ], + curricula=[], + ) + program_list = [program['uuid'], program2['uuid']] + self.expected_options['program_uuids'] = program_list + mock_get_programs.return_value = [program, program2] + call_command(Command(), '--program_uuids', program['uuid'], program2['uuid']) + assert mock_task.called + assert mock_task.call_args[0][0] == self.expected_options + assert mock_task.call_args[0][1].sort() == [course_1_id, course_2_id].sort() + @freeze_time(datetime(2017, 5, 1, 4)) @mock.patch(NOTIFY_CREDENTIALS_TASK) def test_auto_execution(self, mock_task):