diff --git a/openedx/core/djangoapps/credentials/management/__init__.py b/openedx/core/djangoapps/credentials/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/djangoapps/credentials/management/commands/__init__.py b/openedx/core/djangoapps/credentials/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py new file mode 100644 index 0000000000..b06b788c14 --- /dev/null +++ b/openedx/core/djangoapps/credentials/management/commands/notify_credentials.py @@ -0,0 +1,214 @@ +""" +A few places in the LMS want to notify the Credentials service when certain events +happen (like certificates being awarded or grades changing). To do this, they +listen for a signal. Sometimes we want to rebuild the data on these apps +regardless of an actual change in the database, either to recover from a bug or +to bootstrap a new feature we're rolling out for the first time. + +This management command will manually trigger the receivers we care about. +(We don't want to trigger all receivers for these signals, since these are busy +signals.) +""" +from __future__ import print_function +import logging +import time +import sys + +import dateutil.parser +from django.contrib.auth.models import User +from django.core.management.base import BaseCommand, CommandError +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from pytz import UTC + +from lms.djangoapps.certificates.models import GeneratedCertificate +from lms.djangoapps.grades.models import PersistentCourseGrade +from openedx.core.djangoapps.credentials.signals import handle_cert_change, send_grade_if_interesting +from openedx.core.djangoapps.programs.signals import handle_course_cert_awarded, handle_course_cert_changed + + +log = logging.getLogger(__name__) + + +def certstr(cert): + return '{} for user {}'.format(cert.course_id, cert.user.username) + + +def gradestr(grade): + return '{} for user {}'.format(grade.course_id, grade.user_id) + + +def parsetime(timestr): + dt = dateutil.parser.parse(timestr) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + + +class Command(BaseCommand): + """ + Example usage: + + # Process all certs/grades changes for a given course: + $ ./manage.py lms --settings=devstack_docker notify_credentials \ + --courses course-v1:edX+DemoX+Demo_Course + + # Process all certs/grades changes in a given time range: + $ ./manage.py lms --settings=devstack_docker notify_credentials \ + --start-date 2018-06-01 --end-date 2018-07-31 + + A Dry Run will produce output that looks like: + + DRY-RUN: This command would have handled changes for... + 3 Certificates: + course-v1:edX+RecordsSelfPaced+1 for user records_one_cert + course-v1:edX+RecordsSelfPaced+1 for user records + course-v1:edX+RecordsSelfPaced+1 for user records_unverified + 3 Grades: + course-v1:edX+RecordsSelfPaced+1 for user 14 + course-v1:edX+RecordsSelfPaced+1 for user 17 + course-v1:edX+RecordsSelfPaced+1 for user 18 + """ + help = ( + u"Simulate certificate/grade changes without actually modifying database " + u"content. Specifically, trigger the handlers that send data to Credentials." + ) + + def add_arguments(self, parser): + parser.add_argument( + '--dry-run', + action='store_true', + help='Just show a preview of what would happen.', + ) + parser.add_argument( + '--courses', + nargs='+', + help='Send information only for specific courses.', + ) + parser.add_argument( + '--start-date', + type=parsetime, + help='Send information only for certificates or grades that have changed since this date.', + ) + parser.add_argument( + '--end-date', + type=parsetime, + help='Send information only for certificates or grades that have changed before this date.', + ) + parser.add_argument( + '--delay', + type=float, + default=0, + help="Number of seconds to sleep between processing certificates, so that we don't flood our queues.", + ) + + def handle(self, *args, **options): + log.info( + "notify_credentials starting, dry-run=%s, delay=%d seconds", + options['dry_run'], + options['delay'] + ) + + cert_filter_args = {} + grade_filter_args = {} + + if options['courses']: + course_keys = self.get_course_keys(options['courses']) + cert_filter_args['course_id__in'] = course_keys + grade_filter_args['course_id__in'] = course_keys + + if options['start_date']: + cert_filter_args['modified_date__gte'] = options['start_date'] + grade_filter_args['modified__gte'] = options['start_date'] + + if options['end_date']: + cert_filter_args['modified_date__lte'] = options['end_date'] + grade_filter_args['modified__lte'] = options['end_date'] + + if not cert_filter_args: + raise CommandError('You must specify a filter (e.g. --courses= or --start-date)') + + # pylint: disable=no-member + certs = GeneratedCertificate.objects.filter(**cert_filter_args).order_by('modified_date') + grades = PersistentCourseGrade.objects.filter(**grade_filter_args).order_by('modified') + + if options['dry_run']: + self.print_dry_run(list(certs), list(grades)) + else: + self.send_notifications(certs, grades, delay=options['delay']) + + log.info('notify_credentials finished') + + def send_notifications(self, certs, grades, delay=0): + """ Run actual handler commands for the provided certs and grades. """ + + # First, do certs + for i, cert in enumerate(certs, start=1): + log.info( + "Handling credential changes (%d of %d) for certificate %s", + i, len(certs), certstr(cert), + ) + if delay: + time.sleep(delay) + + signal_args = { + 'sender': None, + 'user': cert.user, + 'course_key': cert.course_id, + 'mode': cert.mode, + 'status': cert.status, + } + handle_course_cert_awarded(**signal_args) + handle_course_cert_changed(**signal_args) + handle_cert_change(**signal_args) + + # Then do grades + for i, grade in enumerate(grades, start=1): + log.info( + "Handling grade changes (%d of %d) for grade %s", + i, len(grades), gradestr(grade), + ) + if delay: + time.sleep(delay) + + user = User.objects.get(id=grade.user_id) + send_grade_if_interesting(user, grade.course_id, None, None, grade.letter_grade, grade.percent_grade) + + def get_course_keys(self, courses): + """ + Return a list of CourseKeys that we will emit signals to. + + `courses` is an optional list of strings that can be parsed into + CourseKeys. If `courses` is empty or None, we will default to returning + all courses in the modulestore (which can be very expensive). If one of + the strings passed in the list for `courses` does not parse correctly, + it is a fatal error and will cause us to exit the entire process. + """ + # Use specific courses if specified, but fall back to all courses. + course_keys = [] + log.info("%d courses specified: %s", len(courses), ", ".join(courses)) + for course_id in courses: + try: + course_keys.append(CourseKey.from_string(course_id)) + except InvalidKeyError: + log.fatal("%s is not a parseable CourseKey", course_id) + sys.exit(1) + + return course_keys + + def print_dry_run(self, certs, grades): + """Give a preview of what certs/grades we will handle.""" + print("DRY-RUN: This command would have handled changes for...") + ITEMS_TO_SHOW = 10 + + print(len(certs), "Certificates:") + for cert in certs[:ITEMS_TO_SHOW]: + print(" ", certstr(cert)) + if len(certs) > ITEMS_TO_SHOW: + print(" (+ {} more)".format(len(certs) - ITEMS_TO_SHOW)) + + print(len(grades), "Grades:") + for grade in grades[:ITEMS_TO_SHOW]: + print(" ", gradestr(grade)) + if len(grades) > ITEMS_TO_SHOW: + print(" (+ {} more)".format(len(grades) - ITEMS_TO_SHOW)) diff --git a/openedx/core/djangoapps/credentials/management/commands/tests/__init__.py b/openedx/core/djangoapps/credentials/management/commands/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py b/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py new file mode 100644 index 0000000000..4eebc39965 --- /dev/null +++ b/openedx/core/djangoapps/credentials/management/commands/tests/test_notify_credentials.py @@ -0,0 +1,109 @@ +""" +Tests the ``notify_credentials`` management command. +""" +from __future__ import absolute_import, unicode_literals + +from datetime import datetime +import mock + +from django.core.management import call_command +from django.core.management.base import CommandError +from django.test import TestCase +from freezegun import freeze_time + +from lms.djangoapps.certificates.tests.factories import GeneratedCertificateFactory +from lms.djangoapps.grades.models import PersistentCourseGrade +from openedx.core.djangolib.testing.utils import skip_unless_lms +from student.tests.factories import UserFactory + +from ..notify_credentials import Command + +COMMAND_MODULE = 'openedx.core.djangoapps.credentials.management.commands.notify_credentials' + + +@skip_unless_lms +class TestNotifyCredentials(TestCase): + """ + Tests the ``notify_credentials`` management command. + """ + def setUp(self): + super(TestNotifyCredentials, self).setUp() + self.user = UserFactory.create() + + with freeze_time(datetime(2017, 1, 1)): + self.cert1 = GeneratedCertificateFactory(user=self.user, course_id='course-v1:edX+Test+1') + with freeze_time(datetime(2017, 2, 1)): + self.cert2 = GeneratedCertificateFactory(user=self.user, course_id='course-v1:edX+Test+2') + with freeze_time(datetime(2017, 3, 1)): + self.cert3 = GeneratedCertificateFactory(user=self.user, course_id='course-v1:edX+Test+3') + print('self.cert1.modified_date', self.cert1.modified_date) + + # No factory for these + with freeze_time(datetime(2017, 1, 1)): + self.grade1 = PersistentCourseGrade.objects.create(user_id=self.user.id, course_id='course-v1:edX+Test+1', + percent_grade=1) + with freeze_time(datetime(2017, 2, 1)): + self.grade2 = PersistentCourseGrade.objects.create(user_id=self.user.id, course_id='course-v1:edX+Test+2', + percent_grade=1) + with freeze_time(datetime(2017, 3, 1)): + self.grade3 = PersistentCourseGrade.objects.create(user_id=self.user.id, course_id='course-v1:edX+Test+3', + percent_grade=1) + print('self.grade1.modified', self.grade1.modified) + + @mock.patch(COMMAND_MODULE + '.Command.send_notifications') + def test_course_args(self, mock_send): + call_command(Command(), '--course', 'course-v1:edX+Test+1', 'course-v1:edX+Test+2') + self.assertTrue(mock_send.called) + self.assertEqual(list(mock_send.call_args[0][0]), [self.cert1, self.cert2]) + self.assertEqual(list(mock_send.call_args[0][1]), [self.grade1, self.grade2]) + + @mock.patch(COMMAND_MODULE + '.Command.send_notifications') + def test_date_args(self, mock_send): + call_command(Command(), '--start-date', '2017-01-31') + self.assertTrue(mock_send.called) + self.assertListEqual(list(mock_send.call_args[0][0]), [self.cert2, self.cert3]) + self.assertListEqual(list(mock_send.call_args[0][1]), [self.grade2, self.grade3]) + mock_send.reset_mock() + + call_command(Command(), '--start-date', '2017-02-01', '--end-date', '2017-02-02') + self.assertTrue(mock_send.called) + self.assertListEqual(list(mock_send.call_args[0][0]), [self.cert2]) + self.assertListEqual(list(mock_send.call_args[0][1]), [self.grade2]) + mock_send.reset_mock() + + call_command(Command(), '--end-date', '2017-02-02') + self.assertTrue(mock_send.called) + self.assertListEqual(list(mock_send.call_args[0][0]), [self.cert1, self.cert2]) + self.assertListEqual(list(mock_send.call_args[0][1]), [self.grade1, self.grade2]) + + @mock.patch(COMMAND_MODULE + '.Command.send_notifications') + def test_no_args(self, mock_send): + with self.assertRaisesRegex(CommandError, 'You must specify a filter.*'): + call_command(Command()) + self.assertFalse(mock_send.called) + + @mock.patch(COMMAND_MODULE + '.Command.send_notifications') + def test_dry_run(self, mock_send): + call_command(Command(), '--dry-run', '--start-date', '2017-02-01') + self.assertFalse(mock_send.called) + + @mock.patch(COMMAND_MODULE + '.handle_cert_change') + @mock.patch(COMMAND_MODULE + '.send_grade_if_interesting') + @mock.patch(COMMAND_MODULE + '.handle_course_cert_awarded') + @mock.patch(COMMAND_MODULE + '.handle_course_cert_changed') + def test_hand_off(self, mock_grade_cert_change, mock_grade_interesting, mock_program_awarded, mock_program_changed): + call_command(Command(), '--start-date', '2017-02-01') + self.assertEqual(mock_grade_cert_change.call_count, 2) + self.assertEqual(mock_grade_interesting.call_count, 2) + self.assertEqual(mock_program_awarded.call_count, 2) + self.assertEqual(mock_program_changed.call_count, 2) + + @mock.patch(COMMAND_MODULE + '.time') + def test_delay(self, mock_time): + call_command(Command(), '--start-date', '2017-02-01') + self.assertEqual(mock_time.sleep.call_count, 0) + mock_time.sleep.reset_mock() + + call_command(Command(), '--start-date', '2017-02-01', '--delay', '0.2') + self.assertEqual(mock_time.sleep.call_count, 4) # After each cert and each grade (2 each) + self.assertEqual(mock_time.sleep.call_args[0][0], 0.2)