From 2b80fdbf665eb4fc18807ef7fd1a31ceba733f27 Mon Sep 17 00:00:00 2001 From: Adeel Khan Date: Tue, 19 Nov 2019 08:42:15 +0500 Subject: [PATCH] Automate retry_failed_photo_verification mgt command This patch would enable a user to run management command via jenkins job. Verification ids are injected via a configuration model. PROD-1005 --- lms/djangoapps/verify_student/admin.py | 12 ++++- .../retry_failed_photo_verifications.py | 51 ++++++++++++++++--- .../commands/tests/test_verify_student.py | 45 +++++++++++++++- .../0012_sspverificationretryconfig.py | 31 +++++++++++ lms/djangoapps/verify_student/models.py | 24 ++++++++- 5 files changed, 153 insertions(+), 10 deletions(-) create mode 100644 lms/djangoapps/verify_student/migrations/0012_sspverificationretryconfig.py diff --git a/lms/djangoapps/verify_student/admin.py b/lms/djangoapps/verify_student/admin.py index 81b0ec64cc..971d2d689c 100644 --- a/lms/djangoapps/verify_student/admin.py +++ b/lms/djangoapps/verify_student/admin.py @@ -7,7 +7,9 @@ from __future__ import absolute_import from django.contrib import admin -from lms.djangoapps.verify_student.models import ManualVerification, SoftwareSecurePhotoVerification, SSOVerification +from lms.djangoapps.verify_student.models import ( + ManualVerification, SoftwareSecurePhotoVerification, SSOVerification, + SSPVerificationRetryConfig) @admin.register(SoftwareSecurePhotoVerification) @@ -39,3 +41,11 @@ class ManualVerificationAdmin(admin.ModelAdmin): list_display = ('id', 'user', 'status', 'reason', 'created_at', 'updated_at',) raw_id_fields = ('user',) search_fields = ('user__username', 'reason',) + + +@admin.register(SSPVerificationRetryConfig) +class SSPVerificationRetryAdmin(admin.ModelAdmin): + """ + Admin for the SSPVerificationRetryConfig table. + """ + pass diff --git a/lms/djangoapps/verify_student/management/commands/retry_failed_photo_verifications.py b/lms/djangoapps/verify_student/management/commands/retry_failed_photo_verifications.py index fd4c3cfbe0..3979e5d850 100644 --- a/lms/djangoapps/verify_student/management/commands/retry_failed_photo_verifications.py +++ b/lms/djangoapps/verify_student/management/commands/retry_failed_photo_verifications.py @@ -3,9 +3,13 @@ Django admin commands related to verify_student """ from __future__ import absolute_import, print_function +import logging from django.core.management.base import BaseCommand +from django.core.management.base import CommandError -from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification +from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification, SSPVerificationRetryConfig + +log = logging.getLogger('retry_photo_verification') class Command(BaseCommand): @@ -20,24 +24,59 @@ class Command(BaseCommand): "are in a state of 'must_retry'" ) + def add_arguments(self, parser): + + parser.add_argument( + '--verification-ids', + dest='verification_ids', + action='store', + nargs='+', + type=int, + help='verifications id used to retry verification' + ) + + parser.add_argument( + '--args-from-database', + action='store_true', + help='Use arguments from the SSPVerificationRetryConfig model instead of the command line.', + ) + + def get_args_from_database(self): + """ Returns an options dictionary from the current SSPVerificationRetryConfig model. """ + + sspv_retry_config = SSPVerificationRetryConfig.current() + if not sspv_retry_config.enabled: + raise CommandError('SSPVerificationRetryConfig is disabled, but --args-from-database was requested.') + + # We don't need fancy shell-style whitespace/quote handling - none of our arguments are complicated + argv = sspv_retry_config.arguments.split() + + parser = self.create_parser('manage.py', 'sspv_retry') + return parser.parse_args(argv).__dict__ # we want a dictionary, not a non-iterable Namespace object + def handle(self, *args, **options): + + options = self.get_args_from_database() if options['args_from_database'] else options + args = options.get('verification_ids', None) + if args: attempts_to_retry = SoftwareSecurePhotoVerification.objects.filter( - receipt_id__in=args + receipt_id__in=options['verification_ids'] ) + log.info(u"Fetching retry verification ids from config model") force_must_retry = True else: attempts_to_retry = SoftwareSecurePhotoVerification.objects.filter(status='must_retry') force_must_retry = False - print(u"Attempting to retry {0} failed PhotoVerification submissions".format(len(attempts_to_retry))) + log.info(u"Attempting to retry {0} failed PhotoVerification submissions".format(len(attempts_to_retry))) for index, attempt in enumerate(attempts_to_retry): - print(u"Retrying submission #{0} (ID: {1}, User: {2})".format(index, attempt.id, attempt.user)) + log.info(u"Retrying submission #{0} (ID: {1}, User: {2})".format(index, attempt.id, attempt.user)) # Set the attempts status to 'must_retry' so that we can re-submit it if force_must_retry: attempt.status = 'must_retry' attempt.submit(copy_id_photo_from=attempt.copy_id_photo_from) - print(u"Retry result: {0}".format(attempt.status)) - print("Done resubmitting failed photo verifications") + log.info(u"Retry result: {0}".format(attempt.status)) + log.info("Done resubmitting failed photo verifications") diff --git a/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py b/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py index 5f8fbd9979..986aa2ffe2 100644 --- a/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py +++ b/lms/djangoapps/verify_student/management/commands/tests/test_verify_student.py @@ -8,17 +8,21 @@ from __future__ import absolute_import import boto from django.conf import settings from django.core.management import call_command +from django.core.management.base import CommandError from django.test import TestCase from mock import patch +from testfixtures import LogCapture from common.test.utils import MockS3Mixin -from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification +from lms.djangoapps.verify_student.models import SoftwareSecurePhotoVerification, SSPVerificationRetryConfig from lms.djangoapps.verify_student.tests.test_models import ( FAKE_SETTINGS, mock_software_secure_post, mock_software_secure_post_error ) -from student.tests.factories import UserFactory +from student.tests.factories import UserFactory # pylint: disable=import-error, useless-suppression + +LOGGER_NAME = 'retry_photo_verification' # Lots of patching to stub in our own settings, and HTTP posting @@ -64,3 +68,40 @@ class TestVerifyStudentCommand(MockS3Mixin, TestCase): call_command('retry_failed_photo_verifications') attempts_to_retry = SoftwareSecurePhotoVerification.objects.filter(status='must_retry') assert not attempts_to_retry + + def test_args_from_database(self): + """Test management command arguments injected from config model.""" + # Nothing in the database, should default to disabled + + # pylint: disable=deprecated-method, useless-suppression + with self.assertRaisesRegex(CommandError, 'SSPVerificationRetryConfig is disabled*'): + call_command('retry_failed_photo_verifications', '--args-from-database') + + # Add a config + config = SSPVerificationRetryConfig.current() + config.arguments = '--verification-ids 1 2 3' + config.enabled = True + config.save() + + with patch('lms.djangoapps.verify_student.models.requests.post', new=mock_software_secure_post_error): + self.create_and_submit("RetryRoger") + + with LogCapture(LOGGER_NAME) as log: + call_command('retry_failed_photo_verifications') + + log.check_present( + ( + LOGGER_NAME, 'INFO', + u"Attempting to retry {0} failed PhotoVerification submissions".format(1) + ), + ) + + with LogCapture(LOGGER_NAME) as log: + call_command('retry_failed_photo_verifications', '--args-from-database') + + log.check_present( + ( + LOGGER_NAME, 'INFO', + u"Fetching retry verification ids from config model" + ), + ) diff --git a/lms/djangoapps/verify_student/migrations/0012_sspverificationretryconfig.py b/lms/djangoapps/verify_student/migrations/0012_sspverificationretryconfig.py new file mode 100644 index 0000000000..2f150fdd4e --- /dev/null +++ b/lms/djangoapps/verify_student/migrations/0012_sspverificationretryconfig.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.26 on 2019-12-10 11:19 +from __future__ import unicode_literals + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('verify_student', '0011_add_fields_to_sspv'), + ] + + operations = [ + migrations.CreateModel( + name='SSPVerificationRetryConfig', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')), + ('enabled', models.BooleanField(default=False, verbose_name='Enabled')), + ('arguments', models.TextField(blank=True, default='', help_text='Useful for manually running a Jenkins job. Specify like --verification-ids 1 2 3')), + ('changed_by', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL, verbose_name='Changed by')), + ], + options={ + 'verbose_name': 'sspv retry student argument', + }, + ), + ] diff --git a/lms/djangoapps/verify_student/models.py b/lms/djangoapps/verify_student/models.py index 3f6e3b8ca0..73f2e6b201 100644 --- a/lms/djangoapps/verify_student/models.py +++ b/lms/djangoapps/verify_student/models.py @@ -16,13 +16,14 @@ import functools import json import logging import os.path -import simplejson import uuid from datetime import timedelta from email.utils import formatdate import requests +import simplejson import six +from config_models.models import ConfigurationModel from django.conf import settings from django.contrib.auth.models import User from django.core.files.base import ContentFile @@ -44,6 +45,7 @@ from lms.djangoapps.verify_student.ssencrypt import ( ) from openedx.core.djangoapps.signals.signals import LEARNER_NOW_VERIFIED from openedx.core.storage import get_storage + from .utils import earliest_allowed_verification_date log = logging.getLogger(__name__) @@ -1108,3 +1110,23 @@ class VerificationDeadline(TimeStampedModel): return deadline.deadline except cls.DoesNotExist: return None + + +class SSPVerificationRetryConfig(ConfigurationModel): # pylint: disable=model-missing-unicode, useless-suppression + """ + SSPVerificationRetryConfig used to inject arguments + to retry_failed_photo_verifications management command + """ + + class Meta(object): + app_label = 'verify_student' + verbose_name = 'sspv retry student argument' + + arguments = models.TextField( + blank=True, + help_text='Useful for manually running a Jenkins job. Specify like --verification-ids 1 2 3', + default='' + ) + + def __str__(self): + return six.text_type(self.arguments)