Merge pull request #12421 from edx/renzo/backpopulate-command
Add management command for backpopulating missing program credentials
This commit is contained in:
Renzo Lucioni
@@ -0,0 +1,119 @@
|
||||
"""Management command for backpopulating missing program credentials."""
|
||||
from collections import namedtuple
|
||||
import logging
|
||||
|
||||
from django.core.management import BaseCommand, CommandError
|
||||
from django.db.models import Q
|
||||
from opaque_keys.edx.keys import CourseKey
|
||||
from provider.oauth2.models import Client
|
||||
|
||||
from certificates.models import GeneratedCertificate # pylint: disable=import-error
|
||||
from openedx.core.djangoapps.programs.models import ProgramsApiConfig
|
||||
from openedx.core.djangoapps.programs.tasks.v1.tasks import award_program_certificates
|
||||
from openedx.core.djangoapps.programs.utils import get_programs
|
||||
|
||||
|
||||
# TODO: Log to console, even with debug mode disabled?
|
||||
logger = logging.getLogger(__name__) # pylint: disable=invalid-name
|
||||
RunMode = namedtuple('RunMode', ['course_key', 'mode_slug'])
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""Management command for backpopulating missing program credentials.
|
||||
|
||||
The command's goal is to pass a narrow subset of usernames to an idempotent
|
||||
Celery task for further (parallelized) processing.
|
||||
"""
|
||||
help = 'Backpopulate missing program credentials.'
|
||||
client = None
|
||||
run_modes = None
|
||||
usernames = None
|
||||
|
||||
def add_arguments(self, parser):
|
||||
parser.add_argument(
|
||||
'-c', '--commit',
|
||||
action='store_true',
|
||||
dest='commit',
|
||||
default=False,
|
||||
help='Submit tasks for processing.'
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
programs_config = ProgramsApiConfig.current()
|
||||
self.client = Client.objects.get(name=programs_config.OAUTH2_CLIENT_NAME)
|
||||
|
||||
if self.client.user is None:
|
||||
msg = (
|
||||
'No user is associated with the {} OAuth2 client. '
|
||||
'A service user is necessary to make requests to the Programs API. '
|
||||
'No tasks have been enqueued. '
|
||||
'Associate a user with the client and try again.'
|
||||
).format(programs_config.OAUTH2_CLIENT_NAME)
|
||||
|
||||
raise CommandError(msg)
|
||||
|
||||
self._load_run_modes()
|
||||
|
||||
logger.info('Looking for users who may be eligible for a program certificate.')
|
||||
|
||||
self._load_usernames()
|
||||
|
||||
if options.get('commit'):
|
||||
logger.info('Enqueuing program certification tasks for %d candidates.', len(self.usernames))
|
||||
else:
|
||||
logger.info(
|
||||
'Found %d candidates. To enqueue program certification tasks, pass the -c or --commit flags.',
|
||||
len(self.usernames)
|
||||
)
|
||||
return
|
||||
|
||||
succeeded, failed = 0, 0
|
||||
for username in self.usernames:
|
||||
try:
|
||||
award_program_certificates.delay(username)
|
||||
except: # pylint: disable=bare-except
|
||||
failed += 1
|
||||
logger.exception('Failed to enqueue task for user [%s]', username)
|
||||
else:
|
||||
succeeded += 1
|
||||
logger.debug('Successfully enqueued task for user [%s]', username)
|
||||
|
||||
logger.info(
|
||||
'Done. Successfully enqueued tasks for %d candidates. '
|
||||
'Failed to enqueue tasks for %d candidates.',
|
||||
succeeded,
|
||||
failed
|
||||
)
|
||||
|
||||
def _load_run_modes(self):
|
||||
"""Find all run modes which are part of a program."""
|
||||
programs = get_programs(self.client.user)
|
||||
self.run_modes = self._flatten(programs)
|
||||
|
||||
def _flatten(self, programs):
|
||||
"""Flatten program dicts into a set of run modes."""
|
||||
run_modes = set()
|
||||
for program in programs:
|
||||
for course_code in program['course_codes']:
|
||||
for run in course_code['run_modes']:
|
||||
course_key = CourseKey.from_string(run['course_key'])
|
||||
run_modes.add(
|
||||
RunMode(course_key, run['mode_slug'])
|
||||
)
|
||||
|
||||
return run_modes
|
||||
|
||||
def _load_usernames(self):
|
||||
"""Identify a subset of users who may be eligible for a program certificate.
|
||||
|
||||
This is done by finding users who have earned a certificate in at least one
|
||||
program course code's run mode.
|
||||
"""
|
||||
query = reduce(
|
||||
lambda x, y: x | y,
|
||||
[Q(course_id=r.course_key, mode=r.mode_slug) for r in self.run_modes]
|
||||
)
|
||||
|
||||
# TODO: Filter further, by passing status?
|
||||
username_dicts = GeneratedCertificate.eligible_certificates.filter(query).values('user__username').distinct()
|
||||
self.usernames = [d['user__username'] for d in username_dicts]
|
||||
@@ -0,0 +1,294 @@
|
||||
"""Tests for the backpopulate_program_credentials management command."""
|
||||
import json
|
||||
from unittest import skipUnless
|
||||
|
||||
import ddt
|
||||
from django.conf import settings
|
||||
from django.core.management import call_command, CommandError
|
||||
from django.test import TestCase
|
||||
from edx_oauth2_provider.tests.factories import ClientFactory
|
||||
import httpretty
|
||||
import mock
|
||||
from provider.constants import CONFIDENTIAL
|
||||
|
||||
from lms.djangoapps.certificates.api import MODES
|
||||
from lms.djangoapps.certificates.tests.factories import GeneratedCertificateFactory
|
||||
from openedx.core.djangoapps.programs.models import ProgramsApiConfig
|
||||
from openedx.core.djangoapps.programs.tests import factories
|
||||
from openedx.core.djangoapps.programs.tests.mixins import ProgramsApiConfigMixin
|
||||
from student.tests.factories import UserFactory
|
||||
|
||||
|
||||
COMMAND_MODULE = 'openedx.core.djangoapps.programs.management.commands.backpopulate_program_credentials'
|
||||
|
||||
|
||||
@ddt.ddt
|
||||
@httpretty.activate
|
||||
@mock.patch(COMMAND_MODULE + '.award_program_certificates.delay')
|
||||
@skipUnless(settings.ROOT_URLCONF == 'lms.urls', 'Test only valid in lms')
|
||||
class BackpopulateProgramCredentialsTests(ProgramsApiConfigMixin, TestCase):
|
||||
"""Tests for the backpopulate_program_credentials management command."""
|
||||
course_id, alternate_course_id = 'org/course/run', 'org/alternate/run'
|
||||
|
||||
def setUp(self):
|
||||
super(BackpopulateProgramCredentialsTests, self).setUp()
|
||||
|
||||
self.alice = UserFactory()
|
||||
self.bob = UserFactory()
|
||||
self.oauth2_user = UserFactory()
|
||||
self.oauth2_client = ClientFactory(name=ProgramsApiConfig.OAUTH2_CLIENT_NAME, client_type=CONFIDENTIAL)
|
||||
|
||||
self.create_programs_config()
|
||||
|
||||
def _link_oauth2_user(self):
|
||||
"""Helper to link user and OAuth2 client."""
|
||||
self.oauth2_client.user = self.oauth2_user
|
||||
self.oauth2_client.save() # pylint: disable=no-member
|
||||
|
||||
def _mock_programs_api(self, data):
|
||||
"""Helper for mocking out Programs API URLs."""
|
||||
self.assertTrue(httpretty.is_enabled(), msg='httpretty must be enabled to mock Programs API calls.')
|
||||
|
||||
url = ProgramsApiConfig.current().internal_api_url.strip('/') + '/programs/'
|
||||
body = json.dumps({'results': data})
|
||||
|
||||
httpretty.register_uri(httpretty.GET, url, body=body, content_type='application/json')
|
||||
|
||||
@ddt.data(True, False)
|
||||
def test_handle(self, commit, mock_task):
|
||||
"""Verify that relevant tasks are only enqueued when the commit option is passed."""
|
||||
data = [
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=self.course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
]
|
||||
self._mock_programs_api(data)
|
||||
self._link_oauth2_user()
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.alice,
|
||||
course_id=self.course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.bob,
|
||||
course_id=self.alternate_course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
call_command('backpopulate_program_credentials', commit=commit)
|
||||
|
||||
if commit:
|
||||
mock_task.assert_called_once_with(self.alice.username)
|
||||
else:
|
||||
mock_task.assert_not_called()
|
||||
|
||||
@ddt.data(
|
||||
[
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=alternate_course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
],
|
||||
[
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=course_id),
|
||||
]),
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=alternate_course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
],
|
||||
[
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=course_id),
|
||||
factories.RunMode(course_key=alternate_course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_handle_flatten(self, data, mock_task):
|
||||
"""Verify that program structures are flattened correctly."""
|
||||
self._mock_programs_api(data)
|
||||
self._link_oauth2_user()
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.alice,
|
||||
course_id=self.course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.bob,
|
||||
course_id=self.alternate_course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
call_command('backpopulate_program_credentials', commit=True)
|
||||
|
||||
calls = [
|
||||
mock.call(self.alice.username),
|
||||
mock.call(self.bob.username)
|
||||
]
|
||||
mock_task.assert_has_calls(calls, any_order=True)
|
||||
|
||||
def test_handle_username_dedup(self, mock_task):
|
||||
"""Verify that only one task is enqueued for a user with multiple eligible certs."""
|
||||
data = [
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=self.course_id),
|
||||
factories.RunMode(course_key=self.alternate_course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
]
|
||||
self._mock_programs_api(data)
|
||||
self._link_oauth2_user()
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.alice,
|
||||
course_id=self.course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.alice,
|
||||
course_id=self.alternate_course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
call_command('backpopulate_program_credentials', commit=True)
|
||||
|
||||
mock_task.assert_called_once_with(self.alice.username)
|
||||
|
||||
def test_handle_mode_slugs(self, mock_task):
|
||||
"""Verify that mode slugs are taken into account."""
|
||||
data = [
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(
|
||||
course_key=self.course_id,
|
||||
mode_slug=MODES.honor
|
||||
),
|
||||
]),
|
||||
]
|
||||
),
|
||||
]
|
||||
self._mock_programs_api(data)
|
||||
self._link_oauth2_user()
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.alice,
|
||||
course_id=self.course_id,
|
||||
)
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.bob,
|
||||
course_id=self.course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
call_command('backpopulate_program_credentials', commit=True)
|
||||
|
||||
mock_task.assert_called_once_with(self.alice.username)
|
||||
|
||||
def test_handle_unlinked_oauth2_user(self, mock_task):
|
||||
"""Verify that the command fails when no user is associated with the OAuth2 client."""
|
||||
data = [
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=self.course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
]
|
||||
self._mock_programs_api(data)
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.alice,
|
||||
course_id=self.course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
with self.assertRaises(CommandError):
|
||||
call_command('backpopulate_program_credentials')
|
||||
|
||||
mock_task.assert_not_called()
|
||||
|
||||
@mock.patch(COMMAND_MODULE + '.logger.exception')
|
||||
def test_handle_enqueue_failure(self, mock_log, mock_task):
|
||||
"""Verify that failure to enqueue a task doesn't halt execution."""
|
||||
def side_effect(username):
|
||||
"""Simulate failure to enqueue a task."""
|
||||
if username == self.alice.username:
|
||||
raise Exception
|
||||
|
||||
mock_task.side_effect = side_effect
|
||||
|
||||
data = [
|
||||
factories.Program(
|
||||
organizations=[factories.Organization()],
|
||||
course_codes=[
|
||||
factories.CourseCode(run_modes=[
|
||||
factories.RunMode(course_key=self.course_id),
|
||||
]),
|
||||
]
|
||||
),
|
||||
]
|
||||
self._mock_programs_api(data)
|
||||
self._link_oauth2_user()
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.alice,
|
||||
course_id=self.course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
GeneratedCertificateFactory(
|
||||
user=self.bob,
|
||||
course_id=self.course_id,
|
||||
mode=MODES.verified,
|
||||
)
|
||||
|
||||
call_command('backpopulate_program_credentials', commit=True)
|
||||
|
||||
self.assertTrue(mock_log.called)
|
||||
|
||||
calls = [
|
||||
mock.call(self.alice.username),
|
||||
mock.call(self.bob.username)
|
||||
]
|
||||
mock_task.assert_has_calls(calls, any_order=True)
|
||||
Reference in New Issue
Block a user