Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Backfill and Django Admin for Learning Sequence Outline

Browse Source 
* Adds the backfill_course_outlines management command to contentstore
* Adds a read-only Django admin interface to learning_sequences for the
  support team and debugging.
* Adds two new functions to the learning_sequences public API:
  key_supports_outlines and get_course_keys_with_outlines

The learning_sequences app isn't supposed to know about contentstore or
modulestore, as it's intended to be extracted out of edx-platform in the
long term. Therefore, the backfill_course_outlines command is in
contentstore, and not learning_sequences.

This work was tracked in TNL-7983, but it also fixes a bug where we were
trying to generate course outlines for libraries (TNL-7981).

All Open edX instances upgrading to Lilac should run the
backfill_course_outlines command as part of their upgrade process.
This commit is contained in:
David Ormsbee
2021-03-04 11:58:00 -05:00
parent 6a00878f59
commit f5b74fcf31
12 changed files with 374 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
cms/djangoapps/contentstore/management/commands/backfill_course_outlines.py Normal file
View File

@@ -0,0 +1,66 @@
"""
Management command to create the course outline for all courses that are missing
an outline. Outlines are built automatically on course publish and manually
using the `update_course_outline` command, but they can be backfilled using this
command. People updating to Lilac release should run this command as part of the
upgrade process.
This should be invoked from the Studio process.
"""
import logging
from django.core.management.base import BaseCommand
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from openedx.core.djangoapps.content.learning_sequences.api import (
get_course_keys_with_outlines,
key_supports_outlines,
)
from ...tasks import update_outline_from_modulestore_task
log = logging.getLogger('backfill_course_outlines')
class Command(BaseCommand):
"""
Invoke with:
python manage.py cms backfill_course_outlines
"""
help = (
"Backfill missing course outlines. This will queue a celery task for "
"each course with a missing outline, meaning that the outlines may be "
"generated minutes or hours after this script has finished running."
)
def add_arguments(self, parser):
parser.add_argument(
'--dry',
action='store_true',
help="Show course outlines that will be backfilled, but do not make any changes."
)
def handle(self, *args, **options):
dry_run = options.get('dry', False)
log.info("Starting backfill_course_outlines{}".format(" (dry run)" if dry_run else ""))
all_course_keys_qs = CourseOverview.objects.values_list('id', flat=True)
# .difference() is not supported in MySQL, but this at least does the
# SELECT NOT IN... subquery in the database rather than Python.
missing_outlines_qs = all_course_keys_qs.exclude(
id__in=get_course_keys_with_outlines()
)
num_courses_needing_outlines = len(missing_outlines_qs)
log.info(
"Found %d courses without outlines. Queuing tasks...",
num_courses_needing_outlines
)
for course_key in missing_outlines_qs:
if key_supports_outlines(course_key):
log.info("Queuing outline creation for %s", course_key)
if not dry_run:
update_outline_from_modulestore_task.delay(str(course_key))
else:
log.info("Outlines not supported for %s - skipping", course_key)

117
cms/djangoapps/contentstore/management/commands/tests/test_backfill_course_outlines.py Normal file
View File

@@ -0,0 +1,117 @@
"""
Tests for `backfill_course_outlines` Studio (cms) management command.
"""
from django.core.management import call_command
from opaque_keys.edx.keys import CourseKey
from openedx.core.djangoapps.content.course_overviews.models import CourseOverview
from openedx.core.djangoapps.content.learning_sequences.api import get_course_keys_with_outlines
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from ....outlines import update_outline_from_modulestore
class BackfillCourseOutlinesTest(SharedModuleStoreTestCase):
"""
Test `backfill_orgs_and_org_courses`.
"""
def setUp(self):
"""
Create the CourseOverviews we need for this test case.
There's no publish signal, so we manually create the CourseOverviews.
Without that, backfill_orgs_and_org_courses has no way to figure out
which courses exist, which it needs in order to figure out which ones
need backfilling.
We can't turn on the course_published signal because if we did so, then
the outlines would get generated automatically, and there'd be nothing
to backfill.
"""
super().setUp()
CourseOverview.update_select_courses(self.course_keys, force_update=True)
@classmethod
def setUpClass(cls):
"""
We set up some content here, without publish signals enabled.
"""
super().setUpClass()
course_run_ids = [
"OpenEdX/OutlineCourse/OldMongoRun1",
"course-v1:OpenEdX+OutlineCourse+Run2",
"course-v1:OpenEdX+OutlineCourse+Run3",
]
cls.course_keys = [
CourseKey.from_string(course_run_id) for course_run_id in course_run_ids
]
for course_key in cls.course_keys:
if course_key.deprecated:
store_type = ModuleStoreEnum.Type.mongo
else:
store_type = ModuleStoreEnum.Type.split
with cls.store.default_store(store_type):
course = CourseFactory.create(
org=course_key.org,
number=course_key.course,
run=course_key.run,
display_name=f"Outline Backfill Test Course {course_key.run}"
)
with cls.store.bulk_operations(course_key):
section = ItemFactory.create(
parent_location=course.location,
category="chapter",
display_name="A Section"
)
sequence = ItemFactory.create(
parent_location=section.location,
category="sequential",
display_name="A Sequence"
)
unit = ItemFactory.create(
parent_location=sequence.location,
category="vertical",
display_name="A Unit"
)
ItemFactory.create(
parent_location=unit.location,
category="html",
display_name="An HTML Module"
)
def test_end_to_end(self):
"""Normal invocation, it should skip only the Old Mongo course."""
# In the beginning, we have no outlines...
assert not get_course_keys_with_outlines().exists()
# Run command and outlines appear for Split Mongo courses...
call_command("backfill_course_outlines")
course_keys_with_outlines = set(get_course_keys_with_outlines())
assert course_keys_with_outlines == {
CourseKey.from_string("course-v1:OpenEdX+OutlineCourse+Run2"),
CourseKey.from_string("course-v1:OpenEdX+OutlineCourse+Run3"),
}
def test_partial(self):
"""Also works when we've manually created one in advance."""
course_keys_with_outlines = set(get_course_keys_with_outlines())
assert not get_course_keys_with_outlines().exists()
# Manually create one
update_outline_from_modulestore(
CourseKey.from_string("course-v1:OpenEdX+OutlineCourse+Run2")
)
assert set(get_course_keys_with_outlines()) == {
CourseKey.from_string("course-v1:OpenEdX+OutlineCourse+Run2")
}
# backfill command should fill in the other
call_command("backfill_course_outlines")
course_keys_with_outlines = set(get_course_keys_with_outlines())
assert course_keys_with_outlines == {
CourseKey.from_string("course-v1:OpenEdX+OutlineCourse+Run2"),
CourseKey.from_string("course-v1:OpenEdX+OutlineCourse+Run3"),
}

4
cms/djangoapps/contentstore/signals/handlers.py
View File

@@ -19,6 +19,7 @@ from common.djangoapps.track.event_transaction_utils import get_event_transactio
from common.djangoapps.util.module_utils import yield_dynamic_descriptor_descendants
from lms.djangoapps.grades.api import task_compute_all_grades_for_course
from openedx.core.djangoapps.credit.signals import on_course_publish
from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines
from openedx.core.lib.gating import api as gating_api
from xmodule.modulestore.django import SignalHandler, modulestore
@@ -65,7 +66,8 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable=
# import here, because signal is registered at startup, but items in tasks are not yet able to be loaded
from cms.djangoapps.contentstore.tasks import update_outline_from_modulestore_task, update_search_index
update_outline_from_modulestore_task.delay(str(course_key))
if key_supports_outlines(course_key):
update_outline_from_modulestore_task.delay(str(course_key))
# Finally call into the course search subsystem
# to kick off an indexing action

11
cms/djangoapps/contentstore/tasks.py
View File

@@ -42,6 +42,7 @@ from cms.djangoapps.contentstore.utils import initialize_permissions, reverse_us
from cms.djangoapps.models.settings.course_metadata import CourseMetadata
from common.djangoapps.course_action_state.models import CourseRerunState
from common.djangoapps.student.auth import has_course_author_access
from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines
from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse
from openedx.core.lib.extract_tar import safetar_extractall
from xmodule.contentstore.django import contentstore
@@ -571,6 +572,16 @@ def update_outline_from_modulestore_task(course_key_str):
"""
try:
course_key = CourseKey.from_string(course_key_str)
if not key_supports_outlines(course_key):
LOGGER.warning(
(
"update_outline_from_modulestore_task called for course key"
" %s, which does not support learning_sequence outlines."
),
course_key_str
)
return
update_outline_from_modulestore(course_key)
except Exception: # pylint disable=broad-except
LOGGER.exception("Could not create course outline for course %s", course_key_str)