From 505b4f466c17ac1ca48ec0e7406ebf8bc2efb912 Mon Sep 17 00:00:00 2001 From: Ivan Niedielnitsev <81557788+NiedielnitsevIvan@users.noreply.github.com> Date: Thu, 17 Apr 2025 22:03:46 +0300 Subject: [PATCH] feat: Models for import_from_modulestore (#36515) A new application has been created, described in this ADR: https://github.com/openedx/edx-platform/pull/36545 have been created, as well as related models for mapping original content and new content created during the import process. Python and Django APIs, as well as a Django admin interface, will soon follow. --- .github/workflows/unit-test-shards.json | 1 + .../import_from_modulestore/README.rst | 31 ++++ .../import_from_modulestore/__init__.py | 0 .../import_from_modulestore/admin.py | 35 +++++ .../import_from_modulestore/apps.py | 13 ++ .../import_from_modulestore/data.py | 20 +++ .../migrations/__init__.py | 0 .../import_from_modulestore/models.py | 140 ++++++++++++++++++ cms/envs/common.py | 1 + requirements/constraints.txt | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/testing.txt | 2 +- 14 files changed, 246 insertions(+), 5 deletions(-) create mode 100644 cms/djangoapps/import_from_modulestore/README.rst create mode 100644 cms/djangoapps/import_from_modulestore/__init__.py create mode 100644 cms/djangoapps/import_from_modulestore/admin.py create mode 100644 cms/djangoapps/import_from_modulestore/apps.py create mode 100644 cms/djangoapps/import_from_modulestore/data.py create mode 100644 cms/djangoapps/import_from_modulestore/migrations/__init__.py create mode 100644 cms/djangoapps/import_from_modulestore/models.py diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json index 784f607f06..7184bf917e 100644 --- a/.github/workflows/unit-test-shards.json +++ b/.github/workflows/unit-test-shards.json @@ -238,6 +238,7 @@ "cms/djangoapps/cms_user_tasks/", "cms/djangoapps/course_creators/", "cms/djangoapps/export_course_metadata/", + "cms/djangoapps/import_from_modulestore/", "cms/djangoapps/maintenance/", "cms/djangoapps/models/", "cms/djangoapps/pipeline_js/", diff --git a/cms/djangoapps/import_from_modulestore/README.rst b/cms/djangoapps/import_from_modulestore/README.rst new file mode 100644 index 0000000000..f2725ef422 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/README.rst @@ -0,0 +1,31 @@ +======================== +Import from Modulestore +======================== + +The new Django application `import_from_modulestore` is designed to +automate the process of importing course legacy OLX content from Modulestore +to Content Libraries. The application allows users to easily and quickly +migrate existing course content, minimizing the manual work and potential +errors associated with manual migration. +The new app makes the import process automated and easy to manage. + +The main problems solved by the application: + +* Reducing the time to import course content. +* Ensuring data integrity during the transfer. +* Ability to choose which content to import before the final import. + +------------------------------ +Import from Modulestore Usage +------------------------------ + +* Import course elements at the level of sections, subsections, units, + and xblocks into the Content Libraries. +* Choose the structure of this import, whether it will be only xblocks + from a particular course or full sections/subsections/units. +* Store the history of imports. +* Synchronize the course content with the library content (when re-importing, + the blocks can be updated according to changes in the original course). +* The new import mechanism ensures data integrity at the time of importing + by saving the course in StagedContent. +* Importing the legacy library content into the new Content Libraries. diff --git a/cms/djangoapps/import_from_modulestore/__init__.py b/cms/djangoapps/import_from_modulestore/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/import_from_modulestore/admin.py b/cms/djangoapps/import_from_modulestore/admin.py new file mode 100644 index 0000000000..ed1d7a2023 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/admin.py @@ -0,0 +1,35 @@ +""" +This module contains the admin configuration for the Import model. +""" +from django.contrib import admin + +from .models import Import, PublishableEntityImport, PublishableEntityMapping + + +class ImportAdmin(admin.ModelAdmin): + """ + Admin configuration for the Import model. + """ + + list_display = ( + 'uuid', + 'created', + 'status', + 'source_key', + 'target_change', + ) + list_filter = ( + 'status', + ) + search_fields = ( + 'source_key', + 'target_change', + ) + + raw_id_fields = ('user',) + readonly_fields = ('status',) + + +admin.site.register(Import, ImportAdmin) +admin.site.register(PublishableEntityImport) +admin.site.register(PublishableEntityMapping) diff --git a/cms/djangoapps/import_from_modulestore/apps.py b/cms/djangoapps/import_from_modulestore/apps.py new file mode 100644 index 0000000000..81b4471dac --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/apps.py @@ -0,0 +1,13 @@ +""" +App for importing from the modulestore tools. +""" + +from django.apps import AppConfig + + +class ImportFromModulestoreConfig(AppConfig): + """ + App for importing legacy content from the modulestore. + """ + + name = 'cms.djangoapps.import_from_modulestore' diff --git a/cms/djangoapps/import_from_modulestore/data.py b/cms/djangoapps/import_from_modulestore/data.py new file mode 100644 index 0000000000..7821e463a7 --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/data.py @@ -0,0 +1,20 @@ +""" +This module contains the data models for the import_from_modulestore app. +""" +from django.db.models import TextChoices +from django.utils.translation import gettext_lazy as _ + + +class ImportStatus(TextChoices): + """ + The status of this modulestore-to-learning-core import. + """ + + NOT_STARTED = 'not_started', _('Waiting to stage content') + STAGING = 'staging', _('Staging content for import') + STAGING_FAILED = _('Failed to stage content') + STAGED = 'staged', _('Content is staged and ready for import') + IMPORTING = 'importing', _('Importing staged content') + IMPORTING_FAILED = 'importing_failed', _('Failed to import staged content') + IMPORTED = 'imported', _('Successfully imported content') + CANCELED = 'canceled', _('Canceled') diff --git a/cms/djangoapps/import_from_modulestore/migrations/__init__.py b/cms/djangoapps/import_from_modulestore/migrations/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/cms/djangoapps/import_from_modulestore/models.py b/cms/djangoapps/import_from_modulestore/models.py new file mode 100644 index 0000000000..acbe82fa6d --- /dev/null +++ b/cms/djangoapps/import_from_modulestore/models.py @@ -0,0 +1,140 @@ +""" +Models for the course to library import app. +""" + +import uuid as uuid_tools + +from django.contrib.auth import get_user_model +from django.db import models +from django.utils.translation import gettext_lazy as _ + +from model_utils.models import TimeStampedModel +from opaque_keys.edx.django.models import ( + LearningContextKeyField, + UsageKeyField, +) +from openedx_learning.api.authoring_models import LearningPackage, PublishableEntity + +from .data import ImportStatus + +User = get_user_model() + + +class Import(TimeStampedModel): + """ + Represents the action of a user importing a modulestore-based course or legacy + library into a learning-core based learning package (today, that is always a content library). + """ + + uuid = models.UUIDField(default=uuid_tools.uuid4, editable=False, unique=True) + status = models.CharField( + max_length=100, + choices=ImportStatus.choices, + default=ImportStatus.NOT_STARTED, + db_index=True + ) + user = models.ForeignKey(User, on_delete=models.CASCADE) + + # Note: For now, this will always be a course key. In the future, it may be a legacy library key. + source_key = LearningContextKeyField(help_text=_('The modulestore course'), max_length=255, db_index=True) + target_change = models.ForeignKey(to='oel_publishing.DraftChangeLog', on_delete=models.SET_NULL, null=True) + + class Meta: + verbose_name = _('Import from modulestore') + verbose_name_plural = _('Imports from modulestore') + + def __str__(self): + return f'{self.source_key} → {self.target_change}' + + def set_status(self, status: ImportStatus): + """ + Set import status. + """ + self.status = status + self.save() + if status in [ImportStatus.IMPORTED, ImportStatus.CANCELED]: + self.clean_related_staged_content() + + def clean_related_staged_content(self) -> None: + """ + Clean related staged content. + """ + for staged_content_for_import in self.staged_content_for_import.all(): + staged_content_for_import.staged_content.delete() + + +class PublishableEntityMapping(TimeStampedModel): + """ + Represents a mapping between a source usage key and a target publishable entity. + """ + + source_usage_key = UsageKeyField( + max_length=255, + help_text=_('Original usage key/ID of the thing that has been imported.'), + ) + target_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + target_entity = models.ForeignKey(PublishableEntity, on_delete=models.CASCADE) + + class Meta: + unique_together = ('source_usage_key', 'target_package') + + def __str__(self): + return f'{self.source_usage_key} → {self.target_entity}' + + +class PublishableEntityImport(TimeStampedModel): + """ + Represents a publishableentity version that has been imported into a learning package (e.g. content library) + + This is a many-to-many relationship between a container version and a course to library import. + """ + + import_event = models.ForeignKey(Import, on_delete=models.CASCADE) + resulting_mapping = models.ForeignKey(PublishableEntityMapping, on_delete=models.SET_NULL, null=True, blank=True) + resulting_change = models.OneToOneField( + to='oel_publishing.DraftChangeLogRecord', + # a changelog record can be pruned, which would set this to NULL, but not delete the + # entire import record + null=True, + on_delete=models.SET_NULL, + ) + + class Meta: + unique_together = ( + ('import_event', 'resulting_mapping'), + ) + + def __str__(self): + return f'{self.import_event} → {self.resulting_mapping}' + + +class StagedContentForImport(TimeStampedModel): + """ + Represents m2m relationship between an import and staged content created for that import. + """ + + import_event = models.ForeignKey( + Import, + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + staged_content = models.OneToOneField( + to='content_staging.StagedContent', + on_delete=models.CASCADE, + related_name='staged_content_for_import', + ) + # Since StagedContent stores all the keys of the saved blocks, this field was added to optimize search. + source_usage_key = UsageKeyField( + max_length=255, + help_text=_( + 'The original Usage key of the highest-level component that was saved in StagedContent.' + ), + ) + + class Meta: + unique_together = ( + ('import_event', 'staged_content'), + ) + + def __str__(self): + return f'{self.import_event} → {self.staged_content}' diff --git a/cms/envs/common.py b/cms/envs/common.py index 9be3eb9e09..9dc809001f 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1667,6 +1667,7 @@ INSTALLED_APPS = [ 'openedx.core.djangoapps.course_groups', # not used in cms (yet), but tests run 'cms.djangoapps.xblock_config.apps.XBlockConfig', 'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig', + 'cms.djangoapps.import_from_modulestore.apps.ImportFromModulestoreConfig', # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig', diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 0b11a777e1..ff104b8bc2 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -112,7 +112,7 @@ numpy<2.0.0 # Date: 2023-09-18 # pinning this version to avoid updates while the library is being developed # Issue for unpinning: https://github.com/openedx/edx-platform/issues/35269 -openedx-learning==0.23.0 +openedx-learning==0.23.1 # Date: 2023-11-29 # Open AI version 1.0.0 dropped support for openai.ChatCompletion which is currently in use in enterprise. diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index cd79ea0f6e..c77fd0ee17 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -820,7 +820,7 @@ openedx-filters==2.0.1 # ora2 openedx-forum==0.2.0 # via -r requirements/edx/kernel.in -openedx-learning==0.23.0 +openedx-learning==0.23.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index cec097eaf9..0f0444f32c 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -1383,7 +1383,7 @@ openedx-forum==0.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-learning==0.23.0 +openedx-learning==0.23.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index 8d8771edaf..93d6dbc6c1 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -992,7 +992,7 @@ openedx-filters==2.0.1 # ora2 openedx-forum==0.2.0 # via -r requirements/edx/base.txt -openedx-learning==0.23.0 +openedx-learning==0.23.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index cec8b7bea4..4361fd86e9 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -1050,7 +1050,7 @@ openedx-filters==2.0.1 # ora2 openedx-forum==0.2.0 # via -r requirements/edx/base.txt -openedx-learning==0.23.0 +openedx-learning==0.23.1 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt