Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Preview migration api [FC-0114] (#37818)

Browse Source 
Implements a new API to get the summary preview of a migration given a library key and a source key.
This commit is contained in:
Chris Chávez
2026-01-20 13:50:40 -05:00
committed by GitHub
parent 46272cc0dc
commit 9f48073921
8 changed files with 922 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
cms/djangoapps/modulestore_migrator/api/read_api.py
View File

@@ -5,18 +5,27 @@ from __future__ import annotations
import typing as t
from uuid import UUID
from django.conf import settings
from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locator import (
LibraryLocatorV2, LibraryUsageLocatorV2, LibraryContainerLocator
)
from openedx_learning.api.authoring import get_draft_version
from openedx_learning.api.authoring import get_draft_version, get_all_drafts
from openedx_learning.api.authoring_models import (
PublishableEntityVersion, PublishableEntity, DraftChangeLogRecord
)
from xblock.plugin import PluginMissingError
from openedx.core.djangoapps.content_libraries.api import (
library_component_usage_key, library_container_locator
library_component_usage_key, library_container_locator,
validate_can_add_block_to_library, BlockLimitReachedError,
IncompatibleTypesError, LibraryBlockAlreadyExists,
ContentLibrary
)
from openedx.core.djangoapps.content.search.api import (
fetch_block_types,
get_all_blocks_from_context,
)
from ..data import (
@@ -32,6 +41,7 @@ __all__ = (
'get_forwarding_for_blocks',
'get_migrations',
'get_migration_blocks',
'preview_migration',
)
@@ -242,3 +252,120 @@ def _block_migration_success(
target_title=target_title,
target_version_num=target_version_num,
)
def preview_migration(source_key: SourceContextKey, target_key: LibraryLocatorV2):
"""
Returns a summary preview of the migration given a source key and a target key
on this form:
```
{
"state": "partial",
"unsupported_blocks": 4,
"unsupported_percentage": 25,
"blocks_limit": 1000,
"total_blocks": 20,
"total_components": 10,
"sections": 2,
"subsections": 3,
"units": 5,
}
```
List of states:
- 'success': The migration can be carried out in its entirety
- 'partial': The migration will be partial, because there are unsupported blocks.
- 'block_limit_reached': The migration cannot be performed because the block limit per library has been reached.
This runs Meilisiearch queries to speed up the response, as it's a summary/analysis.
The decision has been made not to run a "migration" for each analysis to obtain this summary.
TODO: For now, the repeat_handling_strategy is not taken into account. This can be taken into
account for a more advanced summary.
"""
# Get all containers and components from the source key
blocks = get_all_blocks_from_context(str(source_key), ["block_type", "block_id"])
unsupported_blocks = []
total_blocks = 0
total_components = 0
sections = 0
subsections = 0
units = 0
blocks_limit = settings.MAX_BLOCKS_PER_CONTENT_LIBRARY
# Builds the summary: counts every container and verify if each component can be added to the library
for block in blocks:
block_type = block["block_type"]
block_id = block["block_id"]
total_blocks += 1
if block_type not in ['chapter', 'sequential', 'vertical']:
total_components += 1
try:
validate_can_add_block_to_library(
target_key,
block_type,
block_id,
)
except BlockLimitReachedError:
return {
"state": "block_limit_reached",
"unsupported_blocks": 0,
"unsupported_percentage": 0,
"blocks_limit": blocks_limit,
"total_blocks": 0,
"total_components": 0,
"sections": 0,
"subsections": 0,
"units": 0,
}
except (IncompatibleTypesError, PluginMissingError):
unsupported_blocks.append(block["usage_key"])
except LibraryBlockAlreadyExists:
# Skip this validation, The block may be repeated in the library, but that's not a bad thing.
pass
elif block_type == "chapter":
sections += 1
elif block_type == "sequential":
subsections += 1
elif block_type == "vertical":
units += 1
# Gets the count of children of unsupported blocks
quoted_keys = ','.join(f'"{key}"' for key in unsupported_blocks)
unsupportedBlocksChildren = fetch_block_types(
[
f'context_key = "{source_key}"',
f'breadcrumbs.usage_key IN [{quoted_keys}]'
],
)
# Final unsupported blocks count
# The unsupported children are subtracted from the totals since they have already been counted in the first query.
unsupported_blocks_count = len(unsupported_blocks)
total_blocks -= unsupportedBlocksChildren["estimatedTotalHits"]
total_components -= unsupportedBlocksChildren["estimatedTotalHits"]
unsupported_percentage = (unsupported_blocks_count / total_blocks) * 100
state = "success"
if unsupported_blocks_count:
state = "partial"
# Checks if this migration reaches the block limit
content_library = ContentLibrary.objects.get_by_key(target_key)
assert content_library.learning_package_id is not None
target_item_counts = get_all_drafts(content_library.learning_package_id).count()
if (target_item_counts + total_blocks - unsupported_blocks_count) > blocks_limit:
state = "block_limit_reached"
return {
"state": state,
"unsupported_blocks": unsupported_blocks_count,
"unsupported_percentage": unsupported_percentage,
"blocks_limit": blocks_limit,
"total_blocks": total_blocks,
"total_components": total_components,
"sections": sections,
"subsections": subsections,
"units": units,
}

15
cms/djangoapps/modulestore_migrator/rest_api/v1/serializers.py
View File

@@ -289,3 +289,18 @@ class BlockMigrationInfoSerializer(serializers.Serializer):
source_key = serializers.CharField()
target_key = serializers.CharField(allow_null=True)
unsupported_reason = serializers.CharField(allow_null=True)
class PreviewMigrationSerializer(serializers.Serializer):
"""
Serializer for the preview migration response.
"""
state = serializers.CharField()
unsupported_blocks = serializers.IntegerField()
unsupported_percentage = serializers.FloatField()
blocks_limit = serializers.IntegerField()
total_blocks = serializers.IntegerField()
total_components = serializers.IntegerField()
sections = serializers.IntegerField()
subsections = serializers.IntegerField()
units = serializers.IntegerField()

2
cms/djangoapps/modulestore_migrator/rest_api/v1/urls.py
View File

@@ -10,6 +10,7 @@ from .views import (
LibraryCourseMigrationViewSet,
MigrationInfoViewSet,
MigrationViewSet,
PreviewMigration,
)
ROUTER = SimpleRouter()
@@ -25,4 +26,5 @@ urlpatterns = [
path('', include(ROUTER.urls)),
path('migration_info/', MigrationInfoViewSet.as_view(), name='migration-info'),
path('migration_blocks/', BlockMigrationInfo.as_view(), name='migration-blocks'),
path('migration_preview/', PreviewMigration.as_view(), name='migration-preview'),
]

86
cms/djangoapps/modulestore_migrator/rest_api/v1/views.py
View File

@@ -41,6 +41,7 @@ from .serializers import (
MigrationInfoResponseSerializer,
ModulestoreMigrationSerializer,
StatusWithModulestoreMigrationsSerializer,
PreviewMigrationSerializer,
)
log = logging.getLogger(__name__)
@@ -668,3 +669,88 @@ class BlockMigrationInfo(APIView):
]
serializer = BlockMigrationInfoSerializer(data, many=True)
return Response(serializer.data)
class PreviewMigration(APIView):
"""
Retrieve the summary preview of the migration given a source key and a target key
It returns the migration block information for each block migrated by a specific task.
API Endpoints
-------------
GET /api/modulestore_migrator/v1/migration_preview/
Retrieve the summary preview of the migration given a source key and a target key
Query parameters:
source_key (str): Source content key
Example: ?source_key=course-v1:UNIX+UX1+2025_T3
target_key (str): target content key
Example: ?target_key=lib:UNIX:CIT1
Example request:
GET /api/modulestore_migrator/v1/migration_blocks/?source_key=course_key&target_key=library_key
Example response:
"""
permission_classes = (IsAuthenticated,)
authentication_classes = (
BearerAuthenticationAllowInactiveUser,
JwtAuthentication,
SessionAuthenticationAllowInactiveUser,
)
@apidocs.schema(
parameters=[
apidocs.string_parameter(
"target_key",
apidocs.ParameterLocation.QUERY,
description="Target key of the migration",
),
apidocs.string_parameter(
"source_key",
apidocs.ParameterLocation.QUERY,
description="Source key of the migration",
),
],
responses={
200: PreviewMigrationSerializer,
400: "Missing required parameter: target_key/source_key",
401: "The requester is not authenticated.",
},
)
def get(self, request: Request):
"""
Handle the migration info `GET` request
"""
target_key: LibraryLocatorV2 | None
if target_key_param := request.query_params.get("target_key"):
try:
target_key = LibraryLocatorV2.from_string(target_key_param)
except InvalidKeyError:
return Response({"error": f"Bad target_key: {target_key_param}"}, status=400)
else:
return Response({"error": "Target key cannot be blank."}, status=400)
source_key: SourceContextKey | None = None
if source_key_param := request.query_params.get("source_key"):
try:
source_key = CourseLocator.from_string(source_key_param)
except InvalidKeyError:
try:
source_key = LibraryLocator.from_string(source_key_param)
except InvalidKeyError:
return Response({"error": f"Bad source: {source_key_param}"}, status=400)
else:
return Response({"error": "Source key cannot be blank."}, status=400)
lib_api.require_permission_for_library_key(
target_key,
request.user,
lib_api.permissions.CAN_EDIT_THIS_CONTENT_LIBRARY,
)
result = migrator_api.preview_migration(source_key, target_key)
serializer = PreviewMigrationSerializer(result)
return Response(serializer.data)

384
cms/djangoapps/modulestore_migrator/tests/test_api.py
View File

@@ -3,7 +3,8 @@ Test cases for the modulestore migrator API.
"""
import pytest
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
from unittest.mock import patch
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2, CourseLocator
from openedx_learning.api import authoring as authoring_api
from organizations.tests.factories import OrganizationFactory
@@ -29,7 +30,11 @@ class TestModulestoreMigratorAPI(ModuleStoreTestCase):
self.user = UserFactory(password=self.user_password, is_staff=True)
self.organization = OrganizationFactory(name="My Org", short_name="myorg")
self.lib_key_v1 = LibraryLocator.from_string("library-v1:myorg+old")
self.lib_key_v1_2 = LibraryLocator.from_string("library-v1:myorg+old2")
self.lib_key_v1_3 = LibraryLocator.from_string("library-v1:myorg+old3")
LibraryFactory.create(org="myorg", library="old", display_name="Old Library", modulestore=self.store)
LibraryFactory.create(org="myorg", library="old2", display_name="Old Library 2", modulestore=self.store)
LibraryFactory.create(org="myorg", library="old3", display_name="Old Library 3", modulestore=self.store)
self.lib_key_v2_1 = LibraryLocatorV2.from_string("lib:myorg:1")
self.lib_key_v2_2 = LibraryLocatorV2.from_string("lib:myorg:2")
lib_api.create_library(org=self.organization, slug="1", title="Test Library 1")
@@ -58,6 +63,90 @@ class TestModulestoreMigratorAPI(ModuleStoreTestCase):
]
# We load this last so that it has an updated list of children.
self.lib_v1 = self.store.get_library(self.lib_key_v1)
self.course_key = CourseLocator.from_string('course-v1:TestOrg+TestCourse+TestRun')
# Create containers and blocks for legacy libraries
# Old Library 2
for c in ["X", "Y", "Z"]:
BlockFactory.create(
display_name=f"Unit {c}",
category="vertical",
location=self.lib_key_v1_2.make_usage_key("vertical", c),
parent_location=self.lib_key_v1_2.make_usage_key("library", "library"),
user_id=self.user.id, publish_item=False,
)
for c in ["X", "Y"]:
BlockFactory.create(
display_name=f"Subsection {c}",
category="sequential",
location=self.lib_key_v1_2.make_usage_key("sequential", c),
parent_location=self.lib_key_v1_2.make_usage_key("library", "library"),
user_id=self.user.id, publish_item=False,
)
BlockFactory.create(
display_name="Section X",
category="chapter",
location=self.lib_key_v1_2.make_usage_key("chapter", "X"),
parent_location=self.lib_key_v1_2.make_usage_key("library", "library"),
user_id=self.user.id, publish_item=False,
)
for c in ["X", "Y", "Z"]:
BlockFactory.create(
display_name=f"HTML {c}",
category="html",
location=self.lib_key_v1_2.make_usage_key("html", c),
parent_location=self.lib_key_v1_2.make_usage_key("vertical", c),
user_id=self.user.id, publish_item=False,
)
# Old Library 3
for c in ["X", "Y", "Z"]:
BlockFactory.create(
display_name=f"Unit {c}",
category="vertical",
location=self.lib_key_v1_3.make_usage_key("vertical", c),
parent_location=self.lib_key_v1_3.make_usage_key("library", "library"),
user_id=self.user.id, publish_item=False,
)
for c in ["X", "Y"]:
BlockFactory.create(
display_name=f"Subsection {c}",
category="sequential",
location=self.lib_key_v1_3.make_usage_key("sequential", c),
parent_location=self.lib_key_v1_3.make_usage_key("library", "library"),
user_id=self.user.id, publish_item=False,
)
BlockFactory.create(
display_name="Section X",
category="chapter",
location=self.lib_key_v1_3.make_usage_key("chapter", "X"),
parent_location=self.lib_key_v1_3.make_usage_key("library", "library"),
user_id=self.user.id, publish_item=False,
)
for c in ["X", "Y", "Z"]:
BlockFactory.create(
display_name=f"Html {c}",
category="html",
location=self.lib_key_v1_3.make_usage_key("html", c),
parent_location=self.lib_key_v1_3.make_usage_key("vertical", c),
user_id=self.user.id, publish_item=False,
)
for c in ["A"]:
BlockFactory.create(
display_name=f"Item Bank {c}",
category="item_bank",
location=self.lib_key_v1_3.make_usage_key("item_bank", c),
parent_location=self.lib_key_v1_3.make_usage_key("vertical", "X"),
user_id=self.user.id, publish_item=False,
)
for c in ["B"]:
BlockFactory.create(
display_name=f"Invalid {c}",
category="invalid",
location=self.lib_key_v1_3.make_usage_key("invalid", c),
parent_location=self.lib_key_v1_3.make_usage_key("vertical", "X"),
user_id=self.user.id, publish_item=False,
)
def test_start_migration_to_library(self):
"""
@@ -566,3 +655,296 @@ class TestModulestoreMigratorAPI(ModuleStoreTestCase):
forwarded_blocks = api.get_forwarding_for_blocks(all_source_usage_keys)
assert forwarded_blocks[self.source_html_keys[1]].target_key.context_key == self.lib_key_v2_1
assert forwarded_blocks[self.source_unit_keys[1]].target_key.context_key == self.lib_key_v2_1
def _get_summary_from_migration(self, migration, expected_state, blocks_limit):
"""
Manually calculate the summary from the migration data
"""
blocks = api.get_migration_blocks(migration.pk)
summary = {
"state": expected_state,
"unsupported_blocks": 0,
"unsupported_percentage": 0,
"blocks_limit": blocks_limit,
"total_blocks": 0,
"total_components": 0,
"sections": 0,
"subsections": 0,
"units": 0,
}
for key, block in blocks.items():
block_type = key.block_type
print(block_type)
summary['total_blocks'] += 1
if block_type not in ['vertical', 'sequential', 'chapter']:
summary['total_components'] += 1
if block.is_failed:
summary['unsupported_blocks'] += 1
elif block_type == 'vertical':
summary['units'] += 1
elif block_type == 'sequential':
summary['subsections'] += 1
elif block_type == 'chapter':
summary['sections'] += 1
if summary['unsupported_blocks']:
summary['unsupported_percentage'] = summary['unsupported_blocks'] * 100 / summary['total_blocks']
return summary
@patch('cms.djangoapps.modulestore_migrator.api.read_api.fetch_block_types')
@patch('cms.djangoapps.modulestore_migrator.api.read_api.get_all_blocks_from_context')
def test_preview_migration_success(self, mock_get_blocks, mock_fetch_block_types):
"""
Test the preview migration summary in the success state
This tests compare the summary generated by `preview_migration` with the
data generated by a migration.
"""
user = UserFactory()
api.start_migration_to_library(
user=user,
source_key=self.lib_key_v1_2,
target_library_key=self.lib_key_v2_1,
target_collection_slug=None,
composition_level=CompositionLevel.Section,
repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
migration = list(api.get_migrations(self.lib_key_v1_2))[0]
summary = self._get_summary_from_migration(migration, "success", 1000)
mock_get_blocks.return_value = [
{
'block_type': 'html',
'block_id': '16',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@16',
},
{
'block_type': 'html',
'block_id': '17',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@17',
},
{
'block_type': 'html',
'block_id': '18',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@18',
},
{
'block_type': 'chapter',
'block_id': '19',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@chapter+block@19',
},
{
'block_type': 'sequential',
'block_id': '20',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@20',
},
{
'block_type': 'sequential',
'block_id': '21',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@21',
},
{
'block_type': 'vertical',
'block_id': '22',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@22',
},
{
'block_type': 'vertical',
'block_id': '23',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@23',
},
{
'block_type': 'vertical',
'block_id': '23',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@2',
},
]
mock_fetch_block_types.return_value = {
"estimatedTotalHits": 0,
}
with self.settings(MAX_BLOCKS_PER_CONTENT_LIBRARY=1000):
results = api.preview_migration(self.course_key, self.lib_key_v2_1)
assert results == summary
@patch('cms.djangoapps.modulestore_migrator.api.read_api.fetch_block_types')
@patch('cms.djangoapps.modulestore_migrator.api.read_api.get_all_blocks_from_context')
def test_preview_migration_partial(self, mock_get_blocks, mock_fetch_block_types):
"""
Test the preview migration summary in the partial state
This tests compare the summary generated by `preview_migration` with the
data generated by a migration.
"""
user = UserFactory()
api.start_migration_to_library(
user=user,
source_key=self.lib_key_v1_3,
target_library_key=self.lib_key_v2_1,
target_collection_slug=None,
composition_level=CompositionLevel.Section,
repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
migration = list(api.get_migrations(self.lib_key_v1_3))[0]
summary = self._get_summary_from_migration(migration, "partial", 1000)
mock_get_blocks.return_value = [
{
'block_type': 'html',
'block_id': '16',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@16',
},
{
'block_type': 'html',
'block_id': '17',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@17',
},
{
'block_type': 'html',
'block_id': '18',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@18',
},
{
'block_type': 'chapter',
'block_id': '19',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@chapter+block@19',
},
{
'block_type': 'sequential',
'block_id': '20',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@20',
},
{
'block_type': 'sequential',
'block_id': '21',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@21',
},
{
'block_type': 'vertical',
'block_id': '22',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@22',
},
{
'block_type': 'vertical',
'block_id': '23',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@23',
},
{
'block_type': 'vertical',
'block_id': '23',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@2',
},
{
'block_type': 'invalid',
'block_id': '24',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@invalid+block@24',
}, # Invalid
{
'block_type': 'item_bank',
'block_id': '25',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@item_bank+block@25',
}, # Invalid
{
'block_type': 'html',
'block_id': '26',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@26',
}, # Child of item bank
{
'block_type': 'html',
'block_id': '27',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@27',
}, # Child of item bank
]
mock_fetch_block_types.return_value = {
"estimatedTotalHits": 2,
}
# The unsupported children are not included in the summary
with self.settings(MAX_BLOCKS_PER_CONTENT_LIBRARY=1000):
results = api.preview_migration(self.course_key, self.lib_key_v2_1)
assert results == summary
@patch('cms.djangoapps.modulestore_migrator.api.read_api.fetch_block_types')
@patch('cms.djangoapps.modulestore_migrator.api.read_api.get_all_blocks_from_context')
def test_preview_migration_block_limit(self, mock_get_blocks, mock_fetch_block_types):
"""
Test the preview migration summary in the block_limit_reached state
This tests compare the summary generated by `preview_migration` with the
data generated by a migration.
"""
user = UserFactory()
api.start_migration_to_library(
user=user,
source_key=self.lib_key_v1_2,
target_library_key=self.lib_key_v2_1,
target_collection_slug=None,
composition_level=CompositionLevel.Section,
repeat_handling_strategy=RepeatHandlingStrategy.Skip,
preserve_url_slugs=True,
forward_source_to_target=False,
)
migration = list(api.get_migrations(self.lib_key_v1_2))[0]
summary = self._get_summary_from_migration(migration, "block_limit_reached", 10)
mock_get_blocks.return_value = [
{
'block_type': 'html',
'block_id': '16',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@16',
},
{
'block_type': 'html',
'block_id': '17',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@17',
},
{
'block_type': 'html',
'block_id': '18',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@html+block@18',
},
{
'block_type': 'chapter',
'block_id': '19',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@chapter+block@19',
},
{
'block_type': 'sequential',
'block_id': '20',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@20',
},
{
'block_type': 'sequential',
'block_id': '21',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@sequential+block@21',
},
{
'block_type': 'vertical',
'block_id': '22',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@22',
},
{
'block_type': 'vertical',
'block_id': '23',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@23',
},
{
'block_type': 'vertical',
'block_id': '23',
'usage_key': 'block-v1:edX+DemoX+Demo_Course+type@vertical+block@2',
},
]
mock_fetch_block_types.return_value = {
"estimatedTotalHits": 0,
}
with self.settings(MAX_BLOCKS_PER_CONTENT_LIBRARY=10):
results = api.preview_migration(self.course_key, self.lib_key_v2_1)
assert results == summary

145
cms/djangoapps/modulestore_migrator/tests/test_rest_api.py
View File

@@ -32,6 +32,7 @@ from cms.djangoapps.modulestore_migrator.rest_api.v1.views import (
BulkMigrationViewSet,
MigrationInfoViewSet,
MigrationViewSet,
PreviewMigration,
)
from openedx.core.djangoapps.content_libraries import api as lib_api
@@ -1109,3 +1110,147 @@ class TestBlockMigrationInfo(TestCase):
response = self.view(request)
assert response.status_code == status.HTTP_403_FORBIDDEN
class TestPreviewMigration(TestCase):
"""
Test the PreviewMigration.get() endpoint.
"""
def setUp(self):
"""Set up test fixtures."""
self.factory = APIRequestFactory()
self.view = PreviewMigration.as_view()
self.user = User.objects.create_user(
username='testuser',
email='testuser@test.com',
password='password'
)
@patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.migrator_api')
@patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api')
def test_preview_migration_success(self, mock_lib_api, mock_migrator_api):
"""
Test successful retrieval of preview migration.
"""
mock_lib_api.require_permission_for_library_key.return_value = None
expected = {
"state": "partial",
"unsupported_blocks": 4,
"unsupported_percentage": 25,
"blocks_limit": 1000,
"total_blocks": 20,
"total_components": 10,
"sections": 2,
"subsections": 3,
"units": 5,
}
mock_migrator_api.preview_migration.return_value = expected
request = self.factory.get(
'/api/modulestore_migrator/v1/migration_preview/',
{
'target_key': 'lib:TestOrg:TestLibrary',
'source_key': 'course-v1:TestOrg+TestCourse+TestRun',
}
)
force_authenticate(request, user=self.user)
response = self.view(request)
assert response.status_code == status.HTTP_200_OK
for key, value in expected.items():
assert response.data[key] == value
@patch('cms.djangoapps.modulestore_migrator.rest_api.v1.views.lib_api')
def test_preview_migration_without_library_access(self, mock_lib_api):
"""
Test that users without library view access get 403 Forbidden.
"""
mock_lib_api.require_permission_for_library_key.side_effect = PermissionDenied(
"User lacks permission to view this library"
)
request = self.factory.get(
'/api/modulestore_migrator/v1/migration_preview/',
{
'target_key': 'lib:TestOrg:TestLibrary',
'source_key': 'course-v1:TestOrg+TestCourse+TestRun',
}
)
force_authenticate(request, user=self.user)
response = self.view(request)
assert response.status_code == status.HTTP_403_FORBIDDEN
def test_preview_migration_missing_target_key(self):
"""
Test that missing target_key parameter returns 400 Bad Request.
"""
request = self.factory.get(
'/api/modulestore_migrator/v1/migration_preview/',
{
'source_key': 'course-v1:TestOrg+TestCourse+TestRun',
}
)
force_authenticate(request, user=self.user)
response = self.view(request)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert 'target' in response.data.get('error', '').lower()
def test_preview_migration_invalid_target_key(self):
"""
Test that invalid target_key returns 400 Bad Request.
"""
request = self.factory.get(
'/api/modulestore_migrator/v1/migration_preview/',
{
'target_key': 'not-a-valid-key',
'source_key': 'course-v1:TestOrg+TestCourse+TestRun',
}
)
force_authenticate(request, user=self.user)
response = self.view(request)
assert response.status_code == status.HTTP_400_BAD_REQUEST
def test_preview_migration_missing_source_key(self):
"""
Test that missing target_key parameter returns 400 Bad Request.
"""
request = self.factory.get(
'/api/modulestore_migrator/v1/migration_preview/',
{
'target_key': 'lib:TestOrg:TestLibrary',
}
)
force_authenticate(request, user=self.user)
response = self.view(request)
assert response.status_code == status.HTTP_400_BAD_REQUEST
assert 'source' in response.data.get('error', '').lower()
def test_preview_migration_invalid_source_key(self):
"""
Test that invalid target_key returns 400 Bad Request.
"""
request = self.factory.get(
'/api/modulestore_migrator/v1/migration_preview/',
{
'target_key': 'lib:TestOrg:TestLibrary',
'source_key': 'not-a-valid-key',
}
)
force_authenticate(request, user=self.user)
response = self.view(request)
assert response.status_code == status.HTTP_400_BAD_REQUEST

98
openedx/core/djangoapps/content/search/api.py
View File

@@ -5,10 +5,11 @@ from __future__ import annotations
import logging
import time
from collections.abc import Iterator
from contextlib import contextmanager, nullcontext
from datetime import datetime, timedelta, timezone
from functools import wraps
from typing import Callable, Generator
from typing import Callable, Generator, cast
from django.conf import settings
from django.contrib.auth import get_user_model
@@ -61,6 +62,8 @@ User = get_user_model()
STUDIO_INDEX_SUFFIX = "studio_content"
Filter = str | list[str | list[str]]
if hasattr(settings, "MEILISEARCH_INDEX_PREFIX"):
STUDIO_INDEX_NAME = settings.MEILISEARCH_INDEX_PREFIX + STUDIO_INDEX_SUFFIX
else:
@@ -981,3 +984,96 @@ def generate_user_token_for_studio_search(request):
"index_name": STUDIO_INDEX_NAME,
"api_key": restricted_api_key,
}
def force_array(extra_filter: Filter | None = None) -> list[str]:
"""
Convert a filter value into a list of strings.
Strings are wrapped in a list, lists are returned as-is (cast to `list[str]`),
and None results in an empty list.
"""
if isinstance(extra_filter, str):
return [extra_filter]
if isinstance(extra_filter, list):
return cast(list[str], extra_filter)
return []
def fetch_block_types(extra_filter: Filter | None = None):
"""
Fetch the block types facet distribution for the search results.
This data may not always be 100% accurate / up to date because it's based
on the search index, so this should only be used for analysis/estimation
purposes.
Params:
- extra_filter: Filters the query. Example: ['context_key = "course-v1:SampleTaxonomyOrg1+CC22+CC22"']
Return example:
{
...
'estimatedTotalHits': 5,
'facetDistribution': {
'block_type': {
'html': 2,
'problem': 1,
'video': 2,
}
},
}
"""
extra_filter_formatted = force_array(extra_filter)
client = _get_meilisearch_client()
index = client.get_index(STUDIO_INDEX_NAME)
response = index.search(
"",
{
"facets": ["block_type"],
"filter": extra_filter_formatted,
"limit": 0,
}
)
return response
def get_all_blocks_from_context(
context_key: str,
extra_attributes_to_retrieve: list[str] | None = None,
) -> Iterator[dict]:
"""
Lazily yields all blocks for a given context key using Meilisearch pagination.
Meilisearch works with limits of 1000 maximum; ensuring we obtain all blocks
requires making several queries.
This data may not always be 100% accurate / up to date because it's based
on the search index, so this should only be used for analysis/estimation
purposes.
"""
limit = 1000
offset = 0
client = _get_meilisearch_client()
index = client.get_index(STUDIO_INDEX_NAME)
while True:
response = index.search(
"",
{
"filter": [f'context_key = "{context_key}"'],
"limit": limit,
"offset": offset,
"attributesToRetrieve": ["usage_key"] + (extra_attributes_to_retrieve or []),
}
)
yield from response["hits"]
if response["estimatedTotalHits"] <= offset + limit:
break
offset += limit

65
openedx/core/djangoapps/content/search/tests/test_api.py
View File

@@ -1233,3 +1233,68 @@ class TestSearchApi(ModuleStoreTestCase):
],
any_order=True,
)
@override_settings(MEILISEARCH_ENABLED=True)
def test_fetch_block_types(self, mock_meilisearch):
from openedx.core.djangoapps.content.search.api import fetch_block_types
mock_index = mock_meilisearch.return_value.get_index.return_value
fetch_block_types('context_key = test')
mock_index.search.assert_called_once_with(
"",
{
"facets": ["block_type"],
"filter": ['context_key = test'],
"limit": 0,
}
)
@override_settings(MEILISEARCH_ENABLED=True)
def test_get_all_blocks_from_context(self, mock_meilisearch):
from openedx.core.djangoapps.content.search.api import get_all_blocks_from_context
mock_index = mock_meilisearch.return_value.get_index.return_value
expected_result = [
{"usage_key": "block-v1:test+type@html+block@1"},
{"usage_key": "block-v1:test+type@video+block@2"},
]
# Simulate two pages: one with results and one empty (while loop ends)
mock_index.search.side_effect = [
{
"hits": expected_result,
"estimatedTotalHits": 1200,
},
{
"hits": [],
"estimatedTotalHits": 1200,
},
]
result = list(get_all_blocks_from_context(
context_key="course-v1:TestOrg+TestCourse+TestRun",
extra_attributes_to_retrieve=["display_name"],
))
assert result == expected_result
assert mock_index.search.call_count == 2
mock_index.search.assert_any_call(
"",
{
"filter": ['context_key = "course-v1:TestOrg+TestCourse+TestRun"'],
"limit": 1000,
"offset": 0,
"attributesToRetrieve": ["usage_key", "display_name"],
}
)
mock_index.search.assert_any_call(
"",
{
"filter": ['context_key = "course-v1:TestOrg+TestCourse+TestRun"'],
"limit": 1000,
"offset": 1000,
"attributesToRetrieve": ["usage_key", "display_name"],
}
)