Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add library copy management command (#32598)

Browse Source 
This PR introduces the "copy" management command, which copies v1 libraries into v2 libraries.
This commit is contained in:
connorhaugh
2023-07-10 15:45:18 -04:00
committed by GitHub
parent 6b19eab038
commit 00b1ce28b9
4 changed files with 303 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

125
cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py Normal file
View File

@@ -0,0 +1,125 @@
"""A Command to Copy or uncopy V1 Content Libraries entires to be stored as v2 content libraries."""
import logging
from textwrap import dedent
from django.core.management import BaseCommand, CommandError
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocator
from xmodule.modulestore.django import modulestore
from celery import group
from cms.djangoapps.contentstore.tasks import create_v2_library_from_v1_library, delete_v2_library_from_v1_library
from .prompt import query_yes_no
log = logging.getLogger(__name__)
class Command(BaseCommand):
"""
Copy or uncopy V1 Content Libraries (default all) entires to be stored as v2 content libraries.
First Specify the uuid for the collection to store the content libraries in.
Specfiy --all for all libraries, library ids for specific libraries,
and -- file followed by the path for a list of libraries from a file.
Example usage:
$ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid' --all
$ ./manage.py cms copy_libraries_from_v1_to_v2
library-v1:edX+DemoX+Demo_Library' 'library-v1:edX+DemoX+Better_Library' -c 'collection_uuid'
$ ./manage.py cms copy_libraries_from_v1_to_v2 --all --uncopy
$ ./manage.py cms copy_libraries_from_v1_to_v2 'library-v1:edX+DemoX+Better_Library' --uncopy
$ ./manage.py cms copy_libraries_from_v1_to_v2
'11111111-2111-4111-8111-111111111111'
'./list_of--library-locators- --file
Note:
This Command Also produces an "output file" which contains the mapping of locators and the status of the copy.
"""
help = dedent(__doc__)
CONFIRMATION_PROMPT = "Reindexing all libraries might be a time consuming operation. Do you want to continue?"
def add_arguments(self, parser):
"""arguements for command"""
parser.add_argument(
'-collection_uuid',
'-c',
nargs=1,
type=str,
help='the uuid for the collection to create the content library in.'
)
parser.add_argument(
'library_ids',
nargs='*',
help='a space-seperated list of v1 library ids to copy'
)
parser.add_argument(
'--all',
action='store_true',
dest='all',
help='Copy all libraries'
)
parser.add_argument(
'--uncopy',
action='store_true',
dest='uncopy',
help='Delete libraries specified'
)
parser.add_argument(
'output_csv',
nargs='?',
default=None,
help='a file path to write the tasks output to. Without this the result is simply logged.'
)
def _parse_library_key(self, raw_value):
""" Parses library key from string """
result = CourseKey.from_string(raw_value)
if not isinstance(result, LibraryLocator):
raise CommandError(f"Argument {raw_value} is not a library key")
return result
def handle(self, *args, **options): # lint-amnesty, pylint: disable=unused-argument
"""Parse args and generate tasks for copying content."""
print(options)
if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']):
raise CommandError("copy_libraries_from_v1_to_v2 requires one or more <library_id>s or the --all flag.")
if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']):
raise CommandError("copy_libraries_from_v1_to_v2 requires one or more <library_id>s or the --all flag.")
if options['all']:
store = modulestore()
if query_yes_no(self.CONFIRMATION_PROMPT, default="no"):
v1_library_keys = [
library.location.library_key.replace(branch=None) for library in store.get_libraries()
]
else:
return
else:
v1_library_keys = list(map(self._parse_library_key, options['library_ids']))
create_library_task_group = group([
delete_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid'][0])
if options['uncopy']
else create_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid'][0])
for v1_library_key in v1_library_keys
])
group_result = create_library_task_group.apply_async().get()
if options['output_csv']:
with open(options['output_csv'][0], 'w', encoding='utf-8', newline='') as output_writer:
output_writer.writerow("v1_library_id", "v2_library_id", "status", "error_msg")
for result in group_result:
output_writer.write(result.keys())
log.info(group_result)

167
cms/djangoapps/contentstore/tasks.py
View File

@@ -19,6 +19,7 @@ from django.conf import settings
from django.contrib.auth import get_user_model
from django.core.exceptions import SuspiciousOperation
from django.core.files import File
from django.db.transaction import atomic
from django.test import RequestFactory
from django.utils.text import get_valid_filename
from edx_django_utils.monitoring import (
@@ -30,9 +31,10 @@ from edx_django_utils.monitoring import (
from olxcleaner.exceptions import ErrorLevel
from olxcleaner.reporting import report_error_summary, report_errors
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocator
from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2
from organizations.api import add_organization_course, ensure_organization
from organizations.models import OrganizationCourse
from organizations.exceptions import InvalidOrganizationException
from organizations.models import Organization, OrganizationCourse
from path import Path as path
from pytz import UTC
from user_tasks.models import UserTaskArtifact, UserTaskStatus
@@ -47,13 +49,17 @@ from cms.djangoapps.contentstore.courseware_index import (
from cms.djangoapps.contentstore.storage import course_import_export_storage
from cms.djangoapps.contentstore.utils import initialize_permissions, reverse_usage_url, translation_language
from cms.djangoapps.models.settings.course_metadata import CourseMetadata
from common.djangoapps.course_action_state.models import CourseRerunState
from common.djangoapps.student.auth import has_course_author_access
from common.djangoapps.student.roles import CourseInstructorRole, CourseStaffRole, LibraryUserRole
from common.djangoapps.util.monitoring import monitor_import_failure
from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines
from openedx.core.djangoapps.content_libraries import api as v2contentlib_api
from openedx.core.djangoapps.course_apps.toggles import exams_ida_enabled
from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks
from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse
from openedx.core.lib.blockstore_api import get_collection
from openedx.core.lib.extract_tar import safetar_extractall
from xmodule.contentstore.django import contentstore # lint-amnesty, pylint: disable=wrong-import-order
from xmodule.course_block import CourseFields # lint-amnesty, pylint: disable=wrong-import-order
@@ -790,7 +796,6 @@ def log_errors_to_artifact(errorstore, status):
def handle_course_import_exception(courselike_key, exception, status, known=True):
"""
Handle course import exception and fail task status.
Arguments:
courselike_key: A locator identifies a course resource.
exception: Exception object
@@ -808,3 +813,159 @@ def handle_course_import_exception(courselike_key, exception, status, known=True
if status.state != UserTaskStatus.FAILED:
status.fail(task_fail_message)
def _parse_organization(org_name):
"""Find a matching organization name, if one does not exist, specify that this is the *unspecfied* organization"""
try:
ensure_organization(org_name)
except InvalidOrganizationException:
return 'None'
return Organization.objects.get(short_name=org_name)
def copy_v1_user_roles_into_v2_library(v2_library_key, v1_library_key):
"""
write the access and edit permissions of a v1 library into a v2 library.
"""
def _get_users_by_access_level(v1_library_key):
"""
Get a permissions object for a library which contains a list of user IDs for every V2 permissions level,
based on V1 library roles.
The following mapping exists for a library:
V1 Library Role -> V2 Permission Level
LibraryUserRole -> READ_LEVEL
CourseStaffRole -> AUTHOR_LEVEL
CourseInstructorRole -> ADMIN_LEVEL
"""
permissions = {}
permissions[v2contentlib_api.AccessLevel.READ_LEVEL] = list(LibraryUserRole(v1_library_key).users_with_role())
permissions[v2contentlib_api.AccessLevel.AUTHOR_LEVEL] = list(CourseStaffRole(v1_library_key).users_with_role())
permissions[v2contentlib_api.AccessLevel.ADMIN_LEVEL] = list(
CourseInstructorRole(v1_library_key).users_with_role()
)
return permissions
permissions = _get_users_by_access_level(v1_library_key)
for access_level in permissions.keys(): # lint-amnesty, pylint: disable=consider-iterating-dictionary
for user in permissions[access_level]:
v2contentlib_api.set_library_user_permissions(v2_library_key, user, access_level)
def _create_copy_content_task(v2_library_key, v1_library_key):
"""
spin up a celery task to import the V1 Library's content into the V2 library.
This utalizes the fact that course and v1 library content is stored almost identically.
"""
return v2contentlib_api.import_blocks_create_task(v2_library_key, v1_library_key)
def _create_metadata(v1_library_key, collection_uuid):
"""instansiate an index for the V2 lib in the collection"""
store = modulestore()
v1_library = store.get_library(v1_library_key)
collection = get_collection(collection_uuid).uuid
# To make it easy, all converted libs are complex, meaning they can contain problems, videos, and text
library_type = 'complex'
org = _parse_organization(v1_library.location.library_key.org)
slug = v1_library.location.library_key.library
title = v1_library.display_name
# V1 libraries do not have descriptions.
description = ''
# permssions & license are most restrictive.
allow_public_learning = False
allow_public_read = False
library_license = '' # '' = ALL_RIGHTS_RESERVED
with atomic():
return v2contentlib_api.create_library(
collection,
library_type,
org,
slug,
title,
description,
allow_public_learning,
allow_public_read,
library_license
)
@shared_task(time_limit=30)
@set_code_owner_attribute
def delete_v2_library_from_v1_library(v1_library_key_string, collection_uuid):
"""
For a V1 Library, delete the matching v2 library, where the library is the result of the copy operation
This method relys on _create_metadata failling for LibraryAlreadyExists in order to obtain the v2 slug.
"""
v1_library_key = CourseKey.from_string(v1_library_key_string)
v2_library_key = LibraryLocatorV2.from_string('lib:' + v1_library_key.org + ':' + v1_library_key.course)
try:
v2contentlib_api.delete_library(v2_library_key)
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": v2_library_key,
"status": "SUCCESS",
"msg": None
}
except Exception as error: # lint-amnesty, pylint: disable=broad-except
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": v2_library_key,
"status": "FAILED",
"msg": f"Exception: {v2_library_key} did not delete: {error}"
}
@shared_task(time_limit=30)
@set_code_owner_attribute
def create_v2_library_from_v1_library(v1_library_key_string, collection_uuid):
"""
write the metadata, permissions, and content of a v1 library into a v2 library in the given collection.
"""
v1_library_key = CourseKey.from_string(v1_library_key_string)
LOGGER.info(f"Copy Library task created for library: {v1_library_key}")
try:
v2_library_metadata = _create_metadata(v1_library_key, collection_uuid)
except v2contentlib_api.LibraryAlreadyExists:
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": None,
"status": "FAILED",
"msg": f"Exception: LibraryAlreadyExists {v1_library_key_string} aleady exists"
}
try:
_create_copy_content_task(v2_library_metadata.key, v1_library_key)
except Exception as error: # lint-amnesty, pylint: disable=broad-except
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": str(v2_library_metadata.key),
"status": "FAILED",
"msg":
f"Could not import content from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}"
}
try:
copy_v1_user_roles_into_v2_library(v2_library_metadata.key, v1_library_key)
except Exception as error: # lint-amnesty, pylint: disable=broad-except
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": str(v2_library_metadata.key),
"status": "FAILED",
"msg":
f"Could not copy permissions from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}"
}
return {
"v1_library_id": v1_library_key_string,
"v2_library_id": str(v2_library_metadata.key),
"status": "SUCCESS",
"msg": None
}

11
openedx/core/djangoapps/content_libraries/api.py
View File

@@ -70,7 +70,13 @@ from django.utils.translation import gettext as _
from elasticsearch.exceptions import ConnectionError as ElasticConnectionError
from lxml import etree
from opaque_keys.edx.keys import LearningContextKey, UsageKey
from opaque_keys.edx.locator import BundleDefinitionLocator, LibraryLocatorV2, LibraryUsageLocatorV2
from opaque_keys.edx.locator import (
BundleDefinitionLocator,
LibraryLocatorV2,
LibraryUsageLocatorV2,
LibraryLocator as LibraryLocatorV1
)
from organizations.models import Organization
from xblock.core import XBlock
from xblock.exceptions import XBlockNotFoundError
@@ -1160,7 +1166,6 @@ class BaseEdxImportClient(abc.ABC):
"""
Import a single modulestore block.
"""
block_data = self.get_block_data(modulestore_key)
# Get or create the block in the library.
@@ -1270,6 +1275,8 @@ class EdxModulestoreImportClient(BaseEdxImportClient):
Retrieve the course from modulestore and traverse its content tree.
"""
course = self.modulestore.get_course(course_key)
if isinstance(course_key, LibraryLocatorV1):
course = self.modulestore.get_library(course_key)
export_keys = set()
blocks_q = collections.deque(course.get_children())
while blocks_q:

9
setup.cfg
View File

@@ -115,11 +115,12 @@ ignore_imports =
# cms.djangoapps.export_course_metadata.tasks
# -> openedx.core.djangoapps.schedules.content_highlights
# -> lms.djangoapps.courseware.block_render & lms.djangoapps.courseware.model_data
openedx.core.djangoapps.content_libraries.* -> lms.djangoapps.grades.api
# cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.models
# -> openedx.core.djangoapps.content_libraries.apps
# -> openedx.core.djangoapps.content_libraries.signal_handlers
openedx.core.djangoapps.content_libraries.* -> lms.djangoapps.*.*
# cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.[various]
# -> lms.djangoapps.grades.api
openedx.core.djangoapps.xblock.*.* -> lms.djangoapps.*.*
# cms.djangoapps.contentstore.tasks -> openedx.core.djangoapps.content_libraries.[various] -> openedx.core.djangoapps.xblock.[various]
# -> lms.djangoapps.courseware & lms.djangoapps.courseware.grades
openedx.core.djangoapps.schedules.content_highlights -> lms.djangoapps.courseware.*
# cms.djangoapps.contentstore.[various]
# -> openedx.core.lib.gating.api