From 15caa9746fe59960409a8daa09f24840f2524952 Mon Sep 17 00:00:00 2001 From: Kyle McCormick Date: Mon, 13 May 2024 09:48:18 -0400 Subject: [PATCH] refactor: Completely remove Blockstore (#34739) Blockstore and all of its (experimental) functionality has been replaced with openedx-learning, aka "Learning Core". This commit uninstalls the now-unused openedx-blockstore package and removes all dangling references to it. Note: This also removes the `copy_library_from_v1_to_v2` management command, which has been broken ever since we switched from Blockstore to Learning Core. Part of this DEPR: https://github.com/openedx/public-engineering/issues/238 --- cms/djangoapps/contentstore/config/waffle.py | 2 +- .../commands/copy_libraries_from_v1_to_v2.py | 121 ----------- cms/djangoapps/contentstore/tasks.py | 113 +--------- cms/envs/common.py | 45 +--- cms/envs/devstack.py | 3 - cms/envs/production.py | 5 - cms/envs/test.py | 15 -- .../content_libraries/xblock_iframe.html | 6 +- common/djangoapps/track/contexts.py | 2 +- docs/concepts/extension_points.rst | 2 +- lms/djangoapps/edxnotes/tests.py | 4 +- lms/envs/common.py | 75 +------ lms/envs/devstack.py | 3 - lms/envs/production.py | 5 - lms/envs/test.py | 17 -- lms/urls.py | 2 +- .../content/search/tests/test_handlers.py | 7 +- .../core/djangoapps/content_libraries/api.py | 24 +-- .../core/djangoapps/content_libraries/apps.py | 2 +- .../djangoapps/content_libraries/constants.py | 3 - .../content_libraries/library_context.py | 2 +- .../commands/content_libraries_import.py | 2 +- .../migrations/0008_auto_20210818_2148.py | 2 +- .../djangoapps/content_libraries/models.py | 10 +- .../content_libraries/permissions.py | 2 +- .../content_libraries/serializers.py | 11 - .../djangoapps/content_libraries/tasks.py | 26 +-- .../content_libraries/tests/base.py | 9 +- .../tests/test_content_libraries.py | 12 +- .../content_libraries/tests/test_runtime.py | 49 +---- .../tests/test_static_assets.py | 8 +- .../content_libraries/tests/test_views_lti.py | 2 +- .../djangoapps/content_libraries/views.py | 13 +- .../content_tagging/tests/test_tasks.py | 2 - openedx/core/djangoapps/olx_rest_api/views.py | 8 +- openedx/core/djangoapps/xblock/README.rst | 5 + openedx/core/djangoapps/xblock/apps.py | 16 +- .../learning_context/learning_context.py | 11 +- .../xblock/runtime/learning_core_runtime.py | 8 +- .../core/djangoapps/xblock/runtime/runtime.py | 10 +- .../core/djangoapps/xblock/runtime/shims.py | 7 +- openedx/core/lib/blockstore_api/__init__.py | 51 ----- openedx/core/lib/blockstore_api/db_routers.py | 60 ------ .../core/lib/blockstore_api/tests/__init__.py | 0 openedx/core/lib/blockstore_api/tests/base.py | 36 ---- .../tests/test_blockstore_api.py | 201 ------------------ openedx/core/lib/xblock_serializer/api.py | 8 +- .../lib/xblock_serializer/block_serializer.py | 12 +- .../core/lib/xblock_serializer/test_api.py | 22 +- openedx/core/lib/xblock_serializer/utils.py | 6 +- requirements/edx/base.txt | 29 +-- requirements/edx/development.txt | 28 --- requirements/edx/doc.txt | 30 +-- requirements/edx/github.in | 10 +- requirements/edx/kernel.in | 1 - requirements/edx/testing.txt | 30 +-- xmodule/README.rst | 6 +- xmodule/html_block.py | 2 +- xmodule/library_tools.py | 2 +- xmodule/raw_block.py | 2 +- xmodule/tests/test_library_tools.py | 2 +- xmodule/video_block/transcripts_utils.py | 52 ++--- xmodule/video_block/video_handlers.py | 2 +- xmodule/xml_block.py | 2 +- 64 files changed, 182 insertions(+), 1083 deletions(-) delete mode 100644 cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py delete mode 100644 openedx/core/lib/blockstore_api/__init__.py delete mode 100644 openedx/core/lib/blockstore_api/db_routers.py delete mode 100644 openedx/core/lib/blockstore_api/tests/__init__.py delete mode 100644 openedx/core/lib/blockstore_api/tests/base.py delete mode 100644 openedx/core/lib/blockstore_api/tests/test_blockstore_api.py diff --git a/cms/djangoapps/contentstore/config/waffle.py b/cms/djangoapps/contentstore/config/waffle.py index b10566bb04..49dbab5715 100644 --- a/cms/djangoapps/contentstore/config/waffle.py +++ b/cms/djangoapps/contentstore/config/waffle.py @@ -35,7 +35,7 @@ SHOW_REVIEW_RULES_FLAG = CourseWaffleFlag( # lint-amnesty, pylint: disable=togg # .. toggle_creation_date: 2020-08-03 # .. toggle_target_removal_date: 2020-12-31 # .. toggle_warning: Also set settings.LIBRARY_AUTHORING_MICROFRONTEND_URL and ENABLE_LIBRARY_AUTHORING_MICROFRONTEND. -# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/COMM/pages/1545011241/BD-14+Blockstore+Powered+Content+Libraries+Taxonomies +# .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/OEPM/pages/4106944527/Libraries+Relaunch+Proposal+For+Product+Review REDIRECT_TO_LIBRARY_AUTHORING_MICROFRONTEND = WaffleFlag( f'{WAFFLE_NAMESPACE}.library_authoring_mfe', __name__, LOG_PREFIX ) diff --git a/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py b/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py deleted file mode 100644 index 3be5cab11c..0000000000 --- a/cms/djangoapps/contentstore/management/commands/copy_libraries_from_v1_to_v2.py +++ /dev/null @@ -1,121 +0,0 @@ -"""A Command to Copy or uncopy V1 Content Libraries entires to be stored as v2 content libraries.""" - -import logging -import csv -from textwrap import dedent - -from django.core.management import BaseCommand, CommandError - -from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryLocator - -from xmodule.modulestore.django import modulestore - - -from celery import group - -from cms.djangoapps.contentstore.tasks import create_v2_library_from_v1_library, delete_v2_library_from_v1_library - -from .prompt import query_yes_no - -log = logging.getLogger(__name__) - - -class Command(BaseCommand): - """ - Copy or uncopy V1 Content Libraries (default all) entires to be stored as v2 content libraries. - First Specify the uuid for the collection to store the content libraries in. - Specfiy --all for all libraries, library ids for specific libraries, - and -- file followed by the path for a list of libraries from a file. - - Example usage: - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid' --all - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid' --all --uncopy - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid 'library-v1:edX+DemoX+Better_Library' - $ ./manage.py cms copy_libraries_from_v1_to_v2 'collection_uuid 'library-v1:edX+DemoX+Better_Library' --uncopy - $ ./manage.py cms copy_libraries_from_v1_to_v2 - '11111111-2111-4111-8111-111111111111' - './list_of--library-locators.csv --all - - Note: - This Command Also produces an "output file" which contains the mapping of locators and the status of the copy. - """ - - help = dedent(__doc__) - CONFIRMATION_PROMPT = "Reindexing all libraries might be a time consuming operation. Do you want to continue?" - - def add_arguments(self, parser): - """arguements for command""" - - parser.add_argument( - 'collection_uuid', - type=str, - help='the uuid for the collection to create the content library in.' - ) - parser.add_argument( - 'output_csv', - type=str, - nargs='?', - default=None, - help='a file path to write the tasks output to. Without this the result is simply logged.' - ) - - parser.add_argument( - '--all', - action='store_true', - dest='all', - help='Copy all libraries' - ) - parser.add_argument( - '--uncopy', - action='store_true', - dest='uncopy', - help='Delete libraries specified' - ) - parser.add_argument( - 'library_ids', - nargs='*', - default=[], - help='a space-seperated list of v1 library ids to copy' - ) - - def _parse_library_key(self, raw_value): - """ Parses library key from string """ - result = CourseKey.from_string(raw_value) - - if not isinstance(result, LibraryLocator): - raise CommandError(f"Argument {raw_value} is not a library key") - return result - - def handle(self, *args, **options): # lint-amnesty, pylint: disable=unused-argument - """Parse args and generate tasks for copying content.""" - - if (not options['library_ids'] and not options['all']) or (options['library_ids'] and options['all']): - raise CommandError("copy_libraries_from_v1_to_v2 requires one or more s or the --all flag.") - - if options['all']: - store = modulestore() - if query_yes_no(self.CONFIRMATION_PROMPT, default="no"): - v1_library_keys = [ - library.location.library_key.replace(branch=None) for library in store.get_libraries() - ] - else: - return - else: - v1_library_keys = list(map(self._parse_library_key, options['library_ids'])) - - create_library_task_group = group([ - delete_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid']) - if options['uncopy'] - else create_v2_library_from_v1_library.s(str(v1_library_key), options['collection_uuid']) - for v1_library_key in v1_library_keys - ]) - - group_result = create_library_task_group.apply_async().get() - if options['output_csv']: - with open(options['output_csv'], 'w', encoding='utf-8', newline='') as file: - output_writer = csv.writer(file) - output_writer.writerow(["v1_library_id", "v2_library_id", "status", "error_msg"]) - for result in group_result: - output_writer.writerow(result.values()) - log.info(group_result) diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index faaf9dc7e1..bb220c3717 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -19,7 +19,6 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.core.exceptions import SuspiciousOperation from django.core.files import File -from django.db.transaction import atomic from django.test import RequestFactory from django.utils.text import get_valid_filename from edx_django_utils.monitoring import ( @@ -31,7 +30,7 @@ from edx_django_utils.monitoring import ( from olxcleaner.exceptions import ErrorLevel from olxcleaner.reporting import report_error_summary, report_errors from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryLocator, LibraryLocatorV2 +from opaque_keys.edx.locator import LibraryLocator from organizations.api import add_organization_course, ensure_organization from organizations.exceptions import InvalidOrganizationException from organizations.models import Organization, OrganizationCourse @@ -66,7 +65,6 @@ from openedx.core.djangoapps.discussions.config.waffle import ENABLE_NEW_STRUCTU from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider from openedx.core.djangoapps.discussions.tasks import update_unit_discussion_state_from_discussion_blocks from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse -from openedx.core.lib.blockstore_api import get_collection from openedx.core.lib.extract_archive import safe_extractall from xmodule.contentstore.django import contentstore from xmodule.course_block import CourseFields @@ -900,115 +898,6 @@ def _create_copy_content_task(v2_library_key, v1_library_key): ) -def _create_metadata(v1_library_key, collection_uuid): - """instansiate an index for the V2 lib in the collection""" - - store = modulestore() - v1_library = store.get_library(v1_library_key) - collection = get_collection(collection_uuid).uuid - # To make it easy, all converted libs are complex, meaning they can contain problems, videos, and text - library_type = 'complex' - org = _parse_organization(v1_library.location.library_key.org) - slug = v1_library.location.library_key.library - title = v1_library.display_name - # V1 libraries do not have descriptions. - description = '' - # permssions & license are most restrictive. - allow_public_learning = False - allow_public_read = False - library_license = '' # '' = ALL_RIGHTS_RESERVED - with atomic(): - return v2contentlib_api.create_library( - org, - slug, - title, - description, - allow_public_learning, - allow_public_read, - library_license, - library_type, - ) - - -@shared_task(time_limit=30) -@set_code_owner_attribute -def delete_v2_library_from_v1_library(v1_library_key_string, collection_uuid): - """ - For a V1 Library, delete the matching v2 library, where the library is the result of the copy operation - This method relys on _create_metadata failling for LibraryAlreadyExists in order to obtain the v2 slug. - """ - v1_library_key = CourseKey.from_string(v1_library_key_string) - v2_library_key = LibraryLocatorV2.from_string('lib:' + v1_library_key.org + ':' + v1_library_key.course) - - try: - v2contentlib_api.delete_library(v2_library_key) - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": v2_library_key, - "status": "SUCCESS", - "msg": None - } - except Exception as error: # lint-amnesty, pylint: disable=broad-except - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": v2_library_key, - "status": "FAILED", - "msg": f"Exception: {v2_library_key} did not delete: {error}" - } - - -@shared_task(time_limit=30) -@set_code_owner_attribute -def create_v2_library_from_v1_library(v1_library_key_string, collection_uuid): - """ - write the metadata, permissions, and content of a v1 library into a v2 library in the given collection. - """ - - v1_library_key = CourseKey.from_string(v1_library_key_string) - - LOGGER.info(f"Copy Library task created for library: {v1_library_key}") - - try: - v2_library_metadata = _create_metadata(v1_library_key, collection_uuid) - - except v2contentlib_api.LibraryAlreadyExists: - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": None, - "status": "FAILED", - "msg": f"Exception: LibraryAlreadyExists {v1_library_key_string} aleady exists" - } - - try: - _create_copy_content_task(v2_library_metadata.key, v1_library_key) - except Exception as error: # lint-amnesty, pylint: disable=broad-except - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": str(v2_library_metadata.key), - "status": "FAILED", - "msg": - f"Could not import content from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}" - } - - try: - copy_v1_user_roles_into_v2_library(v2_library_metadata.key, v1_library_key) - except Exception as error: # lint-amnesty, pylint: disable=broad-except - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": str(v2_library_metadata.key), - "status": "FAILED", - "msg": - f"Could not copy permissions from {v1_library_key_string} into {str(v2_library_metadata.key)}: {str(error)}" - } - - return { - "v1_library_id": v1_library_key_string, - "v2_library_id": str(v2_library_metadata.key), - "status": "SUCCESS", - "msg": None - } - - @shared_task(time_limit=30) @set_code_owner_attribute def delete_v1_library(v1_library_key_string): diff --git a/cms/envs/common.py b/cms/envs/common.py index e505f82457..ebc101801f 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -115,9 +115,6 @@ from lms.envs.common import ( ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_SECRET, ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL, - # Blockstore - BUNDLE_ASSET_STORAGE_SETTINGS, - # Methods to derive settings _make_mako_template_dirs, _make_locale_paths, @@ -444,7 +441,7 @@ FEATURES = { # .. toggle_use_cases: temporary # .. toggle_creation_date: 2020-06-20 # .. toggle_target_removal_date: 2020-12-31 - # .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/COMM/pages/1545011241/BD-14+Blockstore+Powered+Content+Libraries+Taxonomies + # .. toggle_tickets: https://openedx.atlassian.net/wiki/spaces/OEPM/pages/4106944527/Libraries+Relaunch+Proposal+For+Product+Review # .. toggle_warning: Also set settings.LIBRARY_AUTHORING_MICROFRONTEND_URL and see # REDIRECT_TO_LIBRARY_AUTHORING_MICROFRONTEND for rollout. 'ENABLE_LIBRARY_AUTHORING_MICROFRONTEND': False, @@ -1033,6 +1030,11 @@ XBLOCK_EXTRA_MIXINS = () # Paths to wrapper methods which should be applied to every XBlock's FieldData. XBLOCK_FIELD_DATA_WRAPPERS = () +# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE +# .. setting_default: default +# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. +XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' + ############################ ORA 2 ############################################ # By default, don't use a file prefix @@ -1690,7 +1692,7 @@ INSTALLED_APPS = [ 'cms.djangoapps.xblock_config.apps.XBlockConfig', 'cms.djangoapps.export_course_metadata.apps.ExportCourseMetadataConfig', - # New (Blockstore-based) XBlock runtime + # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig', # Maintenance tools @@ -1873,9 +1875,6 @@ INSTALLED_APPS = [ # For edx ace template tags 'edx_ace', - # Blockstore - 'blockstore.apps.bundles', - # alternative swagger generator for CMS API 'drf_spectacular', @@ -2243,25 +2242,11 @@ CUSTOM_RESOURCE_TEMPLATES_DIRECTORY = None DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', - 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', ] ############################ Cache Configuration ############################### CACHES = { - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': ['localhost:11211'], - 'TIMEOUT': '86400', # This data should be long-lived for performance, BundleCache handles invalidation - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'OPTIONS': { - 'no_delay': True, - 'ignore_exc': True, - 'use_pooling': True, - 'connect_timeout': 0.5 - } - }, 'course_structure_cache': { 'KEY_PREFIX': 'course_structure', 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', @@ -2699,22 +2684,6 @@ PROCTORING_BACKENDS = { PROCTORING_SETTINGS = {} -################## BLOCKSTORE RELATED SETTINGS ######################### - -# Which of django's caches to use for storing anonymous user state for XBlocks -# in the blockstore-based XBlock runtime -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' - -# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT -# .. setting_default: 3000 -# .. setting_description: Maximum time-to-live of cached Bundles fetched from -# Blockstore, in seconds. When the values returned from Blockstore have -# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache -# must be lower than the minimum TTL of those values. -# We use a default of 3000s (50mins) because temporary URLs are often -# configured to expire after one hour. -BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 - ###################### LEARNER PORTAL ################################ LEARNER_PORTAL_URL_ROOT = 'https://learner-portal-localhost:18000' diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index 3fe10377d7..faacbc1548 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -218,9 +218,6 @@ IDA_LOGOUT_URI_LIST = [ ENTERPRISE_BACKEND_SERVICE_EDX_OAUTH2_PROVIDER_URL = "http://edx.devstack.lms/oauth2" -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/" - ##################################################################### # pylint: disable=wrong-import-order, wrong-import-position diff --git a/cms/envs/production.py b/cms/envs/production.py index 67524e7608..f65b220432 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -405,11 +405,6 @@ XBLOCK_FIELD_DATA_WRAPPERS = ENV_TOKENS.get( CONTENTSTORE = AUTH_TOKENS.get('CONTENTSTORE', CONTENTSTORE) DOC_STORE_CONFIG = AUTH_TOKENS.get('DOC_STORE_CONFIG', DOC_STORE_CONFIG) -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/" -# Configure an API auth token at (blockstore URL)/admin/authtoken/token/ -BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None) - # Celery Broker CELERY_ALWAYS_EAGER = ENV_TOKENS.get("CELERY_ALWAYS_EAGER", False) CELERY_BROKER_TRANSPORT = ENV_TOKENS.get("CELERY_BROKER_TRANSPORT", "") diff --git a/cms/envs/test.py b/cms/envs/test.py index 118d7e27a7..38b7c78171 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -173,23 +173,8 @@ CACHES = { 'course_structure_cache': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', }, - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': 'edx_loc_mem_cache', - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, } -############################### BLOCKSTORE ##################################### -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - ################################# CELERY ###################################### CELERY_ALWAYS_EAGER = True diff --git a/cms/templates/content_libraries/xblock_iframe.html b/cms/templates/content_libraries/xblock_iframe.html index 84d83de42d..e8eb4c96ea 100644 --- a/cms/templates/content_libraries/xblock_iframe.html +++ b/cms/templates/content_libraries/xblock_iframe.html @@ -228,7 +228,7 @@ // Does the XBlock HTML contain arguments to pass to the InitFunction? let data = {}; [].forEach.call(element.children, (childNode) => { - // The newer/pure/Blockstore runtime uses 'xblock_json_init_args' + // The newer/pure/LearningCore runtime uses 'xblock_json_init_args' // while the LMS runtime uses 'xblock-json-init-args'. if ( childNode.matches('script.xblock_json_init_args') @@ -257,7 +257,7 @@ // Recursively initialize the JavaScript code of each XBlock: function initializeXBlockAndChildren(element, callback) { - // The newer/pure/Blockstore runtime uses the 'data-usage' attribute, while the LMS uses 'data-usage-id' + // The newer/pure/LearningCore runtime uses the 'data-usage' attribute, while the LMS uses 'data-usage-id' const usageId = element.getAttribute('data-usage') || element.getAttribute('data-usage-id'); if (usageId !== null) { element[USAGE_ID_KEY] = usageId; @@ -297,7 +297,7 @@ } // Find the root XBlock node. - // The newer/pure/Blockstore runtime uses '.xblock-v1' while the LMS runtime uses '.xblock'. + // The newer/pure/LearningCore runtime uses '.xblock-v1' while the LMS runtime uses '.xblock'. const rootNode = document.querySelector('.xblock, .xblock-v1'); // will always return the first matching element initializeXBlockAndChildren(rootNode, () => { }); diff --git a/common/djangoapps/track/contexts.py b/common/djangoapps/track/contexts.py index 217a514e4c..0ac8292e25 100644 --- a/common/djangoapps/track/contexts.py +++ b/common/djangoapps/track/contexts.py @@ -48,7 +48,7 @@ def course_context_from_course_id(course_id): """ Creates a course context from a `course_id`. - For newer parts of the system (i.e. Blockstore-based libraries/courses/etc.) + For newer parts of the system (i.e. Learning-Core-based libraries/courses/etc.) use context_dict_for_learning_context instead of this method. Example Returned Context:: diff --git a/docs/concepts/extension_points.rst b/docs/concepts/extension_points.rst index c5ba0d42c0..d4e802baec 100644 --- a/docs/concepts/extension_points.rst +++ b/docs/concepts/extension_points.rst @@ -124,7 +124,7 @@ Here are the different integration points that python plugins can use: - By default, the registration page for each instance of Open edX has fields that ask for information such as a user’s name, country, and highest level of education completed. You can add custom fields to the registration page for your own Open edX instance. These fields can be different types, including text entry fields and drop-down lists. See `Adding Custom Fields to the Registration Page`_. * - Learning Context (``openedx.learning_context``) - Trial, Limited - - A "Learning Context" is a course, a library, a program, a blog, an external site, or some other collection of content where learning happens. If you are trying to build a totally new learning experience that's not a type of course, you may need to implement a new learning context. Learning contexts are a new abstraction and are only supported in the nascent Blockstore-based XBlock runtime. Since existing courses use modulestore instead of Blockstore, they are not yet implemented as learning contexts. However, Blockstore-based content libraries are. See |learning_context.py|_ to learn more. + - A "Learning Context" is a course, a library, a program, a blog, an external site, or some other collection of content where learning happens. If you are trying to build a totally new learning experience that's not a type of course, you may need to implement a new learning context. Learning contexts are a new abstraction and are only supported in the nascent Learning-Core-based XBlock runtime. Since existing courses use modulestore instead of Learning Core, they are not yet implemented as learning contexts. However, Learning-Core-based content libraries are. See |learning_context.py|_ to learn more. * - User partition scheme (``openedx.user_partition_scheme`` and ``openedx.dynamic_partition_generator``) - Unknown, Stable - A user partition scheme is a named way for dividing users in a course into groups, usually to show different content to different users or to run experiments. Partitions may be added to a course manually, or automatically added by a "dynamic partition generator." The core platform includes partition scheme plugins like ``random``, ``cohort``, and ``enrollment_track``. See the |UserPartition docstring|_ to learn more. diff --git a/lms/djangoapps/edxnotes/tests.py b/lms/djangoapps/edxnotes/tests.py index 3cd7bafa5b..688cd54386 100644 --- a/lms/djangoapps/edxnotes/tests.py +++ b/lms/djangoapps/edxnotes/tests.py @@ -167,9 +167,9 @@ class EdxNotesDecoratorTest(ModuleStoreTestCase): self.problem.runtime.is_author_mode = True assert 'original_get_html' == self.problem.get_html() - def test_edxnotes_blockstore_runtime(self): + def test_edxnotes_learning_core_runtime(self): """ - Tests that get_html is not wrapped when problem is rendered by Blockstore runtime. + Tests that get_html is not wrapped when problem is rendered by the learning core runtime. """ del self.problem.block.runtime.modulestore assert 'original_get_html' == self.problem.get_html() diff --git a/lms/envs/common.py b/lms/envs/common.py index 5f8714ca9c..da2bfed626 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1157,26 +1157,12 @@ STATUS_MESSAGE_PATH = ENV_ROOT / "status_message.json" DATABASE_ROUTERS = [ 'openedx.core.lib.django_courseware_routers.StudentModuleHistoryExtendedRouter', - 'openedx.core.lib.blockstore_api.db_routers.BlockstoreRouter', 'edx_django_utils.db.read_replica.ReadReplicaRouter', ] ############################ Cache Configuration ############################### CACHES = { - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': ['localhost:11211'], - 'TIMEOUT': '86400', # This data should be long-lived for performance, BundleCache handles invalidation - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'OPTIONS': { - 'no_delay': True, - 'ignore_exc': True, - 'use_pooling': True, - 'connect_timeout': 0.5 - } - }, 'course_structure_cache': { 'KEY_PREFIX': 'course_structure', 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', @@ -1704,6 +1690,11 @@ XBLOCK_FS_STORAGE_PREFIX = None # for more reference. XBLOCK_SETTINGS = {} +# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE +# .. setting_default: default +# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. +XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' + ############# ModuleStore Configuration ########## MODULESTORE_BRANCH = 'published-only' @@ -3149,7 +3140,7 @@ INSTALLED_APPS = [ # User tours 'lms.djangoapps.user_tours', - # New (Blockstore-based) XBlock runtime + # New (Learning-Core-based) XBlock runtime 'openedx.core.djangoapps.xblock.apps.LmsXBlockAppConfig', # Student support tools @@ -3385,9 +3376,6 @@ INSTALLED_APPS = [ # For edx ace template tags 'edx_ace', - # Blockstore - 'blockstore.apps.bundles', - # MFE API 'lms.djangoapps.mfe_config_api', @@ -5196,57 +5184,6 @@ RATE_LIMIT_FOR_VIDEO_METADATA_API = '10/minute' ########################## MAILCHIMP SETTINGS ################################# MAILCHIMP_NEW_USER_LIST_ID = "" -########################## BLOCKSTORE ##################################### - -# .. setting_name: XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE -# .. setting_default: default -# .. setting_description: The django cache key of the cache to use for storing anonymous user state for XBlocks. -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'default' - -# .. setting_name: BLOCKSTORE_BUNDLE_CACHE_TIMEOUT -# .. setting_default: 3000 -# .. setting_description: Maximum time-to-live of cached Bundles fetched from -# Blockstore, in seconds. When the values returned from Blockstore have -# TTLs of their own (such as signed S3 URLs), the maximum TTL of this cache -# must be lower than the minimum TTL of those values. -# We use a default of 3000s (50mins) because temporary URLs are often -# configured to expire after one hour. -BLOCKSTORE_BUNDLE_CACHE_TIMEOUT = 3000 - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_KEY -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_SECRET` is -# set, and `boto3` is installed, this is used as an AWS IAM access key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_KEY = None - -# .. setting_name: BUNDLE_ASSET_URL_STORAGE_SECRET -# .. setting_default: None -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this is used as an AWS IAM secret key for -# generating signed, read-only URLs for blockstore assets stored in S3. -# Otherwise, URLs are generated based on the default storage configuration. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_URL_STORAGE_SECRET = None - -# .. setting_name: BUNDLE_ASSET_STORAGE_SETTINGS -# .. setting_default: dict, appropriate for file system storage. -# .. setting_description: When this is set, `BUNDLE_ASSET_URL_STORAGE_KEY` is -# set, and `boto3` is installed, this provides the bucket name and location for blockstore assets stored in S3. -# See `blockstore.apps.bundles.storage.LongLivedSignedUrlStorage` for details. -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - # Backend storage - # STORAGE_CLASS='storages.backends.s3boto3.S3Boto3Storage', - # STORAGE_KWARGS=dict(bucket='bundle-asset-bucket', location='/path-to-bundles/'), - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - SYSLOG_SERVER = '' FEEDBACK_SUBMISSION_EMAIL = '' GITHUB_REPO_ROOT = '/edx/var/edxapp/data' diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index b783a276fe..82e9134f45 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -262,9 +262,6 @@ TOKEN_SIGNING.update({ ) }) -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/" - ########################## PROGRAMS LEARNER PORTAL ############################## LEARNER_PORTAL_URL_ROOT = 'http://localhost:8734' diff --git a/lms/envs/production.py b/lms/envs/production.py index 948e81977c..014cf59aa3 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -516,11 +516,6 @@ MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {}) EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is '' EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', '') # django default is '' -############################### BLOCKSTORE ##################################### -BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None) # e.g. "https://blockstore.example.com/api/v1/" -# Configure an API auth token at (blockstore URL)/admin/authtoken/token/ -BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None) - # Analytics API ANALYTICS_API_KEY = AUTH_TOKENS.get("ANALYTICS_API_KEY", ANALYTICS_API_KEY) ANALYTICS_API_URL = ENV_TOKENS.get("ANALYTICS_API_URL", ANALYTICS_API_URL) diff --git a/lms/envs/test.py b/lms/envs/test.py index 14c10e52d3..3c4bb95649 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -215,13 +215,6 @@ CACHES = { 'course_structure_cache': { 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', }, - # Blockstore caching tests require a cache that actually works: - 'blockstore': { - 'KEY_PREFIX': 'blockstore', - 'KEY_FUNCTION': 'common.djangoapps.util.memcache.safe_key', - 'LOCATION': 'edx_loc_mem_cache', - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, } ############################# SECURITY SETTINGS ################################ @@ -546,16 +539,6 @@ add_plugins(__name__, ProjectType.LMS, SettingsType.TEST) derive_settings(__name__) -############################### BLOCKSTORE ##################################### -XBLOCK_RUNTIME_V2_EPHEMERAL_DATA_CACHE = 'blockstore' # This must be set to a working cache for the tests to pass -BUNDLE_ASSET_STORAGE_SETTINGS = dict( - STORAGE_CLASS='django.core.files.storage.FileSystemStorage', - STORAGE_KWARGS=dict( - location=MEDIA_ROOT, - base_url=MEDIA_URL, - ), -) - # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' diff --git a/lms/urls.py b/lms/urls.py index 2d5d9a2003..5ac6283fdd 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -336,7 +336,7 @@ urlpatterns += [ name='xblock_resource_url', ), - # New (Blockstore-based) XBlock REST API + # New (Learning-Core-based) XBlock REST API path('', include(('openedx.core.djangoapps.xblock.rest_api.urls', 'openedx.core.djangoapps.xblock'), namespace='xblock_api')), diff --git a/openedx/core/djangoapps/content/search/tests/test_handlers.py b/openedx/core/djangoapps/content/search/tests/test_handlers.py index 7f209b4ca6..1ce9c57a1a 100644 --- a/openedx/core/djangoapps/content/search/tests/test_handlers.py +++ b/openedx/core/djangoapps/content/search/tests/test_handlers.py @@ -9,7 +9,6 @@ from organizations.tests.factories import OrganizationFactory from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangolib.testing.utils import skip_unless_cms -from openedx.core.lib.blockstore_api.tests.base import BlockstoreAppTestMixin from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase @@ -25,11 +24,7 @@ except RuntimeError: @patch("openedx.core.djangoapps.content.search.api.MeilisearchClient") @override_settings(MEILISEARCH_ENABLED=True) @skip_unless_cms -class TestUpdateIndexHandlers( - ModuleStoreTestCase, - BlockstoreAppTestMixin, - LiveServerTestCase, -): +class TestUpdateIndexHandlers(ModuleStoreTestCase, LiveServerTestCase): """ Test that the search index is updated when XBlocks and Library Blocks are modified """ diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 3e276a37ab..3c47e0d260 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -94,7 +94,7 @@ from xblock.core import XBlock from xblock.exceptions import XBlockNotFoundError from openedx.core.djangoapps.xblock.api import get_component_from_usage_key, xblock_type_display_name -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from xmodule.library_root_xblock import LibraryRoot as LibraryRootV1 from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore @@ -336,6 +336,7 @@ def get_library(library_key): # something that we should remove. It exists to accomodate some complexities # with how Blockstore staged changes, but Learning Core works differently, # and has_unpublished_changes should be sufficient. + # Ref: https://github.com/openedx/edx-platform/issues/34283 has_unpublished_deletes = publishing_api.get_entities_with_unpublished_deletes(learning_package.id) \ .exists() @@ -1012,10 +1013,10 @@ def get_v1_or_v2_library( library = get_library(library_key) if v2_version is not None and library.version != v2_version: raise NotImplementedError( - f"Tried to load version {v2_version} of blockstore-based library {library_key}. " + f"Tried to load version {v2_version} of learning_core-based library {library_key}. " f"Currently, only the latest version ({library.version}) may be loaded. " "This is a known issue. " - "It will be fixed before the production release of blockstore-based (V2) content libraries. " + "It will be fixed before the production release of learning_core-based (V2) content libraries. " ) return library except ContentLibrary.DoesNotExist: @@ -1121,35 +1122,34 @@ class BaseEdxImportClient(abc.ABC): modulestore_key.block_type, block_id, ) - blockstore_key = library_block.usage_key + dest_key = library_block.usage_key except LibraryBlockAlreadyExists: - blockstore_key = LibraryUsageLocatorV2( + dest_key = LibraryUsageLocatorV2( lib_key=self.library.library_key, block_type=modulestore_key.block_type, usage_id=block_id, ) - get_library_block(blockstore_key) + get_library_block(dest_key) log.warning('Library block already exists: Appending static files ' - 'and overwriting OLX: %s', str(blockstore_key)) + 'and overwriting OLX: %s', str(dest_key)) # Handle static files. files = [ f.path for f in - get_library_block_static_asset_files(blockstore_key) + get_library_block_static_asset_files(dest_key) ] for filename, static_file in block_data.get('static_files', {}).items(): if filename in files: # Files already added, move on. continue file_content = self.get_block_static_data(static_file) - add_library_block_static_asset_file( - blockstore_key, filename, file_content) + add_library_block_static_asset_file(dest_key, filename, file_content) files.append(filename) # Import OLX. - set_library_block_olx(blockstore_key, block_data['olx']) + set_library_block_olx(dest_key, block_data['olx']) def import_blocks_from_course(self, course_key, progress_callback): """ @@ -1200,7 +1200,7 @@ class EdxModulestoreImportClient(BaseEdxImportClient): Get block OLX by serializing it from modulestore directly. """ block = self.modulestore.get_item(block_key) - data = serialize_modulestore_block_for_blockstore(block) + data = serialize_modulestore_block_for_learning_core(block) return {'olx': data.olx_str, 'static_files': {s.name: s for s in data.static_files}} diff --git a/openedx/core/djangoapps/content_libraries/apps.py b/openedx/core/djangoapps/content_libraries/apps.py index 685e9259b6..52c3e51797 100644 --- a/openedx/core/djangoapps/content_libraries/apps.py +++ b/openedx/core/djangoapps/content_libraries/apps.py @@ -16,7 +16,7 @@ class ContentLibrariesConfig(AppConfig): """ name = 'openedx.core.djangoapps.content_libraries' - verbose_name = 'Content Libraries (Blockstore-based)' + verbose_name = 'Content Libraries (Learning-Core-based)' # This is designed as a plugin for now so that # the whole thing is self-contained and can easily be enabled/disabled plugin_app = { diff --git a/openedx/core/djangoapps/content_libraries/constants.py b/openedx/core/djangoapps/content_libraries/constants.py index 0a0614e514..9505d52d1c 100644 --- a/openedx/core/djangoapps/content_libraries/constants.py +++ b/openedx/core/djangoapps/content_libraries/constants.py @@ -1,9 +1,6 @@ """ Constants used for the content libraries. """ from django.utils.translation import gettext_lazy as _ -# ./api.py and ./views.py are only used in Studio, so we always work with this draft of any -# content library bundle: -DRAFT_NAME = 'studio_draft' VIDEO = 'video' COMPLEX = 'complex' diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 9408f51e51..93de022474 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -19,7 +19,7 @@ class LibraryContextImpl(LearningContext): """ Implements content libraries as a learning context. - This is the *new* content libraries based on Blockstore, not the old content + This is the *new* content libraries based on Learning Core, not the old content libraries based on modulestore. """ diff --git a/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py index 9b57b42e6a..cd68112d2b 100644 --- a/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py +++ b/openedx/core/djangoapps/content_libraries/management/commands/content_libraries_import.py @@ -89,7 +89,7 @@ class Command(BaseCommand): def handle(self, *args, **options): """ Collect all blocks from a course that are "importable" and write them to the - a blockstore library. + a learning core library. """ # Search for the library. diff --git a/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py b/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py index fb15e4a0fa..6c368dec24 100644 --- a/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py +++ b/openedx/core/djangoapps/content_libraries/migrations/0008_auto_20210818_2148.py @@ -40,7 +40,7 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='ltigradedresource', name='usage_key', - field=opaque_keys.edx.django.models.UsageKeyField(help_text='The usage key string of the blockstore resource serving the content of this launch.', max_length=255), + field=opaque_keys.edx.django.models.UsageKeyField(help_text='The usage key string of the resource serving the content of this launch.', max_length=255), ), migrations.AlterField( model_name='ltiprofile', diff --git a/openedx/core/djangoapps/content_libraries/models.py b/openedx/core/djangoapps/content_libraries/models.py index a4c128c9bc..58a9f6b686 100644 --- a/openedx/core/djangoapps/content_libraries/models.py +++ b/openedx/core/djangoapps/content_libraries/models.py @@ -8,7 +8,7 @@ This module contains the models for new Content Libraries. LTI 1.3 Models ============== -Content Libraries serves blockstore-based content through LTI 1.3 launches. +Content Libraries serves learning-core-based content through LTI 1.3 launches. The interface supports resource link launches and grading services. Two use cases justify the current data model to support LTI launches. They are: @@ -27,7 +27,7 @@ Relationship with LMS's ``lti_provider``` models The data model above is similar to the one provided by the current LTI 1.1 implementation for modulestore and courseware content. But, Content Libraries is orthogonal. Its use-case is to offer standalone, embedded content from a -specific backend (blockstore). As such, it decouples from LTI 1.1. and the +specific backend (learning core). As such, it decouples from LTI 1.1. and the logic assume no relationship or impact across the two applications. The same reasoning applies to steps beyond the data model, such as at the XBlock runtime, authentication, and score handling, etc. @@ -85,9 +85,9 @@ class ContentLibrary(models.Model): """ A Content Library is a collection of content (XBlocks and/or static assets) - All actual content is stored in Blockstore, and any data that we'd want to + All actual content is stored in Learning Core, and any data that we'd want to transfer to another instance if this library were exported and then - re-imported on another Open edX instance should be kept in Blockstore. This + re-imported on another Open edX instance should be kept in Learning Core. This model in Studio should only be used to track settings specific to this Open edX instance, like who has permission to edit this content library. """ @@ -479,7 +479,7 @@ class LtiGradedResource(models.Model): usage_key = UsageKeyField( max_length=255, - help_text=_('The usage key string of the blockstore resource serving the ' + help_text=_('The usage key string of the resource serving the ' 'content of this launch.'), ) diff --git a/openedx/core/djangoapps/content_libraries/permissions.py b/openedx/core/djangoapps/content_libraries/permissions.py index 3c41e0574c..c7da012c9f 100644 --- a/openedx/core/djangoapps/content_libraries/permissions.py +++ b/openedx/core/djangoapps/content_libraries/permissions.py @@ -1,5 +1,5 @@ """ -Permissions for Content Libraries (v2, Blockstore-based) +Permissions for Content Libraries (v2, Learning-Core-based) """ from bridgekeeper import perms, rules from bridgekeeper.rules import Attribute, ManyRelation, Relation, in_current_groups diff --git a/openedx/core/djangoapps/content_libraries/serializers.py b/openedx/core/djangoapps/content_libraries/serializers.py index ee0e48b59c..13c6a756fd 100644 --- a/openedx/core/djangoapps/content_libraries/serializers.py +++ b/openedx/core/djangoapps/content_libraries/serializers.py @@ -14,7 +14,6 @@ from openedx.core.djangoapps.content_libraries.constants import ( from openedx.core.djangoapps.content_libraries.models import ( ContentLibraryPermission, ContentLibraryBlockImportTask ) -from openedx.core.lib import blockstore_api from openedx.core.lib.api.serializers import CourseKeyField @@ -175,16 +174,6 @@ class LibraryXBlockStaticFileSerializer(serializers.Serializer): url = serializers.URLField() size = serializers.IntegerField(min_value=0) - def to_representation(self, instance): - """ - Generate the serialized representation of this static asset file. - """ - result = super().to_representation(instance) - # Make sure the URL is one that will work from the user's browser, - # not one that only works from within a docker container: - result['url'] = blockstore_api.force_browser_url(result['url']) - return result - class LibraryXBlockStaticFilesSerializer(serializers.Serializer): """ diff --git a/openedx/core/djangoapps/content_libraries/tasks.py b/openedx/core/djangoapps/content_libraries/tasks.py index 3714dc55f8..9f4f7aaaf7 100644 --- a/openedx/core/djangoapps/content_libraries/tasks.py +++ b/openedx/core/djangoapps/content_libraries/tasks.py @@ -4,11 +4,11 @@ Celery tasks for Content Libraries. Architecture note: Several functions in this file manage the copying/updating of blocks in modulestore - and blockstore. These operations should only be performed within the context of CMS. + and learning core. These operations should only be performed within the context of CMS. However, due to existing edx-platform code structure, we've had to define the functions in shared source tree (openedx/) and the tasks are registered in both LMS and CMS. - To ensure that we're not accidentally importing things from blockstore in the LMS context, + To ensure that we're not accidentally importing things from learning core in the LMS context, we use ensure_cms throughout this module. A longer-term solution to this issue would be to move the content_libraries app to cms: @@ -39,7 +39,7 @@ from common.djangoapps.student.auth import has_studio_write_access from opaque_keys.edx.keys import CourseKey from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangoapps.xblock.api import load_block -from openedx.core.lib import ensure_cms, blockstore_api +from openedx.core.lib import ensure_cms from xmodule.capa_block import ProblemBlock from xmodule.library_content_block import ANY_CAPA_TYPE_VALUE, LibraryContentBlock from xmodule.library_root_xblock import LibraryRoot as LibraryRootV1 @@ -86,7 +86,7 @@ def import_blocks_from_course(import_task_id, course_key_str, use_course_key_as_ def _import_block(store, user_id, source_block, dest_parent_key): """ - Recursively import a blockstore block and its children.` + Recursively import a learning core block and its children.` """ def generate_block_key(source_key, dest_parent_key): """ @@ -127,7 +127,7 @@ def _import_block(store, user_id, source_block, dest_parent_key): # Prepare a list of this block's static assets; any assets that are referenced as /static/{path} (the # recommended way for referencing them) will stop working, and so we rewrite the url when importing. - # Copying assets not advised because modulestore doesn't namespace assets to each block like blockstore, which + # Copying assets not advised because modulestore doesn't namespace assets to each block like learning core, which # might cause conflicts when the same filename is used across imported blocks. if isinstance(source_key, LibraryUsageLocatorV2): all_assets = library_api.get_library_block_static_asset_files(source_key) @@ -139,12 +139,6 @@ def _import_block(store, user_id, source_block, dest_parent_key): continue # Only copy authored field data if field.is_set_on(source_block) or field.is_set_on(new_block): field_value = getattr(source_block, field_name) - if isinstance(field_value, str): - # If string field (which may also be JSON/XML data), rewrite /static/... URLs to point to blockstore - for asset in all_assets: - field_value = field_value.replace(f'/static/{asset.path}', asset.url) - # Make sure the URL is one that will work from the user's browser when using the docker devstack - field_value = blockstore_api.force_browser_url(field_value) setattr(new_block, field_name, field_value) new_block.save() store.update_item(new_block, user_id) @@ -178,9 +172,9 @@ def _problem_type_filter(store, library, capa_type): return [key for key in library.children if _filter_child(store, key, capa_type)] -def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): +def _import_from_learning_core(user_id, store, dest_block, source_block_ids): """ - Imports a block from a blockstore-based learning context (usually a + Imports a block from a learning-core-based learning context (usually a content library) into modulestore, as a new child of dest_block. Any existing children of dest_block are replaced. """ @@ -190,7 +184,7 @@ def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): if user_id is None: raise ValueError("Cannot check user permissions - LibraryTools user_id is None") - if len(set(blockstore_block_ids)) != len(blockstore_block_ids): + if len(set(source_block_ids)) != len(source_block_ids): # We don't support importing the exact same block twice because it would break the way we generate new IDs # for each block and then overwrite existing copies of blocks when re-importing the same blocks. raise ValueError("One or more library component IDs is a duplicate.") @@ -204,7 +198,7 @@ def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids): # (This could be slow and use lots of memory, except for the fact that LibraryContentBlock which calls this # should be limiting the number of blocks to a reasonable limit. We load them all now instead of one at a # time in order to raise any errors before we start actually copying blocks over.) - orig_blocks = [load_block(UsageKey.from_string(key), user) for key in blockstore_block_ids] + orig_blocks = [load_block(UsageKey.from_string(key), user) for key in source_block_ids] with store.bulk_operations(dest_course_key): child_ids_updated = set() @@ -347,7 +341,7 @@ def _sync_children( str(library_api.LibraryXBlockMetadata.from_component(library_key, component).usage_key) for component in library_api.get_library_components(library_key) ] - _import_from_blockstore(user_id, store, dest_block, source_block_ids) + _import_from_learning_core(user_id, store, dest_block, source_block_ids) dest_block.source_library_version = str(library.version) store.update_item(dest_block, user_id) except Exception as exception: # pylint: disable=broad-except diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index 5f837628c9..2bb94e3d87 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -1,5 +1,5 @@ """ -Tests for Blockstore-based Content Libraries +Tests for Learning-Core-based Content Libraries """ import uuid from contextlib import contextmanager @@ -12,9 +12,6 @@ from rest_framework.test import APITransactionTestCase, APIClient from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries.constants import COMPLEX, ALL_RIGHTS_RESERVED from openedx.core.djangolib.testing.utils import skip_unless_cms -from openedx.core.lib.blockstore_api.tests.base import ( - BlockstoreAppTestMixin, -) # Define the URLs here - don't use reverse() because we want to detect # backwards-incompatible changes like changed URLs. @@ -46,9 +43,9 @@ URL_BLOCK_XBLOCK_HANDLER = '/api/xblock/v2/xblocks/{block_key}/handler/{user_id} @skip_unless_cms # Content Libraries REST API is only available in Studio -class ContentLibrariesRestApiTest(BlockstoreAppTestMixin, APITransactionTestCase): +class ContentLibrariesRestApiTest(APITransactionTestCase): """ - Base class for Blockstore-based Content Libraries test that use the REST API + Base class for Learning-Core-based Content Libraries test that use the REST API These tests use the REST API, which in turn relies on the Python API. Some tests may use the python API directly if necessary to provide diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index c23e728c4b..67933f2964 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -1,5 +1,5 @@ """ -Tests for Blockstore-based Content Libraries +Tests for Learning-Core-based Content Libraries """ from unittest.mock import Mock, patch from unittest import skip @@ -37,7 +37,7 @@ from common.djangoapps.student.tests.factories import UserFactory @ddt.ddt class ContentLibrariesTestCase(ContentLibrariesRestApiTest, OpenEdxEventsTestMixin): """ - General tests for Blockstore-based Content Libraries + General tests for Learning-Core-based Content Libraries These tests use the REST API, which in turn relies on the Python API. Some tests may use the python API directly if necessary to provide @@ -278,7 +278,7 @@ class ContentLibrariesTestCase(ContentLibrariesRestApiTest, OpenEdxEventsTestMix

This is a normal capa problem with unicode πŸ”₯. It has "maximum attempts" set to **5**.

- + XBlock metadata only XBlock data/metadata and associated static asset files @@ -300,7 +300,7 @@ class ContentLibrariesTestCase(ContentLibrariesRestApiTest, OpenEdxEventsTestMix # Now view the XBlock's student_view (including draft changes): fragment = self._render_block_view(block_id, "student_view") assert 'resources' in fragment - assert 'Blockstore is designed to store.' in fragment['content'] + assert 'Learning Core is designed to store.' in fragment['content'] # Also call a handler to make sure that's working: handler_url = self._get_block_handler_url(block_id, "xmodule_handler") + "problem_get" @@ -806,7 +806,7 @@ class ContentLibrariesTestCase(ContentLibrariesRestApiTest, OpenEdxEventsTestMix

This is a normal capa problem with unicode πŸ”₯. It has "maximum attempts" set to **5**.

- + XBlock metadata only XBlock data/metadata and associated static asset files @@ -956,7 +956,7 @@ class ContentLibrariesTestCase(ContentLibrariesRestApiTest, OpenEdxEventsTestMix @ddt.ddt class ContentLibraryXBlockValidationTest(APITestCase): - """Tests only focused on service validation, no Blockstore needed.""" + """Tests only focused on service validation, no Learning Core interactions here.""" @ddt.data( (URL_BLOCK_METADATA_URL, dict(block_key='totally_invalid_key')), diff --git a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py index f35ba7ea7b..89b8cdefd8 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_runtime.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_runtime.py @@ -1,5 +1,5 @@ """ -Test the Blockstore-based XBlock runtime and content libraries together. +Test the Learning-Core-based XBlock runtime and content libraries together. """ import json from gettext import GNUTranslations @@ -14,7 +14,6 @@ from xblock.core import XBlock from lms.djangoapps.courseware.model_data import get_score from openedx.core.djangoapps.content_libraries import api as library_api from openedx.core.djangoapps.content_libraries.tests.base import ( - BlockstoreAppTestMixin, URL_BLOCK_RENDER_VIEW, URL_BLOCK_GET_HANDLER_URL, URL_BLOCK_METADATA_URL, @@ -60,9 +59,9 @@ class ContentLibraryContentTestMixin: ) -class ContentLibraryRuntimeTestMixin(ContentLibraryContentTestMixin): +class ContentLibraryRuntimeTests(ContentLibraryContentTestMixin): """ - Basic tests of the Blockstore-based XBlock runtime using XBlocks in a + Basic tests of the Learning-Core-based XBlock runtime using XBlocks in a content library. """ @@ -95,7 +94,7 @@ class ContentLibraryRuntimeTestMixin(ContentLibraryContentTestMixin):

This is a normal capa problem. It has "maximum attempts" set to **5**.

- + XBlock metadata only XBlock data/metadata and associated static asset files @@ -174,18 +173,10 @@ class ContentLibraryRuntimeTestMixin(ContentLibraryContentTestMixin): assert block_saved.display_name == 'New Display Name' -class ContentLibraryRuntimeTest(ContentLibraryRuntimeTestMixin, BlockstoreAppTestMixin): - """ - Tests XBlock runtime using XBlocks in a content library using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ - - # We can remove the line below to enable this in Studio once we implement a session-backed # field data store which we can use for both studio users and anonymous users @skip_unless_lms -class ContentLibraryXBlockUserStateTestMixin(ContentLibraryContentTestMixin): +class ContentLibraryXBlockUserStateTest(ContentLibraryContentTestMixin): """ Test that the Blockstore-based XBlock runtime can store and retrieve student state for XBlocks when learners access blocks directly in a library context, @@ -389,7 +380,7 @@ class ContentLibraryXBlockUserStateTestMixin(ContentLibraryContentTestMixin):

This is a normal capa problem. It has "maximum attempts" set to **5**.

- + XBlock metadata only XBlock data/metadata and associated static asset files @@ -453,7 +444,7 @@ class ContentLibraryXBlockUserStateTestMixin(ContentLibraryContentTestMixin):

This is a normal capa problem. It has "maximum attempts" set to **5**.

- + XBlock metadata only XBlock data/metadata and associated static asset files @@ -487,19 +478,8 @@ class ContentLibraryXBlockUserStateTestMixin(ContentLibraryContentTestMixin): assert 'Submit' not in dummy_public_view.data['content'] -class ContentLibraryXBlockUserStateTest( # type: ignore[misc] - ContentLibraryXBlockUserStateTestMixin, - BlockstoreAppTestMixin, -): - """ - Tests XBlock user state for XBlocks in a content library using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ - - @skip_unless_lms # No completion tracking in Studio -class ContentLibraryXBlockCompletionTestMixin(ContentLibraryContentTestMixin, CompletionWaffleTestMixin): +class ContentLibraryXBlockCompletionTest(ContentLibraryContentTestMixin, CompletionWaffleTestMixin): """ Test that the Blockstore-based XBlocks can track their completion status using the completion library. @@ -550,16 +530,3 @@ class ContentLibraryXBlockCompletionTestMixin(ContentLibraryContentTestMixin, Co # Now the block is completed assert get_block_completion_status() == 1 - - -class ContentLibraryXBlockCompletionTest( - ContentLibraryXBlockCompletionTestMixin, - CompletionWaffleTestMixin, - BlockstoreAppTestMixin, -): - """ - Test that the Blockstore-based XBlocks can track their completion status - using the installed Blockstore app. - - We run this test with a live server, so that the blockstore asset files can be served. - """ diff --git a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py index 6a75d63110..92ff4c1767 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_static_assets.py @@ -1,5 +1,5 @@ """ -Tests for static asset files in Blockstore-based Content Libraries +Tests for static asset files in Learning-Core-based Content Libraries """ from unittest import skip @@ -26,7 +26,7 @@ I'm Anant Agarwal, I'm the president of edX, @skip("Assets are being reimplemented in Learning Core. Disable until that's ready.") class ContentLibrariesStaticAssetsTest(ContentLibrariesRestApiTest): """ - Tests for static asset files in Blockstore-based Content Libraries + Tests for static asset files in Learning-Core-based Content Libraries WARNING: every test should have a unique library slug, because even though the django/mysql database gets reset for each test case, the lookup between @@ -65,7 +65,7 @@ class ContentLibrariesStaticAssetsTest(ContentLibrariesRestApiTest): def test_video_transcripts(self): """ - Test that video blocks can read transcript files out of blockstore. + Test that video blocks can read transcript files out of learning core. """ library = self._create_library(slug="transcript-test-lib", title="Transcripts Test Library") block = self._add_block_to_library(library["id"], "video", "video1") @@ -104,7 +104,7 @@ class ContentLibrariesStaticAssetsTest(ContentLibrariesRestApiTest): check_sjson() check_download() # Publish the OLX and the transcript file, since published data gets - # served differently by Blockstore and we should test that too. + # served differently by Learning Core and we should test that too. self._commit_library_changes(library["id"]) check_sjson() check_download() diff --git a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py index cb306ebbfe..a25d02761d 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_views_lti.py @@ -87,5 +87,5 @@ class LibraryBlockLtiUrlViewTest( ContentLibrariesRestApiTest, ): """ - Test generating LTI URL for a block in a library, using the installed Blockstore app. + Test generating LTI URL for a block in a library, using the installed Learning Core app. """ diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index 0df670b1ae..38f3e7efd6 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -58,7 +58,8 @@ the api module instead. block. Historical note: These views used to be wrapped with @atomic because we - wanted to make all views that operated on Blockstore data atomic: + wanted to make all views that operated on Blockstore (the predecessor + to Learning Core) atomic: https://github.com/openedx/edx-platform/pull/30456 """ @@ -258,6 +259,7 @@ class LibraryRootView(APIView): # Learning Core. TODO: This can be removed once the frontend stops # sending it to us. This whole bit of deserialization is kind of weird # though, with the renames and such. Look into this later for clennup. + # Ref: https://github.com/openedx/edx-platform/issues/34283 data.pop("collection_uuid", None) try: @@ -708,9 +710,12 @@ class LibraryBlockAssetView(APIView): ) file_wrapper = request.data['content'] if file_wrapper.size > 20 * 1024 * 1024: # > 20 MiB - # In the future, we need a way to use file_wrapper.chunks() to read - # the file in chunks and stream that to Blockstore, but Blockstore - # currently lacks an API for streaming file uploads. + # TODO: This check was written when V2 Libraries were backed by the Blockstore micro-service. + # Now that we're on Learning Core, do we still need it? Here's the original comment: + # In the future, we need a way to use file_wrapper.chunks() to read + # the file in chunks and stream that to Blockstore, but Blockstore + # currently lacks an API for streaming file uploads. + # Ref: https://github.com/openedx/edx-platform/issues/34737 raise ValidationError("File too big") file_content = file_wrapper.read() try: diff --git a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py index 3e396eb754..c14adfcce1 100644 --- a/openedx/core/djangoapps/content_tagging/tests/test_tasks.py +++ b/openedx/core/djangoapps/content_tagging/tests/test_tasks.py @@ -15,7 +15,6 @@ from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangolib.testing.utils import skip_unless_cms from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase from openedx.core.djangoapps.content_libraries.api import create_library, create_library_block, delete_library_block -from openedx.core.lib.blockstore_api.tests.base import BlockstoreAppTestMixin from .. import api from ..models.base import TaxonomyOrg @@ -59,7 +58,6 @@ class LanguageTaxonomyTestMixin: class TestAutoTagging( # type: ignore[misc] LanguageTaxonomyTestMixin, ModuleStoreTestCase, - BlockstoreAppTestMixin, LiveServerTestCase ): """ diff --git a/openedx/core/djangoapps/olx_rest_api/views.py b/openedx/core/djangoapps/olx_rest_api/views.py index 8977b13153..1083ee06f6 100644 --- a/openedx/core/djangoapps/olx_rest_api/views.py +++ b/openedx/core/djangoapps/olx_rest_api/views.py @@ -13,7 +13,7 @@ from common.djangoapps.student.auth import has_studio_read_access from openedx.core.lib.api.view_utils import view_auth_classes from xmodule.modulestore.django import modulestore -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core @api_view(['GET']) @@ -22,7 +22,7 @@ def get_block_olx(request, usage_key_str): """ Given a modulestore XBlock usage ID (block-v1:...), get its OLX and a list of any static asset files it uses. - (There are other APIs for getting the OLX of Blockstore XBlocks.) + (There are other APIs for getting the OLX of Learning Core XBlocks.) """ # Parse the usage key: try: @@ -48,7 +48,7 @@ def get_block_olx(request, usage_key_str): return block = modulestore().get_item(block_key) - serialized_blocks[block_key] = serialize_modulestore_block_for_blockstore(block) + serialized_blocks[block_key] = serialize_modulestore_block_for_learning_core(block) if block.has_children: for child_id in block.children: @@ -103,7 +103,7 @@ def get_block_exportfs_file(request, usage_key_str, path): raise PermissionDenied("You must be a member of the course team in Studio to export OLX using this API.") block = modulestore().get_item(usage_key) - serialized = serialize_modulestore_block_for_blockstore(block) + serialized = serialize_modulestore_block_for_learning_core(block) static_file = None for f in serialized.static_files: if f.name == path: diff --git a/openedx/core/djangoapps/xblock/README.rst b/openedx/core/djangoapps/xblock/README.rst index 0afe70b9d1..245af22599 100644 --- a/openedx/core/djangoapps/xblock/README.rst +++ b/openedx/core/djangoapps/xblock/README.rst @@ -1,3 +1,8 @@ +This README was written back when the new runtime was backed by Blockstore. +Now that the runtime is backed by Learning Core, this README is out of date. +We need to audit and update it as part of +`this task `_. + XBlock App Suite (New) ====================== diff --git a/openedx/core/djangoapps/xblock/apps.py b/openedx/core/djangoapps/xblock/apps.py index afebd2ec71..5ba2361322 100644 --- a/openedx/core/djangoapps/xblock/apps.py +++ b/openedx/core/djangoapps/xblock/apps.py @@ -33,10 +33,7 @@ class XBlockAppConfig(AppConfig): def get_learning_context_params(self): """ Get additional kwargs that are passed to learning context implementations - (LearningContext subclass constructors). For example, this can be used to - specify that the course learning context should load the course's list of - blocks from the _draft_ version of the course in studio, but from the - published version of the course in the LMS. + (LearningContext subclass constructors). """ return {} @@ -68,8 +65,6 @@ class StudioXBlockAppConfig(XBlockAppConfig): Studio-specific configuration of the XBlock Runtime django app. """ - BLOCKSTORE_DRAFT_NAME = "studio_draft" - def get_runtime_system_params(self): """ Get the XBlockRuntimeSystem parameters appropriate for viewing and/or @@ -91,14 +86,9 @@ class StudioXBlockAppConfig(XBlockAppConfig): def get_learning_context_params(self): """ Get additional kwargs that are passed to learning context implementations - (LearningContext subclass constructors). For example, this can be used to - specify that the course learning context should load the course's list of - blocks from the _draft_ version of the course in studio, but from the - published version of the course in the LMS. + (LearningContext subclass constructors). """ - return { - "use_draft": self.BLOCKSTORE_DRAFT_NAME, - } + return {} def get_xblock_app_config(): diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index 72a6a7645e..1ac621ef24 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -53,15 +53,8 @@ class LearningContext: def definition_for_usage(self, usage_key, **kwargs): """ - Given a usage key for an XBlock in this context, return the - BundleDefinitionLocator which specifies the actual XBlock definition - (as a path to an OLX in a specific blockstore bundle). + Given a usage key in this context, return the key indicating the actual XBlock definition. - usage_key: the UsageKeyV2 subclass used for this learning context - - kwargs: optional additional parameters unique to the learning context - - Must return a BundleDefinitionLocator if the XBlock exists in this - context, or None otherwise. + Retuns None if the usage key doesn't exist in this context. """ raise NotImplementedError diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 2875c4b468..dc5a85de3a 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -21,7 +21,7 @@ from xblock.exceptions import NoSuchUsage from xblock.fields import Field, Scope, ScopeIds from xblock.field_data import FieldData -from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_blockstore +from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from ..learning_context.manager import get_learning_context_impl from .runtime import XBlockRuntime @@ -234,11 +234,7 @@ class LearningCoreXBlockRuntime(XBlockRuntime): log.warning("User %s does not have permission to edit %s", self.user.username, block.scope_ids.usage_id) raise RuntimeError("You do not have permission to edit this XBlock") - # We need Blockstore's serialization so we don't have `url_name` showing - # up in all the OLX. TODO: Rename this later, after we figure out what - # other changes we need to make in the serialization as part of the - # Blockstore -> Learning Core conversion. - serialized = serialize_modulestore_block_for_blockstore(block) + serialized = serialize_modulestore_block_for_learning_core(block) now = datetime.now(tz=timezone.utc) usage_key = block.scope_ids.usage_id with atomic(): diff --git a/openedx/core/djangoapps/xblock/runtime/runtime.py b/openedx/core/djangoapps/xblock/runtime/runtime.py index 15e0f3f061..5746af491d 100644 --- a/openedx/core/djangoapps/xblock/runtime/runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/runtime.py @@ -219,14 +219,14 @@ class XBlockRuntime(RuntimeShim, Runtime): def parse_xml_file(self, fileobj): # Deny access to the inherited method - raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime") + raise NotImplementedError("XML Serialization is only supported with LearningCoreXBlockRuntime") def add_node_as_child(self, block, node): """ Called by XBlock.parse_xml to treat a child node as a child block. """ # Deny access to the inherited method - raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime") + raise NotImplementedError("XML Serialization is only supported with LearningCoreXBlockRuntime") def service(self, block: XBlock, service_name: str): """ @@ -261,8 +261,8 @@ class XBlockRuntime(RuntimeShim, Runtime): return DjangoXBlockUserService( self.user, - # The value should be updated to whether the user is staff in the context when Blockstore runtime adds - # support for courses. + # The value should be updated to whether the user is staff in the context when Learning Core runtime + # adds support for courses. user_is_staff=self.user.is_staff, # type: ignore anonymous_user_id=self.anonymous_student_id, # See the docstring of `DjangoXBlockUserService`. @@ -437,7 +437,7 @@ class XBlockRuntimeSystem: student_data_mode: Specifies whether student data should be kept in a temporary in-memory store (e.g. Studio) or persisted forever in the database. - runtime_class: What runtime to use, e.g. BlockstoreXBlockRuntime + runtime_class: What runtime to use, e.g. LearningCoreXBlockRuntime """ self.handler_url = handler_url self.id_reader = id_reader or OpaqueKeyReader() diff --git a/openedx/core/djangoapps/xblock/runtime/shims.py b/openedx/core/djangoapps/xblock/runtime/shims.py index 18aa41eb91..c306b82bdc 100644 --- a/openedx/core/djangoapps/xblock/runtime/shims.py +++ b/openedx/core/djangoapps/xblock/runtime/shims.py @@ -102,7 +102,7 @@ class RuntimeShim: Only used for capa problems. """ - # TODO: load the python code from Blockstore. Ensure it's not publicly accessible. + # TODO: load the python code from Learning Core. Ensure it's not publicly accessible. return None @property @@ -166,9 +166,8 @@ class RuntimeShim: """ A filesystem that XBlocks can use to read large binary assets. """ - # TODO: implement this to serve any static assets that - # self._active_block has in its blockstore "folder". But this API should - # be deprecated and we should instead get compatible XBlocks to use a + # TODO: implement this to serve any static assets that self._active_block has. + # But this API should be deprecated and we should instead get compatible XBlocks to use a # runtime filesystem service. Some initial exploration of that (as well # as of the 'FileField' concept) has been done and is included in the # XBlock repo at xblock.reference.plugins.FSService and is available in diff --git a/openedx/core/lib/blockstore_api/__init__.py b/openedx/core/lib/blockstore_api/__init__.py deleted file mode 100644 index 855d8a1f96..0000000000 --- a/openedx/core/lib/blockstore_api/__init__.py +++ /dev/null @@ -1,51 +0,0 @@ -""" -API Client for Blockstore - -TODO: This should all get ripped out. - -TODO: This wrapper is extraneous now that Blockstore-as-a-service isn't supported. - This whole directory tree should be removed by https://github.com/openedx/blockstore/issues/296. -""" -from blockstore.apps.api.data import ( - BundleFileData, -) -from blockstore.apps.api.exceptions import ( - CollectionNotFound, - BundleNotFound, - DraftNotFound, - BundleVersionNotFound, - BundleFileNotFound, - BundleStorageError, -) -from blockstore.apps.api.methods import ( - # Collections: - get_collection, - create_collection, - update_collection, - delete_collection, - # Bundles: - get_bundles, - get_bundle, - create_bundle, - update_bundle, - delete_bundle, - # Drafts: - get_draft, - get_or_create_bundle_draft, - write_draft_file, - set_draft_link, - commit_draft, - delete_draft, - # Bundles or drafts: - get_bundle_files, - get_bundle_files_dict, - get_bundle_file_metadata, - get_bundle_file_data, - get_bundle_version, - get_bundle_version_files, - # Links: - get_bundle_links, - get_bundle_version_links, - # Misc: - force_browser_url, -) diff --git a/openedx/core/lib/blockstore_api/db_routers.py b/openedx/core/lib/blockstore_api/db_routers.py deleted file mode 100644 index fd0ff50c95..0000000000 --- a/openedx/core/lib/blockstore_api/db_routers.py +++ /dev/null @@ -1,60 +0,0 @@ -""" -Blockstore database router. - -Blockstore started life as an IDA, but is now a Django app plugin within edx-platform. -This router exists to smooth blockstore's transition into edxapp. -""" -from django.conf import settings - - -class BlockstoreRouter: - """ - A Database Router that uses the ``blockstore`` database, if it's configured in settings. - """ - ROUTE_APP_LABELS = {'bundles'} - DATABASE_NAME = 'blockstore' - - def _use_blockstore(self, model): - """ - Return True if the given model should use the blockstore database. - - Ensures that a ``blockstore`` database is configured, and checks the ``model``'s app label. - """ - return (self.DATABASE_NAME in settings.DATABASES) and (model._meta.app_label in self.ROUTE_APP_LABELS) - - def db_for_read(self, model, **hints): # pylint: disable=unused-argument - """ - Use the BlockstoreRouter.DATABASE_NAME when reading blockstore app tables. - """ - if self._use_blockstore(model): - return self.DATABASE_NAME - return None - - def db_for_write(self, model, **hints): # pylint: disable=unused-argument - """ - Use the BlockstoreRouter.DATABASE_NAME when writing to blockstore app tables. - """ - if self._use_blockstore(model): - return self.DATABASE_NAME - return None - - def allow_relation(self, obj1, obj2, **hints): # pylint: disable=unused-argument - """ - Allow relations if both objects are blockstore app models. - """ - if self._use_blockstore(obj1) and self._use_blockstore(obj2): - return True - return None - - def allow_migrate(self, db, app_label, model_name=None, **hints): # pylint: disable=unused-argument - """ - Ensure the blockstore tables only appear in the blockstore database. - """ - if model_name is not None: - model = hints.get('model') - if model is not None and self._use_blockstore(model): - return db == self.DATABASE_NAME - if db == self.DATABASE_NAME: - return False - - return None diff --git a/openedx/core/lib/blockstore_api/tests/__init__.py b/openedx/core/lib/blockstore_api/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/openedx/core/lib/blockstore_api/tests/base.py b/openedx/core/lib/blockstore_api/tests/base.py deleted file mode 100644 index 1d202d7671..0000000000 --- a/openedx/core/lib/blockstore_api/tests/base.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Common code for tests that work with Blockstore -""" -from unittest import mock -from urllib.parse import urlparse - -from django.test.client import RequestFactory - - -class BlockstoreAppTestMixin: - """ - Sets up the environment for tests to be run using the installed Blockstore app. - """ - def setUp(self): - """ - Ensure there's an active request, so that bundle file URLs can be made absolute. - """ - super().setUp() - - # Patch the blockstore get_current_request to use our live_server_url - mock.patch('blockstore.apps.api.methods.get_current_request', - mock.Mock(return_value=self._get_current_request())).start() - self.addCleanup(mock.patch.stopall) - - def _get_current_request(self): - """ - Returns a request object using the live_server_url, if available. - """ - request_args = {} - if hasattr(self, 'live_server_url'): - live_server_url = urlparse(self.live_server_url) - name, port = live_server_url.netloc.split(':') - request_args['SERVER_NAME'] = name - request_args['SERVER_PORT'] = port or '80' - request_args['wsgi.url_scheme'] = live_server_url.scheme - return RequestFactory().request(**request_args) diff --git a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py b/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py deleted file mode 100644 index 859b24e01d..0000000000 --- a/openedx/core/lib/blockstore_api/tests/test_blockstore_api.py +++ /dev/null @@ -1,201 +0,0 @@ -""" -Tests for xblock_utils.py -""" - -from uuid import UUID - -import pytest -from django.test import TestCase - -from openedx.core.lib import blockstore_api as api -from openedx.core.lib.blockstore_api.tests.base import ( - BlockstoreAppTestMixin, -) - -# A fake UUID that won't represent any real bundle/draft/collection: -BAD_UUID = UUID('12345678-0000-0000-0000-000000000000') - - -class BlockstoreApiClientTestMixin: - """ - Tests for the Blockstore API Client. - - The goal of these tests is not to test that Blockstore works correctly, but - that the API client can interact with it and all the API client methods - work. - """ - - # Collections - - def test_nonexistent_collection(self): - """ Request a collection that doesn't exist -> CollectionNotFound """ - with pytest.raises(api.CollectionNotFound): - api.get_collection(BAD_UUID) - - def test_collection_crud(self): - """ Create, Fetch, Update, and Delete a Collection """ - title = "Fire πŸ”₯ Collection" - # Create: - coll = api.create_collection(title) - assert coll.title == title - assert isinstance(coll.uuid, UUID) - # Fetch: - coll2 = api.get_collection(coll.uuid) - assert coll == coll2 - # Update: - new_title = "Air πŸŒ€ Collection" - coll3 = api.update_collection(coll.uuid, title=new_title) - assert coll3.title == new_title - coll4 = api.get_collection(coll.uuid) - assert coll4.title == new_title - # Delete: - api.delete_collection(coll.uuid) - with pytest.raises(api.CollectionNotFound): - api.get_collection(coll.uuid) - - # Bundles - - def test_nonexistent_bundle(self): - """ Request a bundle that doesn't exist -> BundleNotFound """ - with pytest.raises(api.BundleNotFound): - api.get_bundle(BAD_UUID) - - def test_bundle_crud(self): - """ Create, Fetch, Update, and Delete a Bundle """ - coll = api.create_collection("Test Collection") - args = { - "title": "Water πŸ’§ Bundle", - "slug": "h2o", - "description": "Sploosh", - } - # Create: - bundle = api.create_bundle(coll.uuid, **args) - for attr, value in args.items(): - assert getattr(bundle, attr) == value - assert isinstance(bundle.uuid, UUID) - # Fetch: - bundle2 = api.get_bundle(bundle.uuid) - assert bundle == bundle2 - # Update: - new_description = "Water Nation Bending Lessons" - bundle3 = api.update_bundle(bundle.uuid, description=new_description) - assert bundle3.description == new_description - bundle4 = api.get_bundle(bundle.uuid) - assert bundle4.description == new_description - # Delete: - api.delete_bundle(bundle.uuid) - with pytest.raises(api.BundleNotFound): - api.get_bundle(bundle.uuid) - - # Drafts, files, and reading/writing file contents: - - def test_nonexistent_draft(self): - """ Request a draft that doesn't exist -> DraftNotFound """ - with pytest.raises(api.DraftNotFound): - api.get_draft(BAD_UUID) - - def test_drafts_and_files(self): - """ - Test creating, reading, writing, committing, and reverting drafts and - files. - """ - coll = api.create_collection("Test Collection") - bundle = api.create_bundle(coll.uuid, title="Earth πŸ—Ώ Bundle", slug="earth", description="another test bundle") - # Create a draft - draft = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft.bundle_uuid == bundle.uuid - assert draft.name == 'test-draft' - assert draft.updated_at.year >= 2019 - # And retrieve it again: - draft2 = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft") - assert draft == draft2 - # Also test retrieving using get_draft - draft3 = api.get_draft(draft.uuid) - assert draft == draft3 - - # Write a file into the bundle: - api.write_draft_file(draft.uuid, "test.txt", b"initial version") - # Now the file should be visible in the draft: - draft_contents = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents == b'initial version' - api.commit_draft(draft.uuid) - - # Write a new version into the draft: - api.write_draft_file(draft.uuid, "test.txt", b"modified version") - published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt") - assert published_contents == b'initial version' - draft_contents2 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - assert draft_contents2 == b'modified version' - # Now delete the draft: - api.delete_draft(draft.uuid) - draft_contents3 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name) - # Confirm the file is now reset: - assert draft_contents3 == b'initial version' - - # Finaly, test the get_bundle_file* methods: - file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt") - assert file_info1.path == 'test.txt' - assert file_info1.size == len(b'initial version') - assert file_info1.hash_digest == 'a45a5c6716276a66c4005534a51453ab16ea63c4' - - assert list(api.get_bundle_files(bundle.uuid)) == [file_info1] - assert api.get_bundle_files_dict(bundle.uuid) == {'test.txt': file_info1} - - # Links - - def test_links(self): - """ - Test operations involving bundle links. - """ - coll = api.create_collection("Test Collection") - # Create two library bundles and a course bundle: - lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1") - lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft") - lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2") - lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft") - course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course") - course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft") - - # To create links, we need valid BundleVersions, which requires having committed at least one change: - api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world") - api.commit_draft(lib1_draft.uuid) # Creates version 1 - api.write_draft_file(lib2_draft.uuid, "lib2-data.txt", "hello world") - api.commit_draft(lib2_draft.uuid) # Creates version 1 - - # Lib2 has no links: - assert not api.get_bundle_links(lib2_bundle.uuid) - - # Create a link from lib2 to lib1 - link1_name = "lib2_to_lib1" - api.set_draft_link(lib2_draft.uuid, link1_name, lib1_bundle.uuid, version=1) - # Now confirm the link exists in the draft: - lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name) - assert link1_name in lib2_draft_links - assert lib2_draft_links[link1_name].direct.bundle_uuid == lib1_bundle.uuid - assert lib2_draft_links[link1_name].direct.version == 1 - # Now commit the change to lib2: - api.commit_draft(lib2_draft.uuid) # Creates version 2 - - # Now create a link from course to lib2 - link2_name = "course_to_lib2" - api.set_draft_link(course_draft.uuid, link2_name, lib2_bundle.uuid, version=2) - api.commit_draft(course_draft.uuid) - - # And confirm the link exists in the resulting bundle version: - course_links = api.get_bundle_links(course_bundle.uuid) - assert link2_name in course_links - assert course_links[link2_name].direct.bundle_uuid == lib2_bundle.uuid - assert course_links[link2_name].direct.version == 2 - # And since the links go course->lib2->lib1, course has an indirect link to lib1: - assert course_links[link2_name].indirect[0].bundle_uuid == lib1_bundle.uuid - assert course_links[link2_name].indirect[0].version == 1 - - # Finally, test deleting a link from course's draft: - api.set_draft_link(course_draft.uuid, link2_name, None, None) - assert not api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name) - - -class BlockstoreAppApiClientTest(BlockstoreApiClientTestMixin, BlockstoreAppTestMixin, TestCase): - """ - Test the Blockstore API Client, using the installed Blockstore app. - """ diff --git a/openedx/core/lib/xblock_serializer/api.py b/openedx/core/lib/xblock_serializer/api.py index 30dbc8321b..8ac1cd5717 100644 --- a/openedx/core/lib/xblock_serializer/api.py +++ b/openedx/core/lib/xblock_serializer/api.py @@ -2,7 +2,7 @@ Public python API for serializing XBlocks to OLX """ # pylint: disable=unused-import -from .block_serializer import StaticFile, XBlockSerializer, XBlockSerializerForBlockstore +from .block_serializer import StaticFile, XBlockSerializer, XBlockSerializerForLearningCore def serialize_xblock_to_olx(block): @@ -14,10 +14,10 @@ def serialize_xblock_to_olx(block): return XBlockSerializer(block) -def serialize_modulestore_block_for_blockstore(block): +def serialize_modulestore_block_for_learning_core(block): """ This class will serialize an XBlock, producing: - (1) A new definition ID for use in Blockstore + (1) A new definition ID for use in Learning Core (2) an XML string defining the XBlock and referencing the IDs of its children using syntax (which doesn't actually contain the OLX of its children, just refers to them, so you have to @@ -29,4 +29,4 @@ def serialize_modulestore_block_for_blockstore(block): we have around how we should rewrite this (e.g. are we going to remove ?). """ - return XBlockSerializerForBlockstore(block) + return XBlockSerializerForLearningCore(block) diff --git a/openedx/core/lib/xblock_serializer/block_serializer.py b/openedx/core/lib/xblock_serializer/block_serializer.py index af155a3900..966380f250 100644 --- a/openedx/core/lib/xblock_serializer/block_serializer.py +++ b/openedx/core/lib/xblock_serializer/block_serializer.py @@ -137,10 +137,10 @@ class XBlockSerializer: return olx_node -class XBlockSerializerForBlockstore(XBlockSerializer): +class XBlockSerializerForLearningCore(XBlockSerializer): """ This class will serialize an XBlock, producing: - (1) A new definition ID for use in Blockstore + (1) A new definition ID for use in Learning Core (2) an XML string defining the XBlock and referencing the IDs of its children using syntax (which doesn't actually contain the OLX of its children, just refers to them, so you have to @@ -154,7 +154,7 @@ class XBlockSerializerForBlockstore(XBlockSerializer): resulting data in this object. """ super().__init__(block) - self.def_id = utils.blockstore_def_key_from_modulestore_usage_key(self.orig_block_key) + self.def_id = utils.learning_core_def_key_from_modulestore_usage_key(self.orig_block_key) def _serialize_block(self, block) -> etree.Element: """ Serialize an XBlock to OLX/XML. """ @@ -174,12 +174,12 @@ class XBlockSerializerForBlockstore(XBlockSerializer): # the same block to be used in many places (each with a unique # usage key). However, that functionality is not exposed in # Studio (other than via content libraries). So when we import - # into Blockstore, we assume that each usage is unique, don't + # into Learning Core, we assume that each usage is unique, don't # generate a usage key, and create a new "definition key" from # the original usage key. # So modulestore usage key # block-v1:A+B+C+type@html+block@introduction - # will become Blockstore definition key + # will become Learning Core definition key # html+introduction # # If we needed the real definition key, we could get it via @@ -187,7 +187,7 @@ class XBlockSerializerForBlockstore(XBlockSerializer): # child_def_id = str(child.scope_ids.def_id) # and then use # - def_id = utils.blockstore_def_key_from_modulestore_usage_key(child_id) + def_id = utils.learning_core_def_key_from_modulestore_usage_key(child_id) parent_olx_node.append(parent_olx_node.makeelement("xblock-include", {"definition": def_id})) def _transform_olx(self, olx_node, usage_id): diff --git a/openedx/core/lib/xblock_serializer/test_api.py b/openedx/core/lib/xblock_serializer/test_api.py index 39ca6bd675..6b2c0dfeb9 100644 --- a/openedx/core/lib/xblock_serializer/test_api.py +++ b/openedx/core/lib/xblock_serializer/test_api.py @@ -240,17 +240,17 @@ class XBlockSerializationTestCase(SharedModuleStoreTestCase): ), ]) - def test_html_with_static_asset_blockstore(self): + def test_html_with_static_asset_learning_core(self): """ - Test the blockstore-specific serialization of an HTML block + Test the learning-core-specific serialization of an HTML block """ block_id = self.course.id.make_usage_key('html', 'just_img') # see sample_courses.py html_block = modulestore().get_item(block_id) serialized = api.serialize_xblock_to_olx(html_block) - serialized_blockstore = api.serialize_modulestore_block_for_blockstore(html_block) + serialized_learning_core = api.serialize_modulestore_block_for_learning_core(html_block) self.assertXmlEqual( - serialized_blockstore.olx_str, - # For blockstore, OLX should never contain "url_name" as that ID is specified by the filename: + serialized_learning_core.olx_str, + # For learning core, OLX should never contain "url_name" as that ID is specified by the filename: """ @@ -259,9 +259,9 @@ class XBlockSerializationTestCase(SharedModuleStoreTestCase): ) self.assertIn("CDATA", serialized.olx_str) # Static files should be identical: - self.assertEqual(serialized.static_files, serialized_blockstore.static_files) - # This is the only other difference - an extra field with the blockstore-specific definition ID: - self.assertEqual(serialized_blockstore.def_id, "html/just_img") + self.assertEqual(serialized.static_files, serialized_learning_core.static_files) + # This is the only other difference - an extra field with the learning-core-specific definition ID: + self.assertEqual(serialized_learning_core.def_id, "html/just_img") def test_html_with_fields(self): """ Test an HTML Block with non-default fields like editor='raw' """ @@ -299,13 +299,13 @@ class XBlockSerializationTestCase(SharedModuleStoreTestCase): self.assertXmlEqual(serialized.olx_str, EXPECTED_SEQUENTIAL_OLX) - def test_export_sequential_blockstore(self): + def test_export_sequential_learning_core(self): """ - Export a sequential from the toy course, formatted for blockstore. + Export a sequential from the toy course, formatted for learning core. """ sequential_id = self.course.id.make_usage_key('sequential', 'Toy_Videos') # see sample_courses.py sequential = modulestore().get_item(sequential_id) - serialized = api.serialize_modulestore_block_for_blockstore(sequential) + serialized = api.serialize_modulestore_block_for_learning_core(sequential) self.assertXmlEqual(serialized.olx_str, """ diff --git a/openedx/core/lib/xblock_serializer/utils.py b/openedx/core/lib/xblock_serializer/utils.py index 2c736ae299..e78c900b18 100644 --- a/openedx/core/lib/xblock_serializer/utils.py +++ b/openedx/core/lib/xblock_serializer/utils.py @@ -225,17 +225,17 @@ def override_export_fs(block): XmlMixin.export_to_file = old_global_export_to_file -def blockstore_def_key_from_modulestore_usage_key(usage_key): +def learning_core_def_key_from_modulestore_usage_key(usage_key): """ In modulestore, the "definition key" is a MongoDB ObjectID kept in split's definitions table, which theoretically allows the same block to be used in many places (each with a unique usage key). However, that functionality is not exposed in Studio (other than via content libraries). So when we import - into Blockstore, we assume that each usage is unique, don't generate a usage + into learning core, we assume that each usage is unique, don't generate a usage key, and create a new "definition key" from the original usage key. So modulestore usage key block-v1:A+B+C+type@html+block@introduction - will become Blockstore definition key + will become learning core definition key html/introduction """ block_type = usage_key.block_type diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 0c63a94998..ff4319a4fc 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -44,7 +44,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -205,7 +204,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -239,7 +237,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -278,8 +275,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via openedx-blockstore django-fernet-fields-v2==0.9 # via edx-enterprise django-filter==24.2 @@ -287,7 +282,6 @@ django-filter==24.2 # -r requirements/edx/kernel.in # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/kernel.in @@ -368,7 +362,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -381,7 +374,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -392,7 +384,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -402,8 +393,6 @@ done-xblock==2.3.0 # via -r requirements/edx/bundled.in drf-jwt==1.19.2 # via edx-drf-extensions -drf-nested-routers==0.93.5 - # via openedx-blockstore drf-spectacular==0.27.2 # via -r requirements/edx/kernel.in drf-yasg==1.21.5 @@ -417,11 +406,8 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/kernel.in # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/kernel.in - # openedx-blockstore + # via -r requirements/edx/kernel.in edx-braze-client==0.2.5 # via # -r requirements/edx/bundled.in @@ -448,7 +434,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/kernel.in # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/kernel.in edx-django-utils==5.13.0 @@ -464,7 +449,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -738,9 +722,7 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/kernel.in - # openedx-blockstore + # via -r requirements/edx/kernel.in newrelic==9.9.0 # via # -r requirements/edx/bundled.in @@ -770,8 +752,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/kernel.in -openedx-blockstore==1.4.0 - # via -r requirements/edx/kernel.in openedx-calc==3.1.0 # via -r requirements/edx/kernel.in openedx-django-pyfs==3.6.0 @@ -971,7 +951,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1119,9 +1098,7 @@ sortedcontainers==2.4.0 soupsieve==2.5 # via beautifulsoup4 sqlparse==0.5.0 - # via - # django - # openedx-blockstore + # via django staff-graded-xblock==2.3.0 # via -r requirements/edx/bundled.in stevedore==5.2.0 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 53e3f7a1c7..7cf594a15e 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -96,7 +96,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -379,7 +378,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -413,7 +411,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -470,11 +467,6 @@ django-crum==0.7.9 # super-csv django-debug-toolbar==4.3.0 # via -r requirements/edx/development.in -django-environ==0.11.2 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/doc.txt @@ -486,7 +478,6 @@ django-filter==24.2 # -r requirements/edx/testing.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/doc.txt @@ -604,7 +595,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -619,7 +609,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -630,7 +619,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -662,11 +650,6 @@ drf-jwt==1.19.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-drf-extensions -drf-nested-routers==0.93.5 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt - # openedx-blockstore drf-spectacular==0.27.2 # via # -r requirements/edx/doc.txt @@ -687,12 +670,10 @@ edx-api-doc-tools==1.8.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt - # openedx-blockstore edx-braze-client==0.2.5 # via # -r requirements/edx/doc.txt @@ -728,7 +709,6 @@ edx-django-release-util==1.4.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via # -r requirements/edx/doc.txt @@ -747,7 +727,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -1275,7 +1254,6 @@ mysqlclient==2.2.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt - # openedx-blockstore newrelic==9.9.0 # via # -r requirements/edx/doc.txt @@ -1321,10 +1299,6 @@ openedx-atlas==0.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -openedx-blockstore==1.4.0 - # via - # -r requirements/edx/doc.txt - # -r requirements/edx/testing.txt openedx-calc==3.1.0 # via # -r requirements/edx/doc.txt @@ -1743,7 +1717,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -2031,7 +2004,6 @@ sqlparse==0.5.0 # -r requirements/edx/testing.txt # django # django-debug-toolbar - # openedx-blockstore staff-graded-xblock==2.3.0 # via # -r requirements/edx/doc.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index d553ab6ae2..57c0936a09 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -63,7 +63,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -255,7 +254,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -289,7 +287,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -334,10 +331,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via - # -r requirements/edx/base.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/base.txt @@ -347,7 +340,6 @@ django-filter==24.2 # -r requirements/edx/base.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/base.txt @@ -434,7 +426,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -447,7 +438,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -458,7 +448,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -477,10 +466,6 @@ drf-jwt==1.19.2 # via # -r requirements/edx/base.txt # edx-drf-extensions -drf-nested-routers==0.93.5 - # via - # -r requirements/edx/base.txt - # openedx-blockstore drf-spectacular==0.27.2 # via -r requirements/edx/base.txt drf-yasg==1.21.5 @@ -495,11 +480,8 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/base.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt @@ -526,7 +508,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/base.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt edx-django-utils==5.13.0 @@ -542,7 +523,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -869,9 +849,7 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt newrelic==9.9.0 # via # -r requirements/edx/base.txt @@ -905,8 +883,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/base.txt -openedx-blockstore==1.4.0 - # via -r requirements/edx/base.txt openedx-calc==3.1.0 # via -r requirements/edx/base.txt openedx-django-pyfs==3.6.0 @@ -1151,7 +1127,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1363,7 +1338,6 @@ sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django - # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt stevedore==5.2.0 diff --git a/requirements/edx/github.in b/requirements/edx/github.in index ea6d47eec8..6ec36d3a06 100644 --- a/requirements/edx/github.in +++ b/requirements/edx/github.in @@ -29,16 +29,16 @@ # # For example: # -# # https://github.com/openedx/blockstore/issues/212 -# git+https://github.com/openedx/blockstore.git@v1.3.0#egg=openedx-blockstore==1.3.0 +# # https://github.com/openedx/foobar/issues/212 +# git+https://github.com/openedx/foobar.git@v1.3.0#egg=openedx-foobar==1.3.0 # # where: # -# ISSUE-LINK = https://github.com/openedx/blockstore/issues/212 +# ISSUE-LINK = https://github.com/openedx/foobar/issues/212 # OWNER = openedx -# REPO-NAME = blockstore +# REPO-NAME = foobar # TAG-OR-SHA = v1.3.0 -# DIST-NAME = openedx-blockstore +# DIST-NAME = openedx-foobar # VERSION = 1.3.0 # # Rules to follow: diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 47a2435939..1a7abb8364 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -120,7 +120,6 @@ openedx-filters # Open edX Filters from Hooks Extension Fram openedx-learning # Open edX Learning core (experimental) openedx-mongodbproxy openedx-django-wiki -openedx-blockstore path piexif # Exif image metadata manipulation, used in the profile_images app Pillow # Image manipulation library; used for course assets, profile images, invoice PDFs, etc. diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 314f28d1cb..45e2b76578 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -68,7 +68,6 @@ attrs==23.2.0 # edx-ace # jsonschema # lti-consumer-xblock - # openedx-blockstore # openedx-events # openedx-learning # referencing @@ -291,7 +290,6 @@ django==4.2.13 # djangorestframework # done-xblock # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-ace @@ -325,7 +323,6 @@ django==4.2.13 # help-tokens # jsonfield # lti-consumer-xblock - # openedx-blockstore # openedx-django-pyfs # openedx-django-wiki # openedx-events @@ -370,10 +367,6 @@ django-crum==0.7.9 # edx-rbac # edx-toggles # super-csv -django-environ==0.11.2 - # via - # -r requirements/edx/base.txt - # openedx-blockstore django-fernet-fields-v2==0.9 # via # -r requirements/edx/base.txt @@ -383,7 +376,6 @@ django-filter==24.2 # -r requirements/edx/base.txt # edx-enterprise # lti-consumer-xblock - # openedx-blockstore django-ipware==7.0.1 # via # -r requirements/edx/base.txt @@ -470,7 +462,6 @@ django-waffle==4.1.0 # edx-enterprise # edx-proctoring # edx-toggles - # openedx-blockstore django-webpack-loader==0.7.0 # via # -c requirements/edx/../constraints.txt @@ -483,7 +474,6 @@ djangorestframework==3.14.0 # django-config-models # django-user-tasks # drf-jwt - # drf-nested-routers # drf-spectacular # drf-yasg # edx-api-doc-tools @@ -494,7 +484,6 @@ djangorestframework==3.14.0 # edx-organizations # edx-proctoring # edx-submissions - # openedx-blockstore # openedx-learning # ora2 # super-csv @@ -510,10 +499,6 @@ drf-jwt==1.19.2 # via # -r requirements/edx/base.txt # edx-drf-extensions -drf-nested-routers==0.93.5 - # via - # -r requirements/edx/base.txt - # openedx-blockstore drf-spectacular==0.27.2 # via -r requirements/edx/base.txt drf-yasg==1.21.5 @@ -528,11 +513,8 @@ edx-api-doc-tools==1.8.0 # via # -r requirements/edx/base.txt # edx-name-affirmation - # openedx-blockstore edx-auth-backends==4.3.0 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt edx-braze-client==0.2.5 # via # -r requirements/edx/base.txt @@ -559,7 +541,6 @@ edx-django-release-util==1.4.0 # via # -r requirements/edx/base.txt # edxval - # openedx-blockstore edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt edx-django-utils==5.13.0 @@ -575,7 +556,6 @@ edx-django-utils==5.13.0 # edx-toggles # edx-when # event-tracking - # openedx-blockstore # openedx-events # ora2 # super-csv @@ -954,9 +934,7 @@ multidict==6.0.5 # aiohttp # yarl mysqlclient==2.2.4 - # via - # -r requirements/edx/base.txt - # openedx-blockstore + # via -r requirements/edx/base.txt newrelic==9.9.0 # via # -r requirements/edx/base.txt @@ -990,8 +968,6 @@ openai==0.28.1 # edx-enterprise openedx-atlas==0.6.0 # via -r requirements/edx/base.txt -openedx-blockstore==1.4.0 - # via -r requirements/edx/base.txt openedx-calc==3.1.0 # via -r requirements/edx/base.txt openedx-django-pyfs==3.6.0 @@ -1310,7 +1286,6 @@ pytz==2024.1 # icalendar # interchange # olxcleaner - # openedx-blockstore # ora2 # snowflake-connector-python # xblock @@ -1493,7 +1468,6 @@ sqlparse==0.5.0 # via # -r requirements/edx/base.txt # django - # openedx-blockstore staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt starlette==0.37.2 diff --git a/xmodule/README.rst b/xmodule/README.rst index 5096c78c2a..8bbae713b7 100644 --- a/xmodule/README.rst +++ b/xmodule/README.rst @@ -41,14 +41,14 @@ Direction Currently, this directory contains a lot of mission-critical functionality, so continued maintenance and simplification of it is important. Still, we aim to eventually dissolve the directory in favor of more focused & decoupled subsystems: -* ModuleStore is superseded by the `Blockstore`_ storage backend. -* Blockstore-backend content is rendered by a new, simplified `edx-platform XBlock runtime`_. +* ModuleStore is superseded by the `Learning Core`_ storage backend. +* Learning Core-backend content is rendered by a new, simplified `edx-platform XBlock runtime`_. * Navigation, partitioning, and composition of learning content is being re-architected in the `openedx-learning`_ library. * All new XBlocks are implemented in separate repositories, such as `xblock-drag-and-drop-v2`_. To help with this direction, please **do not add new functionality to this directory**. If you feel that you need to add code to this directory, reach out on `the forums`_; it's likely that someone can help you find a different way to implement your change that will be more robust and architecturally sound! -.. _Blockstore: https://github.com/openedx/blockstore/ +.. _Learning Core: https://github.com/openedx/openedx-learning/ .. _edx-platform XBlock runtime: https://github.com/openedx/edx-platform/tree/master/openedx/core/djangoapps/xblock .. _openedx-learning: https://github.com/openedx/openedx-learning .. _xblock-drag-and-drop-v2: https://github.com/openedx/xblock-drag-and-drop-v2 diff --git a/xmodule/html_block.py b/xmodule/html_block.py index 6c883e1322..2db1983601 100644 --- a/xmodule/html_block.py +++ b/xmodule/html_block.py @@ -279,7 +279,7 @@ class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method @classmethod def parse_xml_new_runtime(cls, node, runtime, keys): """ - Parse XML in the new blockstore-based runtime. Since it doesn't yet + Parse XML in the new learning-core-based runtime. Since it doesn't yet support loading separate .html files, the HTML data is assumed to be in a CDATA child or otherwise just inline in the OLX. """ diff --git a/xmodule/library_tools.py b/xmodule/library_tools.py index 7aa0362ea4..2c077a8884 100644 --- a/xmodule/library_tools.py +++ b/xmodule/library_tools.py @@ -25,7 +25,7 @@ class LibraryToolsService: """ Service for LibraryContentBlock. - Allows to interact with libraries in the modulestore and blockstore. + Allows to interact with libraries in the modulestore and learning core. Should only be used in the CMS. """ diff --git a/xmodule/raw_block.py b/xmodule/raw_block.py index 58b21795f2..f24dd7712d 100644 --- a/xmodule/raw_block.py +++ b/xmodule/raw_block.py @@ -66,7 +66,7 @@ class RawMixin: Interpret the parsed XML in `node`, creating a new instance of this module. """ - # In the new/blockstore-based runtime, XModule parsing (from + # In the new/learning-core-based runtime, XModule parsing (from # XmlMixin) is disabled, so definition_from_xml will not be # called, and instead the "normal" XBlock parse_xml will be used. # However, it's not compatible with RawMixin, so we implement diff --git a/xmodule/tests/test_library_tools.py b/xmodule/tests/test_library_tools.py index c6185f8754..f93066cd5c 100644 --- a/xmodule/tests/test_library_tools.py +++ b/xmodule/tests/test_library_tools.py @@ -28,7 +28,7 @@ class ContentLibraryToolsTest(MixedSplitTestCase, ContentLibrariesRestApiTest): """ Tests for LibraryToolsService. - Tests interaction with blockstore-based (V2) and mongo-based (V1) content libraries. + Tests interaction with learning-core-based (V2) and mongo-based (V1) content libraries. """ def setUp(self): super().setUp() diff --git a/xmodule/video_block/transcripts_utils.py b/xmodule/video_block/transcripts_utils.py index 16de851ea5..d82a5d3f47 100644 --- a/xmodule/video_block/transcripts_utils.py +++ b/xmodule/video_block/transcripts_utils.py @@ -15,7 +15,7 @@ import requests import simplejson as json from django.conf import settings from lxml import etree -from opaque_keys.edx.locator import BundleDefinitionLocator +from opaque_keys.edx.keys import UsageKeyV2 from pysrt import SubRipFile, SubRipItem, SubRipTime from pysrt.srtexc import Error @@ -945,7 +945,7 @@ def get_transcript_for_video(video_location, subs_id, file_name, language): """ Get video transcript from content store. This is a lower level function and is used by `get_transcript_from_contentstore`. Prefer that function instead where possible. If you - need to support getting transcripts from VAL or Blockstore as well, use the `get_transcript` + need to support getting transcripts from VAL or Learning Core as well, use the `get_transcript` function instead. NOTE: Transcripts can be searched from content store by two ways: @@ -1033,29 +1033,31 @@ def get_transcript_from_contentstore(video, language, output_format, transcripts return transcript_content, transcript_name, Transcript.mime_types[output_format] -def get_transcript_from_blockstore(video_block, language, output_format, transcripts_info): +def get_transcript_from_learning_core(video_block, language, output_format, transcripts_info): """ - Get video transcript from Blockstore. + Get video transcript from Learning Core. - Blockstore expects video transcripts to be placed into the 'static/' - subfolder of the XBlock's folder in a Blockstore bundle. For example, if the - video XBlock's definition is in the standard location of - video/video1/definition.xml - Then the .srt files should be placed at e.g. - video/video1/static/video1-en.srt - This is the same place where other public static files are placed for other - XBlocks, such as image files used by HTML blocks. + HISTORIC INFORMATION FROM WHEN THIS FUNCTION WAS `get_transcript_from_blockstore`: - Video XBlocks in Blockstore must set the 'transcripts' XBlock field to a - JSON dictionary listing the filename of the transcript for each language: -