Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

feat!: remove Content Libraries V2 index (#33888)

Browse Source 
BREAKING CHANGE: Removes all code, tests, and settings related to
indexing of V2 (blockstore-backed) content libraries in elasticsearch.
This includes indexing of top-level library metadata as well as indexing
of library block metadata. Operators who enabled the experimental
Library Authoring MFE *and* the experimental ENABLE_CONTENT_LIBRARY_INDEX
feature may notice that sorting, filtering, and searching of V2
libraries and their blocks may now be slower and/or less powerful.
The ENABLE_CONTENT_LIBRARY_INDEX feature was already disabled by
default, so most/all operators (including edx.org) should not notice
any difference.

Removed settings include:

* FEATURES['ENABLE_CONTENT_LIBRARY_INDEX']
* ENABLE_ELASTICSEARCH_FOR_TESTS
* TEST_ELASTICSEARCH_USE_SSL
* TEST_ELASTICSEARCH_HOST
* TEST_ELASTICSEARCH_PORT

For rationale, see the updated "Status" section of:
./openedx/core/djangoapps/content_libraries/docs/decisions/0001-index-libraries-in-elasticsearch.rst
This commit is contained in:
Kyle McCormick
2023-12-07 14:57:31 -05:00
committed by GitHub
parent 5ca08fb562
commit 140f85853d
16 changed files with 179 additions and 1046 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cms/djangoapps/contentstore/tests/test_libraries.py
View File

@@ -440,10 +440,6 @@ class TestLibraries(LibraryTestCase):
html_block_2 = modulestore().get_item(lc_block.children[0])
self.assertEqual(html_block_2.data, data2)
@patch(
'openedx.core.djangoapps.content_libraries.tasks.SearchEngine.get_search_engine',
Mock(return_value=None, autospec=True),
)
def test_sync_if_capa_type_changed(self):
""" Tests that children are automatically refreshed if capa type field changes """
name1, name2 = "Option Problem", "Multiple Choice Problem"

3
cms/envs/common.py
View File

@@ -297,9 +297,6 @@ FEATURES = {
# Enable content libraries (modulestore) search functionality
'ENABLE_LIBRARY_INDEX': False,
# Enable content libraries (blockstore) indexing
'ENABLE_CONTENT_LIBRARY_INDEX': False,
# .. toggle_name: FEATURES['ALLOW_COURSE_RERUNS']
# .. toggle_implementation: DjangoSetting
# .. toggle_default: True

1
cms/envs/devstack.py
View File

@@ -147,7 +147,6 @@ XBLOCK_SETTINGS.update({'VideoBlock': {'licensing_enabled': True}})
################################ SEARCH INDEX ################################
FEATURES['ENABLE_COURSEWARE_INDEX'] = True
FEATURES['ENABLE_LIBRARY_INDEX'] = False
FEATURES['ENABLE_CONTENT_LIBRARY_INDEX'] = False
SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"
ELASTIC_SEARCH_CONFIG = [

2
cms/envs/production.py
View File

@@ -510,7 +510,7 @@ PARSE_KEYS = AUTH_TOKENS.get("PARSE_KEYS", {})
# Example: {'CN': 'http://api.xuetangx.com/edx/video?s3_url='}
VIDEO_CDN_URL = ENV_TOKENS.get('VIDEO_CDN_URL', {})
if FEATURES['ENABLE_COURSEWARE_INDEX'] or FEATURES['ENABLE_LIBRARY_INDEX'] or FEATURES['ENABLE_CONTENT_LIBRARY_INDEX']:
if FEATURES['ENABLE_COURSEWARE_INDEX'] or FEATURES['ENABLE_LIBRARY_INDEX']:
# Use ElasticSearch for the search engine
SEARCH_ENGINE = "search.elastic.ElasticSearchEngine"

11
cms/envs/test.py
View File

@@ -259,21 +259,10 @@ VIDEO_CDN_URL = {
# Courseware Search Index
FEATURES['ENABLE_COURSEWARE_INDEX'] = True
FEATURES['ENABLE_LIBRARY_INDEX'] = True
FEATURES['ENABLE_CONTENT_LIBRARY_INDEX'] = False
SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
FEATURES['ENABLE_ENROLLMENT_TRACK_USER_PARTITION'] = True
####################### ELASTICSEARCH TESTS #######################
# Enable this when testing elasticsearch-based code which couldn't be tested using the mock engine
ENABLE_ELASTICSEARCH_FOR_TESTS = os.environ.get(
'EDXAPP_ENABLE_ELASTICSEARCH_FOR_TESTS', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_USE_SSL = os.environ.get(
'EDXAPP_TEST_ELASTICSEARCH_USE_SSL', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_HOST = os.environ.get('EDXAPP_TEST_ELASTICSEARCH_HOST', 'edx.devstack.elasticsearch710')
TEST_ELASTICSEARCH_PORT = int(os.environ.get('EDXAPP_TEST_ELASTICSEARCH_PORT', '9200'))
########################## AUTHOR PERMISSION #######################
FEATURES['ENABLE_CREATOR_GROUP'] = False

10
lms/envs/test.py
View File

@@ -418,16 +418,6 @@ FACEBOOK_APP_SECRET = "Test"
FACEBOOK_APP_ID = "Test"
FACEBOOK_API_VERSION = "v2.8"
####################### ELASTICSEARCH TESTS #######################
# Enable this when testing elasticsearch-based code which couldn't be tested using the mock engine
ENABLE_ELASTICSEARCH_FOR_TESTS = os.environ.get(
'EDXAPP_ENABLE_ELASTICSEARCH_FOR_TESTS', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_USE_SSL = os.environ.get(
'EDXAPP_TEST_ELASTICSEARCH_USE_SSL', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_HOST = os.environ.get('EDXAPP_TEST_ELASTICSEARCH_HOST', 'edx.devstack.elasticsearch710')
TEST_ELASTICSEARCH_PORT = int(os.environ.get('EDXAPP_TEST_ELASTICSEARCH_PORT', '9200'))
######### custom courses #########
INSTALLED_APPS += ['lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig']
FEATURES['CUSTOM_COURSES_EDX'] = True

135
openedx/core/djangoapps/content_libraries/api.py
View File

@@ -67,7 +67,6 @@ from django.core.exceptions import PermissionDenied
from django.core.validators import validate_unicode_slug
from django.db import IntegrityError, transaction
from django.utils.translation import gettext as _
from elasticsearch.exceptions import ConnectionError as ElasticConnectionError
from lxml import etree
from opaque_keys.edx.keys import LearningContextKey, UsageKey
from opaque_keys.edx.locator import (
@@ -94,7 +93,6 @@ from edx_rest_api_client.client import OAuthAPIClient
from openedx.core.djangoapps.content_libraries import permissions
from openedx.core.djangoapps.content_libraries.constants import DRAFT_NAME, COMPLEX
from openedx.core.djangoapps.content_libraries.library_bundle import LibraryBundle
from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryBlockIndexer
from openedx.core.djangoapps.content_libraries.models import (
ContentLibrary,
ContentLibraryPermission,
@@ -290,56 +288,35 @@ def get_libraries_for_user(user, org=None, library_type=None):
return permissions.perms[permissions.CAN_VIEW_THIS_CONTENT_LIBRARY].filter(user, qs)
def get_metadata_from_index(queryset, text_search=None):
def get_metadata(queryset, text_search=None):
"""
Take a list of ContentLibrary objects and return metadata stored in
ContentLibraryIndex.
Take a list of ContentLibrary objects and return metadata from blockstore.
"""
metadata = None
if ContentLibraryIndexer.indexing_is_enabled():
try:
library_keys = [str(lib.library_key) for lib in queryset]
metadata = ContentLibraryIndexer.get_items(library_keys, text_search=text_search)
metadata_dict = {
item["id"]: item
for item in metadata
}
metadata = [
metadata_dict[key]
if key in metadata_dict
else None
for key in library_keys
]
except ElasticConnectionError as e:
log.exception(e)
uuids = [lib.bundle_uuid for lib in queryset]
bundles = get_bundles(uuids=uuids, text_search=text_search)
# If ContentLibraryIndex is not available, we query blockstore for a limited set of metadata
if metadata is None:
uuids = [lib.bundle_uuid for lib in queryset]
bundles = get_bundles(uuids=uuids, text_search=text_search)
if text_search:
# Bundle APIs can't apply text_search on a bundle's org, so including those results here
queryset_org_search = queryset.filter(org__short_name__icontains=text_search)
if queryset_org_search.exists():
uuids_org_search = [lib.bundle_uuid for lib in queryset_org_search]
bundles += get_bundles(uuids=uuids_org_search)
if text_search:
# Bundle APIs can't apply text_search on a bundle's org, so including those results here
queryset_org_search = queryset.filter(org__short_name__icontains=text_search)
if queryset_org_search.exists():
uuids_org_search = [lib.bundle_uuid for lib in queryset_org_search]
bundles += get_bundles(uuids=uuids_org_search)
bundle_dict = {
bundle.uuid: {
'uuid': bundle.uuid,
'title': bundle.title,
'description': bundle.description,
'version': bundle.latest_version,
}
for bundle in bundles
bundle_dict = {
bundle.uuid: {
'uuid': bundle.uuid,
'title': bundle.title,
'description': bundle.description,
'version': bundle.latest_version,
}
metadata = [
bundle_dict[uuid]
if uuid in bundle_dict
else None
for uuid in uuids
]
for bundle in bundles
}
metadata = [
bundle_dict[uuid]
if uuid in bundle_dict
else None
for uuid in uuids
]
libraries = [
ContentLibraryMetadata(
@@ -648,50 +625,28 @@ def get_library_blocks(library_key, text_search=None, block_types=None) -> list[
Returns a list of LibraryXBlockMetadata objects
"""
metadata = None
if LibraryBlockIndexer.indexing_is_enabled():
try:
filter_terms = {
'library_key': [str(library_key)],
'is_child': [False],
}
if block_types:
filter_terms['block_type'] = block_types
metadata = [
{
**item,
"id": LibraryUsageLocatorV2.from_string(item['id']),
}
for item in LibraryBlockIndexer.get_items(filter_terms=filter_terms, text_search=text_search)
if item is not None
]
except ElasticConnectionError as e:
log.exception(e)
metadata = []
ref = ContentLibrary.objects.get_by_key(library_key) # type: ignore[attr-defined]
lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME)
usages = lib_bundle.get_top_level_usages()
# If indexing is disabled, or connection to elastic failed
if metadata is None:
metadata = []
ref = ContentLibrary.objects.get_by_key(library_key) # type: ignore[attr-defined]
lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME)
usages = lib_bundle.get_top_level_usages()
for usage_key in usages:
# For top-level definitions, we can go from definition key to usage key using the following, but this would
# not work for non-top-level blocks as they may have multiple usages. Top level blocks are guaranteed to
# have only a single usage in the library, which is part of the definition of top level block.
def_key = lib_bundle.definition_for_usage(usage_key)
display_name = get_block_display_name(def_key)
text_match = (text_search is None or
text_search.lower() in display_name.lower() or
text_search.lower() in str(usage_key).lower())
type_match = (block_types is None or usage_key.block_type in block_types)
if text_match and type_match:
metadata.append({
"id": usage_key,
"def_key": def_key,
"display_name": display_name,
"has_unpublished_changes": lib_bundle.does_definition_have_unpublished_changes(def_key),
})
for usage_key in usages:
# For top-level definitions, we can go from definition key to usage key using the following, but this would
# not work for non-top-level blocks as they may have multiple usages. Top level blocks are guaranteed to
# have only a single usage in the library, which is part of the definition of top level block.
def_key = lib_bundle.definition_for_usage(usage_key)
display_name = get_block_display_name(def_key)
text_match = (text_search is None or
text_search.lower() in display_name.lower() or
text_search.lower() in str(usage_key).lower())
type_match = (block_types is None or usage_key.block_type in block_types)
if text_match and type_match:
metadata.append({
"id": usage_key,
"def_key": def_key,
"display_name": display_name,
"has_unpublished_changes": lib_bundle.does_definition_have_unpublished_changes(def_key),
})
return [
LibraryXBlockMetadata(

25
openedx/core/djangoapps/content_libraries/docs/decisions/0001-index-libraries-in-elasticsearch.rst
View File

@@ -4,7 +4,30 @@
Status
------
Accepted
**Revoked**
In Dec 2023, we decided to remove the code supporting this decision, because:
* The index is disabled on edx.org, which will initially be the only user
of Content Libraries V2.
* As we migrate libraries from Modulestore to Blockstore and then from
Blockstore to Learning Core, the unused indexing code increases complexity
and decreases certainty.
* With the decision to migrate from Blockstore-the-service to an in-process
storage backend (that is: Blockstore-the-app or Learning Core), it seems
that we will be able to simply use Django ORM in order to filter/sort/search
Content Library V2 metadata for the library listing page.
* Searching Content Library V2 *block* content would still require indexing,
but we would rather implement that in Learning Core than use the current
implementation in the content_libraries app, which is untested, library-
specific, and doesn't take into account library versioning. It always uses
the latest draft, which is good for Library Authoring purposes, but not good for
Course Authoring purposes.
It is possible that we will end up re-instating a modified version of this ADR
future. If that happens, we may re-use and adapt the original library index
code.
Context
-------

346
openedx/core/djangoapps/content_libraries/libraries_index.py
View File

@@ -1,346 +0,0 @@
""" Code to allow indexing content libraries """
import logging
from abc import ABC, abstractmethod
from django.conf import settings
from django.dispatch import receiver
from elasticsearch.exceptions import ConnectionError as ElasticConnectionError
from search.elastic import _translate_hits, RESERVED_CHARACTERS
from search.search_engine_base import SearchEngine
from opaque_keys.edx.locator import LibraryUsageLocatorV2
from openedx_events.content_authoring.data import ContentLibraryData, LibraryBlockData
from openedx_events.content_authoring.signals import (
CONTENT_LIBRARY_CREATED,
CONTENT_LIBRARY_DELETED,
CONTENT_LIBRARY_UPDATED,
LIBRARY_BLOCK_CREATED,
LIBRARY_BLOCK_DELETED,
LIBRARY_BLOCK_UPDATED,
)
from openedx.core.djangoapps.content_libraries.constants import DRAFT_NAME
from openedx.core.djangoapps.content_libraries.library_bundle import LibraryBundle
from openedx.core.djangoapps.content_libraries.models import ContentLibrary
from openedx.core.lib.blockstore_api import get_bundle
log = logging.getLogger(__name__)
MAX_SIZE = 10000 # 10000 is the maximum records elastic is able to return in a single result. Defaults to 10.
class SearchIndexerBase(ABC):
"""
Abstract Base Class for implementing library search indexers.
"""
INDEX_NAME = None
ENABLE_INDEXING_KEY = None
SCHEMA_VERSION = 0
SEARCH_KWARGS = {
# Set this to True or 'wait_for' if immediate refresh is required after any update.
# See elastic docs for more information.
'refresh': False
}
@classmethod
@abstractmethod
def get_item_definition(cls, item):
"""
Returns a serializable dictionary which can be stored in elasticsearch.
"""
@classmethod
def index_items(cls, items):
"""
Index the specified libraries. If they already exist, replace them with new ones.
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
items = [cls.get_item_definition(item) for item in items]
return searcher.index(items, **cls.SEARCH_KWARGS)
@classmethod
def get_items(cls, ids=None, filter_terms=None, text_search=None):
"""
Retrieve a list of items from the index.
Arguments:
ids - List of ids to be searched for in the index
filter_terms - Dictionary of filters to be applied
text_search - String which is used to do a text search in the supported indexes.
"""
if filter_terms is None:
filter_terms = {}
if ids is not None:
filter_terms = {
"id": [str(item) for item in ids],
"schema_version": [cls.SCHEMA_VERSION],
**filter_terms,
}
if text_search:
response = cls._perform_elastic_search(filter_terms, text_search)
else:
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
response = searcher.search(field_dictionary=filter_terms, size=MAX_SIZE)
response = [result["data"] for result in response["results"]]
return sorted(response, key=lambda i: i["id"])
@classmethod
def remove_items(cls, ids):
"""
Remove the provided ids from the index
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
ids_str = [str(i) for i in ids]
searcher.remove(ids_str, **cls.SEARCH_KWARGS)
@classmethod
def remove_all_items(cls):
"""
Remove all items from the index
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
response = searcher.search(filter_dictionary={}, size=MAX_SIZE)
ids = [result["data"]["id"] for result in response["results"]]
searcher.remove(ids, **cls.SEARCH_KWARGS)
@classmethod
def indexing_is_enabled(cls):
"""
Checks to see if the indexing feature is enabled
"""
return settings.FEATURES.get(cls.ENABLE_INDEXING_KEY, False)
@classmethod
def _perform_elastic_search(cls, filter_terms, text_search):
"""
Build a query and search directly on elasticsearch
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
return _translate_hits(searcher._es.search( # pylint: disable=protected-access
index=searcher.index_name,
body=cls.build_elastic_query(filter_terms, text_search),
size=MAX_SIZE
))
@staticmethod
def build_elastic_query(filter_terms, text_search):
"""
Build and return an elastic query for doing text search on a library
"""
# Remove reserved characters (and ") from the text to prevent unexpected errors.
text_search_normalised = text_search.translate(text_search.maketrans('', '', RESERVED_CHARACTERS + '"'))
text_search_normalised = text_search_normalised.replace('-', ' ')
# Wrap with asterix to enable partial matches
text_search_normalised = f"*{text_search_normalised}*"
terms = [
{
'terms': {
item: filter_terms[item]
}
}
for item in filter_terms
]
return {
'query': {
'bool': {
'must': [
{
'query_string': {
'query': text_search_normalised,
"fields": ["content.*"],
'minimum_should_match': '100%',
},
},
],
'filter': {
'bool': {
'must': terms,
}
}
},
},
}
class ContentLibraryIndexer(SearchIndexerBase):
"""
Class to perform indexing for blockstore-based content libraries
"""
INDEX_NAME = "content_library_index"
ENABLE_INDEXING_KEY = "ENABLE_CONTENT_LIBRARY_INDEX"
SCHEMA_VERSION = 0
@classmethod
def get_item_definition(cls, item):
ref = ContentLibrary.objects.get_by_key(item)
lib_bundle = LibraryBundle(item, ref.bundle_uuid, draft_name=DRAFT_NAME)
num_blocks = len(lib_bundle.get_top_level_usages())
last_published = lib_bundle.get_last_published_time()
last_published_str = None
if last_published:
last_published_str = last_published.strftime('%Y-%m-%dT%H:%M:%SZ')
(has_unpublished_changes, has_unpublished_deletes) = lib_bundle.has_changes()
bundle_metadata = get_bundle(ref.bundle_uuid)
# NOTE: Increment ContentLibraryIndexer.SCHEMA_VERSION if the following schema is updated to avoid dealing
# with outdated indexes which might cause errors due to missing/invalid attributes.
return {
"schema_version": ContentLibraryIndexer.SCHEMA_VERSION,
"id": str(item),
"uuid": str(bundle_metadata.uuid),
"title": bundle_metadata.title,
"description": bundle_metadata.description,
"num_blocks": num_blocks,
"version": bundle_metadata.latest_version,
"last_published": last_published_str,
"has_unpublished_changes": has_unpublished_changes,
"has_unpublished_deletes": has_unpublished_deletes,
# only 'content' field is analyzed by elasticsearch, and allows text-search
"content": {
"id": str(item),
"title": bundle_metadata.title,
"description": bundle_metadata.description,
},
}
class LibraryBlockIndexer(SearchIndexerBase):
"""
Class to perform indexing on the XBlocks in content libraries.
"""
INDEX_NAME = "content_library_block_index"
ENABLE_INDEXING_KEY = "ENABLE_CONTENT_LIBRARY_INDEX"
SCHEMA_VERSION = 0
@classmethod
def get_item_definition(cls, item):
from openedx.core.djangoapps.content_libraries.api import get_block_display_name, _lookup_usage_key
def_key, lib_bundle = _lookup_usage_key(item)
is_child = item in lib_bundle.get_bundle_includes().keys()
# NOTE: Increment LibraryBlockIndexer.SCHEMA_VERSION if the following schema is updated to avoid dealing
# with outdated indexes which might cause errors due to missing/invalid attributes.
return {
"schema_version": LibraryBlockIndexer.SCHEMA_VERSION,
"id": str(item),
"library_key": str(lib_bundle.library_key),
"is_child": is_child,
"def_key": str(def_key),
"display_name": get_block_display_name(def_key),
"block_type": def_key.block_type,
"has_unpublished_changes": lib_bundle.does_definition_have_unpublished_changes(def_key),
# only 'content' field is analyzed by elastisearch, and allows text-search
"content": {
"id": str(item),
"display_name": get_block_display_name(def_key),
},
}
@receiver(CONTENT_LIBRARY_CREATED)
@receiver(CONTENT_LIBRARY_UPDATED)
def index_library(**kwargs):
"""
Index library when created or updated, or when its blocks are modified.
"""
content_library = kwargs.get('content_library', None)
if not content_library or not isinstance(content_library, ContentLibraryData):
log.error('Received null or incorrect data for event')
return
library_key = content_library.library_key
update_blocks = content_library.update_blocks
if ContentLibraryIndexer.indexing_is_enabled():
try:
ContentLibraryIndexer.index_items([library_key])
if update_blocks:
blocks = LibraryBlockIndexer.get_items(filter_terms={
'library_key': str(library_key)
})
usage_keys = [LibraryUsageLocatorV2.from_string(block['id']) for block in blocks]
LibraryBlockIndexer.index_items(usage_keys)
except ElasticConnectionError as e:
log.exception(e)
@receiver(LIBRARY_BLOCK_CREATED)
@receiver(LIBRARY_BLOCK_DELETED)
@receiver(LIBRARY_BLOCK_UPDATED)
def index_library_block(**kwargs):
"""
Index library when its blocks are created, modified, or deleted.
"""
library_block = kwargs.get('library_block', None)
if not library_block or not isinstance(library_block, LibraryBlockData):
log.error('Received null or incorrect data for event')
return
library_key = library_block.library_key
if ContentLibraryIndexer.indexing_is_enabled():
try:
ContentLibraryIndexer.index_items([library_key])
except ElasticConnectionError as e:
log.exception(e)
@receiver(CONTENT_LIBRARY_DELETED)
def remove_library_index(**kwargs):
"""
Remove from index when library is deleted
"""
content_library = kwargs.get('content_library', None)
if not content_library or not isinstance(content_library, ContentLibraryData):
log.error('Received null or incorrect data for event')
return
if ContentLibraryIndexer.indexing_is_enabled():
library_key = content_library.library_key
try:
ContentLibraryIndexer.remove_items([library_key])
blocks = LibraryBlockIndexer.get_items(filter_terms={
'library_key': str(library_key)
})
LibraryBlockIndexer.remove_items([block['id'] for block in blocks])
except ElasticConnectionError as e:
log.exception(e)
@receiver(LIBRARY_BLOCK_CREATED)
@receiver(LIBRARY_BLOCK_UPDATED)
def index_block(**kwargs):
"""
Index block metadata when created or updated
"""
library_block = kwargs.get('library_block', None)
if not library_block or not isinstance(library_block, LibraryBlockData):
log.error('Received null or incorrect data for event')
return
usage_key = library_block.usage_key
if LibraryBlockIndexer.indexing_is_enabled():
try:
LibraryBlockIndexer.index_items([usage_key])
except ElasticConnectionError as e:
log.exception(e)
@receiver(LIBRARY_BLOCK_DELETED)
def remove_block_index(**kwargs):
"""
Remove the block from the index when deleted
"""
library_block = kwargs.get('library_block', None)
if not library_block or not isinstance(library_block, LibraryBlockData):
log.error('Received null or incorrect data for LIBRARY_BLOCK_DELETED')
return
usage_key = library_block.usage_key
if LibraryBlockIndexer.indexing_is_enabled():
try:
LibraryBlockIndexer.remove_items([usage_key])
except ElasticConnectionError as e:
log.exception(e)

81
openedx/core/djangoapps/content_libraries/management/commands/reindex_content_library.py
View File

@@ -1,81 +0,0 @@
""" Management command to update content libraries' search index """ # lint-amnesty, pylint: disable=cyclic-import
import logging
from textwrap import dedent
from django.core.management import BaseCommand
from opaque_keys.edx.locator import LibraryLocatorV2
from openedx.core.djangoapps.content_libraries.api import DRAFT_NAME
from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryBlockIndexer
from openedx.core.djangoapps.content_libraries.library_bundle import LibraryBundle
from openedx.core.djangoapps.content_libraries.models import ContentLibrary
from cms.djangoapps.contentstore.management.commands.prompt import query_yes_no
class Command(BaseCommand):
"""
Command to reindex blockstore-based content libraries (single, multiple or all available).
This isn't needed on a regular basis as signals in various library APIs update the index when creating, updating or
deleting libraries.
This is usually required when the schema of the index changes, or if indexes are out of sync due to indexing
being previously disabled or any other reason.
Examples:
./manage.py reindex_content_library lib1 lib2 - reindexes libraries with keys lib1 and lib2
./manage.py reindex_content_library --all - reindexes all available libraries
./manage.py reindex_content_library --clear-all - clear all libraries indexes
"""
help = dedent(__doc__)
CONFIRMATION_PROMPT_CLEAR = "This will clear all indexed libraries from elasticsearch. Do you want to continue?"
CONFIRMATION_PROMPT_ALL = "Reindexing all libraries might be a time consuming operation. Do you want to continue?"
def add_arguments(self, parser):
parser.add_argument(
'--clear-all',
action='store_true',
dest='clear-all',
help='Clear all library indexes'
)
parser.add_argument(
'--all',
action='store_true',
dest='all',
help='Reindex all libraries'
)
parser.add_argument(
'--force',
action='store_true',
dest='force',
help='Run command without user prompt for confirmation'
)
parser.add_argument('library_ids', nargs='*')
def handle(self, *args, **options):
if options['clear-all']:
if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_CLEAR, default="no"):
logging.info("Removing all libraries from the index")
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()
return
if options['all']:
if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_ALL, default="no"):
logging.info("Indexing all libraries")
library_keys = [library.library_key for library in ContentLibrary.objects.all()]
else:
return
else:
logging.info("Indexing libraries: {}".format(options['library_ids']))
library_keys = list(map(LibraryLocatorV2.from_string, options['library_ids']))
ContentLibraryIndexer.index_items(library_keys)
for library_key in library_keys:
ref = ContentLibrary.objects.get_by_key(library_key)
lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME)
LibraryBlockIndexer.index_items(lib_bundle.get_all_usages())

23
openedx/core/djangoapps/content_libraries/tasks.py
View File

@@ -28,10 +28,8 @@ from edx_django_utils.monitoring import set_code_owner_attribute, set_code_owner
from opaque_keys.edx.keys import UsageKey
from opaque_keys.edx.locator import (
BlockUsageLocator,
LibraryUsageLocator,
LibraryUsageLocatorV2
)
from search.search_engine_base import SearchEngine
from user_tasks.tasks import UserTask, UserTaskStatus
from xblock.fields import Scope
@@ -80,11 +78,6 @@ def import_blocks_from_course(import_task_id, course_key_str):
)
def _normalize_key_for_search(library_key):
""" Normalizes library key for use with search indexing """
return library_key.replace(version_guid=None, branch=None)
def _import_block(store, user_id, source_block, dest_parent_key):
"""
Recursively import a blockstore block and its children.`
@@ -168,21 +161,7 @@ def _filter_child(store, usage_key, capa_type):
def _problem_type_filter(store, library, capa_type):
""" Filters library children by capa type."""
try:
search_engine = SearchEngine.get_search_engine(index="library_index")
except: # pylint: disable=bare-except
search_engine = None
if search_engine:
filter_clause = {
"library": str(_normalize_key_for_search(library.location.library_key)),
"content_type": ProblemBlock.INDEX_CONTENT_TYPE,
"problem_types": capa_type
}
search_result = search_engine.search(field_dictionary=filter_clause)
results = search_result.get('results', [])
return [LibraryUsageLocator.from_string(item['data']['id']) for item in results]
else:
return [key for key in library.children if _filter_child(store, key, capa_type)]
return [key for key in library.children if _filter_child(store, key, capa_type)]
def _import_from_blockstore(user_id, store, dest_block, blockstore_block_ids):

44
openedx/core/djangoapps/content_libraries/tests/base.py
View File

@@ -4,17 +4,12 @@ Tests for Blockstore-based Content Libraries
from contextlib import contextmanager
from io import BytesIO
from urllib.parse import urlencode
from unittest import mock
from django.conf import settings
from django.test import LiveServerTestCase
from django.test.utils import override_settings
from organizations.models import Organization
from rest_framework.test import APITestCase, APIClient
from search.search_engine_base import SearchEngine
from common.djangoapps.student.tests.factories import UserFactory
from openedx.core.djangoapps.content_libraries.libraries_index import MAX_SIZE
from openedx.core.djangoapps.content_libraries.constants import COMPLEX, ALL_RIGHTS_RESERVED
from openedx.core.djangolib.testing.utils import skip_unless_cms
from openedx.core.lib import blockstore_api
@@ -51,45 +46,6 @@ URL_BLOCK_FIELDS_URL = '/api/xblock/v2/xblocks/{block_key}/fields/'
URL_BLOCK_XBLOCK_HANDLER = '/api/xblock/v2/xblocks/{block_key}/handler/{user_id}-{secure_token}/{handler_name}/'
def elasticsearch_test(func):
"""
Decorator for tests which connect to elasticsearch when needed
"""
# This is disabled by default. Set to True if the elasticsearch engine is needed to test parts of code.
if settings.ENABLE_ELASTICSEARCH_FOR_TESTS:
func = override_settings(SEARCH_ENGINE="search.elastic.ElasticSearchEngine")(func)
func = override_settings(ELASTIC_SEARCH_CONFIG=[{
'use_ssl': settings.TEST_ELASTICSEARCH_USE_SSL,
'host': settings.TEST_ELASTICSEARCH_HOST,
'port': settings.TEST_ELASTICSEARCH_PORT,
}])(func)
func = mock.patch(
"openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase.SEARCH_KWARGS",
new={
'refresh': 'wait_for'
})(func)
return func
else:
@classmethod
def mock_perform(cls, filter_terms, text_search):
# pylint: disable=no-member
return SearchEngine.get_search_engine(cls.INDEX_NAME).search(
field_dictionary=filter_terms,
query_string=text_search,
size=MAX_SIZE
)
func = mock.patch(
"openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase.SEARCH_KWARGS",
new={}
)(func)
func = mock.patch(
"openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase._perform_elastic_search",
new=mock_perform
)(func)
return func
@skip_unless_cms # Content Libraries REST API is only available in Studio
class _ContentLibrariesRestApiTestMixin:
"""

232
openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py
View File

@@ -5,10 +5,8 @@ from uuid import UUID
from unittest.mock import Mock, patch
import ddt
from django.conf import settings
from django.contrib.auth.models import Group
from django.test.client import Client
from django.test.utils import override_settings
from organizations.models import Organization
from rest_framework.test import APITestCase
@@ -22,10 +20,8 @@ from openedx_events.content_authoring.signals import (
LIBRARY_BLOCK_DELETED,
LIBRARY_BLOCK_UPDATED,
)
from openedx.core.djangoapps.content_libraries.libraries_index import LibraryBlockIndexer, ContentLibraryIndexer
from openedx.core.djangoapps.content_libraries.tests.base import (
ContentLibrariesRestApiTest,
elasticsearch_test,
URL_BLOCK_METADATA_URL,
URL_BLOCK_RENDER_VIEW,
URL_BLOCK_GET_HANDLER_URL,
@@ -61,13 +57,6 @@ class ContentLibrariesTestMixin:
library slug and bundle UUID does not because it's assumed to be immutable
and cached forever.
"""
def setUp(self):
super().setUp()
if settings.ENABLE_ELASTICSEARCH_FOR_TESTS:
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()
def test_library_crud(self):
"""
Test Create, Read, Update, and Delete of a Content Library
@@ -210,89 +199,83 @@ class ContentLibrariesTestMixin:
'slug': ['Enter a valid “slug” consisting of Unicode letters, numbers, underscores, or hyphens.'],
}
@ddt.data(True, False)
@patch("openedx.core.djangoapps.content_libraries.views.LibraryApiPagination.page_size", new=2)
def test_list_library(self, is_indexing_enabled):
def test_list_library(self):
"""
Test the /libraries API and its pagination
"""
with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}):
lib1 = self._create_library(slug="some-slug-1", title="Existing Library")
lib2 = self._create_library(slug="some-slug-2", title="Existing Library")
if not is_indexing_enabled:
lib1['num_blocks'] = lib2['num_blocks'] = None
lib1['last_published'] = lib2['last_published'] = None
lib1['has_unpublished_changes'] = lib2['has_unpublished_changes'] = None
lib1['has_unpublished_deletes'] = lib2['has_unpublished_deletes'] = None
lib1 = self._create_library(slug="some-slug-1", title="Existing Library")
lib2 = self._create_library(slug="some-slug-2", title="Existing Library")
lib1['num_blocks'] = lib2['num_blocks'] = None
lib1['last_published'] = lib2['last_published'] = None
lib1['has_unpublished_changes'] = lib2['has_unpublished_changes'] = None
lib1['has_unpublished_deletes'] = lib2['has_unpublished_deletes'] = None
result = self._list_libraries()
assert len(result) == 2
assert lib1 in result
assert lib2 in result
result = self._list_libraries({'pagination': 'true'})
assert len(result['results']) == 2
assert result['next'] is None
result = self._list_libraries()
assert len(result) == 2
assert lib1 in result
assert lib2 in result
result = self._list_libraries({'pagination': 'true'})
assert len(result['results']) == 2
assert result['next'] is None
# Create another library which causes number of libraries to exceed the page size
self._create_library(slug="some-slug-3", title="Existing Library")
# Verify that if `pagination` param isn't sent, API still honors the max page size.
# This is for maintaining compatibility with older non pagination-aware clients.
result = self._list_libraries()
assert len(result) == 2
# Create another library which causes number of libraries to exceed the page size
self._create_library(slug="some-slug-3", title="Existing Library")
# Verify that if `pagination` param isn't sent, API still honors the max page size.
# This is for maintaining compatibility with older non pagination-aware clients.
result = self._list_libraries()
assert len(result) == 2
# Pagination enabled:
# Verify total elements and valid 'next' in page 1
result = self._list_libraries({'pagination': 'true'})
assert len(result['results']) == 2
assert 'page=2' in result['next']
assert 'pagination=true' in result['next']
# Verify total elements and null 'next' in page 2
result = self._list_libraries({'pagination': 'true', 'page': '2'})
assert len(result['results']) == 1
assert result['next'] is None
# Pagination enabled:
# Verify total elements and valid 'next' in page 1
result = self._list_libraries({'pagination': 'true'})
assert len(result['results']) == 2
assert 'page=2' in result['next']
assert 'pagination=true' in result['next']
# Verify total elements and null 'next' in page 2
result = self._list_libraries({'pagination': 'true', 'page': '2'})
assert len(result['results']) == 1
assert result['next'] is None
@ddt.data(True, False)
def test_library_filters(self, is_indexing_enabled):
def test_library_filters(self):
"""
Test the filters in the list libraries API
"""
suffix = str(is_indexing_enabled)
with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}):
self._create_library(
slug=f"test-lib-filter-{suffix}-1", title="Fob", description=f"Bar-{suffix}", library_type=VIDEO,
)
self._create_library(
slug=f"test-lib-filter-{suffix}-2", title=f"Library-Title-{suffix}-2", description=f"Bar-{suffix}-2",
)
self._create_library(
slug=f"l3{suffix}", title=f"Library-Title-{suffix}-3", description="Description", library_type=VIDEO,
)
self._create_library(
slug="test-lib-filter-1", title="Fob", description="Bar", library_type=VIDEO,
)
self._create_library(
slug="test-lib-filter-2", title="Library-Title-2", description="Bar-2",
)
self._create_library(
slug="l3", title="Library-Title-3", description="Description", library_type=VIDEO,
)
Organization.objects.get_or_create(
short_name=f"org-test-{suffix}",
defaults={"name": "Content Libraries Tachyon Exploration & Survey Team"},
)
self._create_library(
slug=f"l4-{suffix}", title=f"Library-Title-{suffix}-4",
description="Library-Description", org=f'org-test-{suffix}',
library_type=VIDEO,
)
self._create_library(
slug="l5", title=f"Library-Title-{suffix}-5", description="Library-Description",
org=f'org-test-{suffix}',
)
Organization.objects.get_or_create(
short_name="org-test",
defaults={"name": "Content Libraries Tachyon Exploration & Survey Team"},
)
self._create_library(
slug="l4", title="Library-Title-4",
description="Library-Description", org='org-test',
library_type=VIDEO,
)
self._create_library(
slug="l5", title="Library-Title-5", description="Library-Description",
org='org-test',
)
assert len(self._list_libraries()) == 5
assert len(self._list_libraries({'org': f'org-test-{suffix}'})) == 2
assert len(self._list_libraries({'text_search': f'test-lib-filter-{suffix}'})) == 2
assert len(self._list_libraries({'text_search': f'test-lib-filter-{suffix}', 'type': VIDEO})) == 1
assert len(self._list_libraries({'text_search': f'library-title-{suffix}'})) == 4
assert len(self._list_libraries({'text_search': f'library-title-{suffix}', 'type': VIDEO})) == 2
assert len(self._list_libraries({'text_search': f'bar-{suffix}'})) == 2
assert len(self._list_libraries({'text_search': f'org-test-{suffix}'})) == 2
assert len(self._list_libraries({'org': f'org-test-{suffix}',
'text_search': f'library-title-{suffix}-4'})) == 1
assert len(self._list_libraries({'type': VIDEO})) == 3
assert len(self._list_libraries()) == 5
assert len(self._list_libraries({'org': 'org-test'})) == 2
assert len(self._list_libraries({'text_search': 'test-lib-filter'})) == 2
assert len(self._list_libraries({'text_search': 'test-lib-filter', 'type': VIDEO})) == 1
assert len(self._list_libraries({'text_search': 'library-title'})) == 4
assert len(self._list_libraries({'text_search': 'library-title', 'type': VIDEO})) == 2
assert len(self._list_libraries({'text_search': 'bar'})) == 2
assert len(self._list_libraries({'text_search': 'org-test'})) == 2
assert len(self._list_libraries({'org': 'org-test',
'text_search': 'library-title-4'})) == 1
assert len(self._list_libraries({'type': VIDEO})) == 3
# General Content Library XBlock tests:
@@ -439,65 +422,61 @@ class ContentLibrariesTestMixin:
assert 'resources' in fragment
assert 'Hello world!' in fragment['content']
@ddt.data(True, False)
@patch("openedx.core.djangoapps.content_libraries.views.LibraryApiPagination.page_size", new=2)
def test_list_library_blocks(self, is_indexing_enabled):
def test_list_library_blocks(self):
"""
Test the /libraries/{lib_key_str}/blocks API and its pagination
"""
with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}):
lib = self._create_library(slug="list_blocks-slug" + str(is_indexing_enabled), title="Library 1")
block1 = self._add_block_to_library(lib["id"], "problem", "problem1")
block2 = self._add_block_to_library(lib["id"], "unit", "unit1")
lib = self._create_library(slug="list_blocks-slug", title="Library 1")
block1 = self._add_block_to_library(lib["id"], "problem", "problem1")
block2 = self._add_block_to_library(lib["id"], "unit", "unit1")
self._add_block_to_library(lib["id"], "problem", "problem2", parent_block=block2["id"])
self._add_block_to_library(lib["id"], "problem", "problem2", parent_block=block2["id"])
result = self._get_library_blocks(lib["id"])
assert len(result) == 2
assert block1 in result
result = self._get_library_blocks(lib["id"])
assert len(result) == 2
assert block1 in result
result = self._get_library_blocks(lib["id"], {'pagination': 'true'})
assert len(result['results']) == 2
assert result['next'] is None
result = self._get_library_blocks(lib["id"], {'pagination': 'true'})
assert len(result['results']) == 2
assert result['next'] is None
self._add_block_to_library(lib["id"], "problem", "problem3")
# Test pagination
result = self._get_library_blocks(lib["id"])
assert len(result) == 3
result = self._get_library_blocks(lib["id"], {'pagination': 'true'})
assert len(result['results']) == 2
assert 'page=2' in result['next']
assert 'pagination=true' in result['next']
result = self._get_library_blocks(lib["id"], {'pagination': 'true', 'page': '2'})
assert len(result['results']) == 1
assert result['next'] is None
self._add_block_to_library(lib["id"], "problem", "problem3")
# Test pagination
result = self._get_library_blocks(lib["id"])
assert len(result) == 3
result = self._get_library_blocks(lib["id"], {'pagination': 'true'})
assert len(result['results']) == 2
assert 'page=2' in result['next']
assert 'pagination=true' in result['next']
result = self._get_library_blocks(lib["id"], {'pagination': 'true', 'page': '2'})
assert len(result['results']) == 1
assert result['next'] is None
@ddt.data(True, False)
def test_library_blocks_filters(self, is_indexing_enabled):
def test_library_blocks_filters(self):
"""
Test the filters in the list libraries API
"""
with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}):
lib = self._create_library(slug="test-lib-blocks" + str(is_indexing_enabled), title="Title")
block1 = self._add_block_to_library(lib["id"], "problem", "foo-bar")
self._add_block_to_library(lib["id"], "video", "vid-baz")
self._add_block_to_library(lib["id"], "html", "html-baz")
self._add_block_to_library(lib["id"], "problem", "foo-baz")
self._add_block_to_library(lib["id"], "problem", "bar-baz")
lib = self._create_library(slug="test-lib-blocks", title="Title")
block1 = self._add_block_to_library(lib["id"], "problem", "foo-bar")
self._add_block_to_library(lib["id"], "video", "vid-baz")
self._add_block_to_library(lib["id"], "html", "html-baz")
self._add_block_to_library(lib["id"], "problem", "foo-baz")
self._add_block_to_library(lib["id"], "problem", "bar-baz")
self._set_library_block_olx(block1["id"], "<problem display_name=\"DisplayName\"></problem>")
self._set_library_block_olx(block1["id"], "<problem display_name=\"DisplayName\"></problem>")
assert len(self._get_library_blocks(lib['id'])) == 5
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Foo'})) == 2
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Display'})) == 1
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Video'})) == 1
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Foo', 'block_type': 'video'})) == 0
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Baz', 'block_type': 'video'})) == 1
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Baz', 'block_type': ['video', 'html']})) ==\
2
assert len(self._get_library_blocks(lib['id'], {'block_type': 'video'})) == 1
assert len(self._get_library_blocks(lib['id'], {'block_type': 'problem'})) == 3
assert len(self._get_library_blocks(lib['id'], {'block_type': 'squirrel'})) == 0
assert len(self._get_library_blocks(lib['id'])) == 5
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Foo'})) == 2
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Display'})) == 1
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Video'})) == 1
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Foo', 'block_type': 'video'})) == 0
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Baz', 'block_type': 'video'})) == 1
assert len(self._get_library_blocks(lib['id'], {'text_search': 'Baz', 'block_type': ['video', 'html']})) ==\
2
assert len(self._get_library_blocks(lib['id'], {'block_type': 'video'})) == 1
assert len(self._get_library_blocks(lib['id'], {'block_type': 'problem'})) == 3
assert len(self._get_library_blocks(lib['id'], {'block_type': 'squirrel'})) == 0
@ddt.data(
('video-problem', VIDEO, 'problem', 400),
@@ -1231,7 +1210,6 @@ class ContentLibrariesTestMixin:
)
@elasticsearch_test
class ContentLibrariesTest(
ContentLibrariesTestMixin,
ContentLibrariesRestApiTest,

302
openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py
View File

@@ -1,302 +0,0 @@
"""
Testing indexing of blockstore based content libraries
"""
from unittest.mock import patch
from django.conf import settings
from django.core.management import call_command
from django.test.utils import override_settings
from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2
from search.search_engine_base import SearchEngine
from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryBlockIndexer
from openedx.core.djangoapps.content_libraries.tests.base import (
ContentLibrariesRestApiTest,
elasticsearch_test,
)
class ContentLibraryIndexerTestMixin:
"""
Tests the operation of ContentLibraryIndexer
"""
@elasticsearch_test
def setUp(self):
super().setUp()
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()
self.searcher = SearchEngine.get_search_engine(ContentLibraryIndexer.INDEX_NAME)
def test_index_libraries(self):
"""
Test if libraries are being indexed correctly
"""
result1 = self._create_library(slug="test-lib-index-1", title="Title 1", description="Description")
result2 = self._create_library(slug="test-lib-index-2", title="Title 2", description="Description")
for result in [result1, result2]:
library_key = LibraryLocatorV2.from_string(result['id'])
response = ContentLibraryIndexer.get_items([library_key])[0]
assert response['id'] == result['id']
assert response['title'] == result['title']
assert response['description'] == result['description']
assert response['uuid'] == result['bundle_uuid']
assert response['num_blocks'] == 0
assert response['version'] == result['version']
assert response['last_published'] is None
assert response['has_unpublished_changes'] is False
assert response['has_unpublished_deletes'] is False
def test_schema_updates(self):
"""
Test that outdated indexes aren't retrieved
"""
with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION",
new=0):
result = self._create_library(slug="test-lib-schemaupdates-1", title="Title 1", description="Description")
library_key = LibraryLocatorV2.from_string(result['id'])
assert len(ContentLibraryIndexer.get_items([library_key])) == 1
with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION",
new=1):
assert len(ContentLibraryIndexer.get_items([library_key])) == 0
call_command("reindex_content_library", all=True, force=True)
assert len(ContentLibraryIndexer.get_items([library_key])) == 1
def test_remove_all_libraries(self):
"""
Test if remove_all_items() deletes all libraries
"""
lib1 = self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description")
lib2 = self._create_library(slug="test-lib-rm-all-2", title="Title 2", description="Description")
library_key1 = LibraryLocatorV2.from_string(lib1['id'])
library_key2 = LibraryLocatorV2.from_string(lib2['id'])
assert len(ContentLibraryIndexer.get_items([library_key1, library_key2])) == 2
ContentLibraryIndexer.remove_all_items()
assert len(ContentLibraryIndexer.get_items()) == 0
def test_update_libraries(self):
"""
Test if indexes are updated when libraries are updated
"""
lib = self._create_library(slug="test-lib-update", title="Title", description="Description")
library_key = LibraryLocatorV2.from_string(lib['id'])
self._update_library(lib['id'], title="New Title", description="New Title")
response = ContentLibraryIndexer.get_items([library_key])[0]
assert response['id'] == lib['id']
assert response['title'] == 'New Title'
assert response['description'] == 'New Title'
assert response['uuid'] == lib['bundle_uuid']
assert response['num_blocks'] == 0
assert response['version'] == lib['version']
assert response['last_published'] is None
assert response['has_unpublished_changes'] is False
assert response['has_unpublished_deletes'] is False
self._delete_library(lib['id'])
assert ContentLibraryIndexer.get_items([library_key]) == []
ContentLibraryIndexer.get_items([library_key])
def test_update_library_blocks(self):
"""
Test if indexes are updated when blocks in libraries are updated
"""
def commit_library_and_verify(library_key):
"""
Commit library changes, and verify that there are no uncommited changes anymore
"""
last_published = ContentLibraryIndexer.get_items([library_key])[0]['last_published']
self._commit_library_changes(str(library_key))
response = ContentLibraryIndexer.get_items([library_key])[0]
assert response['has_unpublished_changes'] is False
assert response['has_unpublished_deletes'] is False
assert response['last_published'] >= last_published
return response
def verify_uncommitted_libraries(library_key, has_unpublished_changes, has_unpublished_deletes):
"""
Verify uncommitted changes and deletes in the index
"""
response = ContentLibraryIndexer.get_items([library_key])[0]
assert response['has_unpublished_changes'] == has_unpublished_changes
assert response['has_unpublished_deletes'] == has_unpublished_deletes
return response
lib = self._create_library(slug="test-lib-update-block", title="Title", description="Description")
library_key = LibraryLocatorV2.from_string(lib['id'])
# Verify uncommitted new blocks
block = self._add_block_to_library(lib['id'], "problem", "problem1")
response = verify_uncommitted_libraries(library_key, True, False)
assert response['last_published'] is None
assert response['num_blocks'] == 1
# Verify committed new blocks
self._commit_library_changes(lib['id'])
response = verify_uncommitted_libraries(library_key, False, False)
assert response['num_blocks'] == 1
# Verify uncommitted deleted blocks
self._delete_library_block(block['id'])
response = verify_uncommitted_libraries(library_key, True, True)
assert response['num_blocks'] == 0
# Verify committed deleted blocks
self._commit_library_changes(lib['id'])
response = verify_uncommitted_libraries(library_key, False, False)
assert response['num_blocks'] == 0
block = self._add_block_to_library(lib['id'], "problem", "problem1")
self._commit_library_changes(lib['id'])
# Verify changes to blocks
# Verify OLX updates on blocks
self._set_library_block_olx(block["id"], "<problem/>")
verify_uncommitted_libraries(library_key, True, False)
commit_library_and_verify(library_key)
# Verify asset updates on blocks
self._set_library_block_asset(block["id"], "whatever.png", b"data")
verify_uncommitted_libraries(library_key, True, False)
commit_library_and_verify(library_key)
self._delete_library_block_asset(block["id"], "whatever.png")
verify_uncommitted_libraries(library_key, True, False)
commit_library_and_verify(library_key)
lib2 = self._create_library(slug="test-lib-update-block-2", title="Title 2", description="Description")
self._add_block_to_library(lib2["id"], "problem", "problem1")
self._commit_library_changes(lib2["id"])
#Verify new links on libraries
self._link_to_library(lib["id"], "library_2", lib2["id"])
verify_uncommitted_libraries(library_key, True, False)
#Verify reverting uncommitted changes
self._revert_library_changes(lib["id"])
verify_uncommitted_libraries(library_key, False, False)
@override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True})
@elasticsearch_test
class ContentLibraryIndexerTest(
ContentLibraryIndexerTestMixin,
ContentLibrariesRestApiTest,
):
"""
Tests the operation of ContentLibraryIndexer using the installed Blockstore app.
"""
class LibraryBlockIndexerTestMixin:
"""
Tests the operation of LibraryBlockIndexer
"""
@elasticsearch_test
def setUp(self):
super().setUp()
ContentLibraryIndexer.remove_all_items()
LibraryBlockIndexer.remove_all_items()
self.searcher = SearchEngine.get_search_engine(LibraryBlockIndexer.INDEX_NAME)
def test_index_block(self):
"""
Test if libraries are being indexed correctly
"""
lib = self._create_library(slug="test-lib-index-1", title="Title 1", description="Description")
block1 = self._add_block_to_library(lib['id'], "problem", "problem1")
block2 = self._add_block_to_library(lib['id'], "problem", "problem2")
assert len(LibraryBlockIndexer.get_items()) == 2
for block in [block1, block2]:
usage_key = LibraryUsageLocatorV2.from_string(block['id'])
response = LibraryBlockIndexer.get_items([usage_key])[0]
assert response['id'] == block['id']
assert response['def_key'] == block['def_key']
assert response['block_type'] == block['block_type']
assert response['display_name'] == block['display_name']
assert response['has_unpublished_changes'] == block['has_unpublished_changes']
def test_schema_updates(self):
"""
Test that outdated indexes aren't retrieved
"""
lib = self._create_library(slug="test-lib--block-schemaupdates-1", title="Title 1", description="Description")
with patch("openedx.core.djangoapps.content_libraries.libraries_index.LibraryBlockIndexer.SCHEMA_VERSION",
new=0):
block = self._add_block_to_library(lib['id'], "problem", "problem1")
assert len(LibraryBlockIndexer.get_items([block['id']])) == 1
with patch("openedx.core.djangoapps.content_libraries.libraries_index.LibraryBlockIndexer.SCHEMA_VERSION",
new=1):
assert len(LibraryBlockIndexer.get_items([block['id']])) == 0
call_command("reindex_content_library", all=True, force=True)
assert len(LibraryBlockIndexer.get_items([block['id']])) == 1
def test_remove_all_items(self):
"""
Test if remove_all_items() deletes all libraries
"""
lib1 = self._create_library(slug="test-lib-rm-all", title="Title 1", description="Description")
self._add_block_to_library(lib1['id'], "problem", "problem1")
self._add_block_to_library(lib1['id'], "problem", "problem2")
assert len(LibraryBlockIndexer.get_items()) == 2
LibraryBlockIndexer.remove_all_items()
assert len(LibraryBlockIndexer.get_items()) == 0
def test_crud_block(self):
"""
Test that CRUD operations on blocks are reflected in the index
"""
lib = self._create_library(slug="test-lib-crud-block", title="Title", description="Description")
block = self._add_block_to_library(lib['id'], "problem", "problem1")
# Update OLX, verify updates in index
self._set_library_block_olx(block["id"], '<problem display_name="new_name"/>')
response = LibraryBlockIndexer.get_items([block['id']])[0]
assert response['display_name'] == 'new_name'
assert response['has_unpublished_changes'] is True
# Verify has_unpublished_changes after committing library
self._commit_library_changes(lib['id'])
response = LibraryBlockIndexer.get_items([block['id']])[0]
assert response['has_unpublished_changes'] is False
# Verify has_unpublished_changes after reverting library
self._set_library_block_asset(block["id"], "whatever.png", b"data")
response = LibraryBlockIndexer.get_items([block['id']])[0]
assert response['has_unpublished_changes'] is True
self._revert_library_changes(lib['id'])
response = LibraryBlockIndexer.get_items([block['id']])[0]
assert response['has_unpublished_changes'] is False
# Verify that deleting block removes it from index
self._delete_library_block(block['id'])
assert LibraryBlockIndexer.get_items([block['id']]) == []
# Verify that deleting a library removes its blocks from index too
self._add_block_to_library(lib['id'], "problem", "problem1")
LibraryBlockIndexer.get_items([block['id']])
self._delete_library(lib['id'])
assert LibraryBlockIndexer.get_items([block['id']]) == []
@override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True})
@elasticsearch_test
class LibraryBlockIndexerTest(
LibraryBlockIndexerTestMixin,
ContentLibrariesRestApiTest,
):
"""
Tests the operation of LibraryBlockIndexer using the installed Blockstore app.
"""

4
openedx/core/djangoapps/content_libraries/views.py
View File

@@ -172,12 +172,12 @@ class LibraryRootView(APIView):
paginator = LibraryApiPagination()
queryset = api.get_libraries_for_user(request.user, org=org, library_type=library_type)
if text_search:
result = api.get_metadata_from_index(queryset, text_search=text_search)
result = api.get_metadata(queryset, text_search=text_search)
result = paginator.paginate_queryset(result, request)
else:
# We can paginate queryset early and prevent fetching unneeded metadata
paginated_qs = paginator.paginate_queryset(queryset, request)
result = api.get_metadata_from_index(paginated_qs)
result = api.get_metadata(paginated_qs)
serializer = ContentLibraryMetadataSerializer(result, many=True)
# Verify `pagination` param to maintain compatibility with older

2
xmodule/library_tools.py
View File

@@ -185,7 +185,7 @@ class LibraryToolsService:
for lib in self.store.get_library_summaries()
]
v2_query = library_api.get_libraries_for_user(user)
v2_libs_with_meta = library_api.get_metadata_from_index(v2_query)
v2_libs_with_meta = library_api.get_metadata(v2_query)
v2_libs = [(lib.key, lib.title) for lib in v2_libs_with_meta]
if settings.FEATURES.get('ENABLE_LIBRARY_AUTHORING_MICROFRONTEND'):