From 67f3c0343fbf1052e5d7cb071194a33295091989 Mon Sep 17 00:00:00 2001 From: Sid Verma Date: Thu, 6 Aug 2020 21:12:08 +0530 Subject: [PATCH] Simplify and optimize index searching --- .../core/djangoapps/content_libraries/api.py | 46 +++-- .../content_libraries/libraries_index.py | 157 ++++++++---------- .../content_libraries/tests/base.py | 22 ++- .../tests/test_content_libraries.py | 17 +- .../tests/test_libraries_index.py | 61 ++++--- 5 files changed, 164 insertions(+), 139 deletions(-) diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 0f74998081..a8cd06bba2 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -58,11 +58,7 @@ from xblock.exceptions import XBlockNotFoundError from openedx.core.djangoapps.content_libraries import permissions from openedx.core.djangoapps.content_libraries.constants import DRAFT_NAME from openedx.core.djangoapps.content_libraries.library_bundle import LibraryBundle -from openedx.core.djangoapps.content_libraries.libraries_index import ( - ContentLibraryIndexer, - LibraryBlockIndexer, - ItemNotIndexedException, -) +from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryBlockIndexer from openedx.core.djangoapps.content_libraries.models import ContentLibrary, ContentLibraryPermission from openedx.core.djangoapps.content_libraries.signals import ( CONTENT_LIBRARY_CREATED, @@ -243,9 +239,19 @@ def get_metadata_from_index(queryset, text_search=None): metadata = None if ContentLibraryIndexer.indexing_is_enabled(): try: - library_keys = [lib.library_key for lib in queryset] + library_keys = [str(lib.library_key) for lib in queryset] metadata = ContentLibraryIndexer.get_items(library_keys, text_search=text_search) - except (ItemNotIndexedException, KeyError, ElasticConnectionError) as e: + metadata_dict = { + item["id"]: item + for item in metadata + } + metadata = [ + metadata_dict[key] + if key in metadata_dict + else None + for key in library_keys + ] + except ElasticConnectionError as e: log.exception(e) # If ContentLibraryIndex is not available, we query blockstore for a limited set of metadata @@ -256,7 +262,7 @@ def get_metadata_from_index(queryset, text_search=None): if text_search: # Bundle APIs can't apply text_search on a bundle's org, so including those results here queryset_org_search = queryset.filter(org__short_name__icontains=text_search) - if len(queryset_org_search): + if queryset_org_search.exists(): uuids_org_search = [lib.bundle_uuid for lib in queryset_org_search] bundles += get_bundles(uuids=uuids_org_search) @@ -508,30 +514,34 @@ def get_library_blocks(library_key, text_search=None): Returns a list of LibraryXBlockMetadata objects """ metadata = None - ref = ContentLibrary.objects.get_by_key(library_key) - lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME) - usages = lib_bundle.get_top_level_usages() - if LibraryBlockIndexer.indexing_is_enabled(): try: + filter_terms = { + 'library_key': [str(library_key)], + 'is_child': [False], + } metadata = [ { **item, "id": LibraryUsageLocatorV2.from_string(item['id']), } - for item in LibraryBlockIndexer.get_items(usages, text_search=text_search) + for item in LibraryBlockIndexer.get_items(filter_terms=filter_terms, text_search=text_search) if item is not None ] - except (ItemNotIndexedException, KeyError, ConnectionError) as e: + except (ConnectionError) as e: log.exception(e) # If indexing is disabled, or connection to elastic failed if metadata is None: metadata = [] + ref = ContentLibrary.objects.get_by_key(library_key) + lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME) + usages = lib_bundle.get_top_level_usages() + for usage_key in usages: - # For top-level definitions, we can go from definition key to usage key using the following, but this would not - # work for non-top-level blocks as they may have multiple usages. Top level blocks are guaranteed to have only - # a single usage in the library, which is part of the definition of top level block. + # For top-level definitions, we can go from definition key to usage key using the following, but this would + # not work for non-top-level blocks as they may have multiple usages. Top level blocks are guaranteed to + # have only a single usage in the library, which is part of the definition of top level block. def_key = lib_bundle.definition_for_usage(usage_key) display_name = get_block_display_name(def_key) if (text_search is None or @@ -740,6 +750,8 @@ def create_library_block_child(parent_usage_key, block_type, definition_id): include_data = XBlockInclude(link_id=None, block_type=block_type, definition_id=definition_id, usage_hint=None) parent_block.runtime.add_child_include(parent_block, include_data) parent_block.save() + ref = ContentLibrary.objects.get_by_key(parent_usage_key.context_key) + LIBRARY_BLOCK_UPDATED.send(sender=None, library_key=ref.library_key, usage_key=metadata.usage_key) return metadata diff --git a/openedx/core/djangoapps/content_libraries/libraries_index.py b/openedx/core/djangoapps/content_libraries/libraries_index.py index ec3ba6db2c..591f788e12 100644 --- a/openedx/core/djangoapps/content_libraries/libraries_index.py +++ b/openedx/core/djangoapps/content_libraries/libraries_index.py @@ -6,7 +6,7 @@ from abc import ABC, abstractmethod from django.conf import settings from django.dispatch import receiver from elasticsearch.exceptions import ConnectionError as ElasticConnectionError -from search.elastic import ElasticSearchEngine, _translate_hits, _process_field_filters, RESERVED_CHARACTERS +from search.elastic import _translate_hits, RESERVED_CHARACTERS from search.search_engine_base import SearchEngine from opaque_keys.edx.locator import LibraryUsageLocatorV2 @@ -28,16 +28,14 @@ log = logging.getLogger(__name__) MAX_SIZE = 10000 # 10000 is the maximum records elastic is able to return in a single result. Defaults to 10. -class ItemNotIndexedException(Exception): - """ - Item wasn't indexed in ElasticSearch - """ - - class SearchIndexerBase(ABC): + """ + Abstract Base Class for implementing library search indexers. + """ INDEX_NAME = None DOCUMENT_TYPE = None ENABLE_INDEXING_KEY = None + SCHEMA_VERSION = 0 SEARCH_KWARGS = { # Set this to True or 'wait_for' if immediate refresh is required after any update. # See elastic docs for more information. @@ -61,67 +59,31 @@ class SearchIndexerBase(ABC): return searcher.index(cls.DOCUMENT_TYPE, items, **cls.SEARCH_KWARGS) @classmethod - def search(cls, **kwargs): + def get_items(cls, ids=None, filter_terms=None, text_search=None): """ - Search the index with the given kwargs + Retrieve a list of items from the index. + Arguments: + ids - List of ids to be searched for in the index + filter_terms - Dictionary of filters to be applied + text_search - String which is used to do a text search in the supported indexes. """ - searcher = SearchEngine.get_search_engine(cls.INDEX_NAME) - response = searcher.search(doc_type=cls.DOCUMENT_TYPE, field_dictionary=kwargs, size=MAX_SIZE) - return sorted(response["results"], key=lambda i: i['data']["id"]) - - @classmethod - def get_items(cls, ids, text_search=None): - """ - Retrieve a list of items from the index - """ - searcher = SearchEngine.get_search_engine(cls.INDEX_NAME) - ids_str = [str(i) for i in ids] - - response = searcher.search( - doc_type=cls.DOCUMENT_TYPE, - field_dictionary={ - "id": ids_str, - "schema_version": cls.SCHEMA_VERSION - }, - size=MAX_SIZE, - ) - if len(response["results"]) != len(ids_str): - missing = set(ids_str) - set([result["data"]["id"] for result in response["results"]]) - missing = set(ids_str) - set([result["data"]["id"] for result in response["results"]]) - raise ItemNotIndexedException("Keys not found in index: {}".format(missing)) + if filter_terms is None: + filter_terms = {} + if ids is not None: + filter_terms = { + "id": [str(item) for item in ids], + "schema_version": [cls.SCHEMA_VERSION], + **filter_terms, + } if text_search: - # Elastic is hit twice if text_search is valid - # Once above to identify unindexed libraries, and now to filter with text_search - if isinstance(searcher, ElasticSearchEngine): - response = _translate_hits(searcher._es.search( - doc_type=cls.DOCUMENT_TYPE, - index=searcher.index_name, - body=cls.build_elastic_query(ids_str, text_search), - size=MAX_SIZE - )) - else: - # This is used only for running tests. TODO: Remove this and use elasticsearch in tests. - response = searcher.search( - doc_type=cls.DOCUMENT_TYPE, - field_dictionary={"id": ids_str}, - query_string=text_search, - size=MAX_SIZE - ) + response = cls._perform_elastic_search(filter_terms, text_search) + else: + searcher = SearchEngine.get_search_engine(cls.INDEX_NAME) + response = searcher.search(doc_type=cls.DOCUMENT_TYPE, field_dictionary=filter_terms, size=MAX_SIZE) - # Search results may not retain the original order of keys - we use this - # dict to construct a list in the original order of ids - response_dict = { - result["data"]["id"]: result["data"] - for result in response["results"] - } - - return [ - response_dict[key] - if key in response_dict - else None - for key in ids_str - ] + response = [result["data"] for result in response["results"]] + return sorted(response, key=lambda i: i["id"]) @classmethod def remove_items(cls, ids): @@ -149,16 +111,37 @@ class SearchIndexerBase(ABC): """ return settings.FEATURES.get(cls.ENABLE_INDEXING_KEY, False) + @classmethod + def _perform_elastic_search(cls, filter_terms, text_search): + """ + Build a query and search directly on elasticsearch + """ + searcher = SearchEngine.get_search_engine(cls.INDEX_NAME) + return _translate_hits(searcher._es.search( # pylint: disable=protected-access + doc_type=cls.DOCUMENT_TYPE, + index=searcher.index_name, + body=cls.build_elastic_query(filter_terms, text_search), + size=MAX_SIZE + )) + @staticmethod - def build_elastic_query(ids_str, text_search): + def build_elastic_query(filter_terms, text_search): """ Build and return an elastic query for doing text search on a library """ # Remove reserved characters (and ") from the text to prevent unexpected errors. text_search_normalised = text_search.translate(text_search.maketrans('', '', RESERVED_CHARACTERS + '"')) - text_search_normalised = text_search.replace('-',' ') + text_search_normalised = text_search.replace('-', ' ') # Wrap with asterix to enable partial matches text_search_normalised = "*{}*".format(text_search_normalised) + terms = [ + { + 'terms': { + item: filter_terms[item] + } + } + for item in filter_terms + ] return { 'query': { 'filtered': { @@ -172,8 +155,8 @@ class SearchIndexerBase(ABC): "minimum_should_match": "100%", }, }, - # Add a special wildcard search for id, as it contains a ":" character which is filtered out - # in query_string + # Add a special wildcard search for id, as it contains a ":" character which is + # filtered out in query_string { 'wildcard': { 'id': { @@ -185,8 +168,8 @@ class SearchIndexerBase(ABC): }, }, 'filter': { - 'terms': { - 'id': ids_str + 'bool': { + 'must': terms } } }, @@ -205,9 +188,9 @@ class ContentLibraryIndexer(SearchIndexerBase): SCHEMA_VERSION = 0 @classmethod - def get_item_definition(cls, library_key): - ref = ContentLibrary.objects.get_by_key(library_key) - lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME) + def get_item_definition(cls, item): + ref = ContentLibrary.objects.get_by_key(item) + lib_bundle = LibraryBundle(item, ref.bundle_uuid, draft_name=DRAFT_NAME) num_blocks = len(lib_bundle.get_top_level_usages()) last_published = lib_bundle.get_last_published_time() last_published_str = None @@ -221,7 +204,7 @@ class ContentLibraryIndexer(SearchIndexerBase): # with outdated indexes which might cause errors due to missing/invalid attributes. return { "schema_version": ContentLibraryIndexer.SCHEMA_VERSION, - "id": str(library_key), + "id": str(item), "uuid": str(bundle_metadata.uuid), "title": bundle_metadata.title, "description": bundle_metadata.description, @@ -232,7 +215,7 @@ class ContentLibraryIndexer(SearchIndexerBase): "has_unpublished_deletes": has_unpublished_deletes, # only 'content' field is analyzed by elastisearch, and allows text-search "content": { - "id": str(library_key), + "id": str(item), "title": bundle_metadata.title, "description": bundle_metadata.description, }, @@ -250,17 +233,17 @@ class LibraryBlockIndexer(SearchIndexerBase): SCHEMA_VERSION = 0 @classmethod - def get_item_definition(cls, usage_key): + def get_item_definition(cls, item): from openedx.core.djangoapps.content_libraries.api import get_block_display_name, _lookup_usage_key - def_key, lib_bundle = _lookup_usage_key(usage_key) - is_child = usage_key in lib_bundle.get_bundle_includes().keys() + def_key, lib_bundle = _lookup_usage_key(item) + is_child = item in lib_bundle.get_bundle_includes().keys() - # NOTE: Increment ContentLibraryIndexer.SCHEMA_VERSION if the following schema is updated to avoid dealing + # NOTE: Increment LibraryBlockIndexer.SCHEMA_VERSION if the following schema is updated to avoid dealing # with outdated indexes which might cause errors due to missing/invalid attributes. return { - "schema_version": ContentLibraryIndexer.SCHEMA_VERSION, - "id": str(usage_key), + "schema_version": LibraryBlockIndexer.SCHEMA_VERSION, + "id": str(item), "library_key": str(lib_bundle.library_key), "is_child": is_child, "def_key": str(def_key), @@ -269,7 +252,7 @@ class LibraryBlockIndexer(SearchIndexerBase): "has_unpublished_changes": lib_bundle.does_definition_have_unpublished_changes(def_key), # only 'content' field is analyzed by elastisearch, and allows text-search "content": { - "id": str(usage_key), + "id": str(item), "display_name": get_block_display_name(def_key), }, } @@ -288,8 +271,10 @@ def index_library(sender, library_key, **kwargs): # pylint: disable=unused-argu try: ContentLibraryIndexer.index_items([library_key]) if kwargs.get('update_blocks', False): - blocks = LibraryBlockIndexer.search(library_key=str(library_key)) - usage_keys = [LibraryUsageLocatorV2.from_string(block['data']['id']) for block in blocks] + blocks = LibraryBlockIndexer.get_items(filter_terms={ + 'library_key': str(library_key) + }) + usage_keys = [LibraryUsageLocatorV2.from_string(block['id']) for block in blocks] LibraryBlockIndexer.index_items(usage_keys) except ElasticConnectionError as e: log.exception(e) @@ -303,8 +288,10 @@ def remove_library_index(sender, library_key, **kwargs): # pylint: disable=unus if ContentLibraryIndexer.indexing_is_enabled(): try: ContentLibraryIndexer.remove_items([library_key]) - blocks = LibraryBlockIndexer.search(library_key=str(library_key)) - LibraryBlockIndexer.remove_items([block['data']['id'] for block in blocks]) + blocks = LibraryBlockIndexer.get_items(filter_terms={ + 'library_key': str(library_key) + }) + LibraryBlockIndexer.remove_items([block['id'] for block in blocks]) except ElasticConnectionError as e: log.exception(e) diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index b5f15e8b10..53dde620da 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -12,8 +12,10 @@ from django.conf import settings from django.test.utils import override_settings from organizations.models import Organization from rest_framework.test import APITestCase, APIClient +from search.search_engine_base import SearchEngine from student.tests.factories import UserFactory +from openedx.core.djangoapps.content_libraries.libraries_index import MAX_SIZE from openedx.core.djangolib.testing.utils import skip_unless_cms from openedx.core.lib import blockstore_api @@ -61,7 +63,25 @@ def elasticsearch_test(func): })(func) return func else: - return patch("openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase.SEARCH_KWARGS", new={})(func) + @classmethod + def mock_perform(cls, filter_terms, text_search): + # pylint: disable=no-member + return SearchEngine.get_search_engine(cls.INDEX_NAME).search( + doc_type=cls.DOCUMENT_TYPE, + field_dictionary=filter_terms, + query_string=text_search, + size=MAX_SIZE + ) + + func = patch( + "openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase.SEARCH_KWARGS", + new={} + )(func) + func = patch( + "openedx.core.djangoapps.content_libraries.libraries_index.SearchIndexerBase._perform_elastic_search", + new=mock_perform + )(func) + return func @requires_blockstore diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index b0d406575f..14229e7a91 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -134,9 +134,7 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest): """ Test the filters in the list libraries API """ - features = settings.FEATURES - features['ENABLE_CONTENT_LIBRARY_INDEX'] = is_indexing_enabled - with override_settings(FEATURES=features): + with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}): self._create_library(slug="test-lib1", title="Foo", description="Bar") self._create_library(slug="test-lib2", title="Library-Title-2", description="Bar2") self._create_library(slug="l3", title="Library-Title-3", description="Description") @@ -253,12 +251,11 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest): """ Test the /libraries/{lib_key_str}/blocks API and its pagination """ - features = settings.FEATURES - features['ENABLE_CONTENT_LIBRARY_INDEX'] = is_indexing_enabled - with override_settings(FEATURES=features): + with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}): lib = self._create_library(slug="list_blocks-slug" + str(is_indexing_enabled), title="Library 1") block1 = self._add_block_to_library(lib["id"], "problem", "problem1") block2 = self._add_block_to_library(lib["id"], "unit", "unit1") + self._add_block_to_library(lib["id"], "problem", "problem2", parent_block=block2["id"]) result = self._get_library_blocks(lib["id"]) @@ -286,13 +283,11 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest): """ Test the filters in the list libraries API """ - features = settings.FEATURES - features['ENABLE_CONTENT_LIBRARY_INDEX'] = is_indexing_enabled - with override_settings(FEATURES=features): + with override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': is_indexing_enabled}): lib = self._create_library(slug="test-lib-blocks" + str(is_indexing_enabled), title="Title") block1 = self._add_block_to_library(lib["id"], "problem", "foo-bar") - block2 = self._add_block_to_library(lib["id"], "problem", "foo-baz") - block3 = self._add_block_to_library(lib["id"], "problem", "bar-baz") + self._add_block_to_library(lib["id"], "problem", "foo-baz") + self._add_block_to_library(lib["id"], "problem", "bar-baz") self._set_library_block_olx(block1["id"], "") diff --git a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py index 2c7cf5bcc7..89fb660ed5 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py @@ -9,17 +9,13 @@ from mock import patch from opaque_keys.edx.locator import LibraryLocatorV2, LibraryUsageLocatorV2 from search.search_engine_base import SearchEngine -from openedx.core.djangoapps.content_libraries.libraries_index import ( - ContentLibraryIndexer, - LibraryBlockIndexer, - ItemNotIndexedException, -) +from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryBlockIndexer from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test @override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True}) @elasticsearch_test -class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest): +class ContentLibraryIndexerTest(ContentLibrariesRestApiTest): """ Tests the operation of ContentLibraryIndexer """ @@ -56,34 +52,33 @@ class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest): """ Test that outdated indexes aren't retrieved """ - result = self._create_library(slug="test-lib-schemaupdates-1", title="Title 1", description="Description") - library_key = LibraryLocatorV2.from_string(result['id']) - - ContentLibraryIndexer.get_items([library_key]) + with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION", + new=0): + result = self._create_library(slug="test-lib-schemaupdates-1", title="Title 1", description="Description") + library_key = LibraryLocatorV2.from_string(result['id']) + self.assertEqual(len(ContentLibraryIndexer.get_items([library_key])), 1) with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION", new=1): - with self.assertRaises(ItemNotIndexedException): - ContentLibraryIndexer.get_items([library_key]) + self.assertEqual(len(ContentLibraryIndexer.get_items([library_key])), 0) call_command("reindex_content_library", all=True, force=True) - ContentLibraryIndexer.get_items([library_key]) + self.assertEqual(len(ContentLibraryIndexer.get_items([library_key])), 1) def test_remove_all_libraries(self): """ Test if remove_all_items() deletes all libraries """ lib1 = self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description") - lib2 = self._create_library(slug="test-lib-rm-all-2", title="Title 1", description="Description") + lib2 = self._create_library(slug="test-lib-rm-all-2", title="Title 2", description="Description") library_key1 = LibraryLocatorV2.from_string(lib1['id']) library_key2 = LibraryLocatorV2.from_string(lib2['id']) self.assertEqual(len(ContentLibraryIndexer.get_items([library_key1, library_key2])), 2) ContentLibraryIndexer.remove_all_items() - with self.assertRaises(ItemNotIndexedException): - ContentLibraryIndexer.get_items([library_key1, library_key2]) + self.assertEqual(len(ContentLibraryIndexer.get_items()), 0) def test_update_libraries(self): """ @@ -107,8 +102,8 @@ class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest): self.assertEqual(response['has_unpublished_deletes'], False) self._delete_library(lib['id']) - with self.assertRaises(ItemNotIndexedException): - ContentLibraryIndexer.get_items([library_key]) + self.assertEqual(ContentLibraryIndexer.get_items([library_key]), []) + ContentLibraryIndexer.get_items([library_key]) def test_update_library_blocks(self): """ @@ -206,7 +201,7 @@ class LibraryBlockIndexerTest(ContentLibrariesRestApiTest): block1 = self._add_block_to_library(lib['id'], "problem", "problem1") block2 = self._add_block_to_library(lib['id'], "problem", "problem2") - self.assertEqual(len(LibraryBlockIndexer.search()), 2) + self.assertEqual(len(LibraryBlockIndexer.get_items()), 2) for block in [block1, block2]: usage_key = LibraryUsageLocatorV2.from_string(block['id']) @@ -218,6 +213,24 @@ class LibraryBlockIndexerTest(ContentLibrariesRestApiTest): self.assertEqual(response['display_name'], block['display_name']) self.assertEqual(response['has_unpublished_changes'], block['has_unpublished_changes']) + def test_schema_updates(self): + """ + Test that outdated indexes aren't retrieved + """ + lib = self._create_library(slug="test-lib--block-schemaupdates-1", title="Title 1", description="Description") + with patch("openedx.core.djangoapps.content_libraries.libraries_index.LibraryBlockIndexer.SCHEMA_VERSION", + new=0): + block = self._add_block_to_library(lib['id'], "problem", "problem1") + self.assertEqual(len(LibraryBlockIndexer.get_items([block['id']])), 1) + + with patch("openedx.core.djangoapps.content_libraries.libraries_index.LibraryBlockIndexer.SCHEMA_VERSION", + new=1): + self.assertEqual(len(LibraryBlockIndexer.get_items([block['id']])), 0) + + call_command("reindex_content_library", all=True, force=True) + + self.assertEqual(len(LibraryBlockIndexer.get_items([block['id']])), 1) + def test_remove_all_items(self): """ Test if remove_all_items() deletes all libraries @@ -225,10 +238,10 @@ class LibraryBlockIndexerTest(ContentLibrariesRestApiTest): lib1 = self._create_library(slug="test-lib-rm-all", title="Title 1", description="Description") self._add_block_to_library(lib1['id'], "problem", "problem1") self._add_block_to_library(lib1['id'], "problem", "problem2") - self.assertEqual(len(LibraryBlockIndexer.search()), 2) + self.assertEqual(len(LibraryBlockIndexer.get_items()), 2) LibraryBlockIndexer.remove_all_items() - self.assertEqual(len(LibraryBlockIndexer.search()), 0) + self.assertEqual(len(LibraryBlockIndexer.get_items()), 0) def test_crud_block(self): """ @@ -259,12 +272,10 @@ class LibraryBlockIndexerTest(ContentLibrariesRestApiTest): # Verify that deleting block removes it from index self._delete_library_block(block['id']) - with self.assertRaises(ItemNotIndexedException): - LibraryBlockIndexer.get_items([block['id']]) + self.assertEqual(LibraryBlockIndexer.get_items([block['id']]), []) # Verify that deleting a library removes its blocks from index too self._add_block_to_library(lib['id'], "problem", "problem1") LibraryBlockIndexer.get_items([block['id']]) self._delete_library(lib['id']) - with self.assertRaises(ItemNotIndexedException): - LibraryBlockIndexer.get_items([block['id']]) + self.assertEqual(LibraryBlockIndexer.get_items([block['id']]), [])