From 4198eba6c19328781c9360b767b04746bd9c3d91 Mon Sep 17 00:00:00 2001 From: Sid Verma Date: Fri, 21 Aug 2020 20:35:18 +0530 Subject: [PATCH] Add schema version in library indexes, improve reindexing command --- .../content_libraries/libraries_index.py | 10 ++++++++- .../commands/reindex_content_library.py | 15 +++++++++++-- .../tests/test_libraries_index.py | 22 ++++++++++++++++++- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/openedx/core/djangoapps/content_libraries/libraries_index.py b/openedx/core/djangoapps/content_libraries/libraries_index.py index 5a9c7a7a55..8124e61b6e 100644 --- a/openedx/core/djangoapps/content_libraries/libraries_index.py +++ b/openedx/core/djangoapps/content_libraries/libraries_index.py @@ -39,6 +39,8 @@ class ContentLibraryIndexer: INDEX_NAME = "content_library_index" LIBRARY_DOCUMENT_TYPE = "content_library" + SCHEMA_VERSION = 0 + @classmethod def index_libraries(cls, library_keys): """ @@ -60,7 +62,10 @@ class ContentLibraryIndexer: bundle_metadata = get_bundle(ref.bundle_uuid) + # NOTE: Increment ContentLibraryIndexer.SCHEMA_VERSION if the following schema is updated to avoid dealing + # with outdated indexes which might cause errors due to missing/invalid attributes. library_dict = { + "schema_version": ContentLibraryIndexer.SCHEMA_VERSION, "id": str(library_key), "uuid": str(bundle_metadata.uuid), "title": bundle_metadata.title, @@ -84,7 +89,10 @@ class ContentLibraryIndexer: library_keys_str = [str(key) for key in library_keys] response = searcher.search( doc_type=cls.LIBRARY_DOCUMENT_TYPE, - field_dictionary={"id": library_keys_str}, + field_dictionary={ + "id": library_keys_str, + "schema_version": ContentLibraryIndexer.SCHEMA_VERSION + }, size=MAX_SIZE, ) diff --git a/openedx/core/djangoapps/content_libraries/management/commands/reindex_content_library.py b/openedx/core/djangoapps/content_libraries/management/commands/reindex_content_library.py index b26b013ae6..205ebae552 100644 --- a/openedx/core/djangoapps/content_libraries/management/commands/reindex_content_library.py +++ b/openedx/core/djangoapps/content_libraries/management/commands/reindex_content_library.py @@ -1,6 +1,8 @@ """ Management command to update content libraries' search index """ +import logging + from textwrap import dedent from django.core.management import BaseCommand @@ -43,20 +45,29 @@ class Command(BaseCommand): dest='all', help='Reindex all libraries' ) + parser.add_argument( + '--force', + action='store_true', + dest='force', + help='Run command without user prompt for confirmation' + ) parser.add_argument('library_ids', nargs='*') def handle(self, *args, **options): if options['clear-all']: - if query_yes_no(self.CONFIRMATION_PROMPT_CLEAR, default="no"): + if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_CLEAR, default="no"): + logging.info("Removing all libraries from the index") ContentLibraryIndexer.remove_all_libraries() return if options['all']: - if query_yes_no(self.CONFIRMATION_PROMPT_ALL, default="no"): + if options['force'] or query_yes_no(self.CONFIRMATION_PROMPT_ALL, default="no"): + logging.info("Indexing all libraries") library_keys = [library.library_key for library in ContentLibrary.objects.all()] else: return else: + logging.info("Indexing libraries: {}".format(options['library_ids'])) library_keys = list(map(LibraryLocatorV2.from_string, options['library_ids'])) ContentLibraryIndexer.index_libraries(library_keys) diff --git a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py index 2fc573d9fc..876a3c2ca6 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py @@ -3,9 +3,11 @@ Testing indexing of blockstore based content libraries """ from django.conf import settings +from django.core.management import call_command from django.test.utils import override_settings -from search.search_engine_base import SearchEngine +from mock import patch from opaque_keys.edx.locator import LibraryLocatorV2 +from search.search_engine_base import SearchEngine from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryNotIndexedException from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest @@ -47,6 +49,24 @@ class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest): self.assertEqual(response['has_unpublished_changes'], False) self.assertEqual(response['has_unpublished_deletes'], False) + def test_schema_updates(self): + """ + Test that outdated indexes aren't retrieved + """ + result = self._create_library(slug="test-lib-schemaupdates-1", title="Title 1", description="Description") + library_key = LibraryLocatorV2.from_string(result['id']) + + ContentLibraryIndexer.get_libraries([library_key]) + + with patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SCHEMA_VERSION", + new=1): + with self.assertRaises(LibraryNotIndexedException): + ContentLibraryIndexer.get_libraries([library_key]) + + call_command("reindex_content_library", all=True, quiet=True) + + ContentLibraryIndexer.get_libraries([library_key]) + def test_remove_all_libraries(self): """ Test if remove_all_libraries() deletes all libraries