diff --git a/cms/envs/test.py b/cms/envs/test.py index dc56e0e177..bb387639aa 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -250,6 +250,16 @@ SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine" FEATURES['ENABLE_ENROLLMENT_TRACK_USER_PARTITION'] = True +####################### ELASTICSEARCH TESTS ####################### +# Enable this when testing elasticsearch-based code which couldn't be tested using the mock engine +ENABLE_ELASTICSEARCH_FOR_TESTS = os.environ.get( + 'EDXAPP_ENABLE_ELASTICSEARCH_FOR_TESTS', 'no').lower() in ('true', 'yes', '1') + +TEST_ELASTICSEARCH_USE_SSL = os.environ.get( + 'EDXAPP_TEST_ELASTICSEARCH_USE_SSL', 'no').lower() in ('true', 'yes', '1') +TEST_ELASTICSEARCH_HOST = os.environ.get('EDXAPP_TEST_ELASTICSEARCH_HOST', 'edx.devstack.elasticsearch') +TEST_ELASTICSEARCH_PORT = int(os.environ.get('EDXAPP_TEST_ELASTICSEARCH_PORT', '9200')) + ########################## AUTHOR PERMISSION ####################### FEATURES['ENABLE_CREATOR_GROUP'] = False diff --git a/lms/envs/test.py b/lms/envs/test.py index f633e68e8c..4f384233e6 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -458,6 +458,16 @@ FACEBOOK_APP_SECRET = "Test" FACEBOOK_APP_ID = "Test" FACEBOOK_API_VERSION = "v2.8" +####################### ELASTICSEARCH TESTS ####################### +# Enable this when testing elasticsearch-based code which couldn't be tested using the mock engine +ENABLE_ELASTICSEARCH_FOR_TESTS = os.environ.get( + 'EDXAPP_ENABLE_ELASTICSEARCH_FOR_TESTS', 'no').lower() in ('true', 'yes', '1') + +TEST_ELASTICSEARCH_USE_SSL = os.environ.get( + 'EDXAPP_TEST_ELASTICSEARCH_USE_SSL', 'no').lower() in ('true', 'yes', '1') +TEST_ELASTICSEARCH_HOST = os.environ.get('EDXAPP_TEST_ELASTICSEARCH_HOST', 'edx.devstack.elasticsearch') +TEST_ELASTICSEARCH_PORT = int(os.environ.get('EDXAPP_TEST_ELASTICSEARCH_PORT', '9200')) + ######### custom courses ######### INSTALLED_APPS += ['lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig'] FEATURES['CUSTOM_COURSES_EDX'] = True diff --git a/openedx/core/djangoapps/content_libraries/libraries_index.py b/openedx/core/djangoapps/content_libraries/libraries_index.py index 9d9321a6a8..bbdbddd970 100644 --- a/openedx/core/djangoapps/content_libraries/libraries_index.py +++ b/openedx/core/djangoapps/content_libraries/libraries_index.py @@ -39,6 +39,11 @@ class ContentLibraryIndexer: INDEX_NAME = "content_library_index" LIBRARY_DOCUMENT_TYPE = "content_library" + SEARCH_KWARGS = { + # Set this to True or 'wait_for' if immediate refresh is required after any update. + # See elastic docs for more information. + 'refresh': False + } SCHEMA_VERSION = 0 @@ -84,8 +89,7 @@ class ContentLibraryIndexer: }, } library_dicts.append(library_dict) - - return searcher.index(cls.LIBRARY_DOCUMENT_TYPE, library_dicts) + return searcher.index(cls.LIBRARY_DOCUMENT_TYPE, library_dicts, **cls.SEARCH_KWARGS) @classmethod def get_libraries(cls, library_keys, text_search=None): @@ -144,7 +148,7 @@ class ContentLibraryIndexer: """ searcher = SearchEngine.get_search_engine(cls.INDEX_NAME) ids_str = [str(key) for key in library_keys] - searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids_str) + searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids_str, **cls.SEARCH_KWARGS) @classmethod def remove_all_libraries(cls): @@ -154,7 +158,7 @@ class ContentLibraryIndexer: searcher = SearchEngine.get_search_engine(cls.INDEX_NAME) response = searcher.search(doc_type=cls.LIBRARY_DOCUMENT_TYPE, filter_dictionary={}, size=MAX_SIZE) ids = [result["data"]["id"] for result in response["results"]] - searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids) + searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids, **cls.SEARCH_KWARGS) @classmethod def indexing_is_enabled(cls): @@ -198,6 +202,7 @@ def build_elastic_query(library_keys_str, text_search): """ # Remove reserved characters (and ") from the text to prevent unexpected errors. text_search_normalised = text_search.translate(text_search.maketrans('', '', RESERVED_CHARACTERS + '"')) + text_search_normalised = text_search.replace('-',' ') # Wrap with asterix to enable partial matches text_search_normalised = "*{}*".format(text_search_normalised) return { diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index 2f0f908d73..6881f22d46 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -4,10 +4,12 @@ Tests for Blockstore-based Content Libraries """ from contextlib import contextmanager from io import BytesIO +from mock import patch from urllib.parse import urlencode import unittest from django.conf import settings +from django.test.utils import override_settings from organizations.models import Organization from rest_framework.test import APITestCase, APIClient @@ -42,6 +44,26 @@ URL_BLOCK_METADATA_URL = '/api/xblock/v2/xblocks/{block_key}/' requires_blockstore = unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server") +def elasticsearch_test(func): + """ + Decorator for tests which connect to elasticsearch when needed + """ + # This is disabled by default. Set to True if the elasticsearch engine is needed to test parts of code. + if settings.ENABLE_ELASTICSEARCH_FOR_TESTS: + func = override_settings(SEARCH_ENGINE="search.elastic.ElasticSearchEngine")(func) + func = override_settings(ELASTIC_SEARCH_CONFIG=[{ + 'use_ssl': settings.TEST_ELASTICSEARCH_USE_SSL, + 'host': settings.TEST_ELASTICSEARCH_HOST, + 'port': settings.TEST_ELASTICSEARCH_PORT, + }])(func) + func = patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SEARCH_KWARGS", new={ + 'refresh': 'wait_for' + })(func) + return func + else: + return patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SEARCH_KWARGS", new={})(func) + + @requires_blockstore @skip_unless_cms # Content Libraries REST API is only available in Studio class ContentLibrariesRestApiTest(APITestCase): diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index 493ce4ad69..cc95e6eda9 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -12,12 +12,13 @@ from django.test.utils import override_settings from mock import patch from organizations.models import Organization -from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest +from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test from openedx.core.djangoapps.content_libraries.api import BlockLimitReachedError from student.tests.factories import UserFactory @ddt.ddt +@elasticsearch_test class ContentLibrariesTest(ContentLibrariesRestApiTest): """ General tests for Blockstore-based Content Libraries @@ -89,7 +90,6 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest): @ddt.data(True, False) @patch("openedx.core.djangoapps.content_libraries.views.LibraryRootPagination.page_size", new=2) - @override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine") def test_list_library(self, is_indexing_enabled): """ Test the /libraries API and its pagination @@ -105,7 +105,8 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest): result = self._list_libraries() self.assertEqual(len(result), 2) - self.assertEqual(result[0], lib1) + self.assertIn(lib1, result) + self.assertIn(lib2, result) result = self._list_libraries({'pagination': 'true'}) self.assertEqual(len(result['results']), 2) self.assertEqual(result['next'], None) diff --git a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py index a8fb06d239..e06a98c28d 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py @@ -10,16 +10,17 @@ from opaque_keys.edx.locator import LibraryLocatorV2 from search.search_engine_base import SearchEngine from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryNotIndexedException -from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest +from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test @override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True}) -@override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine") +@elasticsearch_test class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest): """ Tests the operation of ContentLibraryIndexer """ + @elasticsearch_test def setUp(self): super().setUp() ContentLibraryIndexer.remove_all_libraries() @@ -32,9 +33,6 @@ class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest): result1 = self._create_library(slug="test-lib-index-1", title="Title 1", description="Description") result2 = self._create_library(slug="test-lib-index-2", title="Title 2", description="Description") - response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={}) - self.assertEqual(response['total'], 2) - for result in [result1, result2]: library_key = LibraryLocatorV2.from_string(result['id']) response = ContentLibraryIndexer.get_libraries([library_key])[0] @@ -71,15 +69,16 @@ class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest): """ Test if remove_all_libraries() deletes all libraries """ - self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description") - self._create_library(slug="test-lib-rm-all-2", title="Title 2", description="Description") + lib1 = self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description") + lib2 = self._create_library(slug="test-lib-rm-all-2", title="Title 1", description="Description") + library_key1 = LibraryLocatorV2.from_string(lib1['id']) + library_key2 = LibraryLocatorV2.from_string(lib2['id']) - response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={}) - self.assertEqual(response['total'], 2) + ContentLibraryIndexer.get_libraries([library_key1, library_key2]) ContentLibraryIndexer.remove_all_libraries() - response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={}) - self.assertEqual(response['total'], 0) + with self.assertRaises(LibraryNotIndexedException): + ContentLibraryIndexer.get_libraries([library_key1, library_key2]) def test_update_libraries(self): """