Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add elasticsearch tests

Browse Source
This commit is contained in:
Sid Verma
2020-09-03 12:14:49 +05:30
committed by Kyle McCormick
parent 78045115ab
commit 735bc412d4
6 changed files with 65 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
cms/envs/test.py
View File

@@ -250,6 +250,16 @@ SEARCH_ENGINE = "search.tests.mock_search_engine.MockSearchEngine"
FEATURES['ENABLE_ENROLLMENT_TRACK_USER_PARTITION'] = True
####################### ELASTICSEARCH TESTS #######################
# Enable this when testing elasticsearch-based code which couldn't be tested using the mock engine
ENABLE_ELASTICSEARCH_FOR_TESTS = os.environ.get(
'EDXAPP_ENABLE_ELASTICSEARCH_FOR_TESTS', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_USE_SSL = os.environ.get(
'EDXAPP_TEST_ELASTICSEARCH_USE_SSL', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_HOST = os.environ.get('EDXAPP_TEST_ELASTICSEARCH_HOST', 'edx.devstack.elasticsearch')
TEST_ELASTICSEARCH_PORT = int(os.environ.get('EDXAPP_TEST_ELASTICSEARCH_PORT', '9200'))
########################## AUTHOR PERMISSION #######################
FEATURES['ENABLE_CREATOR_GROUP'] = False

10
lms/envs/test.py
View File

@@ -458,6 +458,16 @@ FACEBOOK_APP_SECRET = "Test"
FACEBOOK_APP_ID = "Test"
FACEBOOK_API_VERSION = "v2.8"
####################### ELASTICSEARCH TESTS #######################
# Enable this when testing elasticsearch-based code which couldn't be tested using the mock engine
ENABLE_ELASTICSEARCH_FOR_TESTS = os.environ.get(
'EDXAPP_ENABLE_ELASTICSEARCH_FOR_TESTS', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_USE_SSL = os.environ.get(
'EDXAPP_TEST_ELASTICSEARCH_USE_SSL', 'no').lower() in ('true', 'yes', '1')
TEST_ELASTICSEARCH_HOST = os.environ.get('EDXAPP_TEST_ELASTICSEARCH_HOST', 'edx.devstack.elasticsearch')
TEST_ELASTICSEARCH_PORT = int(os.environ.get('EDXAPP_TEST_ELASTICSEARCH_PORT', '9200'))
######### custom courses #########
INSTALLED_APPS += ['lms.djangoapps.ccx', 'openedx.core.djangoapps.ccxcon.apps.CCXConnectorConfig']
FEATURES['CUSTOM_COURSES_EDX'] = True

13
openedx/core/djangoapps/content_libraries/libraries_index.py
View File

@@ -39,6 +39,11 @@ class ContentLibraryIndexer:
INDEX_NAME = "content_library_index"
LIBRARY_DOCUMENT_TYPE = "content_library"
SEARCH_KWARGS = {
# Set this to True or 'wait_for' if immediate refresh is required after any update.
# See elastic docs for more information.
'refresh': False
}
SCHEMA_VERSION = 0
@@ -84,8 +89,7 @@ class ContentLibraryIndexer:
},
}
library_dicts.append(library_dict)
return searcher.index(cls.LIBRARY_DOCUMENT_TYPE, library_dicts)
return searcher.index(cls.LIBRARY_DOCUMENT_TYPE, library_dicts, **cls.SEARCH_KWARGS)
@classmethod
def get_libraries(cls, library_keys, text_search=None):
@@ -144,7 +148,7 @@ class ContentLibraryIndexer:
"""
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
ids_str = [str(key) for key in library_keys]
searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids_str)
searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids_str, **cls.SEARCH_KWARGS)
@classmethod
def remove_all_libraries(cls):
@@ -154,7 +158,7 @@ class ContentLibraryIndexer:
searcher = SearchEngine.get_search_engine(cls.INDEX_NAME)
response = searcher.search(doc_type=cls.LIBRARY_DOCUMENT_TYPE, filter_dictionary={}, size=MAX_SIZE)
ids = [result["data"]["id"] for result in response["results"]]
searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids)
searcher.remove(cls.LIBRARY_DOCUMENT_TYPE, ids, **cls.SEARCH_KWARGS)
@classmethod
def indexing_is_enabled(cls):
@@ -198,6 +202,7 @@ def build_elastic_query(library_keys_str, text_search):
"""
# Remove reserved characters (and ") from the text to prevent unexpected errors.
text_search_normalised = text_search.translate(text_search.maketrans('', '', RESERVED_CHARACTERS + '"'))
text_search_normalised = text_search.replace('-',' ')
# Wrap with asterix to enable partial matches
text_search_normalised = "*{}*".format(text_search_normalised)
return {

22
openedx/core/djangoapps/content_libraries/tests/base.py
View File

@@ -4,10 +4,12 @@ Tests for Blockstore-based Content Libraries
"""
from contextlib import contextmanager
from io import BytesIO
from mock import patch
from urllib.parse import urlencode
import unittest
from django.conf import settings
from django.test.utils import override_settings
from organizations.models import Organization
from rest_framework.test import APITestCase, APIClient
@@ -42,6 +44,26 @@ URL_BLOCK_METADATA_URL = '/api/xblock/v2/xblocks/{block_key}/'
requires_blockstore = unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server")
def elasticsearch_test(func):
"""
Decorator for tests which connect to elasticsearch when needed
"""
# This is disabled by default. Set to True if the elasticsearch engine is needed to test parts of code.
if settings.ENABLE_ELASTICSEARCH_FOR_TESTS:
func = override_settings(SEARCH_ENGINE="search.elastic.ElasticSearchEngine")(func)
func = override_settings(ELASTIC_SEARCH_CONFIG=[{
'use_ssl': settings.TEST_ELASTICSEARCH_USE_SSL,
'host': settings.TEST_ELASTICSEARCH_HOST,
'port': settings.TEST_ELASTICSEARCH_PORT,
}])(func)
func = patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SEARCH_KWARGS", new={
'refresh': 'wait_for'
})(func)
return func
else:
return patch("openedx.core.djangoapps.content_libraries.libraries_index.ContentLibraryIndexer.SEARCH_KWARGS", new={})(func)
@requires_blockstore
@skip_unless_cms # Content Libraries REST API is only available in Studio
class ContentLibrariesRestApiTest(APITestCase):

7
openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py
View File

@@ -12,12 +12,13 @@ from django.test.utils import override_settings
from mock import patch
from organizations.models import Organization
from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest
from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test
from openedx.core.djangoapps.content_libraries.api import BlockLimitReachedError
from student.tests.factories import UserFactory
@ddt.ddt
@elasticsearch_test
class ContentLibrariesTest(ContentLibrariesRestApiTest):
"""
General tests for Blockstore-based Content Libraries
@@ -89,7 +90,6 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest):
@ddt.data(True, False)
@patch("openedx.core.djangoapps.content_libraries.views.LibraryRootPagination.page_size", new=2)
@override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine")
def test_list_library(self, is_indexing_enabled):
"""
Test the /libraries API and its pagination
@@ -105,7 +105,8 @@ class ContentLibrariesTest(ContentLibrariesRestApiTest):
result = self._list_libraries()
self.assertEqual(len(result), 2)
self.assertEqual(result[0], lib1)
self.assertIn(lib1, result)
self.assertIn(lib2, result)
result = self._list_libraries({'pagination': 'true'})
self.assertEqual(len(result['results']), 2)
self.assertEqual(result['next'], None)

21
openedx/core/djangoapps/content_libraries/tests/test_libraries_index.py
View File

@@ -10,16 +10,17 @@ from opaque_keys.edx.locator import LibraryLocatorV2
from search.search_engine_base import SearchEngine
from openedx.core.djangoapps.content_libraries.libraries_index import ContentLibraryIndexer, LibraryNotIndexedException
from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest
from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest, elasticsearch_test
@override_settings(FEATURES={**settings.FEATURES, 'ENABLE_CONTENT_LIBRARY_INDEX': True})
@override_settings(SEARCH_ENGINE="search.tests.mock_search_engine.MockSearchEngine")
@elasticsearch_test
class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest):
"""
Tests the operation of ContentLibraryIndexer
"""
@elasticsearch_test
def setUp(self):
super().setUp()
ContentLibraryIndexer.remove_all_libraries()
@@ -32,9 +33,6 @@ class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest):
result1 = self._create_library(slug="test-lib-index-1", title="Title 1", description="Description")
result2 = self._create_library(slug="test-lib-index-2", title="Title 2", description="Description")
response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={})
self.assertEqual(response['total'], 2)
for result in [result1, result2]:
library_key = LibraryLocatorV2.from_string(result['id'])
response = ContentLibraryIndexer.get_libraries([library_key])[0]
@@ -71,15 +69,16 @@ class ContentLibraryIndexerIndexer(ContentLibrariesRestApiTest):
"""
Test if remove_all_libraries() deletes all libraries
"""
self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description")
self._create_library(slug="test-lib-rm-all-2", title="Title 2", description="Description")
lib1 = self._create_library(slug="test-lib-rm-all-1", title="Title 1", description="Description")
lib2 = self._create_library(slug="test-lib-rm-all-2", title="Title 1", description="Description")
library_key1 = LibraryLocatorV2.from_string(lib1['id'])
library_key2 = LibraryLocatorV2.from_string(lib2['id'])
response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={})
self.assertEqual(response['total'], 2)
ContentLibraryIndexer.get_libraries([library_key1, library_key2])
ContentLibraryIndexer.remove_all_libraries()
response = self.searcher.search(doc_type=ContentLibraryIndexer.LIBRARY_DOCUMENT_TYPE, filter_dictionary={})
self.assertEqual(response['total'], 0)
with self.assertRaises(LibraryNotIndexedException):
ContentLibraryIndexer.get_libraries([library_key1, library_key2])
def test_update_libraries(self):
"""