Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #7313 from edx/mjames/SOL-174

Browse Source 
SOL-174 - Indexing within celery task
This commit is contained in:
Martyn James
2015-04-10 15:22:20 -04:00
parent d54cb3b962 e37a98d1be
commit 953581e62f
16 changed files with 916 additions and 295 deletions
Download Patch File Download Diff File Expand all files Collapse all files

176
common/lib/xmodule/xmodule/modulestore/courseware_index.py
View File

@@ -1,176 +0,0 @@
""" Code to allow module store to interface with courseware index """
from __future__ import absolute_import
import logging
from django.utils.translation import ugettext as _
from opaque_keys.edx.locator import CourseLocator
from search.search_engine_base import SearchEngine
from eventtracking import tracker
from . import ModuleStoreEnum
from .exceptions import ItemNotFoundError
# Use default index and document names for now
INDEX_NAME = "courseware_index"
DOCUMENT_TYPE = "courseware_content"
log = logging.getLogger('edx.modulestore')
class SearchIndexingError(Exception):
""" Indicates some error(s) occured during indexing """
def __init__(self, message, error_list):
super(SearchIndexingError, self).__init__(message)
self.error_list = error_list
class CoursewareSearchIndexer(object):
"""
Class to perform indexing for courseware search from different modulestores
"""
@staticmethod
def add_to_search_index(modulestore, location, delete=False, raise_on_error=False):
"""
Add to courseware search index from given location and its children
"""
error_list = []
indexed_count = 0
# TODO - inline for now, need to move this out to a celery task
searcher = SearchEngine.get_search_engine(INDEX_NAME)
if not searcher:
return
if isinstance(location, CourseLocator):
course_key = location
else:
course_key = location.course_key
location_info = {
"course": unicode(course_key),
}
def _fetch_item(item_location):
""" Fetch the item from the modulestore location, log if not found, but continue """
try:
if isinstance(item_location, CourseLocator):
item = modulestore.get_course(item_location)
else:
item = modulestore.get_item(item_location, revision=ModuleStoreEnum.RevisionOption.published_only)
except ItemNotFoundError:
log.warning('Cannot find: %s', item_location)
return None
return item
def index_item_location(item_location, current_start_date):
""" add this item to the search index """
item = _fetch_item(item_location)
if not item:
return
is_indexable = hasattr(item, "index_dictionary")
# if it's not indexable and it does not have children, then ignore
if not is_indexable and not item.has_children:
return
# if it has a defined start, then apply it and to it's children
if item.start and (not current_start_date or item.start > current_start_date):
current_start_date = item.start
if item.has_children:
for child_loc in item.children:
index_item_location(child_loc, current_start_date)
item_index = {}
item_index_dictionary = item.index_dictionary() if is_indexable else None
# if it has something to add to the index, then add it
if item_index_dictionary:
try:
item_index.update(location_info)
item_index.update(item_index_dictionary)
item_index['id'] = unicode(item.scope_ids.usage_id)
if current_start_date:
item_index['start_date'] = current_start_date
searcher.index(DOCUMENT_TYPE, item_index)
except Exception as err: # pylint: disable=broad-except
# broad exception so that index operation does not fail on one item of many
log.warning('Could not index item: %s - %s', item_location, unicode(err))
error_list.append(_('Could not index item: {}').format(item_location))
def remove_index_item_location(item_location):
""" remove this item from the search index """
item = _fetch_item(item_location)
if item:
if item.has_children:
for child_loc in item.children:
remove_index_item_location(child_loc)
searcher.remove(DOCUMENT_TYPE, unicode(item.scope_ids.usage_id))
try:
if delete:
remove_index_item_location(location)
else:
index_item_location(location, None)
indexed_count += 1
except Exception as err: # pylint: disable=broad-except
# broad exception so that index operation does not prevent the rest of the application from working
log.exception(
"Indexing error encountered, courseware index may be out of date %s - %s",
course_key,
unicode(err)
)
error_list.append(_('General indexing error occurred'))
if raise_on_error and error_list:
raise SearchIndexingError(_('Error(s) present during indexing'), error_list)
return indexed_count
@classmethod
def do_publish_index(cls, modulestore, location, delete=False, raise_on_error=False):
"""
Add to courseware search index published section and children
"""
indexed_count = cls.add_to_search_index(modulestore, location, delete, raise_on_error)
cls._track_index_request('edx.course.index.published', indexed_count, str(location))
return indexed_count
@classmethod
def do_course_reindex(cls, modulestore, course_key):
"""
(Re)index all content within the given course
"""
indexed_count = cls.add_to_search_index(modulestore, course_key, delete=False, raise_on_error=True)
cls._track_index_request('edx.course.index.reindexed', indexed_count)
return indexed_count
@staticmethod
def _track_index_request(event_name, indexed_count, location=None):
"""Track content index requests.
Arguments:
location (str): The ID of content to be indexed.
event_name (str): Name of the event to be logged.
Returns:
None
"""
data = {
"indexed_count": indexed_count,
'category': 'courseware_index',
}
if location:
data['location_id'] = location
tracker.emit(
event_name,
data
)

8
common/lib/xmodule/xmodule/modulestore/mongo/draft.py
View File

@@ -12,7 +12,6 @@ import logging
from opaque_keys.edx.locations import Location
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
from xmodule.modulestore.exceptions import (
ItemNotFoundError, DuplicateItemError, DuplicateCourseError, InvalidBranchSetting
)
@@ -565,10 +564,6 @@ class DraftModuleStore(MongoModuleStore):
)
self._delete_subtree(location, as_functions)
# Remove this location from the courseware search index so that searches
# will refrain from showing it as a result
CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
def _delete_subtree(self, location, as_functions, draft_only=False):
"""
Internal method for deleting all of the subtree whose revisions match the as_functions
@@ -745,9 +740,6 @@ class DraftModuleStore(MongoModuleStore):
self._flag_publish_event(course_key)
# Now it's been published, add the object to the courseware search index so that it appears in search results
CoursewareSearchIndexer.do_publish_index(self, location)
return self.get_item(as_published(location))
def unpublish(self, location, user_id, **kwargs):

8
common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
View File

@@ -5,7 +5,6 @@ Module for the dual-branch fall-back Draft->Published Versioning ModuleStore
from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore, EXCLUDE_ALL
from xmodule.exceptions import InvalidVersionError
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.courseware_index import CoursewareSearchIndexer
from xmodule.modulestore.exceptions import InsufficientSpecificationError, ItemNotFoundError
from xmodule.modulestore.draft_and_published import (
ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES, UnsupportedRevisionError
@@ -217,10 +216,6 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
if branch == ModuleStoreEnum.BranchName.draft and branched_location.block_type in DIRECT_ONLY_CATEGORIES:
self.publish(parent_loc.version_agnostic(), user_id, blacklist=EXCLUDE_ALL, **kwargs)
# Remove this location from the courseware search index so that searches
# will refrain from showing it as a result
CoursewareSearchIndexer.add_to_search_index(self, location, delete=True)
def _map_revision_to_branch(self, key, revision=None):
"""
Maps RevisionOptions to BranchNames, inserting them into the key
@@ -366,9 +361,6 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
self._flag_publish_event(location.course_key)
# Now it's been published, add the object to the courseware search index so that it appears in search results
CoursewareSearchIndexer.do_publish_index(self, location)
return self.get_item(location.for_branch(ModuleStoreEnum.BranchName.published), **kwargs)
def unpublish(self, location, user_id, **kwargs):

20
common/lib/xmodule/xmodule/vertical_block.py
View File

@@ -128,3 +128,23 @@ class VerticalBlock(SequenceFields, XModuleFields, StudioEditableBlock, XmlParse
# TODO: Remove this when studio better supports editing of pure XBlocks.
fragment.add_javascript('VerticalBlock = XModule.Descriptor;')
return fragment
def index_dictionary(self):
"""
Return dictionary prepared with module content and type for indexing.
"""
# return key/value fields in a Python dict object
# values may be numeric / string or dict
# default implementation is an empty dict
xblock_body = super(VerticalBlock, self).index_dictionary()
index_body = {
"display_name": self.display_name,
}
if "content" in xblock_body:
xblock_body["content"].update(index_body)
else:
xblock_body["content"] = index_body
# We use "Sequence" for sequentials and verticals
xblock_body["content_type"] = "Sequence"
return xblock_body

68
common/test/acceptance/tests/lms/test_lms_courseware_search.py
View File

@@ -84,17 +84,11 @@ class CoursewareSearchTest(UniqueCourseTest):
AutoAuthPage(self.browser, username=username, email=email,
course_id=self.course_id, staff=staff).visit()
def test_page_existence(self):
"""
Make sure that the page is accessible.
"""
self._auto_auth(self.USERNAME, self.EMAIL, False)
self.courseware_search_page.visit()
def _studio_publish_content(self, section_index):
"""
Publish content on studio course page under specified section
"""
self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
self.course_outline.visit()
subsection = self.course_outline.section_at(section_index).subsection_at(0)
subsection.expand_subsection()
@@ -105,6 +99,7 @@ class CoursewareSearchTest(UniqueCourseTest):
"""
Edit chapter name on studio course page under specified section
"""
self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
self.course_outline.visit()
section = self.course_outline.section_at(section_index)
section.change_name(self.EDITED_CHAPTER_NAME)
@@ -114,6 +109,7 @@ class CoursewareSearchTest(UniqueCourseTest):
Add content on studio course page under specified section
"""
self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
# create a unit in course outline
self.course_outline.visit()
subsection = self.course_outline.section_at(section_index).subsection_at(0)
@@ -140,30 +136,44 @@ class CoursewareSearchTest(UniqueCourseTest):
self.course_outline.start_reindex()
self.course_outline.wait_for_ajax()
def _search_for_content(self, search_term):
"""
Login and search for specific content
Arguments:
search_term - term to be searched for
Returns:
(bool) True if search term is found in resulting content; False if not found
"""
self._auto_auth(self.USERNAME, self.EMAIL, False)
self.courseware_search_page.visit()
self.courseware_search_page.search_for_term(search_term)
return search_term in self.courseware_search_page.search_results.html[0]
def test_page_existence(self):
"""
Make sure that the page is accessible.
"""
self._auto_auth(self.USERNAME, self.EMAIL, False)
self.courseware_search_page.visit()
def test_search(self):
"""
Make sure that you can search for something.
"""
# Create content in studio without publishing.
self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
self._studio_add_content(0)
# Do a search, there should be no results shown.
self._auto_auth(self.USERNAME, self.EMAIL, False)
self.courseware_search_page.visit()
self.courseware_search_page.search_for_term(self.SEARCH_STRING)
assert self.SEARCH_STRING not in self.courseware_search_page.search_results.html[0]
self.assertFalse(self._search_for_content(self.SEARCH_STRING))
# Publish in studio to trigger indexing.
self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
self._studio_publish_content(0)
# Do the search again, this time we expect results.
self._auto_auth(self.USERNAME, self.EMAIL, False)
self.courseware_search_page.visit()
self.courseware_search_page.search_for_term(self.SEARCH_STRING)
assert self.SEARCH_STRING in self.courseware_search_page.search_results.html[0]
self.assertTrue(self._search_for_content(self.SEARCH_STRING))
def test_reindex(self):
"""
@@ -171,24 +181,24 @@ class CoursewareSearchTest(UniqueCourseTest):
"""
# Create content in studio without publishing.
self._auto_auth(self.STAFF_USERNAME, self.STAFF_EMAIL, True)
self._studio_add_content(1)
# Do a search, there should be no results shown.
self.assertFalse(self._search_for_content(self.EDITED_SEARCH_STRING))
# Publish in studio to trigger indexing, and edit chapter name afterwards.
self._studio_publish_content(1)
# Do a ReIndex from studio to ensure that our stuff is updated before the next stage of the test
self._studio_reindex()
# Search after publish, there should still be no results shown.
self.assertFalse(self._search_for_content(self.EDITED_SEARCH_STRING))
self._studio_edit_chapter_name(1)
# Do a search, there should be no results shown.
self._auto_auth(self.USERNAME, self.EMAIL, False)
self.courseware_search_page.visit()
self.courseware_search_page.search_for_term(self.EDITED_SEARCH_STRING)
assert self.EDITED_SEARCH_STRING not in self.courseware_search_page.search_results.html[0]
# Do a ReIndex from studio, to add edited chapter name
# Do a ReIndex from studio to ensure that our stuff is updated before the next stage of the test
self._studio_reindex()
# Do the search again, this time we expect results.
self._auto_auth(self.USERNAME, self.EMAIL, False)
self.courseware_search_page.visit()
self.courseware_search_page.search_for_term(self.EDITED_SEARCH_STRING)
assert self.EDITED_SEARCH_STRING in self.courseware_search_page.search_results.html[0]
self.assertTrue(self._search_for_content(self.EDITED_SEARCH_STRING))