Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #10455 from edx/search/optimization

Browse Source 
Search performance (SOL-1030, SOL-1031)
This commit is contained in:
Davorin Šego
2015-11-11 10:41:33 +01:00
parent ce99ca63fa a62e403ed8
commit a8cca47cc9
7 changed files with 54 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
lms/lib/courseware_search/lms_filter_generator.py
View File

@@ -5,15 +5,9 @@ This file contains implementation override of SearchFilterGenerator which will a
from microsite_configuration import microsite
from student.models import CourseEnrollment
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.modulestore.django import modulestore
from search.filter_generator import SearchFilterGenerator
from openedx.core.djangoapps.user_api.partition_schemes import RandomUserPartitionScheme
from openedx.core.djangoapps.course_groups.partition_scheme import CohortPartitionScheme
from courseware.access import get_user_role
INCLUDE_SCHEMES = [CohortPartitionScheme, RandomUserPartitionScheme, ]
@@ -31,67 +25,6 @@ class LmsSearchFilterGenerator(SearchFilterGenerator):
self._user_enrollments[user] = CourseEnrollment.enrollments_for_user(user)
return self._user_enrollments[user]
def filter_dictionary(self, **kwargs):
""" LMS implementation, adds filtering by user partition, course id and user """
def get_group_for_user_partition(user_partition, course_key, user):
""" Returns the specified user's group for user partition """
if user_partition.scheme in SCHEME_SUPPORTS_ASSIGNMENT:
return user_partition.scheme.get_group_for_user(
course_key,
user,
user_partition,
assign=False,
)
else:
return user_partition.scheme.get_group_for_user(
course_key,
user,
user_partition,
)
def get_group_ids_for_user(course, user):
""" Collect user partition group ids for user for this course """
partition_groups = []
for user_partition in course.user_partitions:
if user_partition.scheme in INCLUDE_SCHEMES:
group = get_group_for_user_partition(user_partition, course.id, user)
if group:
partition_groups.append(group)
partition_group_ids = [unicode(partition_group.id) for partition_group in partition_groups]
return partition_group_ids if partition_group_ids else None
filter_dictionary = super(LmsSearchFilterGenerator, self).filter_dictionary(**kwargs)
if 'user' in kwargs:
user = kwargs['user']
if 'course_id' in kwargs and kwargs['course_id']:
try:
course_key = CourseKey.from_string(kwargs['course_id'])
except InvalidKeyError:
course_key = SlashSeparatedCourseKey.from_deprecated_string(kwargs['course_id'])
# Staff user looking at course as staff user
if get_user_role(user, course_key) in ('instructor', 'staff'):
return filter_dictionary
# Need to check course exist (if course gets deleted enrollments don't get cleaned up)
course = modulestore().get_course(course_key)
if course:
filter_dictionary['content_groups'] = get_group_ids_for_user(course, user)
else:
user_enrollments = self._enrollments_for_user(user)
content_groups = []
for enrollment in user_enrollments:
course = modulestore().get_course(enrollment.course_id)
if course:
enrollment_group_ids = get_group_ids_for_user(course, user)
if enrollment_group_ids:
content_groups.extend(enrollment_group_ids)
filter_dictionary['content_groups'] = content_groups if content_groups else None
return filter_dictionary
def field_dictionary(self, **kwargs):
""" add course if provided otherwise add courses in which the user is enrolled in """
field_dictionary = super(LmsSearchFilterGenerator, self).field_dictionary(**kwargs)

30
lms/lib/courseware_search/lms_result_processor.py
View File

@@ -8,18 +8,16 @@ from django.core.urlresolvers import reverse
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from search.result_processor import SearchResultProcessor
from xmodule.modulestore.django import modulestore
from courseware.access import has_access
from lms.djangoapps.course_blocks.api import get_course_blocks
from lms.djangoapps.courseware.access import has_access
class LmsSearchResultProcessor(SearchResultProcessor):
""" SearchResultProcessor for LMS Search """
_course_key = None
_course_name = None
_usage_key = None
_module_store = None
_module_temp_dictionary = {}
_course_blocks = {}
def get_course_key(self):
""" fetch course key object from string representation - retain result for subsequent uses """
@@ -39,11 +37,13 @@ class LmsSearchResultProcessor(SearchResultProcessor):
self._module_store = modulestore()
return self._module_store
def get_item(self, usage_key):
""" fetch item from the modulestore - don't refetch if we've already retrieved it beforehand """
if usage_key not in self._module_temp_dictionary:
self._module_temp_dictionary[usage_key] = self.get_module_store().get_item(usage_key)
return self._module_temp_dictionary[usage_key]
def get_course_blocks(self, user):
""" fetch cached blocks for course - retain for subsequent use """
course_key = self.get_course_key()
if course_key not in self._course_blocks:
root_block_usage_key = self.get_module_store().make_course_usage_key(course_key)
self._course_blocks[course_key] = get_course_blocks(user, root_block_usage_key)
return self._course_blocks[course_key]
@property
def url(self):
@@ -60,10 +60,6 @@ class LmsSearchResultProcessor(SearchResultProcessor):
def should_remove(self, user):
""" Test to see if this result should be removed due to access restriction """
user_has_access = has_access(
user,
"load",
self.get_item(self.get_usage_key()),
self.get_course_key()
)
return not user_has_access
if has_access(user, 'staff', self.get_course_key()):
return False
return self.get_usage_key() not in self.get_course_blocks(user).get_block_keys()

287
lms/lib/courseware_search/test/test_lms_filter_generator.py
View File

@@ -3,20 +3,10 @@ Tests for the lms_filter_generator
"""
from mock import patch, Mock
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.modulestore.tests.factories import ItemFactory, CourseFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from student.tests.factories import UserFactory
from student.models import CourseEnrollment
from xmodule.partitions.partitions import Group, UserPartition
from openedx.core.djangoapps.course_groups.partition_scheme import CohortPartitionScheme
from openedx.core.djangoapps.user_api.partition_schemes import RandomUserPartitionScheme
from openedx.core.djangoapps.course_groups.tests.helpers import CohortFactory, config_course_cohorts
from openedx.core.djangoapps.course_groups.cohorts import add_user_to_cohort
from openedx.core.djangoapps.course_groups.views import link_cohort_to_partition_group
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey
from lms.lib.courseware_search.lms_filter_generator import LmsSearchFilterGenerator
@@ -58,10 +48,6 @@ class LmsSearchFilterGeneratorTestCase(ModuleStoreTestCase):
publish_item=True,
)
self.groups = [Group(1, 'Group 1'), Group(2, 'Group 2')]
self.content_groups = [1, 2]
def setUp(self):
super(LmsSearchFilterGeneratorTestCase, self).setUp()
self.build_courses()
@@ -150,274 +136,3 @@ class LmsSearchFilterGeneratorTestCase(ModuleStoreTestCase):
self.assertNotIn('org', exclude_dictionary)
self.assertIn('org', field_dictionary)
self.assertEqual('TestMicrosite3', field_dictionary['org'])
class LmsSearchFilterGeneratorGroupsTestCase(LmsSearchFilterGeneratorTestCase):
"""
Test case class to test search result processor
with content and user groups present within the course
"""
def setUp(self):
super(LmsSearchFilterGeneratorGroupsTestCase, self).setUp()
self.user_partition = None
self.split_test_user_partition = None
self.first_cohort = None
self.second_cohort = None
def add_seq_with_content_groups(self, groups=None):
"""
Adds sequential and two content groups to first course in courses list.
"""
config_course_cohorts(self.courses[0], is_cohorted=True)
if groups is None:
groups = self.groups
self.user_partition = UserPartition(
id=0,
name='Partition 1',
description='This is partition 1',
groups=groups,
scheme=CohortPartitionScheme
)
self.user_partition.scheme.name = "cohort"
ItemFactory.create(
parent_location=self.chapter.location,
category='sequential',
display_name="Lesson 1",
publish_item=True,
metadata={u"user_partitions": [self.user_partition.to_json()]}
)
self.first_cohort, self.second_cohort = [
CohortFactory(course_id=self.courses[0].id) for _ in range(2)
]
self.courses[0].user_partitions = [self.user_partition]
self.courses[0].save()
modulestore().update_item(self.courses[0], self.user.id)
def add_user_to_cohort_group(self):
"""
adds user to cohort and links cohort to content group
"""
add_user_to_cohort(self.first_cohort, self.user.username)
link_cohort_to_partition_group(
self.first_cohort,
self.user_partition.id,
self.groups[0].id,
)
self.courses[0].save()
modulestore().update_item(self.courses[0], self.user.id)
def add_split_test(self, groups=None):
"""
Adds split test and two content groups to second course in courses list.
"""
if groups is None:
groups = self.groups
self.split_test_user_partition = UserPartition(
id=0,
name='Partition 2',
description='This is partition 2',
groups=groups,
scheme=RandomUserPartitionScheme
)
self.split_test_user_partition.scheme.name = "random"
sequential = ItemFactory.create(
parent_location=self.chapter.location,
category='sequential',
display_name="Lesson 2",
publish_item=True,
)
vertical = ItemFactory.create(
parent_location=sequential.location,
category='vertical',
display_name='Subsection 3',
publish_item=True,
)
split_test_unit = ItemFactory.create(
parent_location=vertical.location,
category='split_test',
user_partition_id=0,
display_name="Test Content Experiment 1",
)
condition_1_vertical = ItemFactory.create(
parent_location=split_test_unit.location,
category="vertical",
display_name="Group ID 1",
)
condition_2_vertical = ItemFactory.create(
parent_location=split_test_unit.location,
category="vertical",
display_name="Group ID 2",
)
ItemFactory.create(
parent_location=condition_1_vertical.location,
category="html",
display_name="Group A",
publish_item=True,
)
ItemFactory.create(
parent_location=condition_2_vertical.location,
category="html",
display_name="Group B",
publish_item=True,
)
self.courses[1].user_partitions = [self.split_test_user_partition]
self.courses[1].save()
modulestore().update_item(self.courses[1], self.user.id)
def add_user_to_splittest_group(self):
"""
adds user to a random split test group
"""
self.split_test_user_partition.scheme.get_group_for_user(
CourseKey.from_string(unicode(self.courses[1].id)),
self.user,
self.split_test_user_partition,
assign=True,
)
self.courses[1].save()
modulestore().update_item(self.courses[1], self.user.id)
def test_content_group_id_provided(self):
"""
Tests that we get the content group ID when course is assigned to cohort
which is assigned content group.
"""
self.add_seq_with_content_groups()
self.add_user_to_cohort_group()
field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
user=self.user,
course_id=unicode(self.courses[0].id)
)
self.assertTrue('start_date' in filter_dictionary)
self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
self.assertEqual([unicode(self.content_groups[0])], filter_dictionary['content_groups'])
def test_content_multiple_groups_id_provided(self):
"""
Tests that we get content groups IDs when course is assigned to cohort
which is assigned to multiple content groups.
"""
self.add_seq_with_content_groups()
self.add_user_to_cohort_group()
# Second cohort link
link_cohort_to_partition_group(
self.second_cohort,
self.user_partition.id,
self.groups[0].id,
)
self.courses[0].save()
modulestore().update_item(self.courses[0], self.user.id)
field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
user=self.user,
course_id=unicode(self.courses[0].id)
)
self.assertTrue('start_date' in filter_dictionary)
self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
# returns only first group, relevant to current user
self.assertEqual([unicode(self.content_groups[0])], filter_dictionary['content_groups'])
def test_content_group_id_not_provided(self):
"""
Tests that we don't get content group ID when course is assigned a cohort
but cohort is not assigned to content group.
"""
self.add_seq_with_content_groups(groups=[])
field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
user=self.user,
course_id=unicode(self.courses[0].id)
)
self.assertTrue('start_date' in filter_dictionary)
self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
self.assertEqual(None, filter_dictionary['content_groups'])
def test_content_group_with_cohort_not_provided(self):
"""
Tests that we don't get content group ID when course has no cohorts
"""
self.add_seq_with_content_groups()
field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
user=self.user,
course_id=unicode(self.courses[0].id)
)
self.assertTrue('start_date' in filter_dictionary)
self.assertEqual(unicode(self.courses[0].id), field_dictionary['course'])
self.assertEqual(None, filter_dictionary['content_groups'])
def test_split_test_with_user_groups_user_not_assigned(self):
"""
Tests that we don't get user group ID when user is not assigned to a split test group
"""
self.add_split_test()
field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
user=self.user,
course_id=unicode(self.courses[1].id)
)
self.assertTrue('start_date' in filter_dictionary)
self.assertEqual(unicode(self.courses[1].id), field_dictionary['course'])
self.assertEqual(None, filter_dictionary['content_groups'])
def test_split_test_with_user_groups_user_assigned(self):
"""
Tests that we get user group ID when user is assigned to a split test group
"""
self.add_split_test()
self.add_user_to_splittest_group()
field_dictionary, filter_dictionary, _ = LmsSearchFilterGenerator.generate_field_filters(
user=self.user,
course_id=unicode(self.courses[1].id)
)
partition_group = self.split_test_user_partition.scheme.get_group_for_user(
CourseKey.from_string(unicode(self.courses[1].id)),
self.user,
self.split_test_user_partition,
assign=False,
)
self.assertTrue('start_date' in filter_dictionary)
self.assertEqual(unicode(self.courses[1].id), field_dictionary['course'])
self.assertEqual([unicode(partition_group.id)], filter_dictionary['content_groups'])
def test_invalid_course_key(self):
"""
Test system raises an error if no course found.
"""
self.add_seq_with_content_groups()
with self.assertRaises(InvalidKeyError):
LmsSearchFilterGenerator.generate_field_filters(
user=self.user,
course_id='this_is_false_course_id'
)

151
lms/lib/courseware_search/test/test_lms_search_initializer.py
View File

@@ -1,151 +0,0 @@
"""
Tests for the lms_search_initializer
"""
from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory
from xmodule.partitions.partitions import Group, UserPartition
from xmodule.modulestore.django import modulestore
from courseware.tests.factories import UserFactory
from courseware.tests.test_masquerade import StaffMasqueradeTestCase
from courseware.masquerade import handle_ajax
from lms.lib.courseware_search.lms_search_initializer import LmsSearchInitializer
from lms.lib.courseware_search.lms_filter_generator import LmsSearchFilterGenerator
class LmsSearchInitializerTestCase(StaffMasqueradeTestCase):
""" Test case class to test search initializer """
def build_course(self):
"""
Build up a course tree with an html control
"""
self.global_staff = UserFactory(is_staff=True)
self.course = CourseFactory.create(
org='Elasticsearch',
course='ES101',
run='test_run',
display_name='Elasticsearch test course',
)
self.section = ItemFactory.create(
parent=self.course,
category='chapter',
display_name='Test Section',
)
self.subsection = ItemFactory.create(
parent=self.section,
category='sequential',
display_name='Test Subsection',
)
self.vertical = ItemFactory.create(
parent=self.subsection,
category='vertical',
display_name='Test Unit',
)
self.html = ItemFactory.create(
parent=self.vertical,
category='html',
display_name='Test Html control 1',
)
self.html = ItemFactory.create(
parent=self.vertical,
category='html',
display_name='Test Html control 2',
)
def setUp(self):
super(LmsSearchInitializerTestCase, self).setUp()
self.build_course()
self.user_partition = UserPartition(
id=0,
name='Test User Partition',
description='',
groups=[Group(0, 'Group 1'), Group(1, 'Group 2')],
scheme_id='cohort'
)
self.course.user_partitions.append(self.user_partition)
modulestore().update_item(self.course, self.global_staff.id) # pylint: disable=no-member
def test_staff_masquerading_added_to_group(self):
"""
Tests that initializer sets masquerading for a staff user in a group.
"""
# Verify that there is no masquerading group initially
_, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters( # pylint: disable=unused-variable
user=self.global_staff,
course_id=unicode(self.course.id)
)
# User is staff by default, no content groups filter is set - see all
self.assertNotIn('content_groups', filter_directory)
# Install a masquerading group
request = self._create_mock_json_request(
self.global_staff,
body='{"role": "student", "user_partition_id": 0, "group_id": 1}'
)
handle_ajax(request, unicode(self.course.id))
# Call initializer
LmsSearchInitializer.set_search_enviroment(
request=request,
course_id=unicode(self.course.id)
)
# Verify that there is masquerading group after masquerade
_, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters( # pylint: disable=unused-variable
user=self.global_staff,
course_id=unicode(self.course.id)
)
self.assertEqual(filter_directory['content_groups'], [unicode(1)])
def test_staff_masquerading_as_a_staff_user(self):
"""
Tests that initializer sets masquerading for a staff user as staff.
"""
# Install a masquerading group
request = self._create_mock_json_request(
self.global_staff,
body='{"role": "staff"}'
)
handle_ajax(request, unicode(self.course.id))
# Call initializer
LmsSearchInitializer.set_search_enviroment(
request=request,
course_id=unicode(self.course.id)
)
# Verify that there is masquerading group after masquerade
_, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters( # pylint: disable=unused-variable
user=self.global_staff,
course_id=unicode(self.course.id)
)
self.assertNotIn('content_groups', filter_directory)
def test_staff_masquerading_as_a_student_user(self):
"""
Tests that initializer sets masquerading for a staff user as student.
"""
# Install a masquerading group
request = self._create_mock_json_request(
self.global_staff,
body='{"role": "student"}'
)
handle_ajax(request, unicode(self.course.id))
# Call initializer
LmsSearchInitializer.set_search_enviroment(
request=request,
course_id=unicode(self.course.id)
)
# Verify that there is masquerading group after masquerade
_, filter_directory, _ = LmsSearchFilterGenerator.generate_field_filters( # pylint: disable=unused-variable
user=self.global_staff,
course_id=unicode(self.course.id)
)
self.assertEqual(filter_directory['content_groups'], None)