Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #14213 from edx/beryl/mongocrazy

Browse Source 
Pre-load Definitions in Split Modulestore when accessing all blocks
This commit is contained in:
Nimisha Asthagiri
2017-01-03 09:37:40 -05:00
committed by GitHub
parent 0d222b3ed4 99e6d53939
commit 1de87f9112
13 changed files with 287 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
lms/djangoapps/course_api/blocks/tests/test_api.py
View File

@@ -2,12 +2,14 @@
Tests for Blocks api.py
"""
import ddt
from django.test.client import RequestFactory
from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache
from student.tests.factories import UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.tests.factories import SampleCourseFactory
from xmodule.modulestore.tests.factories import SampleCourseFactory, check_mongo_calls
from ..api import get_blocks
@@ -19,7 +21,8 @@ class TestGetBlocks(SharedModuleStoreTestCase):
@classmethod
def setUpClass(cls):
super(TestGetBlocks, cls).setUpClass()
cls.course = SampleCourseFactory.create()
with cls.store.default_store(ModuleStoreEnum.Type.split):
cls.course = SampleCourseFactory.create()
# hide the html block
cls.html_block = cls.store.get_item(cls.course.id.make_usage_key('html', 'html_x1a_1'))
@@ -95,3 +98,47 @@ class TestGetBlocks(SharedModuleStoreTestCase):
self.assertEquals(len(blocks['blocks']), 3)
for block in blocks['blocks'].itervalues():
self.assertEqual(block['type'], 'problem')
@ddt.ddt
class TestGetBlocksQueryCounts(SharedModuleStoreTestCase):
"""
Tests query counts for the get_blocks function.
"""
def setUp(self):
super(TestGetBlocksQueryCounts, self).setUp()
self.user = UserFactory.create()
self.request = RequestFactory().get("/dummy")
self.request.user = self.user
def _create_course(self, store_type):
"""
Creates the sample course in the given store type.
"""
with self.store.default_store(store_type):
return SampleCourseFactory.create()
def _get_blocks(self, course, expected_mongo_queries):
"""
Verifies the number of expected queries when calling
get_blocks on the given course.
"""
with check_mongo_calls(expected_mongo_queries):
with self.assertNumQueries(2):
get_blocks(self.request, course.location, self.user)
@ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
def test_query_counts_cached(self, store_type):
course = self._create_course(store_type)
self._get_blocks(course, expected_mongo_queries=0)
@ddt.data(
(ModuleStoreEnum.Type.mongo, 5),
(ModuleStoreEnum.Type.split, 3),
)
@ddt.unpack
def test_query_counts_uncached(self, store_type, expected_mongo_queries):
course = self._create_course(store_type)
clear_course_from_cache(course.id)
self._get_blocks(course, expected_mongo_queries)

42
lms/djangoapps/courseware/tests/test_views.py
View File

@@ -189,6 +189,48 @@ class TestJumpTo(ModuleStoreTestCase):
self.assertEqual(response.status_code, 404)
@attr(shard=2)
@ddt.ddt
class IndexQueryTestCase(ModuleStoreTestCase):
"""
Tests for query count.
"""
CREATE_USER = False
NUM_PROBLEMS = 20
@ddt.data(
(ModuleStoreEnum.Type.mongo, 8),
(ModuleStoreEnum.Type.split, 4),
)
@ddt.unpack
def test_index_query_counts(self, store_type, expected_query_count):
with self.store.default_store(store_type):
course = CourseFactory.create()
with self.store.bulk_operations(course.id):
chapter = ItemFactory.create(category='chapter', parent_location=course.location)
section = ItemFactory.create(category='sequential', parent_location=chapter.location)
vertical = ItemFactory.create(category='vertical', parent_location=section.location)
for _ in range(self.NUM_PROBLEMS):
ItemFactory.create(category='problem', parent_location=vertical.location)
password = 'test'
self.user = UserFactory(password=password)
self.client.login(username=self.user.username, password=password)
CourseEnrollment.enroll(self.user, course.id)
with check_mongo_calls(expected_query_count):
url = reverse(
'courseware_section',
kwargs={
'course_id': unicode(course.id),
'chapter': unicode(chapter.location.name),
'section': unicode(section.location.name),
}
)
response = self.client.get(url)
self.assertEqual(response.status_code, 200)
@attr(shard=2)
@ddt.ddt
class ViewsTestCase(ModuleStoreTestCase):

2
lms/djangoapps/courseware/views/index.py
View File

@@ -348,7 +348,7 @@ class CoursewareIndex(View):
sets up the runtime, which binds the request user to the section.
"""
# Pre-fetch all descendant data
self.section = modulestore().get_item(self.section.location, depth=None)
self.section = modulestore().get_item(self.section.location, depth=None, lazy=False)
self.field_data_cache.add_descriptor_descendents(self.section, depth=None)
# Bind section to user

72
lms/djangoapps/grades/tasks.py
View File

@@ -158,43 +158,45 @@ def _update_subsection_grades(
that those subsection grades were updated.
"""
student = User.objects.get(id=user_id)
course_structure = get_course_blocks(student, modulestore().make_course_usage_key(course_key))
subsections_to_update = course_structure.get_transformer_block_field(
scored_block_usage_key,
GradesTransformer,
'subsections',
set(),
)
course = modulestore().get_course(course_key, depth=0)
subsection_grade_factory = SubsectionGradeFactory(student, course, course_structure)
try:
for subsection_usage_key in subsections_to_update:
if subsection_usage_key in course_structure:
subsection_grade = subsection_grade_factory.update(
course_structure[subsection_usage_key],
only_if_higher,
)
SUBSECTION_SCORE_CHANGED.send(
sender=recalculate_subsection_grade,
course=course,
course_structure=course_structure,
user=student,
subsection_grade=subsection_grade,
)
except DatabaseError as exc:
raise _retry_recalculate_subsection_grade(
user_id,
course_id,
usage_id,
only_if_higher,
expected_modified_time,
score_deleted,
exc,
store = modulestore()
with store.bulk_operations(course_key):
course_structure = get_course_blocks(student, store.make_course_usage_key(course_key))
subsections_to_update = course_structure.get_transformer_block_field(
scored_block_usage_key,
GradesTransformer,
'subsections',
set(),
)
course = store.get_course(course_key, depth=0)
subsection_grade_factory = SubsectionGradeFactory(student, course, course_structure)
try:
for subsection_usage_key in subsections_to_update:
if subsection_usage_key in course_structure:
subsection_grade = subsection_grade_factory.update(
course_structure[subsection_usage_key],
only_if_higher,
)
SUBSECTION_SCORE_CHANGED.send(
sender=recalculate_subsection_grade,
course=course,
course_structure=course_structure,
user=student,
subsection_grade=subsection_grade,
)
except DatabaseError as exc:
raise _retry_recalculate_subsection_grade(
user_id,
course_id,
usage_id,
only_if_higher,
expected_modified_time,
score_deleted,
exc,
)
def _retry_recalculate_subsection_grade(
user_id,

45
lms/djangoapps/grades/tests/test_tasks.py
View File

@@ -121,16 +121,17 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
self.assertTrue(mock_subsection_signal.called)
@ddt.data(
(ModuleStoreEnum.Type.mongo, 1),
(ModuleStoreEnum.Type.split, 0),
(ModuleStoreEnum.Type.mongo, 1, 23),
(ModuleStoreEnum.Type.split, 3, 22),
)
@ddt.unpack
def test_subsection_grade_updated(self, default_store, added_queries):
def test_subsection_grade_updated(self, default_store, num_mongo_calls, num_sql_calls):
with self.store.default_store(default_store):
self.set_up_course()
self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
with check_mongo_calls(2) and self.assertNumQueries(22 + added_queries):
self._apply_recalculate_subsection_grade()
with check_mongo_calls(num_mongo_calls):
with self.assertNumQueries(num_sql_calls):
self._apply_recalculate_subsection_grade()
@patch('lms.djangoapps.grades.signals.signals.SUBSECTION_SCORE_CHANGED.send')
def test_other_inaccessible_subsection(self, mock_subsection_signal):
@@ -166,27 +167,38 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
self._apply_recalculate_subsection_grade()
self.assertEquals(mock_block_structure_create.call_count, 1)
# TODO (TNL-6225) Fix the number of SQL queries so they
# don't grow linearly with the number of sequentials.
@ddt.data(
(ModuleStoreEnum.Type.mongo, 1),
(ModuleStoreEnum.Type.split, 0),
(ModuleStoreEnum.Type.mongo, 1, 46),
(ModuleStoreEnum.Type.split, 3, 45),
)
@ddt.unpack
def test_query_count_does_not_change_with_more_problems(self, default_store, added_queries):
def test_query_count_does_not_change_with_more_content(self, default_store, num_mongo_calls, num_sql_calls):
with self.store.default_store(default_store):
self.set_up_course()
self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
ItemFactory.create(parent=self.sequential, category='problem', display_name='problem2')
ItemFactory.create(parent=self.sequential, category='problem', display_name='problem3')
with check_mongo_calls(2) and self.assertNumQueries(22 + added_queries):
self._apply_recalculate_subsection_grade()
num_problems = 10
for _ in range(num_problems):
ItemFactory.create(parent=self.sequential, category='problem')
num_sequentials = 10
for _ in range(num_sequentials):
ItemFactory.create(parent=self.chapter, category='sequential')
with check_mongo_calls(num_mongo_calls):
with self.assertNumQueries(num_sql_calls):
self._apply_recalculate_subsection_grade()
@ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split)
def test_subsection_grades_not_enabled_on_course(self, default_store):
with self.store.default_store(default_store):
self.set_up_course(enable_subsection_grades=False)
self.assertFalse(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
with check_mongo_calls(2) and self.assertNumQueries(0):
self._apply_recalculate_subsection_grade()
with check_mongo_calls(0):
with self.assertNumQueries(0):
self._apply_recalculate_subsection_grade()
@skip("Pending completion of TNL-5089")
@ddt.data(
@@ -200,8 +212,9 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
PersistentGradesEnabledFlag.objects.create(enabled=feature_flag)
with self.store.default_store(default_store):
self.set_up_course()
with check_mongo_calls(0) and self.assertNumQueries(3 if feature_flag else 2):
self._apply_recalculate_subsection_grade()
with check_mongo_calls(0):
with self.assertNumQueries(3 if feature_flag else 2):
self._apply_recalculate_subsection_grade()
@patch('lms.djangoapps.grades.tasks.recalculate_subsection_grade_v2.retry')
@patch('lms.djangoapps.grades.new.subsection_grade.SubsectionGradeFactory.update')

17
lms/djangoapps/grades/tests/test_transformer.py
View File

@@ -17,7 +17,7 @@ from xmodule.modulestore.tests.factories import check_mongo_calls
from lms.djangoapps.course_blocks.api import get_course_blocks
from lms.djangoapps.course_blocks.transformers.tests.helpers import CourseStructureTestCase
from openedx.core.djangoapps.content.block_structure.api import get_cache
from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache
from ..transformer import GradesTransformer
@@ -390,6 +390,7 @@ class GradesTransformerTestCase(CourseStructureTestCase):
)
@ddt.ddt
class MultiProblemModulestoreAccessTestCase(CourseStructureTestCase, SharedModuleStoreTestCase):
"""
Test mongo usage in GradesTransformer.
@@ -403,7 +404,12 @@ class MultiProblemModulestoreAccessTestCase(CourseStructureTestCase, SharedModul
self.student = UserFactory.create(is_staff=False, username=u'test_student', password=password)
self.client.login(username=self.student.username, password=password)
def test_modulestore_performance(self):
@ddt.data(
(ModuleStoreEnum.Type.split, 3),
(ModuleStoreEnum.Type.mongo, 2),
)
@ddt.unpack
def test_modulestore_performance(self, store_type, expected_mongo_queries):
"""
Test that a constant number of mongo calls are made regardless of how
many grade-related blocks are in the course.
@@ -437,7 +443,8 @@ class MultiProblemModulestoreAccessTestCase(CourseStructureTestCase, SharedModul
</problem>'''.format(number=problem_number),
}
)
blocks = self.build_course(course)
get_cache().clear()
with check_mongo_calls(2):
with self.store.default_store(store_type):
blocks = self.build_course(course)
clear_course_from_cache(blocks[u'course'].id)
with check_mongo_calls(expected_mongo_queries):
get_course_blocks(self.student, blocks[u'course'].location, self.transformers)