2051c90924
There are a number of Django Signals that are on the modulestore's
SignalHandler class, such as SignalHandler.course_published. These
signals can trigger very expensive processes to occur, such as course
overview or block structures generation. Most of the time, the test
author doesn't care about these side-effects.
This commit does a few things:
* Converts the signals on SignalHandler to be instances of a new
SwitchedSignal class, that allows signal sending to be disabled.
* Creates a SignalIsolationMixin helper similar in spirit to the
CacheIsolationMixin, and adds it to the ModuleStoreIsolationMixin
(and thus to ModuleStoreTestCase and SharedModuleStoreTestCase).
* Converts our various tests to use this new mechanism. In some cases,
this means adjusting query counts downwards because they no longer
have to account for publishing listener actions.
Modulestore generated signals are now muted by default during test runs.
Calls to send() them will result in no-ops. You can choose to enable
specific signals for a given subclass of ModuleStoreTestCase or
SharedModuleStoreTestCase by specifying an ENABLED_SIGNALS class
attribute, like the following example:
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
class MyPublishTestCase(ModuleStoreTestCase):
ENABLED_SIGNALS = ['course_published', 'pre_publish']
You should take great care when disabling signals outside of a
ModuleStoreTestCase or SharedModuleStoreTestCase, since they can leak
out into other tests. Be sure to always clean up, and never disable
signals outside of testing. Because signals are essentially process
globals, it can have a lot of unpleasant side-effects if we start
mucking around with them during live requests.
Overall, this change has cut the total test execution time for
edx-platform by a bit over a third, though we still spend a lot in
pre-test setup during our test builds.
[PERF-413]
453 lines
18 KiB
Python
453 lines
18 KiB
Python
"""
|
|
Test the behavior of the GradesTransformer
|
|
"""
|
|
|
|
import datetime
|
|
import pytz
|
|
import random
|
|
|
|
import ddt
|
|
from copy import deepcopy
|
|
|
|
from student.tests.factories import UserFactory
|
|
from xmodule.modulestore import ModuleStoreEnum
|
|
from xmodule.modulestore.django import modulestore
|
|
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
|
|
from xmodule.modulestore.tests.factories import check_mongo_calls
|
|
|
|
from lms.djangoapps.course_blocks.api import get_course_blocks
|
|
from lms.djangoapps.course_blocks.transformers.tests.helpers import CourseStructureTestCase
|
|
from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache
|
|
from ..transformer import GradesTransformer
|
|
|
|
|
|
@ddt.ddt
|
|
class GradesTransformerTestCase(CourseStructureTestCase):
|
|
"""
|
|
Verify behavior of the GradesTransformer
|
|
"""
|
|
|
|
TRANSFORMER_CLASS_TO_TEST = GradesTransformer
|
|
|
|
ENABLED_SIGNALS = ['course_published']
|
|
|
|
problem_metadata = {
|
|
u'graded': True,
|
|
u'weight': 1,
|
|
u'due': datetime.datetime(2099, 3, 15, 12, 30, 0, tzinfo=pytz.utc),
|
|
}
|
|
|
|
def setUp(self):
|
|
super(GradesTransformerTestCase, self).setUp()
|
|
password = u'test'
|
|
self.student = UserFactory.create(is_staff=False, username=u'test_student', password=password)
|
|
self.client.login(username=self.student.username, password=password)
|
|
|
|
def _update_course_grading_policy(self, course, grading_policy):
|
|
"""
|
|
Helper to update a course's grading policy in the modulestore.
|
|
"""
|
|
course.set_grading_policy(grading_policy)
|
|
modulestore().update_item(course, self.user.id)
|
|
|
|
def _validate_grading_policy_hash(self, course_location, grading_policy_hash):
|
|
"""
|
|
Helper to retrieve the course at the given course_location and
|
|
assert that its hashed grading policy (from the grades transformer)
|
|
is as expected.
|
|
"""
|
|
block_structure = get_course_blocks(self.student, course_location, self.transformers)
|
|
self.assert_collected_transformer_block_fields(
|
|
block_structure,
|
|
course_location,
|
|
self.TRANSFORMER_CLASS_TO_TEST,
|
|
grading_policy_hash=grading_policy_hash,
|
|
)
|
|
|
|
def assert_collected_xblock_fields(self, block_structure, usage_key, **expectations):
|
|
"""
|
|
Given a block structure, a block usage key, and a list of keyword
|
|
arguments representing XBlock fields, verify that the block structure
|
|
has the specified values for each XBlock field.
|
|
"""
|
|
self.assertGreater(len(expectations), 0)
|
|
for field in expectations:
|
|
# Append our custom message to the default assertEqual error message
|
|
self.longMessage = True # pylint: disable=invalid-name
|
|
self.assertEqual(
|
|
expectations[field],
|
|
block_structure.get_xblock_field(usage_key, field),
|
|
msg=u'in field {},'.format(repr(field)),
|
|
)
|
|
self.assertIsNotNone(
|
|
block_structure.get_xblock_field(usage_key, u'subtree_edited_on'),
|
|
)
|
|
|
|
def assert_collected_transformer_block_fields(self, block_structure, usage_key, transformer_class, **expectations):
|
|
"""
|
|
Given a block structure, a block usage key, a transformer, and a list
|
|
of keyword arguments representing transformer block fields, verify that
|
|
the block structure has the specified values for each transformer block
|
|
field.
|
|
"""
|
|
self.assertGreater(len(expectations), 0)
|
|
# Append our custom message to the default assertEqual error message
|
|
self.longMessage = True # pylint: disable=invalid-name
|
|
for field in expectations:
|
|
self.assertEqual(
|
|
expectations[field],
|
|
block_structure.get_transformer_block_field(usage_key, transformer_class, field),
|
|
msg=u'in {} and field {}'.format(transformer_class, repr(field)),
|
|
)
|
|
|
|
def build_course_with_problems(self, data='<problem></problem>', metadata=None):
|
|
"""
|
|
Create a test course with the requested problem `data` and `metadata` values.
|
|
|
|
Appropriate defaults are provided when either argument is omitted.
|
|
"""
|
|
metadata = metadata or self.problem_metadata
|
|
|
|
# Special structure-related keys start with '#'. The rest get passed as
|
|
# kwargs to Factory.create. See docstring at
|
|
# `CourseStructureTestCase.build_course` for details.
|
|
return self.build_course([
|
|
{
|
|
u'org': u'GradesTestOrg',
|
|
u'course': u'GB101',
|
|
u'run': u'cannonball',
|
|
u'metadata': {u'format': u'homework'},
|
|
u'#type': u'course',
|
|
u'#ref': u'course',
|
|
u'#children': [
|
|
{
|
|
u'metadata': metadata,
|
|
u'#type': u'problem',
|
|
u'#ref': u'problem',
|
|
u'data': data,
|
|
}
|
|
]
|
|
}
|
|
])
|
|
|
|
def build_complicated_hypothetical_course(self):
|
|
"""
|
|
Create a test course with a very odd structure as a stress-test for various methods.
|
|
|
|
Current design is to test containing_subsection logic in collect_unioned_set_field.
|
|
I can't reasonably draw this in ascii art (due to intentional complexities), so here's an overview:
|
|
We have 1 course, containing 1 chapter, containing 2 subsections.
|
|
|
|
From here, it starts to get hairy. Call our subsections A and B.
|
|
Subsection A contains 3 verticals (call them 1, 2, and 3), and another subsection (C)
|
|
Subsection B contains vertical 3 and subsection C
|
|
Subsection C contains 1 problem (b)
|
|
Vertical 1 contains 1 vertical (11)
|
|
Vertical 2 contains no children
|
|
Vertical 3 contains no children
|
|
Vertical 11 contains 1 problem (aa) and vertical 2
|
|
Problem b contains no children
|
|
"""
|
|
return self.build_course([
|
|
{
|
|
u'org': u'GradesTestOrg',
|
|
u'course': u'GB101',
|
|
u'run': u'cannonball',
|
|
u'metadata': {u'format': u'homework'},
|
|
u'#type': u'course',
|
|
u'#ref': u'course',
|
|
u'#children': [
|
|
{
|
|
u'#type': u'chapter',
|
|
u'#ref': u'chapter',
|
|
u'#children': [
|
|
{
|
|
u'#type': u'sequential',
|
|
u'#ref': 'sub_A',
|
|
u'#children': [
|
|
{
|
|
u'#type': u'vertical',
|
|
u'#ref': 'vert_1',
|
|
u'#children': [
|
|
{
|
|
u'#type': u'vertical',
|
|
u'#ref': u'vert_A11',
|
|
u'#children': [{u'#type': u'problem', u'#ref': u'prob_A1aa'}]
|
|
},
|
|
]
|
|
},
|
|
{u'#type': u'vertical', u'#ref': 'vert_2', '#parents': [u'vert_A11']},
|
|
]
|
|
},
|
|
{
|
|
u'#type': u'sequential',
|
|
u'#ref': u'sub_B',
|
|
u'#children': [
|
|
{u'#type': u'vertical', u'#ref': 'vert_3', '#parents': ['sub_A']},
|
|
{
|
|
u'#type': u'sequential',
|
|
u'#ref': 'sub_C',
|
|
'#parents': ['sub_A'],
|
|
u'#children': [{u'#type': u'problem', u'#ref': u'prob_BCb'}]
|
|
},
|
|
]
|
|
},
|
|
]
|
|
}
|
|
]
|
|
}
|
|
])
|
|
|
|
def test_collect_containing_subsection(self):
|
|
expected_subsections = {
|
|
'course': set(),
|
|
'chapter': set(),
|
|
'sub_A': {'sub_A'},
|
|
'sub_B': {'sub_B'},
|
|
'sub_C': {'sub_A', 'sub_B', 'sub_C'},
|
|
'vert_1': {'sub_A'},
|
|
'vert_2': {'sub_A'},
|
|
'vert_3': {'sub_A', 'sub_B'},
|
|
'vert_A11': {'sub_A'},
|
|
'prob_A1aa': {'sub_A'},
|
|
'prob_BCb': {'sub_A', 'sub_B', 'sub_C'},
|
|
}
|
|
blocks = self.build_complicated_hypothetical_course()
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
for block_ref, expected_subsections in expected_subsections.iteritems():
|
|
actual_subsections = block_structure.get_transformer_block_field(
|
|
blocks[block_ref].location,
|
|
self.TRANSFORMER_CLASS_TO_TEST,
|
|
'subsections',
|
|
)
|
|
self.assertEqual(actual_subsections, {blocks[sub].location for sub in expected_subsections})
|
|
|
|
def test_unscored_block_collection(self):
|
|
blocks = self.build_course_with_problems()
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
self.assert_collected_xblock_fields(
|
|
block_structure,
|
|
blocks[u'course'].location,
|
|
weight=None,
|
|
graded=False,
|
|
has_score=False,
|
|
due=None,
|
|
format=u'homework',
|
|
)
|
|
self.assert_collected_transformer_block_fields(
|
|
block_structure,
|
|
blocks[u'course'].location,
|
|
self.TRANSFORMER_CLASS_TO_TEST,
|
|
max_score=None,
|
|
explicit_graded=None,
|
|
)
|
|
|
|
def test_grades_collected_basic(self):
|
|
|
|
blocks = self.build_course_with_problems()
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
|
|
self.assert_collected_xblock_fields(
|
|
block_structure,
|
|
blocks[u'problem'].location,
|
|
weight=self.problem_metadata[u'weight'],
|
|
graded=self.problem_metadata[u'graded'],
|
|
has_score=True,
|
|
due=self.problem_metadata[u'due'],
|
|
format=None,
|
|
)
|
|
self.assert_collected_transformer_block_fields(
|
|
block_structure,
|
|
blocks[u'problem'].location,
|
|
self.TRANSFORMER_CLASS_TO_TEST,
|
|
max_score=0,
|
|
explicit_graded=True,
|
|
)
|
|
|
|
@ddt.data(True, False, None)
|
|
def test_graded_at_problem(self, graded):
|
|
problem_metadata = {
|
|
u'has_score': True,
|
|
}
|
|
if graded is not None:
|
|
problem_metadata[u'graded'] = graded
|
|
blocks = self.build_course_with_problems(metadata=problem_metadata)
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
self.assert_collected_transformer_block_fields(
|
|
block_structure,
|
|
blocks[u'problem'].location,
|
|
self.TRANSFORMER_CLASS_TO_TEST,
|
|
explicit_graded=graded,
|
|
)
|
|
|
|
def test_collecting_staff_only_problem(self):
|
|
# Demonstrate that the problem data can by collected by the SystemUser
|
|
# even if the block has access restrictions placed on it.
|
|
problem_metadata = {
|
|
u'graded': True,
|
|
u'weight': 1,
|
|
u'due': datetime.datetime(2016, 10, 16, 0, 4, 0, tzinfo=pytz.utc),
|
|
u'visible_to_staff_only': True,
|
|
}
|
|
|
|
blocks = self.build_course_with_problems(metadata=problem_metadata)
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
|
|
self.assert_collected_xblock_fields(
|
|
block_structure,
|
|
blocks[u'problem'].location,
|
|
weight=problem_metadata[u'weight'],
|
|
graded=problem_metadata[u'graded'],
|
|
has_score=True,
|
|
due=problem_metadata[u'due'],
|
|
format=None,
|
|
)
|
|
|
|
def test_max_score_collection(self):
|
|
problem_data = u'''
|
|
<problem>
|
|
<numericalresponse answer="2">
|
|
<textline label="1+1" trailing_text="%" />
|
|
</numericalresponse>
|
|
</problem>
|
|
'''
|
|
|
|
blocks = self.build_course_with_problems(data=problem_data)
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
|
|
self.assert_collected_transformer_block_fields(
|
|
block_structure,
|
|
blocks[u'problem'].location,
|
|
self.TRANSFORMER_CLASS_TO_TEST,
|
|
max_score=1,
|
|
)
|
|
|
|
def test_max_score_for_multiresponse_problem(self):
|
|
problem_data = u'''
|
|
<problem>
|
|
<numericalresponse answer="27">
|
|
<textline label="3^3" />
|
|
</numericalresponse>
|
|
<numericalresponse answer="13.5">
|
|
<textline label="and then half of that?" />
|
|
</numericalresponse>
|
|
</problem>
|
|
'''
|
|
|
|
blocks = self.build_course_with_problems(problem_data)
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
|
|
self.assert_collected_transformer_block_fields(
|
|
block_structure,
|
|
blocks[u'problem'].location,
|
|
self.TRANSFORMER_CLASS_TO_TEST,
|
|
max_score=2,
|
|
)
|
|
|
|
def test_course_version_not_collected_in_old_mongo(self):
|
|
blocks = self.build_course_with_problems()
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
self.assertIsNone(block_structure.get_xblock_field(blocks[u'course'].location, u'course_version'))
|
|
|
|
def test_course_version_collected_in_split(self):
|
|
with self.store.default_store(ModuleStoreEnum.Type.split):
|
|
blocks = self.build_course_with_problems()
|
|
block_structure = get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|
|
self.assertIsNotNone(block_structure.get_xblock_field(blocks[u'course'].location, u'course_version'))
|
|
self.assertEqual(
|
|
block_structure.get_xblock_field(blocks[u'problem'].location, u'course_version'),
|
|
block_structure.get_xblock_field(blocks[u'course'].location, u'course_version')
|
|
)
|
|
|
|
def test_grading_policy_collected(self):
|
|
# the calculated hash of the original and updated grading policies of the test course
|
|
original_grading_policy_hash = u'ChVp0lHGQGCevD0t4njna/C44zQ='
|
|
updated_grading_policy_hash = u'TsbX04qWOy1WRnC0NHy+94upPd4='
|
|
|
|
blocks = self.build_course_with_problems()
|
|
course_block = blocks[u'course']
|
|
self._validate_grading_policy_hash(
|
|
course_block.location,
|
|
original_grading_policy_hash
|
|
)
|
|
|
|
# make sure the hash changes when the course grading policy is edited
|
|
grading_policy_with_updates = course_block.grading_policy
|
|
original_grading_policy = deepcopy(grading_policy_with_updates)
|
|
for section in grading_policy_with_updates['GRADER']:
|
|
self.assertNotEqual(section['weight'], 0.25)
|
|
section['weight'] = 0.25
|
|
|
|
self._update_course_grading_policy(course_block, grading_policy_with_updates)
|
|
self._validate_grading_policy_hash(
|
|
course_block.location,
|
|
updated_grading_policy_hash
|
|
)
|
|
|
|
# reset the grading policy and ensure the hash matches the original
|
|
self._update_course_grading_policy(course_block, original_grading_policy)
|
|
self._validate_grading_policy_hash(
|
|
course_block.location,
|
|
original_grading_policy_hash
|
|
)
|
|
|
|
|
|
@ddt.ddt
|
|
class MultiProblemModulestoreAccessTestCase(CourseStructureTestCase, SharedModuleStoreTestCase):
|
|
"""
|
|
Test mongo usage in GradesTransformer.
|
|
"""
|
|
|
|
TRANSFORMER_CLASS_TO_TEST = GradesTransformer
|
|
|
|
def setUp(self):
|
|
super(MultiProblemModulestoreAccessTestCase, self).setUp()
|
|
password = u'test'
|
|
self.student = UserFactory.create(is_staff=False, username=u'test_student', password=password)
|
|
self.client.login(username=self.student.username, password=password)
|
|
|
|
@ddt.data(
|
|
(ModuleStoreEnum.Type.split, 3),
|
|
(ModuleStoreEnum.Type.mongo, 2),
|
|
)
|
|
@ddt.unpack
|
|
def test_modulestore_performance(self, store_type, expected_mongo_queries):
|
|
"""
|
|
Test that a constant number of mongo calls are made regardless of how
|
|
many grade-related blocks are in the course.
|
|
"""
|
|
course = [
|
|
{
|
|
u'org': u'GradesTestOrg',
|
|
u'course': u'GB101',
|
|
u'run': u'cannonball',
|
|
u'metadata': {u'format': u'homework'},
|
|
u'#type': u'course',
|
|
u'#ref': u'course',
|
|
u'#children': [],
|
|
},
|
|
]
|
|
for problem_number in xrange(random.randrange(10, 20)):
|
|
course[0][u'#children'].append(
|
|
{
|
|
u'metadata': {
|
|
u'graded': True,
|
|
u'weight': 1,
|
|
u'due': datetime.datetime(2099, 3, 15, 12, 30, 0, tzinfo=pytz.utc),
|
|
},
|
|
u'#type': u'problem',
|
|
u'#ref': u'problem_{}'.format(problem_number),
|
|
u'data': u'''
|
|
<problem>
|
|
<numericalresponse answer="{number}">
|
|
<textline label="1*{number}" />
|
|
</numericalresponse>
|
|
</problem>'''.format(number=problem_number),
|
|
}
|
|
)
|
|
with self.store.default_store(store_type):
|
|
blocks = self.build_course(course)
|
|
clear_course_from_cache(blocks[u'course'].id)
|
|
with check_mongo_calls(expected_mongo_queries):
|
|
get_course_blocks(self.student, blocks[u'course'].location, self.transformers)
|