378 lines
18 KiB
Python
378 lines
18 KiB
Python
"""
|
|
Unit tests for LMS instructor-initiated background tasks.
|
|
|
|
Runs tasks on answers to course problems to validate that code
|
|
paths actually work.
|
|
|
|
"""
|
|
import json
|
|
from uuid import uuid4
|
|
from unittest import skip
|
|
|
|
from mock import Mock, patch
|
|
|
|
from celery.states import SUCCESS, FAILURE
|
|
|
|
from xmodule.modulestore.exceptions import ItemNotFoundError
|
|
|
|
from courseware.model_data import StudentModule
|
|
from courseware.tests.factories import StudentModuleFactory
|
|
from student.tests.factories import UserFactory
|
|
|
|
from instructor_task.models import InstructorTask
|
|
from instructor_task.tests.test_base import InstructorTaskModuleTestCase
|
|
from instructor_task.tests.factories import InstructorTaskFactory
|
|
from instructor_task.tasks import rescore_problem, reset_problem_attempts, delete_problem_state
|
|
from instructor_task.tasks_helper import UpdateProblemModuleStateError, update_problem_module_state
|
|
|
|
|
|
PROBLEM_URL_NAME = "test_urlname"
|
|
|
|
|
|
class TestTaskFailure(Exception):
|
|
pass
|
|
|
|
|
|
class TestInstructorTasks(InstructorTaskModuleTestCase):
|
|
def setUp(self):
|
|
super(InstructorTaskModuleTestCase, self).setUp()
|
|
self.initialize_course()
|
|
self.instructor = self.create_instructor('instructor')
|
|
self.problem_url = InstructorTaskModuleTestCase.problem_location(PROBLEM_URL_NAME)
|
|
|
|
def _create_input_entry(self, student_ident=None):
|
|
"""Creates a InstructorTask entry for testing."""
|
|
task_id = str(uuid4())
|
|
task_input = {'problem_url': self.problem_url}
|
|
if student_ident is not None:
|
|
task_input['student'] = student_ident
|
|
|
|
instructor_task = InstructorTaskFactory.create(course_id=self.course.id,
|
|
requester=self.instructor,
|
|
task_input=json.dumps(task_input),
|
|
task_key='dummy value',
|
|
task_id=task_id)
|
|
return instructor_task
|
|
|
|
def _get_xmodule_instance_args(self):
|
|
"""
|
|
Calculate dummy values for parameters needed for instantiating xmodule instances.
|
|
"""
|
|
return {'xqueue_callback_url_prefix': 'dummy_value',
|
|
'request_info': {},
|
|
}
|
|
|
|
def _run_task_with_mock_celery(self, task_function, entry_id, task_id, expected_failure_message=None):
|
|
"""Submit a task and mock how celery provides a current_task."""
|
|
self.current_task = Mock()
|
|
self.current_task.request = Mock()
|
|
self.current_task.request.id = task_id
|
|
self.current_task.update_state = Mock()
|
|
if expected_failure_message is not None:
|
|
self.current_task.update_state.side_effect = TestTaskFailure(expected_failure_message)
|
|
with patch('instructor_task.tasks_helper._get_current_task') as mock_get_task:
|
|
mock_get_task.return_value = self.current_task
|
|
return task_function(entry_id, self._get_xmodule_instance_args())
|
|
|
|
def _test_missing_current_task(self, task_function):
|
|
"""Check that a task_function fails when celery doesn't provide a current_task."""
|
|
task_entry = self._create_input_entry()
|
|
with self.assertRaises(UpdateProblemModuleStateError):
|
|
task_function(task_entry.id, self._get_xmodule_instance_args())
|
|
|
|
def test_rescore_missing_current_task(self):
|
|
self._test_missing_current_task(rescore_problem)
|
|
|
|
def test_reset_missing_current_task(self):
|
|
self._test_missing_current_task(reset_problem_attempts)
|
|
|
|
def test_delete_missing_current_task(self):
|
|
self._test_missing_current_task(delete_problem_state)
|
|
|
|
def _test_undefined_problem(self, task_function):
|
|
"""Run with celery, but no problem defined."""
|
|
task_entry = self._create_input_entry()
|
|
with self.assertRaises(ItemNotFoundError):
|
|
self._run_task_with_mock_celery(task_function, task_entry.id, task_entry.task_id)
|
|
|
|
def test_rescore_undefined_problem(self):
|
|
self._test_undefined_problem(rescore_problem)
|
|
|
|
def test_reset_undefined_problem(self):
|
|
self._test_undefined_problem(reset_problem_attempts)
|
|
|
|
def test_delete_undefined_problem(self):
|
|
self._test_undefined_problem(delete_problem_state)
|
|
|
|
def _test_run_with_task(self, task_function, action_name, expected_num_updated):
|
|
"""Run a task and check the number of StudentModules processed."""
|
|
task_entry = self._create_input_entry()
|
|
status = self._run_task_with_mock_celery(task_function, task_entry.id, task_entry.task_id)
|
|
# check return value
|
|
self.assertEquals(status.get('attempted'), expected_num_updated)
|
|
self.assertEquals(status.get('updated'), expected_num_updated)
|
|
self.assertEquals(status.get('total'), expected_num_updated)
|
|
self.assertEquals(status.get('action_name'), action_name)
|
|
self.assertGreater('duration_ms', 0)
|
|
# compare with entry in table:
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
self.assertEquals(json.loads(entry.task_output), status)
|
|
self.assertEquals(entry.task_state, SUCCESS)
|
|
|
|
def _test_run_with_no_state(self, task_function, action_name):
|
|
"""Run with no StudentModules defined for the current problem."""
|
|
self.define_option_problem(PROBLEM_URL_NAME)
|
|
self._test_run_with_task(task_function, action_name, 0)
|
|
|
|
def test_rescore_with_no_state(self):
|
|
self._test_run_with_no_state(rescore_problem, 'rescored')
|
|
|
|
def test_reset_with_no_state(self):
|
|
self._test_run_with_no_state(reset_problem_attempts, 'reset')
|
|
|
|
def test_delete_with_no_state(self):
|
|
self._test_run_with_no_state(delete_problem_state, 'deleted')
|
|
|
|
def _create_students_with_state(self, num_students, state=None):
|
|
"""Create students, a problem, and StudentModule objects for testing"""
|
|
self.define_option_problem(PROBLEM_URL_NAME)
|
|
students = [
|
|
UserFactory.create(username='robot%d' % i, email='robot+test+%d@edx.org' % i)
|
|
for i in xrange(num_students)
|
|
]
|
|
for student in students:
|
|
StudentModuleFactory.create(course_id=self.course.id,
|
|
module_state_key=self.problem_url,
|
|
student=student,
|
|
state=state)
|
|
return students
|
|
|
|
def _assert_num_attempts(self, students, num_attempts):
|
|
"""Check the number attempts for all students is the same"""
|
|
for student in students:
|
|
module = StudentModule.objects.get(course_id=self.course.id,
|
|
student=student,
|
|
module_state_key=self.problem_url)
|
|
state = json.loads(module.state)
|
|
self.assertEquals(state['attempts'], num_attempts)
|
|
|
|
def test_reset_with_some_state(self):
|
|
initial_attempts = 3
|
|
input_state = json.dumps({'attempts': initial_attempts})
|
|
num_students = 10
|
|
students = self._create_students_with_state(num_students, input_state)
|
|
# check that entries were set correctly
|
|
self._assert_num_attempts(students, initial_attempts)
|
|
# run the task
|
|
self._test_run_with_task(reset_problem_attempts, 'reset', num_students)
|
|
# check that entries were reset
|
|
self._assert_num_attempts(students, 0)
|
|
|
|
def test_delete_with_some_state(self):
|
|
# This will create StudentModule entries -- we don't have to worry about
|
|
# the state inside them.
|
|
num_students = 10
|
|
students = self._create_students_with_state(num_students)
|
|
# check that entries were created correctly
|
|
for student in students:
|
|
StudentModule.objects.get(course_id=self.course.id,
|
|
student=student,
|
|
module_state_key=self.problem_url)
|
|
self._test_run_with_task(delete_problem_state, 'deleted', num_students)
|
|
# confirm that no state can be found anymore:
|
|
for student in students:
|
|
with self.assertRaises(StudentModule.DoesNotExist):
|
|
StudentModule.objects.get(course_id=self.course.id,
|
|
student=student,
|
|
module_state_key=self.problem_url)
|
|
|
|
def _test_reset_with_student(self, use_email):
|
|
"""Run a reset task for one student, with several StudentModules for the problem defined."""
|
|
num_students = 10
|
|
initial_attempts = 3
|
|
input_state = json.dumps({'attempts': initial_attempts})
|
|
students = self._create_students_with_state(num_students, input_state)
|
|
# check that entries were set correctly
|
|
for student in students:
|
|
module = StudentModule.objects.get(course_id=self.course.id,
|
|
student=student,
|
|
module_state_key=self.problem_url)
|
|
state = json.loads(module.state)
|
|
self.assertEquals(state['attempts'], initial_attempts)
|
|
|
|
if use_email:
|
|
student_ident = students[3].email
|
|
else:
|
|
student_ident = students[3].username
|
|
task_entry = self._create_input_entry(student_ident)
|
|
|
|
status = self._run_task_with_mock_celery(reset_problem_attempts, task_entry.id, task_entry.task_id)
|
|
# check return value
|
|
self.assertEquals(status.get('attempted'), 1)
|
|
self.assertEquals(status.get('updated'), 1)
|
|
self.assertEquals(status.get('total'), 1)
|
|
self.assertEquals(status.get('action_name'), 'reset')
|
|
self.assertGreater('duration_ms', 0)
|
|
# compare with entry in table:
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
self.assertEquals(json.loads(entry.task_output), status)
|
|
self.assertEquals(entry.task_state, SUCCESS)
|
|
# check that the correct entry was reset
|
|
for index, student in enumerate(students):
|
|
module = StudentModule.objects.get(course_id=self.course.id,
|
|
student=student,
|
|
module_state_key=self.problem_url)
|
|
state = json.loads(module.state)
|
|
if index == 3:
|
|
self.assertEquals(state['attempts'], 0)
|
|
else:
|
|
self.assertEquals(state['attempts'], initial_attempts)
|
|
|
|
def test_reset_with_student_username(self):
|
|
self._test_reset_with_student(False)
|
|
|
|
def test_reset_with_student_email(self):
|
|
self._test_reset_with_student(True)
|
|
|
|
def _test_run_with_failure(self, task_function, expected_message):
|
|
"""Run a task and trigger an artificial failure with give message."""
|
|
task_entry = self._create_input_entry()
|
|
self.define_option_problem(PROBLEM_URL_NAME)
|
|
with self.assertRaises(TestTaskFailure):
|
|
self._run_task_with_mock_celery(task_function, task_entry.id, task_entry.task_id, expected_message)
|
|
# compare with entry in table:
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
self.assertEquals(entry.task_state, FAILURE)
|
|
output = json.loads(entry.task_output)
|
|
self.assertEquals(output['exception'], 'TestTaskFailure')
|
|
self.assertEquals(output['message'], expected_message)
|
|
|
|
def test_rescore_with_failure(self):
|
|
self._test_run_with_failure(rescore_problem, 'We expected this to fail')
|
|
|
|
def test_reset_with_failure(self):
|
|
self._test_run_with_failure(reset_problem_attempts, 'We expected this to fail')
|
|
|
|
def test_delete_with_failure(self):
|
|
self._test_run_with_failure(delete_problem_state, 'We expected this to fail')
|
|
|
|
def _test_run_with_long_error_msg(self, task_function):
|
|
"""
|
|
Run with an error message that is so long it will require
|
|
truncation (as well as the jettisoning of the traceback).
|
|
"""
|
|
task_entry = self._create_input_entry()
|
|
self.define_option_problem(PROBLEM_URL_NAME)
|
|
expected_message = "x" * 1500
|
|
with self.assertRaises(TestTaskFailure):
|
|
self._run_task_with_mock_celery(task_function, task_entry.id, task_entry.task_id, expected_message)
|
|
# compare with entry in table:
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
self.assertEquals(entry.task_state, FAILURE)
|
|
self.assertGreater(1023, len(entry.task_output))
|
|
output = json.loads(entry.task_output)
|
|
self.assertEquals(output['exception'], 'TestTaskFailure')
|
|
self.assertEquals(output['message'], expected_message[:len(output['message']) - 3] + "...")
|
|
self.assertTrue('traceback' not in output)
|
|
|
|
def test_rescore_with_long_error_msg(self):
|
|
self._test_run_with_long_error_msg(rescore_problem)
|
|
|
|
def test_reset_with_long_error_msg(self):
|
|
self._test_run_with_long_error_msg(reset_problem_attempts)
|
|
|
|
def test_delete_with_long_error_msg(self):
|
|
self._test_run_with_long_error_msg(delete_problem_state)
|
|
|
|
def _test_run_with_short_error_msg(self, task_function):
|
|
"""
|
|
Run with an error message that is short enough to fit
|
|
in the output, but long enough that the traceback won't.
|
|
Confirm that the traceback is truncated.
|
|
"""
|
|
task_entry = self._create_input_entry()
|
|
self.define_option_problem(PROBLEM_URL_NAME)
|
|
expected_message = "x" * 900
|
|
with self.assertRaises(TestTaskFailure):
|
|
self._run_task_with_mock_celery(task_function, task_entry.id, task_entry.task_id, expected_message)
|
|
# compare with entry in table:
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
self.assertEquals(entry.task_state, FAILURE)
|
|
self.assertGreater(1023, len(entry.task_output))
|
|
output = json.loads(entry.task_output)
|
|
self.assertEquals(output['exception'], 'TestTaskFailure')
|
|
self.assertEquals(output['message'], expected_message)
|
|
self.assertEquals(output['traceback'][-3:], "...")
|
|
|
|
def test_rescore_with_short_error_msg(self):
|
|
self._test_run_with_short_error_msg(rescore_problem)
|
|
|
|
def test_reset_with_short_error_msg(self):
|
|
self._test_run_with_short_error_msg(reset_problem_attempts)
|
|
|
|
def test_delete_with_short_error_msg(self):
|
|
self._test_run_with_short_error_msg(delete_problem_state)
|
|
|
|
def test_successful_result_too_long(self):
|
|
# while we don't expect the existing tasks to generate output that is too
|
|
# long, we can test the framework will handle such an occurrence.
|
|
task_entry = self._create_input_entry()
|
|
self.define_option_problem(PROBLEM_URL_NAME)
|
|
action_name = 'x' * 1000
|
|
update_fcn = lambda(_module_descriptor, _student_module, _xmodule_instance_args): True
|
|
task_function = (lambda entry_id, xmodule_instance_args:
|
|
update_problem_module_state(entry_id,
|
|
update_fcn, action_name, filter_fcn=None,
|
|
xmodule_instance_args=None))
|
|
|
|
with self.assertRaises(ValueError):
|
|
self._run_task_with_mock_celery(task_function, task_entry.id, task_entry.task_id)
|
|
# compare with entry in table:
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
self.assertEquals(entry.task_state, FAILURE)
|
|
self.assertGreater(1023, len(entry.task_output))
|
|
output = json.loads(entry.task_output)
|
|
self.assertEquals(output['exception'], 'ValueError')
|
|
self.assertTrue("Length of task output is too long" in output['message'])
|
|
self.assertTrue('traceback' not in output)
|
|
|
|
@skip
|
|
def test_rescoring_unrescorable(self):
|
|
# TODO: this test needs to have Mako templates initialized
|
|
# to make sure that the creation of an XModule works.
|
|
input_state = json.dumps({'done': True})
|
|
num_students = 1
|
|
self._create_students_with_state(num_students, input_state)
|
|
task_entry = self._create_input_entry()
|
|
with self.assertRaises(UpdateProblemModuleStateError):
|
|
self._run_task_with_mock_celery(rescore_problem, task_entry.id, task_entry.task_id)
|
|
# check values stored in table:
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
output = json.loads(entry.task_output)
|
|
self.assertEquals(output['exception'], "UpdateProblemModuleStateError")
|
|
self.assertEquals(output['message'], "Specified problem does not support rescoring.")
|
|
self.assertGreater(len(output['traceback']), 0)
|
|
|
|
@skip
|
|
def test_rescoring_success(self):
|
|
# TODO: this test needs to have Mako templates initialized
|
|
# to make sure that the creation of an XModule works.
|
|
input_state = json.dumps({'done': True})
|
|
num_students = 10
|
|
self._create_students_with_state(num_students, input_state)
|
|
task_entry = self._create_input_entry()
|
|
mock_instance = Mock()
|
|
mock_instance.rescore_problem = Mock({'success': 'correct'})
|
|
# TODO: figure out why this mock is not working....
|
|
with patch('courseware.module_render.get_module_for_descriptor_internal') as mock_get_module:
|
|
mock_get_module.return_value = mock_instance
|
|
self._run_task_with_mock_celery(rescore_problem, task_entry.id, task_entry.task_id)
|
|
# check return value
|
|
entry = InstructorTask.objects.get(id=task_entry.id)
|
|
output = json.loads(entry.task_output)
|
|
self.assertEquals(output.get('attempted'), num_students)
|
|
self.assertEquals(output.get('updated'), num_students)
|
|
self.assertEquals(output.get('total'), num_students)
|
|
self.assertEquals(output.get('action_name'), 'rescored')
|
|
self.assertGreater('duration_ms', 0)
|