From 637f55414f5fa5e3fc537aaf2ed04f228949c592 Mon Sep 17 00:00:00 2001 From: Jason Bau Date: Fri, 29 Aug 2014 12:36:27 -0700 Subject: [PATCH] capa custom response support for decimal grades --- common/lib/capa/capa/responsetypes.py | 39 ++++++-- common/lib/capa/capa/tests/test_correctmap.py | 4 +- .../lib/capa/capa/tests/test_responsetypes.py | 90 ++++++++++++++++++- .../xmodule/xmodule/tests/test_capa_module.py | 34 +++++-- 4 files changed, 153 insertions(+), 14 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 6f8e7596ad..caf2d08452 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -1598,11 +1598,17 @@ class CustomResponse(LoncapaResponse): correct = self.context['correct'] messages = self.context['messages'] overall_message = self.clean_message_html(self.context['overall_message']) + grade_decimals = self.context.get('grade_decimals') + correct_map = CorrectMap() correct_map.set_overall_message(overall_message) for k in range(len(idset)): - npoints = self.maxpoints[idset[k]] if correct[k] == 'correct' else 0 + max_points = self.maxpoints[idset[k]] + if grade_decimals: + npoints = max_points * grade_decimals[k] + else: + npoints = max_points if correct[k] == 'correct' else 0 correct_map.set(idset[k], correct[k], msg=messages[k], npoints=npoints) return correct_map @@ -1643,7 +1649,9 @@ class CustomResponse(LoncapaResponse): ) if isinstance(ret, dict): # One kind of dictionary the check function can return has the - # form {'ok': BOOLEAN, 'msg': STRING} + # form {'ok': BOOLEAN, 'msg': STRING, 'grade_decimal' (optional): FLOAT (between 0.0 and 1.0)} + # 'ok' will control the checkmark, while grade_decimal, if present, will scale + # the score the student receives on the response. # If there are multiple inputs, they all get marked # to the same correct/incorrect value if 'ok' in ret: @@ -1658,28 +1666,49 @@ class CustomResponse(LoncapaResponse): else: self.context['messages'][0] = msg + if 'grade_decimal' in ret: + decimal = ret['grade_decimal'] + else: + decimal = 1.0 if ret['ok'] else 0.0 + grade_decimals = [decimal] * len(idset) + self.context['grade_decimals'] = grade_decimals + # Another kind of dictionary the check function can return has # the form: - # {'overall_message': STRING, - # 'input_list': [{ 'ok': BOOLEAN, 'msg': STRING }, ...] } + # { 'overall_message': STRING, + # 'input_list': [ + # { 'ok': BOOLEAN, 'msg': STRING, 'grade_decimal' (optional): FLOAT (between 0.0 and 1.0)}, + # ... + # ] + # } + # 'ok' will control the checkmark, while grade_decimal, if present, will scale + # the score the student receives on the response. # # This allows the function to return an 'overall message' # that applies to the entire problem, as well as correct/incorrect - # status and messages for individual inputs + # status, scaled grades, and messages for individual inputs elif 'input_list' in ret: overall_message = ret.get('overall_message', '') input_list = ret['input_list'] correct = [] messages = [] + grade_decimals = [] for input_dict in input_list: correct.append('correct' if input_dict['ok'] else 'incorrect') msg = (self.clean_message_html(input_dict['msg']) if 'msg' in input_dict else None) messages.append(msg) + if 'grade_decimal' in input_dict: + decimal = input_dict['grade_decimal'] + else: + decimal = 1.0 if input_dict['ok'] else 0.0 + grade_decimals.append(decimal) + self.context['messages'] = messages self.context['overall_message'] = overall_message + self.context['grade_decimals'] = grade_decimals # Otherwise, we do not recognize the dictionary # Raise an exception diff --git a/common/lib/capa/capa/tests/test_correctmap.py b/common/lib/capa/capa/tests/test_correctmap.py index c5e49edecb..e51cc1854b 100644 --- a/common/lib/capa/capa/tests/test_correctmap.py +++ b/common/lib/capa/capa/tests/test_correctmap.py @@ -85,7 +85,7 @@ class CorrectMapTest(unittest.TestCase): self.cmap.set( answer_id='1_2_1', correctness='correct', - npoints=5 + npoints=5.3 ) self.cmap.set( @@ -116,7 +116,7 @@ class CorrectMapTest(unittest.TestCase): # If points assigned --> npoints # If no points assigned and correct --> 1 point # If no points assigned and incorrect --> 0 points - self.assertEqual(self.cmap.get_npoints('1_2_1'), 5) + self.assertEqual(self.cmap.get_npoints('1_2_1'), 5.3) self.assertEqual(self.cmap.get_npoints('2_2_1'), 1) self.assertEqual(self.cmap.get_npoints('3_2_1'), 5) self.assertEqual(self.cmap.get_npoints('4_2_1'), 0) diff --git a/common/lib/capa/capa/tests/test_responsetypes.py b/common/lib/capa/capa/tests/test_responsetypes.py index b6f15eac90..934dac93f4 100644 --- a/common/lib/capa/capa/tests/test_responsetypes.py +++ b/common/lib/capa/capa/tests/test_responsetypes.py @@ -1399,7 +1399,7 @@ class CustomResponseTest(ResponseTest): # or an ordered list of answers (if there are multiple inputs) # # The function should return a dict of the form - # { 'ok': BOOL, 'msg': STRING } + # { 'ok': BOOL, 'msg': STRING } (no 'grade_decimal' key to test that it's optional) # script = textwrap.dedent(""" def check_func(expect, answer_given): @@ -1414,9 +1414,11 @@ class CustomResponseTest(ResponseTest): correctness = correct_map.get_correctness('1_2_1') msg = correct_map.get_msg('1_2_1') + npoints = correct_map.get_npoints('1_2_1') self.assertEqual(correctness, 'correct') self.assertEqual(msg, "Message text") + self.assertEqual(npoints, 1) # Incorrect answer input_dict = {'1_2_1': '0'} @@ -1424,9 +1426,45 @@ class CustomResponseTest(ResponseTest): correctness = correct_map.get_correctness('1_2_1') msg = correct_map.get_msg('1_2_1') + npoints = correct_map.get_npoints('1_2_1') self.assertEqual(correctness, 'incorrect') self.assertEqual(msg, "Message text") + self.assertEqual(npoints, 0) + + def test_function_code_single_input_decimal_score(self): + # For function code, we pass in these arguments: + # + # 'expect' is the expect attribute of the + # + # 'answer_given' is the answer the student gave (if there is just one input) + # or an ordered list of answers (if there are multiple inputs) + # + # The function should return a dict of the form + # { 'ok': BOOL, 'msg': STRING, 'grade_decimal': FLOAT } + # + script = textwrap.dedent(""" + def check_func(expect, answer_given): + return { + 'ok': answer_given == expect, + 'msg': 'Message text', + 'grade_decimal': 0.9 if answer_given == expect else 0.1, + } + """) + + problem = self.build_problem(script=script, cfn="check_func", expect="42") + + # Correct answer + input_dict = {'1_2_1': '42'} + correct_map = problem.grade_answers(input_dict) + self.assertEqual(correct_map.get_npoints('1_2_1'), 0.9) + self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct') + + # Incorrect answer + input_dict = {'1_2_1': '43'} + correct_map = problem.grade_answers(input_dict) + self.assertEqual(correct_map.get_npoints('1_2_1'), 0.1) + self.assertEqual(correct_map.get_correctness('1_2_1'), 'incorrect') def test_function_code_multiple_input_no_msg(self): @@ -1469,7 +1507,7 @@ class CustomResponseTest(ResponseTest): # the check function can return a dict of the form: # # {'overall_message': STRING, - # 'input_list': [{'ok': BOOL, 'msg': STRING}, ...] } + # 'input_list': [{'ok': BOOL, 'msg': STRING}, ...] } (no grade_decimal to test it's optional) # # 'overall_message' is displayed at the end of the response # @@ -1502,11 +1540,59 @@ class CustomResponseTest(ResponseTest): self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct') self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct') + # Expect that the inputs were given correct npoints + self.assertEqual(correct_map.get_npoints('1_2_1'), 0) + self.assertEqual(correct_map.get_npoints('1_2_2'), 1) + self.assertEqual(correct_map.get_npoints('1_2_3'), 1) + # Expect that we received messages for each individual input self.assertEqual(correct_map.get_msg('1_2_1'), 'Feedback 1') self.assertEqual(correct_map.get_msg('1_2_2'), 'Feedback 2') self.assertEqual(correct_map.get_msg('1_2_3'), 'Feedback 3') + def test_function_code_multiple_inputs_decimal_score(self): + + # If the has multiple inputs associated with it, + # the check function can return a dict of the form: + # + # {'overall_message': STRING, + # 'input_list': [{'ok': BOOL, 'msg': STRING, 'grade_decimal': FLOAT}, ...] } + # # + # 'input_list' contains dictionaries representing the correctness + # and message for each input. + script = textwrap.dedent(""" + def check_func(expect, answer_given): + check1 = (int(answer_given[0]) == 1) + check2 = (int(answer_given[1]) == 2) + check3 = (int(answer_given[2]) == 3) + score1 = 0.9 if check1 else 0.1 + score2 = 0.9 if check2 else 0.1 + score3 = 0.9 if check3 else 0.1 + return { + 'input_list': [ + {'ok': check1, 'grade_decimal': score1, 'msg': 'Feedback 1'}, + {'ok': check2, 'grade_decimal': score2, 'msg': 'Feedback 2'}, + {'ok': check3, 'grade_decimal': score3, 'msg': 'Feedback 3'}, + ] + } + """) + + problem = self.build_problem(script=script, cfn="check_func", num_inputs=3) + + # Grade the inputs (one input incorrect) + input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3'} + correct_map = problem.grade_answers(input_dict) + + # Expect that the inputs were graded individually + self.assertEqual(correct_map.get_correctness('1_2_1'), 'incorrect') + self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct') + self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct') + + # Expect that the inputs were given correct npoints + self.assertEqual(correct_map.get_npoints('1_2_1'), 0.1) + self.assertEqual(correct_map.get_npoints('1_2_2'), 0.9) + self.assertEqual(correct_map.get_npoints('1_2_3'), 0.9) + def test_function_code_with_extra_args(self): script = textwrap.dedent("""\ def check_func(expect, answer_given, options, dynamath): diff --git a/common/lib/xmodule/xmodule/tests/test_capa_module.py b/common/lib/xmodule/xmodule/tests/test_capa_module.py index 1e0cf84b7f..2bf73ec1a2 100644 --- a/common/lib/xmodule/xmodule/tests/test_capa_module.py +++ b/common/lib/xmodule/xmodule/tests/test_capa_module.py @@ -83,6 +83,7 @@ class CapaFactory(object): problem_state=None, correct=False, xml=None, + override_get_score=True, **kwargs ): """ @@ -130,11 +131,12 @@ class CapaFactory(object): ScopeIds(None, None, location, location), ) - if correct: - # TODO: probably better to actually set the internal state properly, but... - module.get_score = lambda: {'score': 1, 'total': 1} - else: - module.get_score = lambda: {'score': 0, 'total': 1} + if override_get_score: + if correct: + # TODO: probably better to actually set the internal state properly, but... + module.get_score = lambda: {'score': 1, 'total': 1} + else: + module.get_score = lambda: {'score': 0, 'total': 1} return module @@ -211,6 +213,28 @@ class CapaModuleTest(unittest.TestCase): other_module = CapaFactory.create(correct=True) self.assertEqual(other_module.get_score()['score'], 1) + def test_get_score(self): + """ + Do 1 test where the internals of get_score are properly set + + @jbau Note: this obviously depends on a particular implementation of get_score, but I think this is actually + useful as unit-code coverage for this current implementation. I don't see a layer where LoncapaProblem + is tested directly + """ + from capa.correctmap import CorrectMap + student_answers = {'1_2_1': 'abcd'} + correct_map = CorrectMap(answer_id='1_2_1', correctness="correct", npoints=0.9) + module = CapaFactory.create(correct=True, override_get_score=False) + module.lcp.correct_map = correct_map + module.lcp.student_answers = student_answers + self.assertEqual(module.get_score()['score'], 0.9) + + other_correct_map = CorrectMap(answer_id='1_2_1', correctness="incorrect", npoints=0.1) + other_module = CapaFactory.create(correct=False, override_get_score=False) + other_module.lcp.correct_map = other_correct_map + other_module.lcp.student_answers = student_answers + self.assertEqual(other_module.get_score()['score'], 0.1) + def test_showanswer_default(self): """ Make sure the show answer logic does the right thing.