From aba25c2de7dbb8ec60c276bacf9dcd8019da7c26 Mon Sep 17 00:00:00 2001 From: Waheed Ahmed Date: Mon, 25 Nov 2013 15:35:51 +0000 Subject: [PATCH] In OpenEndedModule latest_score and all_score calculate sum of rubric scores every time for ML grading. ORA-72 --- .../combined_open_ended_modulev1.py | 19 +- .../open_ended_module.py | 40 +- .../xmodule/tests/test_combined_open_ended.py | 182 ++++- .../xmodule/tests/test_util_open_ended.py | 670 ++++++++++++++++++ 4 files changed, 903 insertions(+), 8 deletions(-) diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py index 72915eb7b3..3158ff291e 100644 --- a/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py +++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/combined_open_ended_modulev1.py @@ -258,8 +258,23 @@ class CombinedOpenEndedV1Module(): if not task_states: return (0, 0, state_values[OpenEndedChild.INITIAL], idx) - final_child_state = json.loads(task_states[-1]) - scores = [attempt.get('score', 0) for attempt in final_child_state.get('child_history', [])] + final_task_xml = self.task_xml[-1] + final_child_state_json = task_states[-1] + final_child_state = json.loads(final_child_state_json) + + tag_name = self.get_tag_name(final_task_xml) + children = self.child_modules() + task_descriptor = children['descriptors'][tag_name](self.system) + task_parsed_xml = task_descriptor.definition_from_xml(etree.fromstring(final_task_xml), self.system) + task = children['modules'][tag_name]( + self.system, + self.location, + task_parsed_xml, + task_descriptor, + self.static_data, + instance_state=final_child_state_json, + ) + scores = task.all_scores() if scores: best_score = max(scores) else: diff --git a/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py b/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py index d020987fdf..35c1c361cc 100644 --- a/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py +++ b/common/lib/xmodule/xmodule/open_ended_grading_classes/open_ended_module.py @@ -679,7 +679,7 @@ class OpenEndedModule(openendedchild.OpenEndedChild): return { 'success': success, 'error': error_message, - 'student_response': data['student_answer'].replace("\n","
") + 'student_response': data['student_answer'].replace("\n", "
") } def update_score(self, data, system): @@ -738,6 +738,44 @@ class OpenEndedModule(openendedchild.OpenEndedChild): html = system.render_template('{0}/open_ended.html'.format(self.TEMPLATE_DIR), context) return html + def latest_score(self): + """None if not available""" + if not self.child_history: + return None + return self.score_for_attempt(-1) + + def all_scores(self): + """None if not available""" + if not self.child_history: + return None + return [self.score_for_attempt(index) for index in xrange(0, len(self.child_history))] + + def score_for_attempt(self, index): + """ + Return sum of rubric scores for ML grading otherwise return attempt["score"]. + """ + attempt = self.child_history[index] + score = attempt.get('score') + post_assessment_data = self._parse_score_msg(attempt.get('post_assessment'), self.system) + grader_types = post_assessment_data.get('grader_types') + + # According to _parse_score_msg in ML grading there should be only one grader type. + if len(grader_types) == 1 and grader_types[0] == 'ML': + rubric_scores = post_assessment_data.get("rubric_scores") + + # Similarly there should be only one list of rubric scores. + if len(rubric_scores) == 1: + rubric_scores_sum = sum(rubric_scores[0]) + log.debug("""Score normalized for location={loc}, old_score={old_score}, + new_score={new_score}, rubric_score={rubric_score}""".format( + loc=self.location_string, + old_score=score, + new_score=rubric_scores_sum, + rubric_score=rubric_scores + )) + return rubric_scores_sum + return score + class OpenEndedDescriptor(): """ diff --git a/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py b/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py index e1b2a4ebe2..6498c82c4a 100644 --- a/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py +++ b/common/lib/xmodule/xmodule/tests/test_combined_open_ended.py @@ -27,7 +27,9 @@ from xmodule.progress import Progress from xmodule.tests.test_util_open_ended import ( DummyModulestore, TEST_STATE_SA_IN, MOCK_INSTANCE_STATE, TEST_STATE_SA, TEST_STATE_AI, TEST_STATE_AI2, TEST_STATE_AI2_INVALID, - TEST_STATE_SINGLE, TEST_STATE_PE_SINGLE, MockUploadedFile + TEST_STATE_SINGLE, TEST_STATE_PE_SINGLE, MockUploadedFile, INSTANCE_INCONSISTENT_STATE, + INSTANCE_INCONSISTENT_STATE2, INSTANCE_INCONSISTENT_STATE3, INSTANCE_INCONSISTENT_STATE4, + INSTANCE_INCONSISTENT_STATE5 ) from xblock.field_data import DictFieldData @@ -358,7 +360,7 @@ class OpenEndedModuleTest(unittest.TestCase): # Create a module with no state yet. Important that this start off as a blank slate. test_module = OpenEndedModule(self.test_system, self.location, - self.definition, self.descriptor, self.static_data, self.metadata) + self.definition, self.descriptor, self.static_data, self.metadata) saved_response = "Saved response." submitted_response = "Submitted response." @@ -369,7 +371,7 @@ class OpenEndedModuleTest(unittest.TestCase): self.assertEqual(test_module.get_display_answer(), "") # Now, store an answer in the module. - test_module.handle_ajax("store_answer", {'student_answer' : saved_response}, get_test_system()) + test_module.handle_ajax("store_answer", {'student_answer': saved_response}, get_test_system()) # The stored answer should now equal our response. self.assertEqual(test_module.stored_answer, saved_response) self.assertEqual(test_module.get_display_answer(), saved_response) @@ -387,6 +389,7 @@ class OpenEndedModuleTest(unittest.TestCase): # Confirm that the answer is stored properly. self.assertEqual(test_module.latest_answer(), submitted_response) + class CombinedOpenEndedModuleTest(unittest.TestCase): """ Unit tests for the combined open ended xmodule @@ -610,7 +613,6 @@ class CombinedOpenEndedModuleTest(unittest.TestCase): metadata=self.metadata, instance_state={'task_states': TEST_STATE_SA_IN}) - def test_get_score_realistic(self): """ Try to parse the correct score from a json instance state @@ -717,6 +719,175 @@ class CombinedOpenEndedModuleTest(unittest.TestCase): self.ai_state_success(TEST_STATE_PE_SINGLE, iscore=0, tasks=[self.task_xml2]) +class CombinedOpenEndedModuleConsistencyTest(unittest.TestCase): + """ + Unit tests for the combined open ended xmodule rubric scores consistency. + """ + + # location, definition_template, prompt, rubric, max_score, metadata, oeparam, task_xml1, task_xml2 + # All these variables are used to construct the xmodule descriptor. + location = Location(["i4x", "edX", "open_ended", "combinedopenended", + "SampleQuestion"]) + definition_template = """ + + {rubric} + {prompt} + + {task1} + + + {task2} + + + """ + prompt = "This is a question prompt" + rubric = ''' + + Response Quality + + + + ''' + max_score = 10 + + metadata = {'attempts': '10', 'max_score': max_score} + + oeparam = etree.XML(''' + + Enter essay here. + This is the answer. + {"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"} + + ''') + + task_xml1 = ''' + + + What hint about this problem would you give to someone? + + + Save Succcesful. Thanks for participating! + + + ''' + task_xml2 = ''' + + + Enter essay here. + This is the answer. + {"grader_settings" : "ml_grading.conf", "problem_id" : "6.002x/Welcome/OETest"} + + ''' + + + static_data = { + 'max_attempts': 20, + 'prompt': prompt, + 'rubric': rubric, + 'max_score': max_score, + 'display_name': 'Name', + 'accept_file_upload': False, + 'close_date': "", + 's3_interface': test_util_open_ended.S3_INTERFACE, + 'open_ended_grading_interface': test_util_open_ended.OPEN_ENDED_GRADING_INTERFACE, + 'skip_basic_checks': False, + 'graded': True, + } + + definition = {'prompt': etree.XML(prompt), 'rubric': etree.XML(rubric), 'task_xml': [task_xml1, task_xml2]} + full_definition = definition_template.format(prompt=prompt, rubric=rubric, task1=task_xml1, task2=task_xml2) + descriptor = Mock(data=full_definition) + test_system = get_test_system() + test_system.open_ended_grading_interface = None + combinedoe_container = CombinedOpenEndedModule( + descriptor=descriptor, + runtime=test_system, + field_data=DictFieldData({ + 'data': full_definition, + 'weight': '1', + }), + scope_ids=ScopeIds(None, None, None, None), + ) + + def setUp(self): + self.combinedoe = CombinedOpenEndedV1Module(self.test_system, + self.location, + self.definition, + self.descriptor, + static_data=self.static_data, + metadata=self.metadata, + instance_state=json.loads(INSTANCE_INCONSISTENT_STATE)) + + def test_get_score(self): + """ + If grader type is ML score should be updated from rubric scores. Aggregate rubric scores = sum([3])*5. + """ + score_dict = self.combinedoe.get_score() + self.assertEqual(score_dict['score'], 15.0) + self.assertEqual(score_dict['total'], 5.0) + + def test_get_score_with_pe_grader(self): + """ + If grader type is PE score should not be updated from rubric scores. Aggregate rubric scores = sum([3])*5. + """ + combinedoe = CombinedOpenEndedV1Module(self.test_system, + self.location, + self.definition, + self.descriptor, + static_data=self.static_data, + metadata=self.metadata, + instance_state=json.loads(INSTANCE_INCONSISTENT_STATE2)) + score_dict = combinedoe.get_score() + self.assertNotEqual(score_dict['score'], 15.0) + + def test_get_score_with_different_score_value_in_rubric(self): + """ + If grader type is ML score should be updated from rubric scores. Aggregate rubric scores = sum([5])*5. + """ + combinedoe = CombinedOpenEndedV1Module(self.test_system, + self.location, + self.definition, + self.descriptor, + static_data=self.static_data, + metadata=self.metadata, + instance_state=json.loads(INSTANCE_INCONSISTENT_STATE3)) + score_dict = combinedoe.get_score() + self.assertEqual(score_dict['score'], 25.0) + self.assertEqual(score_dict['total'], 5.0) + + def test_get_score_with_old_task_states(self): + """ + If grader type is ML and old_task_states are present in instance inconsistent state score should be updated + from rubric scores. Aggregate rubric scores = sum([3])*5. + """ + combinedoe = CombinedOpenEndedV1Module(self.test_system, + self.location, + self.definition, + self.descriptor, + static_data=self.static_data, + metadata=self.metadata, + instance_state=json.loads(INSTANCE_INCONSISTENT_STATE4)) + score_dict = combinedoe.get_score() + self.assertEqual(score_dict['score'], 15.0) + self.assertEqual(score_dict['total'], 5.0) + + def test_get_score_with_score_missing(self): + """ + If grader type is ML and score field is missing in instance inconsistent state score should be updated from + rubric scores. Aggregate rubric scores = sum([3])*5. + """ + combinedoe = CombinedOpenEndedV1Module(self.test_system, + self.location, + self.definition, + self.descriptor, + static_data=self.static_data, + metadata=self.metadata, + instance_state=json.loads(INSTANCE_INCONSISTENT_STATE5)) + score_dict = combinedoe.get_score() + self.assertEqual(score_dict['score'], 15.0) + self.assertEqual(score_dict['total'], 5.0) + + class OpenEndedModuleXmlTest(unittest.TestCase, DummyModulestore): """ Test the student flow in the combined open ended xmodule @@ -948,6 +1119,7 @@ class OpenEndedModuleXmlAttemptTest(unittest.TestCase, DummyModulestore): reset_data = json.loads(self._handle_ajax("reset", {})) self.assertEqual(reset_data['success'], False) + class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore): """ Test if student is able to upload images properly. @@ -1018,7 +1190,7 @@ class OpenEndedModuleXmlImageUploadTest(unittest.TestCase, DummyModulestore): # Simulate a student saving an answer with a link. response = module.handle_ajax("save_answer", { "student_answer": "{0} {1}".format(self.answer_text, self.answer_link) - }) + }) response = json.loads(response) diff --git a/common/lib/xmodule/xmodule/tests/test_util_open_ended.py b/common/lib/xmodule/xmodule/tests/test_util_open_ended.py index bbb0653512..f4a607aecf 100644 --- a/common/lib/xmodule/xmodule/tests/test_util_open_ended.py +++ b/common/lib/xmodule/xmodule/tests/test_util_open_ended.py @@ -1,3 +1,5 @@ +import json +from textwrap import dedent from xmodule.modulestore import Location from xmodule.modulestore.xml import XMLModuleStore from xmodule.tests import DATA_DIR, get_test_system @@ -98,12 +100,680 @@ class DummyModulestore(object): descriptor.xmodule_runtime = self.get_module_system(descriptor) return descriptor +def serialize_child_history(task_state): + """ + To json serialize feedback and post_assessment in child_history of task state. + """ + child_history = task_state.get("child_history", []) + for i, attempt in enumerate(child_history): + if "post_assessment" in attempt: + if "feedback" in attempt["post_assessment"]: + attempt["post_assessment"]["feedback"] = json.dumps(attempt["post_assessment"].get("feedback")) + task_state["child_history"][i]["post_assessment"] = json.dumps(attempt["post_assessment"]) + +def serialize_open_ended_instance_state(json_str): + """ + To json serialize task_states and old_task_states in instance state. + """ + json_data = json.loads(json_str) + task_states = json_data.get('task_states', []) + for i, task_state in enumerate(task_states): + serialize_child_history(task_state) + json_data['task_states'][i] = json.dumps(task_state) + + old_task_states = json_data.get('old_task_states', []) + for i, old_task in enumerate(old_task_states): + for j, task_state in enumerate(old_task): + old_task[j] = json.dumps(task_state) + json_data['old_task_states'][i] = old_task + + return json.dumps(json_data) + + # Task state for a module with self assessment then instructor assessment. TEST_STATE_SA_IN = ["{\"child_created\": false, \"child_attempts\": 2, \"version\": 1, \"child_history\": [{\"answer\": \"However venture pursuit he am mr cordial. Forming musical am hearing studied be luckily. Ourselves for determine attending how led gentleman sincerity. Valley afford uneasy joy she thrown though bed set. In me forming general prudent on country carried. Behaved an or suppose justice. Seemed whence how son rather easily and change missed. Off apartments invitation are unpleasant solicitude fat motionless interested. Hardly suffer wisdom wishes valley as an. As friendship advantages resolution it alteration stimulated he or increasing. \\r

Now led tedious shy lasting females off. Dashwood marianne in of entrance be on wondered possible building. Wondered sociable he carriage in speedily margaret. Up devonshire of he thoroughly insensible alteration. An mr settling occasion insisted distance ladyship so. Not attention say frankness intention out dashwoods now curiosity. Stronger ecstatic as no judgment daughter speedily thoughts. Worse downs nor might she court did nay forth these. \", \"post_assessment\": \"[3, 3, 2, 2, 2]\", \"score\": 12}, {\"answer\": \"Delightful remarkably mr on announcing themselves entreaties favourable. About to in so terms voice at. Equal an would is found seems of. The particular friendship one sufficient terminated frequently themselves. It more shed went up is roof if loud case. Delay music in lived noise an. Beyond genius really enough passed is up. \\r

John draw real poor on call my from. May she mrs furnished discourse extremely. Ask doubt noisy shade guest did built her him. Ignorant repeated hastened it do. Consider bachelor he yourself expenses no. Her itself active giving for expect vulgar months. Discovery commanded fat mrs remaining son she principle middleton neglected. Be miss he in post sons held. No tried is defer do money scale rooms. \", \"post_assessment\": \"[3, 3, 2, 2, 2]\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"done\"}", "{\"child_created\": false, \"child_attempts\": 0, \"version\": 1, \"child_history\": [{\"answer\": \"However venture pursuit he am mr cordial. Forming musical am hearing studied be luckily. Ourselves for determine attending how led gentleman sincerity. Valley afford uneasy joy she thrown though bed set. In me forming general prudent on country carried. Behaved an or suppose justice. Seemed whence how son rather easily and change missed. Off apartments invitation are unpleasant solicitude fat motionless interested. Hardly suffer wisdom wishes valley as an. As friendship advantages resolution it alteration stimulated he or increasing. \\r

Now led tedious shy lasting females off. Dashwood marianne in of entrance be on wondered possible building. Wondered sociable he carriage in speedily margaret. Up devonshire of he thoroughly insensible alteration. An mr settling occasion insisted distance ladyship so. Not attention say frankness intention out dashwoods now curiosity. Stronger ecstatic as no judgment daughter speedily thoughts. Worse downs nor might she court did nay forth these. \", \"post_assessment\": \"{\\\"submission_id\\\": 1460, \\\"score\\\": 12, \\\"feedback\\\": \\\"{\\\\\\\"feedback\\\\\\\": \\\\\\\"\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 5413, \\\"grader_type\\\": \\\"IN\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"\\\\nIdeas\\\\n3\\\\nContent\\\\n3\\\\nOrganization\\\\n2\\\\nStyle\\\\n2\\\\nVoice\\\\n2\\\"}\", \"score\": 12}, {\"answer\": \"Delightful remarkably mr on announcing themselves entreaties favourable. About to in so terms voice at. Equal an would is found seems of. The particular friendship one sufficient terminated frequently themselves. It more shed went up is roof if loud case. Delay music in lived noise an. Beyond genius really enough passed is up. \\r

John draw real poor on call my from. May she mrs furnished discourse extremely. Ask doubt noisy shade guest did built her him. Ignorant repeated hastened it do. Consider bachelor he yourself expenses no. Her itself active giving for expect vulgar months. Discovery commanded fat mrs remaining son she principle middleton neglected. Be miss he in post sons held. No tried is defer do money scale rooms. \", \"post_assessment\": \"{\\\"submission_id\\\": 1462, \\\"score\\\": 12, \\\"feedback\\\": \\\"{\\\\\\\"feedback\\\\\\\": \\\\\\\"\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 5418, \\\"grader_type\\\": \\\"IN\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"\\\\nIdeas\\\\n3\\\\nContent\\\\n3\\\\nOrganization\\\\n2\\\\nStyle\\\\n2\\\\nVoice\\\\n2\\\"}\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"post_assessment\"}"] # Mock instance state. Should receive a score of 15. MOCK_INSTANCE_STATE = r"""{"ready_to_reset": false, "skip_spelling_checks": true, "current_task_number": 1, "weight": 5.0, "graceperiod": "1 day 12 hours 59 minutes 59 seconds", "graded": "True", "task_states": ["{\"child_created\": false, \"child_attempts\": 4, \"version\": 1, \"child_history\": [{\"answer\": \"After 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\\r\\nStarting Mass (g)\\tEnding Mass (g)\\tDifference in Mass (g)\\r\\nMarble\\t 9.8\\t 9.4\\t\\u20130.4\\r\\nLimestone\\t10.4\\t 9.1\\t\\u20131.3\\r\\nWood\\t11.2\\t11.2\\t 0.0\\r\\nPlastic\\t 7.2\\t 7.1\\t\\u20130.1\\r\\nAfter reading the\", \"post_assessment\": \"[3]\", \"score\": 3}, {\"answer\": \"To replicate the experiment, the procedure would require more detail. One piece of information that is omitted is the amount of vinegar used in the experiment. It is also important to know what temperature the experiment was kept at during the 24 hours. Finally, the procedure needs to include details about the experiment, for example if the whole sample must be submerged.\", \"post_assessment\": \"[3]\", \"score\": 3}, {\"answer\": \"e the mass of four different samples.\\r\\nPour vinegar in each of four separate, but identical, containers.\\r\\nPlace a sample of one material into one container and label. Repeat with remaining samples, placing a single sample into a single container.\\r\\nAfter 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\", \"post_assessment\": \"[3]\", \"score\": 3}, {\"answer\": \"\", \"post_assessment\": \"[3]\", \"score\": 3}], \"max_score\": 3, \"child_state\": \"done\"}", "{\"child_created\": false, \"child_attempts\": 0, \"version\": 1, \"child_history\": [{\"answer\": \"The students\\u2019 data are recorded in the table below.\\r\\n\\r\\nStarting Mass (g)\\tEnding Mass (g)\\tDifference in Mass (g)\\r\\nMarble\\t 9.8\\t 9.4\\t\\u20130.4\\r\\nLimestone\\t10.4\\t 9.1\\t\\u20131.3\\r\\nWood\\t11.2\\t11.2\\t 0.0\\r\\nPlastic\\t 7.2\\t 7.1\\t\\u20130.1\\r\\nAfter reading the group\\u2019s procedure, describe what additional information you would need in order to replicate the expe\", \"post_assessment\": \"{\\\"submission_id\\\": 3097, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: More grammar errors than average.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"the students data are recorded in the table below . starting mass g ending mass g difference in mass g marble . . . limestone . . . wood . . . plastic . . . after reading the groups procedure , describe what additional information you would need in order to replicate the expe\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3233, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"Response Quality0\\\"}\", \"score\": 0}, {\"answer\": \"After 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\\r\\nStarting Mass (g)\\tEnding Mass (g)\\tDifference in Mass (g)\\r\\nMarble\\t 9.8\\t 9.4\\t\\u20130.4\\r\\nLimestone\\t10.4\\t 9.1\\t\\u20131.3\\r\\nWood\\t11.2\\t11.2\\t 0.0\\r\\nPlastic\\t 7.2\\t 7.1\\t\\u20130.1\\r\\nAfter reading the\", \"post_assessment\": \"{\\\"submission_id\\\": 3098, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"after hours , remove the samples from the containers and rinse each sample with distilled water . allow the samples to sit and dry for minutes . determine the mass of each sample . the students data are recorded in the table below . starting mass g ending mass g difference in mass g marble . . . limestone . . . wood . . . plastic . . . after reading the\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3235, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"Response Quality0\\\"}\", \"score\": 0}, {\"answer\": \"To replicate the experiment, the procedure would require more detail. One piece of information that is omitted is the amount of vinegar used in the experiment. It is also important to know what temperature the experiment was kept at during the 24 hours. Finally, the procedure needs to include details about the experiment, for example if the whole sample must be submerged.\", \"post_assessment\": \"{\\\"submission_id\\\": 3099, \\\"score\\\": 3, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"to replicate the experiment , the procedure would require more detail . one piece of information that is omitted is the amount of vinegar used in the experiment . it is also important to know what temperature the experiment was kept at during the hours . finally , the procedure needs to include details about the experiment , for example if the whole sample must be submerged .\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3237, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"Response Quality3\\\"}\", \"score\": 3}, {\"answer\": \"e the mass of four different samples.\\r\\nPour vinegar in each of four separate, but identical, containers.\\r\\nPlace a sample of one material into one container and label. Repeat with remaining samples, placing a single sample into a single container.\\r\\nAfter 24 hours, remove the samples from the containers and rinse each sample with distilled water.\\r\\nAllow the samples to sit and dry for 30 minutes.\\r\\nDetermine the mass of each sample.\\r\\nThe students\\u2019 data are recorded in the table below.\\r\\n\", \"post_assessment\": \"{\\\"submission_id\\\": 3100, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"e the mass of four different samples . pour vinegar in each of four separate , but identical , containers . place a sample of one material into one container and label . repeat with remaining samples , placing a single sample into a single container . after hours , remove the samples from the containers and rinse each sample with distilled water . allow the samples to sit and dry for minutes . determine the mass of each sample . the students data are recorded in the table below . \\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3239, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"Response Quality0\\\"}\", \"score\": 0}, {\"answer\": \"\", \"post_assessment\": \"{\\\"submission_id\\\": 3101, \\\"score\\\": 0, \\\"feedback\\\": \\\"{\\\\\\\"spelling\\\\\\\": \\\\\\\"Spelling: Ok.\\\\\\\", \\\\\\\"grammar\\\\\\\": \\\\\\\"Grammar: Ok.\\\\\\\", \\\\\\\"markup-text\\\\\\\": \\\\\\\"invalid essay .\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 3241, \\\"grader_type\\\": \\\"ML\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"Response Quality0\\\"}\", \"score\": 0}], \"max_score\": 3, \"child_state\": \"done\"}"], "attempts": "10000", "student_attempts": 0, "due": null, "state": "done", "accept_file_upload": false, "display_name": "Science Question -- Machine Assessed"}""" +# Instance state. To test the rubric scores are consistent. Should receive a score of 15. +INSTANCE_INCONSISTENT_STATE = serialize_open_ended_instance_state(""" +{ "accept_file_upload" : false, + "attempts" : "10000", + "current_task_number" : 1, + "display_name" : "Science Question -- Machine Assessed", + "due" : null, + "graceperiod" : "1 day 12 hours 59 minutes 59 seconds", + "graded" : "True", + "ready_to_reset" : false, + "skip_spelling_checks" : true, + "state" : "done", + "student_attempts" : 0, + "task_states" : [ { "child_attempts" : 4, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : [ 3 ], + "score" : 1 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : [ 3 ], + "score" : 1 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : [ 3 ], + "score" : 1 + }, + { "answer" : "", + "post_assessment" : [ 3 ], + "score" : 1 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + }, + { "child_attempts" : 0, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3233, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3097, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3235, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3098, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3237, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality3", + "score" : 2, + "submission_id" : 3099, + "success" : true + }, + "score" : 2 + }, + { "answer" : "Student answer 4th attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3239, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3100, + "success" : true + }, + "score" : 0 + }, + { "answer" : "", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "invalid essay .", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3241, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3101, + "success" : true + }, + "score" : 0 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + } + ], + "weight" : 5.0 +} + """) + +# Instance state. Should receive a score of 10 if grader type is PE. +INSTANCE_INCONSISTENT_STATE2 = serialize_open_ended_instance_state(""" +{ "accept_file_upload" : false, + "attempts" : "10000", + "current_task_number" : 1, + "display_name" : "Science Question -- Machine Assessed", + "due" : null, + "graceperiod" : "1 day 12 hours 59 minutes 59 seconds", + "graded" : "True", + "ready_to_reset" : false, + "skip_spelling_checks" : true, + "state" : "done", + "student_attempts" : 0, + "task_states" : [ { "child_attempts" : 4, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "", + "post_assessment" : [3], + "score" : 1 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + }, + { "child_attempts" : 0, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3233, + "grader_type" : "PE", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3097, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3235, + "grader_type" : "PE", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3098, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3237, + "grader_type" : "PE", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality5", + "score" : 2, + "submission_id" : 3099, + "success" : true + }, + "score" : 2 + }, + { "answer" : "Student answer 4th attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3239, + "grader_type" : "PE", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3100, + "success" : true + }, + "score" : 0 + }, + { "answer" : "", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "invalid essay .", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3241, + "grader_type" : "PE", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3101, + "success" : true + }, + "score" : 0 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + } + ], + "weight" : 5.0 +} + """) + +# Instance state. To test score if sum of rubric score is different from score value. Should receive score of 25. +INSTANCE_INCONSISTENT_STATE3 = serialize_open_ended_instance_state(""" +{ "accept_file_upload" : false, + "attempts" : "10000", + "current_task_number" : 1, + "display_name" : "Science Question -- Machine Assessed", + "due" : null, + "graceperiod" : "1 day 12 hours 59 minutes 59 seconds", + "graded" : "True", + "ready_to_reset" : false, + "skip_spelling_checks" : true, + "state" : "done", + "student_attempts" : 0, + "task_states" : [ { "child_attempts" : 4, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "", + "post_assessment" : [3], + "score" : 1 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + }, + { "child_attempts" : 0, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3233, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality2", + "score" : 0, + "submission_id" : 3097, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3235, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3098, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3237, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality5", + "score" : 2, + "submission_id" : 3099, + "success" : true + }, + "score" : 2 + }, + { "answer" : "Student answer 4th attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3239, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3100, + "success" : true + }, + "score" : 0 + }, + { "answer" : "", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "invalid essay .", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3241, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3101, + "success" : true + }, + "score" : 0 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + } + ], + "weight" : 5.0 +} +""") + +# Instance state. To test score if old task states are available. Should receive a score of 15. +INSTANCE_INCONSISTENT_STATE4 = serialize_open_ended_instance_state(""" +{ "accept_file_upload" : false, + "attempts" : "10000", + "current_task_number" : 0, + "display_name" : "Science Question -- Machine Assessed", + "due" : null, + "graceperiod" : "1 day 12 hours 59 minutes 59 seconds", + "graded" : "True", + "old_task_states" : [ [ { "child_attempts" : 4, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : "[3]", + "score" : 1 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : "[3]", + "score" : 1 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assesssment" : "[3]", + "score" : 1 + }, + { "answer" : "", + "post_assessment" : "[3]", + "score" : 1 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + } ] ], + "ready_to_reset" : false, + "skip_spelling_checks" : true, + "state" : "assessing", + "student_attempts" : 0, + "task_states" : [ { "child_attempts" : 4, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "", + "post_assessment" : [3], + "score" : 1 + } + ], + "child_state" : "done", + "max_score" : 3, + "stored_answer" : null, + "version" : 1 + }, + { "child_attempts" : 0, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3233, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3097, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3235, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3098, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3237, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality3", + "score" : 2, + "submission_id" : 3099, + "success" : true + }, + "score" : 2 + }, + { "answer" : "Student answer 4th attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3239, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3100, + "success" : true + }, + "score" : 0 + }, + { "answer" : "", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "invalid essay .", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3241, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3101, + "success" : true + }, + "score" : 0 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + } + ], + "weight" : 5.0 +} +""") + +# Instance state. To test score if rubric scores are available but score is missing. Should receive a score of 15. +INSTANCE_INCONSISTENT_STATE5 = serialize_open_ended_instance_state(""" +{ "accept_file_upload" : false, + "attempts" : "10000", + "current_task_number" : 1, + "display_name" : "Science Question -- Machine Assessed", + "due" : null, + "graceperiod" : "1 day 12 hours 59 minutes 59 seconds", + "graded" : "True", + "ready_to_reset" : false, + "skip_spelling_checks" : true, + "state" : "done", + "student_attempts" : 0, + "task_states" : [ { "child_attempts" : 4, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : [3], + "score" : 1 + }, + { "answer" : "", + "post_assessment" : [3], + "score" : 1 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + }, + { "child_attempts" : 0, + "child_created" : false, + "child_history" : [ { "answer" : "Student answer 1st attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: More grammar errors than average.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3233, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3097, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 2nd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3235, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3098, + "success" : true + }, + "score" : 0 + }, + { "answer" : "Student answer 3rd attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3237, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality3", + "score" : 2, + "submission_id" : 3099, + "success" : true + } + }, + { "answer" : "Student answer 4th attempt.", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "valid essay", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3239, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3100, + "success" : true + }, + "score" : 0 + }, + { "answer" : "", + "post_assessment" : { "feedback" : { "grammar" : "Grammar: Ok.", + "markup-text" : "invalid essay .", + "spelling" : "Spelling: Ok." + }, + "grader_id" : 3241, + "grader_type" : "ML", + "rubric_scores_complete" : true, + "rubric_xml" : "Response Quality0", + "score" : 0, + "submission_id" : 3101, + "success" : true + }, + "score" : 0 + } + ], + "child_state" : "done", + "max_score" : 3, + "version" : 1 + } + ], + "weight" : 5.0 +} +""") + # Task state with self assessment only. TEST_STATE_SA = ["{\"child_created\": false, \"child_attempts\": 1, \"version\": 1, \"child_history\": [{\"answer\": \"Censorship in the Libraries\\r
'All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us.' --Katherine Paterson, Author\\r

Write a persuasive essay to a newspaper reflecting your vies on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.\", \"post_assessment\": \"[3, 3, 2, 2, 2]\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"done\"}", "{\"child_created\": false, \"child_attempts\": 0, \"version\": 1, \"child_history\": [{\"answer\": \"Censorship in the Libraries\\r
'All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us.' --Katherine Paterson, Author\\r

Write a persuasive essay to a newspaper reflecting your vies on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.\", \"post_assessment\": \"{\\\"submission_id\\\": 1461, \\\"score\\\": 12, \\\"feedback\\\": \\\"{\\\\\\\"feedback\\\\\\\": \\\\\\\"\\\\\\\"}\\\", \\\"success\\\": true, \\\"grader_id\\\": 5414, \\\"grader_type\\\": \\\"IN\\\", \\\"rubric_scores_complete\\\": true, \\\"rubric_xml\\\": \\\"\\\\nIdeas\\\\n3\\\\nContent\\\\n3\\\\nOrganization\\\\n2\\\\nStyle\\\\n2\\\\nVoice\\\\n2\\\"}\", \"score\": 12}], \"max_score\": 12, \"child_state\": \"post_assessment\"}"]