diff --git a/common/lib/capa/capa/tests/test_input_templates.py b/common/lib/capa/capa/tests/test_input_templates.py new file mode 100644 index 0000000000..7bb32acd10 --- /dev/null +++ b/common/lib/capa/capa/tests/test_input_templates.py @@ -0,0 +1,269 @@ +"""Tests for the logic in input type mako templates.""" + +import unittest +import capa +import os.path +from lxml import etree +from mako.template import Template as MakoTemplate + + +class TemplateTestCase(unittest.TestCase): + """Utilitites for testing templates""" + + # Subclasses override this to specify the file name of the template + # to be loaded from capa/templates. + # The template name should include the .html extension: + # for example: choicegroup.html + TEMPLATE_NAME = None + + def setUp(self): + """Load the template""" + capa_path = capa.__path__[0] + self.template_path = os.path.join(capa_path, + 'templates', + self.TEMPLATE_NAME) + template_file = open(self.template_path) + self.template = MakoTemplate(template_file.read()) + template_file.close() + + def render_to_xml(self, context_dict): + """Render the template using the `context_dict` dict. + + Returns an `etree` XML element.""" + xml_str = self.template.render_unicode(**context_dict) + return etree.fromstring(xml_str) + + def assert_has_xpath(self, xml_root, xpath, context_dict, exact_num=1): + """Asserts that the xml tree has an element satisfying `xpath`. + + `xml_root` is an etree XML element + `xpath` is an XPath string, such as `'/foo/bar'` + `context` is used to print a debugging message + `exact_num` is the exact number of matches to expect. + """ + message = ("XML does not have %d match(es) for xpath '%s'\nXML: %s\nContext: %s" + % (exact_num, str(xpath), etree.tostring(xml_root), str(context_dict))) + + self.assertEqual(len(xml_root.xpath(xpath)), exact_num, msg=message) + + def assert_no_xpath(self, xml_root, xpath, context_dict): + """Asserts that the xml tree does NOT have an element + satisfying `xpath`. + + `xml_root` is an etree XML element + `xpath` is an XPath string, such as `'/foo/bar'` + `context` is used to print a debugging message + """ + self.assert_has_xpath(xml_root, xpath, context_dict, exact_num=0) + + +class TestChoiceGroupTemplate(TemplateTestCase): + """Test mako template for `` input""" + + TEMPLATE_NAME = 'choicegroup.html' + + def setUp(self): + choices = [('1', 'choice 1'), ('2', 'choice 2'), ('3', 'choice 3')] + self.context = {'id': '1', + 'choices': choices, + 'status': 'correct', + 'input_type': 'checkbox', + 'name_array_suffix': '1', + 'value': '3'} + super(TestChoiceGroupTemplate, self).setUp() + + def test_problem_marked_correct(self): + """Test conditions under which the entire problem + (not a particular option) is marked correct""" + + self.context['status'] = 'correct' + self.context['input_type'] = 'checkbox' + self.context['value'] = ['1', '2'] + + # Should mark the entire problem correct + xml = self.render_to_xml(self.context) + xpath = "//div[@class='indicator_container']/span[@class='correct']" + self.assert_has_xpath(xml, xpath, self.context) + + # Should NOT mark individual options + self.assert_no_xpath(xml, "//label[@class='choicegroup_incorrect']", + self.context) + + self.assert_no_xpath(xml, "//label[@class='choicegroup_correct']", + self.context) + + def test_problem_marked_incorrect(self): + """Test all conditions under which the entire problem + (not a particular option) is marked incorrect""" + conditions = [ + {'status': 'incorrect', 'input_type': 'radio', 'value': ''}, + {'status': 'incorrect', 'input_type': 'checkbox', 'value': []}, + {'status': 'incorrect', 'input_type': 'checkbox', 'value': ['2']}, + {'status': 'incorrect', 'input_type': 'checkbox', 'value': ['2', '3']}, + {'status': 'incomplete', 'input_type': 'radio', 'value': ''}, + {'status': 'incomplete', 'input_type': 'checkbox', 'value': []}, + {'status': 'incomplete', 'input_type': 'checkbox', 'value': ['2']}, + {'status': 'incomplete', 'input_type': 'checkbox', 'value': ['2', '3']}] + + for test_conditions in conditions: + self.context.update(test_conditions) + xml = self.render_to_xml(self.context) + xpath = "//div[@class='indicator_container']/span[@class='incorrect']" + self.assert_has_xpath(xml, xpath, self.context) + + # Should NOT mark individual options + self.assert_no_xpath(xml, + "//label[@class='choicegroup_incorrect']", + self.context) + + self.assert_no_xpath(xml, + "//label[@class='choicegroup_correct']", + self.context) + + def test_problem_marked_unanswered(self): + """Test all conditions under which the entire problem + (not a particular option) is marked unanswered""" + conditions = [ + {'status': 'unsubmitted', 'input_type': 'radio', 'value': ''}, + {'status': 'unsubmitted', 'input_type': 'radio', 'value': []}, + {'status': 'unsubmitted', 'input_type': 'checkbox', 'value': []}, + {'input_type': 'radio', 'value': ''}, + {'input_type': 'radio', 'value': []}, + {'input_type': 'checkbox', 'value': []}, + {'input_type': 'checkbox', 'value': ['1']}, + {'input_type': 'checkbox', 'value': ['1', '2']}] + + self.context['status'] = 'unanswered' + + for test_conditions in conditions: + self.context.update(test_conditions) + xml = self.render_to_xml(self.context) + xpath = "//div[@class='indicator_container']/span[@class='unanswered']" + self.assert_has_xpath(xml, xpath, self.context) + + # Should NOT mark individual options + self.assert_no_xpath(xml, + "//label[@class='choicegroup_incorrect']", + self.context) + + self.assert_no_xpath(xml, + "//label[@class='choicegroup_correct']", + self.context) + + def test_option_marked_correct(self): + """Test conditions under which a particular option + (not the entire problem) is marked correct.""" + conditions = [ + {'input_type': 'radio', 'value': '2'}, + {'input_type': 'radio', 'value': ['2']}] + + self.context['status'] = 'correct' + + for test_conditions in conditions: + self.context.update(test_conditions) + xml = self.render_to_xml(self.context) + xpath = "//label[@class='choicegroup_correct']" + self.assert_has_xpath(xml, xpath, self.context) + + # Should NOT mark the whole problem + xpath = "//div[@class='indicator_container']/span" + self.assert_no_xpath(xml, xpath, self.context) + + def test_option_marked_incorrect(self): + """Test conditions under which a particular option + (not the entire problem) is marked incorrect.""" + conditions = [ + {'input_type': 'radio', 'value': '2'}, + {'input_type': 'radio', 'value': ['2']}] + + self.context['status'] = 'incorrect' + + for test_conditions in conditions: + self.context.update(test_conditions) + xml = self.render_to_xml(self.context) + xpath = "//label[@class='choicegroup_incorrect']" + self.assert_has_xpath(xml, xpath, self.context) + + # Should NOT mark the whole problem + xpath = "//div[@class='indicator_container']/span" + self.assert_no_xpath(xml, xpath, self.context) + + def test_never_show_correctness(self): + """Test conditions under which we tell the template to + NOT show correct/incorrect, but instead show a message. + + This is used, for example, by the Justice course to ask + questions without specifying a correct answer. When + the student responds, the problem displays "Thank you + for your response" + """ + + conditions = [ + {'input_type': 'radio', 'status': 'correct', 'value': ''}, + {'input_type': 'radio', 'status': 'correct', 'value': '2'}, + {'input_type': 'radio', 'status': 'correct', 'value': ['2']}, + {'input_type': 'radio', 'status': 'incorrect', 'value': '2'}, + {'input_type': 'radio', 'status': 'incorrect', 'value': []}, + {'input_type': 'radio', 'status': 'incorrect', 'value': ['2']}, + {'input_type': 'checkbox', 'status': 'correct', 'value': []}, + {'input_type': 'checkbox', 'status': 'correct', 'value': ['2']}, + {'input_type': 'checkbox', 'status': 'incorrect', 'value': []}, + {'input_type': 'checkbox', 'status': 'incorrect', 'value': ['2']}] + + self.context['show_correctness'] = 'never' + self.context['submitted_message'] = 'Test message' + + for test_conditions in conditions: + self.context.update(test_conditions) + xml = self.render_to_xml(self.context) + + # Should NOT mark the entire problem correct/incorrect + xpath = "//div[@class='indicator_container']/span[@class='correct']" + self.assert_no_xpath(xml, xpath, self.context) + + xpath = "//div[@class='indicator_container']/span[@class='incorrect']" + self.assert_no_xpath(xml, xpath, self.context) + + # Should NOT mark individual options + self.assert_no_xpath(xml, + "//label[@class='choicegroup_incorrect']", + self.context) + + self.assert_no_xpath(xml, + "//label[@class='choicegroup_correct']", + self.context) + + # Expect to see the message + message_elements = xml.xpath("//div[@class='capa_alert']") + self.assertEqual(len(message_elements), 1) + self.assertEqual(message_elements[0].text, + self.context['submitted_message']) + + def test_no_message_before_submission(self): + """Ensure that we don't show the `submitted_message` + before submitting""" + + conditions = [ + {'input_type': 'radio', 'status': 'unsubmitted', 'value': ''}, + {'input_type': 'radio', 'status': 'unsubmitted', 'value': []}, + {'input_type': 'checkbox', 'status': 'unsubmitted', 'value': []}, + + # These tests expose bug #365 + # When the bug is fixed, uncomment these cases. + #{'input_type': 'radio', 'status': 'unsubmitted', 'value': '2'}, + #{'input_type': 'radio', 'status': 'unsubmitted', 'value': ['2']}, + #{'input_type': 'radio', 'status': 'unsubmitted', 'value': '2'}, + #{'input_type': 'radio', 'status': 'unsubmitted', 'value': ['2']}, + #{'input_type': 'checkbox', 'status': 'unsubmitted', 'value': ['2']}, + #{'input_type': 'checkbox', 'status': 'unsubmitted', 'value': ['2']}] + ] + + self.context['show_correctness'] = 'never' + self.context['submitted_message'] = 'Test message' + + for test_conditions in conditions: + self.context.update(test_conditions) + xml = self.render_to_xml(self.context) + + # Expect that we do NOT see the message yet + self.assert_no_xpath(xml, "//div[@class='capa_alert']", self.context) diff --git a/common/lib/xmodule/xmodule/tests/test_capa_module.py b/common/lib/xmodule/xmodule/tests/test_capa_module.py index bc5d342646..f948f5bdfe 100644 --- a/common/lib/xmodule/xmodule/tests/test_capa_module.py +++ b/common/lib/xmodule/xmodule/tests/test_capa_module.py @@ -1,17 +1,19 @@ +"""Tests of the Capa XModule""" +#pylint: disable=C0111 +#pylint: disable=R0904 +#pylint: disable=C0103 +#pylint: disable=C0302 + import datetime -import json -from mock import Mock, MagicMock, patch -from pprint import pprint +from mock import Mock, patch import unittest import random import xmodule -import capa from capa.responsetypes import StudentInputError, \ - LoncapaProblemError, ResponseError + LoncapaProblemError, ResponseError from xmodule.capa_module import CapaModule from xmodule.modulestore import Location -from lxml import etree from django.http import QueryDict @@ -384,7 +386,7 @@ class CapaModuleTest(unittest.TestCase): # what the input is, by patching CorrectMap.is_correct() # Also simulate rendering the HTML # TODO: pep8 thinks the following line has invalid syntax - with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct,\ + with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct, \ patch('xmodule.capa_module.CapaModule.get_problem_html') as mock_html: mock_is_correct.return_value = True mock_html.return_value = "Test HTML" @@ -435,32 +437,38 @@ class CapaModuleTest(unittest.TestCase): self.assertEqual(module.attempts, 3) def test_check_problem_resubmitted_with_randomize(self): - # Randomize turned on - module = CapaFactory.create(rerandomize='always', attempts=0) + rerandomize_values = ['always', 'true'] - # Simulate that the problem is completed - module.done = True + for rerandomize in rerandomize_values: + # Randomize turned on + module = CapaFactory.create(rerandomize=rerandomize, attempts=0) - # Expect that we cannot submit - with self.assertRaises(xmodule.exceptions.NotFoundError): - get_request_dict = {CapaFactory.input_key(): '3.14'} - module.check_problem(get_request_dict) + # Simulate that the problem is completed + module.done = True - # Expect that number of attempts NOT incremented - self.assertEqual(module.attempts, 0) + # Expect that we cannot submit + with self.assertRaises(xmodule.exceptions.NotFoundError): + get_request_dict = {CapaFactory.input_key(): '3.14'} + module.check_problem(get_request_dict) + + # Expect that number of attempts NOT incremented + self.assertEqual(module.attempts, 0) def test_check_problem_resubmitted_no_randomize(self): - # Randomize turned off - module = CapaFactory.create(rerandomize='never', attempts=0, done=True) + rerandomize_values = ['never', 'false', 'per_student'] - # Expect that we can submit successfully - get_request_dict = {CapaFactory.input_key(): '3.14'} - result = module.check_problem(get_request_dict) + for rerandomize in rerandomize_values: + # Randomize turned off + module = CapaFactory.create(rerandomize=rerandomize, attempts=0, done=True) - self.assertEqual(result['success'], 'correct') + # Expect that we can submit successfully + get_request_dict = {CapaFactory.input_key(): '3.14'} + result = module.check_problem(get_request_dict) - # Expect that number of attempts IS incremented - self.assertEqual(module.attempts, 1) + self.assertEqual(result['success'], 'correct') + + # Expect that number of attempts IS incremented + self.assertEqual(module.attempts, 1) def test_check_problem_queued(self): module = CapaFactory.create(attempts=1) @@ -615,24 +623,34 @@ class CapaModuleTest(unittest.TestCase): self.assertTrue('success' in result and not result['success']) def test_save_problem_submitted_with_randomize(self): - module = CapaFactory.create(rerandomize='always', done=True) - # Try to save - get_request_dict = {CapaFactory.input_key(): '3.14'} - result = module.save_problem(get_request_dict) + # Capa XModule treats 'always' and 'true' equivalently + rerandomize_values = ['always', 'true'] - # Expect that we cannot save - self.assertTrue('success' in result and not result['success']) + for rerandomize in rerandomize_values: + module = CapaFactory.create(rerandomize=rerandomize, done=True) + + # Try to save + get_request_dict = {CapaFactory.input_key(): '3.14'} + result = module.save_problem(get_request_dict) + + # Expect that we cannot save + self.assertTrue('success' in result and not result['success']) def test_save_problem_submitted_no_randomize(self): - module = CapaFactory.create(rerandomize='never', done=True) - # Try to save - get_request_dict = {CapaFactory.input_key(): '3.14'} - result = module.save_problem(get_request_dict) + # Capa XModule treats 'false' and 'per_student' equivalently + rerandomize_values = ['never', 'false', 'per_student'] - # Expect that we succeed - self.assertTrue('success' in result and result['success']) + for rerandomize in rerandomize_values: + module = CapaFactory.create(rerandomize=rerandomize, done=True) + + # Try to save + get_request_dict = {CapaFactory.input_key(): '3.14'} + result = module.save_problem(get_request_dict) + + # Expect that we succeed + self.assertTrue('success' in result and result['success']) def test_check_button_name(self): @@ -681,21 +699,30 @@ class CapaModuleTest(unittest.TestCase): # If user submitted a problem but hasn't reset, # do NOT show the check button - # Note: we can only reset when rerandomize="always" + # Note: we can only reset when rerandomize="always" or "true" module = CapaFactory.create(rerandomize="always", done=True) self.assertFalse(module.should_show_check_button()) + module = CapaFactory.create(rerandomize="true", done=True) + self.assertFalse(module.should_show_check_button()) + # Otherwise, DO show the check button module = CapaFactory.create() self.assertTrue(module.should_show_check_button()) # If the user has submitted the problem # and we do NOT have a reset button, then we can show the check button - # Setting rerandomize to "never" ensures that the reset button + # Setting rerandomize to "never" or "false" ensures that the reset button # is not shown module = CapaFactory.create(rerandomize="never", done=True) self.assertTrue(module.should_show_check_button()) + module = CapaFactory.create(rerandomize="false", done=True) + self.assertTrue(module.should_show_check_button()) + + module = CapaFactory.create(rerandomize="per_student", done=True) + self.assertTrue(module.should_show_check_button()) + def test_should_show_reset_button(self): attempts = random.randint(1, 10) @@ -712,6 +739,14 @@ class CapaModuleTest(unittest.TestCase): module = CapaFactory.create(rerandomize="never", done=True) self.assertFalse(module.should_show_reset_button()) + # If we're NOT randomizing, then do NOT show the reset button + module = CapaFactory.create(rerandomize="per_student", done=True) + self.assertFalse(module.should_show_reset_button()) + + # If we're NOT randomizing, then do NOT show the reset button + module = CapaFactory.create(rerandomize="false", done=True) + self.assertFalse(module.should_show_reset_button()) + # If the user hasn't submitted an answer yet, # then do NOT show the reset button module = CapaFactory.create(done=False) @@ -742,13 +777,19 @@ class CapaModuleTest(unittest.TestCase): module = CapaFactory.create(rerandomize="always", done=True) self.assertFalse(module.should_show_save_button()) + module = CapaFactory.create(rerandomize="true", done=True) + self.assertFalse(module.should_show_save_button()) + # If the user has unlimited attempts and we are not randomizing, # then do NOT show a save button # because they can keep using "Check" module = CapaFactory.create(max_attempts=None, rerandomize="never", done=False) self.assertFalse(module.should_show_save_button()) - module = CapaFactory.create(max_attempts=None, rerandomize="never", done=True) + module = CapaFactory.create(max_attempts=None, rerandomize="false", done=True) + self.assertFalse(module.should_show_save_button()) + + module = CapaFactory.create(max_attempts=None, rerandomize="per_student", done=True) self.assertFalse(module.should_show_save_button()) # Otherwise, DO show the save button @@ -759,6 +800,12 @@ class CapaModuleTest(unittest.TestCase): module = CapaFactory.create(rerandomize="never", max_attempts=2, done=True) self.assertTrue(module.should_show_save_button()) + module = CapaFactory.create(rerandomize="false", max_attempts=2, done=True) + self.assertTrue(module.should_show_save_button()) + + module = CapaFactory.create(rerandomize="per_student", max_attempts=2, done=True) + self.assertTrue(module.should_show_save_button()) + # If survey question for capa (max_attempts = 0), # DO show the save button module = CapaFactory.create(max_attempts=0, done=False) @@ -788,9 +835,15 @@ class CapaModuleTest(unittest.TestCase): done=True) self.assertTrue(module.should_show_save_button()) + module = CapaFactory.create(force_save_button="true", + rerandomize="true", + done=True) + self.assertTrue(module.should_show_save_button()) + def test_no_max_attempts(self): module = CapaFactory.create(max_attempts='') html = module.get_problem_html() + self.assertTrue(html is not None) # assert that we got here without exploding def test_get_problem_html(self): @@ -875,6 +928,8 @@ class CapaModuleTest(unittest.TestCase): # Try to render the module with DEBUG turned off html = module.get_problem_html() + self.assertTrue(html is not None) + # Check the rendering context render_args, _ = module.system.render_template.call_args context = render_args[1] @@ -886,7 +941,9 @@ class CapaModuleTest(unittest.TestCase): def test_random_seed_no_change(self): # Run the test for each possible rerandomize value - for rerandomize in ['never', 'per_student', 'always', 'onreset']: + for rerandomize in ['false', 'never', + 'per_student', 'always', + 'true', 'onreset']: module = CapaFactory.create(rerandomize=rerandomize) # Get the seed @@ -896,8 +953,9 @@ class CapaModuleTest(unittest.TestCase): # If we're not rerandomizing, the seed is always set # to the same value (1) - if rerandomize == 'never': - self.assertEqual(seed, 1) + if rerandomize in ['never']: + self.assertEqual(seed, 1, + msg="Seed should always be 1 when rerandomize='%s'" % rerandomize) # Check the problem get_request_dict = {CapaFactory.input_key(): '3.14'} @@ -947,7 +1005,8 @@ class CapaModuleTest(unittest.TestCase): return success # Run the test for each possible rerandomize value - for rerandomize in ['never', 'per_student', 'always', 'onreset']: + for rerandomize in ['never', 'false', 'per_student', + 'always', 'true', 'onreset']: module = CapaFactory.create(rerandomize=rerandomize) # Get the seed @@ -959,7 +1018,7 @@ class CapaModuleTest(unittest.TestCase): # is set to 'never' -- it should still be 1 # The seed also stays the same if we're randomizing # 'per_student': the same student should see the same problem - if rerandomize in ['never', 'per_student']: + if rerandomize in ['never', 'false', 'per_student']: self.assertEqual(seed, _reset_and_get_seed(module)) # Otherwise, we expect the seed to change @@ -969,10 +1028,8 @@ class CapaModuleTest(unittest.TestCase): # Since there's a small chance we might get the # same seed again, give it 5 chances # to generate a different seed - success = _retry_and_check(5, - lambda: _reset_and_get_seed(module) != seed) + success = _retry_and_check(5, lambda: _reset_and_get_seed(module) != seed) - # TODO: change this comparison to module.seed is not None? - self.assertTrue(module.seed != None) + self.assertTrue(module.seed is not None) msg = 'Could not get a new seed from reset after 5 tries' self.assertTrue(success, msg) diff --git a/lms/djangoapps/courseware/features/problems.feature b/lms/djangoapps/courseware/features/problems.feature index dc8495af60..266ffa3680 100644 --- a/lms/djangoapps/courseware/features/problems.feature +++ b/lms/djangoapps/courseware/features/problems.feature @@ -15,6 +15,7 @@ Feature: Answer problems | drop down | | multiple choice | | checkbox | + | radio | | string | | numerical | | formula | @@ -33,6 +34,7 @@ Feature: Answer problems | drop down | | multiple choice | | checkbox | + | radio | | string | | numerical | | formula | @@ -50,6 +52,7 @@ Feature: Answer problems | drop down | | multiple choice | | checkbox | + | radio | | string | | numerical | | formula | @@ -71,6 +74,8 @@ Feature: Answer problems | multiple choice | incorrect | | checkbox | correct | | checkbox | incorrect | + | radio | correct | + | radio | incorrect | | string | correct | | string | incorrect | | numerical | correct | diff --git a/lms/djangoapps/courseware/features/problems.py b/lms/djangoapps/courseware/features/problems.py index b25d606c4e..3d538d7ae1 100644 --- a/lms/djangoapps/courseware/features/problems.py +++ b/lms/djangoapps/courseware/features/problems.py @@ -42,7 +42,13 @@ PROBLEM_FACTORY_DICT = { 'choice_type': 'checkbox', 'choices': [True, False, True, False, False], 'choice_names': ['Choice 1', 'Choice 2', 'Choice 3', 'Choice 4']}}, - + 'radio': { + 'factory': ChoiceResponseXMLFactory(), + 'kwargs': { + 'question_text': 'The correct answer is Choice 3', + 'choice_type': 'radio', + 'choices': [False, False, True, False], + 'choice_names': ['Choice 1', 'Choice 2', 'Choice 3', 'Choice 4']}}, 'string': { 'factory': StringResponseXMLFactory(), 'kwargs': { @@ -174,6 +180,12 @@ def answer_problem(step, problem_type, correctness): else: inputfield('checkbox', choice='choice_3').check() + elif problem_type == 'radio': + if correctness == 'correct': + inputfield('radio', choice='choice_2').check() + else: + inputfield('radio', choice='choice_1').check() + elif problem_type == 'string': textvalue = 'correct string' if correctness == 'correct' \ else 'incorrect' @@ -252,6 +264,14 @@ def assert_problem_has_answer(step, problem_type, answer_class): else: assert_checked('checkbox', []) + elif problem_type == "radio": + if answer_class == 'correct': + assert_checked('radio', ['choice_2']) + elif answer_class == 'incorrect': + assert_checked('radio', ['choice_1']) + else: + assert_checked('radio', []) + elif problem_type == 'string': if answer_class == 'blank': expected = '' @@ -298,6 +318,7 @@ CORRECTNESS_SELECTORS = { 'correct': {'drop down': ['span.correct'], 'multiple choice': ['label.choicegroup_correct'], 'checkbox': ['span.correct'], + 'radio': ['label.choicegroup_correct'], 'string': ['div.correct'], 'numerical': ['div.correct'], 'formula': ['div.correct'], @@ -308,6 +329,8 @@ CORRECTNESS_SELECTORS = { 'multiple choice': ['label.choicegroup_incorrect', 'span.incorrect'], 'checkbox': ['span.incorrect'], + 'radio': ['label.choicegroup_incorrect', + 'span.incorrect'], 'string': ['div.incorrect'], 'numerical': ['div.incorrect'], 'formula': ['div.incorrect'], @@ -317,6 +340,7 @@ CORRECTNESS_SELECTORS = { 'unanswered': {'drop down': ['span.unanswered'], 'multiple choice': ['span.unanswered'], 'checkbox': ['span.unanswered'], + 'radio': ['span.unanswered'], 'string': ['div.unanswered'], 'numerical': ['div.unanswered'], 'formula': ['div.unanswered'],