Merge pull request #1563 from MITx/fix/will/capa_response_unit_tests

Fix/will/capa response unit tests

common/lib/capa/capa/tests/response_xml_factory.py

@@ -0,0 +1,668 @@
+from lxml import etree
+from abc import ABCMeta, abstractmethod
+
+class ResponseXMLFactory(object):
+    """ Abstract base class for capa response XML factories.
+    Subclasses override create_response_element and
+    create_input_element to produce XML of particular response types"""
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def create_response_element(self, **kwargs):
+        """ Subclasses override to return an etree element
+        representing the capa response XML
+        (e.g. <numericalresponse>).
+        
+        The tree should NOT contain any input elements
+        (such as <textline />) as these will be added later."""
+        return None
+
+    @abstractmethod
+    def create_input_element(self, **kwargs):
+        """ Subclasses override this to return an etree element
+        representing the capa input XML (such as <textline />)"""
+        return None
+
+    def build_xml(self, **kwargs):
+        """ Construct an XML string for a capa response 
+        based on **kwargs.
+
+        **kwargs is a dictionary that will be passed
+        to create_response_element() and create_input_element().
+        See the subclasses below for other keyword arguments
+        you can specify.
+
+        For all response types, **kwargs can contain:
+
+        *question_text*: The text of the question to display,
+            wrapped in <p> tags.
+        
+        *explanation_text*: The detailed explanation that will
+            be shown if the user answers incorrectly.
+
+        *script*: The embedded Python script (a string)
+
+        *num_responses*: The number of responses to create [DEFAULT: 1]
+
+        *num_inputs*: The number of input elements
+            to create [DEFAULT: 1]
+
+        Returns a string representation of the XML tree.
+        """
+
+        # Retrieve keyward arguments
+        question_text = kwargs.get('question_text', '')
+        explanation_text = kwargs.get('explanation_text', '')
+        script = kwargs.get('script', None)
+        num_responses = kwargs.get('num_responses', 1)
+        num_inputs = kwargs.get('num_inputs', 1)
+
+        # The root is <problem>
+        root = etree.Element("problem")
+
+        # Add a script if there is one
+        if script:
+            script_element = etree.SubElement(root, "script")
+            script_element.set("type", "loncapa/python")
+            script_element.text = str(script)
+
+        # The problem has a child <p> with question text
+        question = etree.SubElement(root, "p")
+        question.text = question_text
+
+        # Add the response(s)
+        for i in range(0, int(num_responses)):
+            response_element = self.create_response_element(**kwargs)
+            root.append(response_element)
+            
+            # Add input elements
+            for j in range(0, int(num_inputs)):
+                input_element = self.create_input_element(**kwargs)
+                if not (None == input_element):
+                    response_element.append(input_element)
+
+            # The problem has an explanation of the solution
+            if explanation_text:
+                explanation = etree.SubElement(root, "solution")
+                explanation_div = etree.SubElement(explanation, "div")
+                explanation_div.set("class", "detailed-solution")
+                explanation_div.text = explanation_text
+
+        return etree.tostring(root)
+
+    @staticmethod
+    def textline_input_xml(**kwargs):
+        """ Create a <textline/> XML element
+
+        Uses **kwargs:
+
+        *math_display*: If True, then includes a MathJax display of user input
+
+        *size*: An integer representing the width of the text line
+        """
+        math_display = kwargs.get('math_display', False)
+        size = kwargs.get('size', None)
+
+        input_element = etree.Element('textline')
+
+        if math_display:
+            input_element.set('math', '1')
+
+        if size:
+            input_element.set('size', str(size))
+
+        return input_element
+
+    @staticmethod
+    def choicegroup_input_xml(**kwargs):
+        """ Create a <choicegroup> XML element
+
+        Uses **kwargs:
+
+        *choice_type*: Can be "checkbox", "radio", or "multiple"
+
+        *choices*: List of True/False values indicating whether
+                            a particular choice is correct or not.
+                            Users must choose *all* correct options in order
+                            to be marked correct.
+                            DEFAULT: [True]
+
+        *choice_names": List of strings identifying the choices.
+                        If specified, you must ensure that
+                        len(choice_names) == len(choices)
+        """
+        # Names of group elements
+        group_element_names = {'checkbox': 'checkboxgroup',
+                                'radio': 'radiogroup',
+                                'multiple': 'choicegroup' }
+
+        # Retrieve **kwargs
+        choices = kwargs.get('choices', [True])
+        choice_type = kwargs.get('choice_type', 'multiple')
+        choice_names = kwargs.get('choice_names', [None] * len(choices))
+
+        # Create the <choicegroup>, <checkboxgroup>, or <radiogroup> element
+        assert(choice_type in group_element_names)
+        group_element = etree.Element(group_element_names[choice_type])
+
+        # Create the <choice> elements
+        for (correct_val, name) in zip(choices, choice_names):
+            choice_element = etree.SubElement(group_element, "choice")
+            choice_element.set("correct", "true" if correct_val else "false")
+
+            # Add some text describing the choice
+            etree.SubElement(choice_element, "startouttext")
+            etree.text = "Choice description"
+            etree.SubElement(choice_element, "endouttext")
+
+            # Add a name identifying the choice, if one exists
+            if name:
+                choice_element.set("name", str(name))
+
+        return group_element
+
+
+class NumericalResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <numericalresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <numericalresponse> XML element.
+        Uses **kwarg keys:
+
+        *answer*: The correct answer (e.g. "5")
+
+        *tolerance*: The tolerance within which a response
+        is considered correct.  Can be a decimal (e.g. "0.01")
+        or percentage (e.g. "2%")
+        """
+
+        answer = kwargs.get('answer', None)
+        tolerance = kwargs.get('tolerance', None)
+
+        response_element = etree.Element('numericalresponse')
+
+        if answer:
+            response_element.set('answer', str(answer))
+
+        if tolerance:
+            responseparam_element = etree.SubElement(response_element, 'responseparam')
+            responseparam_element.set('type', 'tolerance')
+            responseparam_element.set('default', str(tolerance))
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class CustomResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <customresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <customresponse> XML element.
+
+        Uses **kwargs:
+
+        *cfn*: the Python code to run.  Can be inline code,
+        or the name of a function defined in earlier <script> tags.
+
+        Should have the form: cfn(expect, answer_given, student_answers)
+        where expect is a value (see below),
+        answer_given is a single value (for 1 input)
+        or a list of values (for multiple inputs),
+        and student_answers is a dict of answers by input ID.
+
+        *expect*: The value passed to the function cfn
+
+        *answer*: Inline script that calculates the answer
+        """
+        
+        # Retrieve **kwargs
+        cfn = kwargs.get('cfn', None)
+        expect = kwargs.get('expect', None)
+        answer = kwargs.get('answer', None)
+
+        # Create the response element
+        response_element = etree.Element("customresponse")
+
+        if cfn:
+            response_element.set('cfn', str(cfn))
+
+        if expect:
+            response_element.set('expect', str(expect))
+
+        if answer:
+            answer_element = etree.SubElement(response_element, "answer")
+            answer_element.text = str(answer)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class SchematicResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <schematicresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <schematicresponse> XML element.
+        
+        Uses *kwargs*:
+
+        *answer*: The Python script used to evaluate the answer.
+        """
+        answer_script = kwargs.get('answer', None)
+
+        # Create the <schematicresponse> element
+        response_element = etree.Element("schematicresponse")
+
+        # Insert the <answer> script if one is provided
+        if answer_script:
+            answer_element = etree.SubElement(response_element, "answer")
+            answer_element.set("type", "loncapa/python")
+            answer_element.text = str(answer_script)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        """ Create the <schematic> XML element.
+
+        Although <schematic> can have several attributes,
+        (*height*, *width*, *parts*, *analyses*, *submit_analysis*, and *initial_value*),
+        none of them are used in the capa module.
+        For testing, we create a bare-bones version of <schematic>."""
+        return etree.Element("schematic")
+
+class CodeResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <coderesponse> XML trees """
+
+    def build_xml(self, **kwargs):
+        # Since we are providing an <answer> tag,
+        # we should override the default behavior
+        # of including a <solution> tag as well
+        kwargs['explanation_text'] = None
+        return super(CodeResponseXMLFactory, self).build_xml(**kwargs)
+
+    def create_response_element(self, **kwargs):
+        """ Create a <coderesponse> XML element:
+            
+            Uses **kwargs:
+                
+            *initial_display*: The code that initially appears in the textbox
+                                [DEFAULT: "Enter code here"]
+            *answer_display*: The answer to display to the student
+                                [DEFAULT: "This is the correct answer!"]
+            *grader_payload*: A JSON-encoded string sent to the grader
+                                [DEFAULT: empty dict string]
+        """
+        # Get **kwargs
+        initial_display = kwargs.get("initial_display", "Enter code here")
+        answer_display = kwargs.get("answer_display", "This is the correct answer!")
+        grader_payload = kwargs.get("grader_payload", '{}')
+
+        # Create the <coderesponse> element
+        response_element = etree.Element("coderesponse")
+        codeparam_element = etree.SubElement(response_element, "codeparam")
+
+        # Set the initial display text
+        initial_element = etree.SubElement(codeparam_element, "initial_display")
+        initial_element.text = str(initial_display)
+
+        # Set the answer display text
+        answer_element = etree.SubElement(codeparam_element, "answer_display")
+        answer_element.text = str(answer_display)
+
+        # Set the grader payload string
+        grader_element = etree.SubElement(codeparam_element, "grader_payload")
+        grader_element.text = str(grader_payload)
+
+        # Create the input within the response
+        input_element = etree.SubElement(response_element, "textbox")
+        input_element.set("mode", "python")
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        # Since we create this in create_response_element(),
+        # return None here
+        return None
+
+class ChoiceResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <choiceresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <choiceresponse> element """
+        return etree.Element("choiceresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create a <checkboxgroup> element."""
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+
+class FormulaResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <formularesponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <formularesponse> element.
+
+        *sample_dict*: A dictionary of the form:
+                        { VARIABLE_NAME: (MIN, MAX), ....}
+
+                        This specifies the range within which
+                        to numerically sample each variable to check
+                        student answers.
+                        [REQUIRED]
+
+        *num_samples*: The number of times to sample the student's answer
+                        to numerically compare it to the correct answer.
+        
+        *tolerance*: The tolerance within which answers will be accepted
+                        [DEFAULT: 0.01] 
+
+        *answer*: The answer to the problem.  Can be a formula string
+                    or a Python variable defined in a script 
+                    (e.g. "$calculated_answer" for a Python variable 
+                    called calculated_answer)
+                    [REQUIRED]
+
+        *hints*: List of (hint_prompt, hint_name, hint_text) tuples
+                Where *hint_prompt* is the formula for which we show the hint,
+                *hint_name* is an internal identifier for the hint,
+                and *hint_text* is the text we show for the hint.
+        """
+        # Retrieve kwargs
+        sample_dict = kwargs.get("sample_dict", None)
+        num_samples = kwargs.get("num_samples", None)
+        tolerance = kwargs.get("tolerance", 0.01)
+        answer = kwargs.get("answer", None)
+        hint_list = kwargs.get("hints", None)
+
+        assert(answer)
+        assert(sample_dict and num_samples)
+
+        # Create the <formularesponse> element
+        response_element = etree.Element("formularesponse")
+
+        # Set the sample information
+        sample_str = self._sample_str(sample_dict, num_samples, tolerance)
+        response_element.set("samples", sample_str)
+                
+
+        # Set the tolerance
+        responseparam_element = etree.SubElement(response_element, "responseparam")
+        responseparam_element.set("type", "tolerance")
+        responseparam_element.set("default", str(tolerance))
+
+        # Set the answer
+        response_element.set("answer", str(answer))
+
+        # Include hints, if specified
+        if hint_list:
+            hintgroup_element = etree.SubElement(response_element, "hintgroup")
+
+            for (hint_prompt, hint_name, hint_text) in hint_list:
+
+                # For each hint, create a <formulahint> element
+                formulahint_element = etree.SubElement(hintgroup_element, "formulahint")
+
+                # We could sample a different range, but for simplicity,
+                # we use the same sample string for the hints
+                # that we used previously.  
+                formulahint_element.set("samples", sample_str)
+
+                formulahint_element.set("answer", str(hint_prompt))
+                formulahint_element.set("name", str(hint_name))
+
+                # For each hint, create a <hintpart> element
+                # corresponding to the <formulahint>
+                hintpart_element = etree.SubElement(hintgroup_element, "hintpart")
+                hintpart_element.set("on", str(hint_name))
+                text_element = etree.SubElement(hintpart_element, "text")
+                text_element.text = str(hint_text)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+    def _sample_str(self, sample_dict, num_samples, tolerance):
+        # Loncapa uses a special format for sample strings:
+        # "x,y,z@4,5,3:10,12,8#4" means plug in values for (x,y,z)
+        # from within the box defined by points (4,5,3) and (10,12,8)
+        # The "#4" means to repeat 4 times.
+        variables = [str(v) for v in sample_dict.keys()]
+        low_range_vals = [str(f[0]) for f in sample_dict.values()]
+        high_range_vals = [str(f[1]) for f in sample_dict.values()]
+        sample_str = (",".join(sample_dict.keys()) + "@" +
+                        ",".join(low_range_vals) + ":" +
+                        ",".join(high_range_vals) + 
+                        "#" + str(num_samples))
+        return sample_str
+
+class ImageResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <imageresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <imageresponse> element."""
+        return etree.Element("imageresponse")
+
+
+    def create_input_element(self, **kwargs):
+        """ Create the <imageinput> element.
+        
+        Uses **kwargs:
+            
+        *src*: URL for the image file [DEFAULT: "/static/image.jpg"]
+
+        *width*: Width of the image [DEFAULT: 100]
+
+        *height*: Height of the image [DEFAULT: 100]
+
+        *rectangle*: String representing the rectangles the user should select.
+
+                    Take the form "(x1,y1)-(x2,y2)", where the two (x,y)
+                    tuples define the corners of the rectangle.
+
+                    Can include multiple rectangles separated by a semicolon, e.g.
+                    "(490,11)-(556,98);(242,202)-(296,276)"
+
+        *regions*: String representing the regions a user can select
+
+                    Take the form "[ [[x1,y1], [x2,y2], [x3,y3]],
+                                    [[x1,y1], [x2,y2], [x3,y3]] ]"
+                    (Defines two regions, each with 3 points)
+
+        REQUIRED: Either *rectangle* or *region* (or both)
+        """
+
+        # Get the **kwargs
+        src = kwargs.get("src", "/static/image.jpg")
+        width = kwargs.get("width", 100)
+        height = kwargs.get("height", 100)
+        rectangle = kwargs.get('rectangle', None)
+        regions = kwargs.get('regions', None)
+
+        assert(rectangle or regions)
+
+        # Create the <imageinput> element
+        input_element = etree.Element("imageinput")
+        input_element.set("src", str(src))
+        input_element.set("width", str(width))
+        input_element.set("height", str(height))
+        
+        if rectangle:
+            input_element.set("rectangle", rectangle)
+
+        if regions:
+            input_element.set("regions", regions)
+
+        return input_element
+
+class JavascriptResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <javascriptresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <javascriptresponse> element.
+
+        Uses **kwargs:
+
+        *generator_src*: Name of the JS file to generate the problem.
+        *grader_src*: Name of the JS file to grade the problem.
+        *display_class*: Name of the class used to display the problem
+        *display_src*: Name of the JS file used to display the problem
+        *param_dict*: Dictionary of parameters to pass to the JS
+        """
+        # Get **kwargs
+        generator_src = kwargs.get("generator_src", None)
+        grader_src = kwargs.get("grader_src", None)
+        display_class = kwargs.get("display_class", None)
+        display_src = kwargs.get("display_src", None)
+        param_dict = kwargs.get("param_dict", {})
+
+        # Both display_src and display_class given,
+        # or neither given
+        assert((display_src and display_class) or 
+                (not display_src and not display_class))
+
+        # Create the <javascriptresponse> element
+        response_element = etree.Element("javascriptresponse")
+
+        if generator_src:
+            generator_element = etree.SubElement(response_element, "generator")
+            generator_element.set("src", str(generator_src))
+
+        if grader_src:
+            grader_element = etree.SubElement(response_element, "grader")
+            grader_element.set("src", str(grader_src))
+
+        if display_class and display_src:
+            display_element = etree.SubElement(response_element, "display")
+            display_element.set("class", str(display_class))
+            display_element.set("src", str(display_src))
+
+        for (param_name, param_val) in param_dict.items():
+            responseparam_element = etree.SubElement(response_element, "responseparam")
+            responseparam_element.set("name", str(param_name))
+            responseparam_element.set("value", str(param_val))
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        """ Create the <javascriptinput> element """
+        return etree.Element("javascriptinput")
+
+class MultipleChoiceResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <multiplechoiceresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <multiplechoiceresponse> element"""
+        return etree.Element('multiplechoiceresponse')
+
+    def create_input_element(self, **kwargs):
+        """ Create the <choicegroup> element"""
+        kwargs['choice_type'] = 'multiple'
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+class TrueFalseResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <truefalseresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <truefalseresponse> element"""
+        return etree.Element('truefalseresponse')
+
+    def create_input_element(self, **kwargs):
+        """ Create the <choicegroup> element"""
+        kwargs['choice_type'] = 'multiple'
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+class OptionResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <optionresponse> XML"""
+
+    def create_response_element(self, **kwargs):
+        """ Create the <optionresponse> element"""
+        return etree.Element("optionresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create the <optioninput> element.
+
+        Uses **kwargs:
+
+        *options*: a list of possible options the user can choose from [REQUIRED]
+                    You must specify at least 2 options.
+        *correct_option*: the correct choice from the list of options [REQUIRED]
+        """
+
+        options_list = kwargs.get('options', None)
+        correct_option = kwargs.get('correct_option', None)
+
+        assert(options_list and correct_option)
+        assert(len(options_list) > 1)
+        assert(correct_option in options_list)
+
+        # Create the <optioninput> element
+        optioninput_element = etree.Element("optioninput")
+
+        # Set the "options" attribute
+        # Format: "('first', 'second', 'third')"
+        options_attr_string = ",".join(["'%s'" % str(o) for o in options_list])
+        options_attr_string = "(%s)" % options_attr_string
+        optioninput_element.set('options', options_attr_string)
+
+        # Set the "correct" attribute
+        optioninput_element.set('correct', str(correct_option))
+
+        return optioninput_element
+
+
+class StringResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <stringresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <stringresponse> XML element.
+        
+            Uses **kwargs:
+
+            *answer*: The correct answer (a string) [REQUIRED]
+
+            *case_sensitive*: Whether the response is case-sensitive (True/False)
+                            [DEFAULT: True]
+
+            *hints*: List of (hint_prompt, hint_name, hint_text) tuples
+                Where *hint_prompt* is the string for which we show the hint,
+                *hint_name* is an internal identifier for the hint,
+                and *hint_text* is the text we show for the hint.
+        """
+        # Retrieve the **kwargs
+        answer = kwargs.get("answer", None)
+        case_sensitive = kwargs.get("case_sensitive", True)
+        hint_list = kwargs.get('hints', None)
+        assert(answer)
+
+        # Create the <stringresponse> element
+        response_element = etree.Element("stringresponse")
+
+        # Set the answer attribute 
+        response_element.set("answer", str(answer))
+
+        # Set the case sensitivity
+        response_element.set("type", "cs" if case_sensitive else "ci")
+
+        # Add the hints if specified
+        if hint_list:
+            hintgroup_element = etree.SubElement(response_element, "hintgroup")
+            for (hint_prompt, hint_name, hint_text) in hint_list:
+                stringhint_element = etree.SubElement(hintgroup_element, "stringhint")
+                stringhint_element.set("answer", str(hint_prompt))
+                stringhint_element.set("name", str(hint_name))
+
+                hintpart_element = etree.SubElement(hintgroup_element, "hintpart")
+                hintpart_element.set("on", str(hint_name))
+
+                hint_text_element = etree.SubElement(hintpart_element, "text")
+                hint_text_element.text = str(hint_text)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)

common/lib/capa/capa/tests/test_files/choiceresponse_checkbox.xml

@@ -1,59 +0,0 @@
-<problem>
-<choiceresponse>
-  <checkboxgroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </checkboxgroup>
-</choiceresponse>
-<choiceresponse>
-  <checkboxgroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </checkboxgroup>
-</choiceresponse>
-<choiceresponse>
-  <checkboxgroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </checkboxgroup>
-</choiceresponse>
-</problem>

common/lib/capa/capa/tests/test_files/choiceresponse_radio.xml

@@ -1,40 +0,0 @@
-<problem>
-<choiceresponse>
-  <radiogroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </radiogroup>
-</choiceresponse>
-<choiceresponse>
-  <radiogroup>
-    <choice correct="false">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="true">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </radiogroup>
-</choiceresponse>
-</problem>

common/lib/capa/capa/tests/test_files/coderesponse.xml

@@ -1,33 +0,0 @@
-<problem>
-<text>
-<h2>Code response</h2>
-
-<p>
-</p>
-
-<text>
-Write a program to compute the square of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <codeparam>
-      <initial_display>def square(x):</initial_display>
-      <answer_display>answer</answer_display>
-      <grader_payload>grader stuff</grader_payload>
-    </codeparam>
-  </coderesponse>
-</text>
-
-<text>
-Write a program to compute the square of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <codeparam>
-      <initial_display>def square(x):</initial_display>
-      <answer_display>answer</answer_display>
-      <grader_payload>grader stuff</grader_payload>
-    </codeparam>
-  </coderesponse>
-</text>
-
-</text>
-</problem>

common/lib/capa/capa/tests/test_files/coderesponse_externalresponseformat.xml

@@ -1,101 +0,0 @@
-<problem>
-<text>
-<h2>Code response</h2>
-
-<p>
-</p>
-
-<text>
-Write a program to compute the square of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def square(n):
-"""
-
-answer = """
-def square(n):
-    return n**2    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testSquare(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: square(%d)'%n
-    return str(square(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testSquare(0))
-   elif test == 2: f.write(testSquare(1))
-   else:  f.write(testSquare())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
-  </coderesponse>
-</text>
-
-<text>
-Write a program to compute the cube of a number
-  <coderesponse tests="repeat:2,generate">
-    <textbox rows="10" cols="70"  mode="python"/>
-    <answer><![CDATA[
-initial_display = """
-def cube(n):
-"""
-
-answer = """
-def cube(n):
-    return n**3    
-"""
-
-preamble = """ 
-import sys, time
-"""
-
-test_program = """
-import random
-import operator
-
-def testCube(n = None):
-    if n is None:
-       n = random.randint(2, 20)
-    print 'Test is: cube(%d)'%n
-    return str(cube(n))
-
-def main():
-   f = os.fdopen(3,'w')
-   test = int(sys.argv[1])
-   rndlist = map(int,os.getenv('rndlist').split(','))
-   random.seed(rndlist[0])
-   if test == 1: f.write(testCube(0))
-   elif test == 2: f.write(testCube(1))
-   else:  f.write(testCube())
-   f.close()
-
-main()
-sys.exit(0)
-"""
-]]>
-    </answer>
-  </coderesponse>
-</text>
-
-</text>
-</problem>

common/lib/capa/capa/tests/test_files/filename_convert_test.txt

@@ -0,0 +1 @@
+This file is used to test converting file handles to filenames in the capa utility module

common/lib/capa/capa/tests/test_files/formularesponse_with_hint.xml

@@ -1,45 +0,0 @@
-<problem>
-<script type="loncapa/python">
-# from loncapa import *
-x1 = 4 # lc_random(2,4,1)
-y1 = 5 # lc_random(3,7,1)
-
-x2 = 10 # lc_random(x1+1,9,1)
-y2 = 20 # lc_random(y1+1,15,1)
-
-m = (y2-y1)/(x2-x1)
-b = y1 - m*x1
-answer = "%s*x+%s" % (m,b)
-answer = answer.replace('+-','-')
-
-inverted_m = (x2-x1)/(y2-y1)
-inverted_b = b
-wrongans = "%s*x+%s" % (inverted_m,inverted_b)
-wrongans = wrongans.replace('+-','-')
-</script>
-
-<text>
-<p>Hints can be provided to students, based on the last response given, as well as the history of responses given.  Here is an example of a hint produced by a Formula Response problem.</p>
-
-<p>
-What is the equation of the line which passess through ($x1,$y1) and
-($x2,$y2)?</p>
-
-<p>The correct answer is <tt>$answer</tt>. A common error is to invert the equation for the slope.  Enter <tt>
-$wrongans</tt> to see a hint.</p>
-
-</text>
-
-<formularesponse samples="x@-5:5#11" id="11" answer="$answer">
- <responseparam description="Numerical Tolerance" type="tolerance" default="0.001" name="tol" /> 
- <text>y = <textline size="25" /></text>
-  <hintgroup>
-   <formulahint samples="x@-5:5#11" answer="$wrongans" name="inversegrad">
-   </formulahint>
-   <hintpart on="inversegrad">
-    <text>You have inverted the slope in the question.</text>
-   </hintpart>
-  </hintgroup>     
-</formularesponse>
-</problem>
-

common/lib/capa/capa/tests/test_files/imageresponse.xml

@@ -1,40 +0,0 @@
-<problem>
-<text><p>
-Two skiers are on frictionless black diamond ski slopes.
-Hello</p></text>
-
-<imageresponse max="1" loncapaid="11">
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)"/>
-<text>Click on the image where the top skier will stop momentarily if the top skier starts from rest.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(242,202)-(296,276)"/>
-<text>Click on the image where the lower skier will stop momentarily if the lower skier starts from rest.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98);(242,202)-(296,276)"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98);(242,202)-(296,276)"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98);(242,202)-(296,276)"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<hintgroup showoncorrect="no">
-<text><p>Use conservation of energy.</p></text>
-</hintgroup>
-</imageresponse>
-
-
-<imageresponse max="1" loncapaid="12">
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" rectangle="(490,11)-(556,98)" regions='[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]'/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[[10,10], [20,10], [20, 30]]]"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [15, 15]]"/>
-<imageinput src="/static/Physics801/Figures/Skier-conservation of energy.jpg" width="560" height="388" regions="[[10,10], [30,30], [10, 30], [30, 10]]"/>
-<text>Click on either of the two positions as discussed previously.</text>
-<hintgroup showoncorrect="no">
-<text><p>Use conservation of energy.</p></text>
-</hintgroup>
-</imageresponse>
-
-
-</problem>

common/lib/capa/capa/tests/test_files/javascriptresponse.xml

@@ -1,13 +0,0 @@
-<problem>
-
-<javascriptresponse>
-  <generator src="test_problem_generator.js"/>
-  <grader src="test_problem_grader.js"/>
-  <display class="TestProblemDisplay" src="test_problem_display.js"/>
-  <responseparam name="value" value="4"/>
-  <javascriptinput>
-  </javascriptinput>
-</javascriptresponse>
-
-</problem>
-

common/lib/capa/capa/tests/test_files/js/compiled/c9a9cd4242d84c924fe5f8324e9ae79d.js

@@ -1,50 +0,0 @@
-// Generated by CoffeeScript 1.3.3
-(function() {
-  var MinimaxProblemDisplay, root,
-    __hasProp = {}.hasOwnProperty,
-    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
-
-  MinimaxProblemDisplay = (function(_super) {
-
-    __extends(MinimaxProblemDisplay, _super);
-
-    function MinimaxProblemDisplay(state, submission, evaluation, container, submissionField, parameters) {
-      this.state = state;
-      this.submission = submission;
-      this.evaluation = evaluation;
-      this.container = container;
-      this.submissionField = submissionField;
-      this.parameters = parameters != null ? parameters : {};
-      MinimaxProblemDisplay.__super__.constructor.call(this, this.state, this.submission, this.evaluation, this.container, this.submissionField, this.parameters);
-    }
-
-    MinimaxProblemDisplay.prototype.render = function() {};
-
-    MinimaxProblemDisplay.prototype.createSubmission = function() {
-      var id, value, _ref, _results;
-      this.newSubmission = {};
-      if (this.submission != null) {
-        _ref = this.submission;
-        _results = [];
-        for (id in _ref) {
-          value = _ref[id];
-          _results.push(this.newSubmission[id] = value);
-        }
-        return _results;
-      }
-    };
-
-    MinimaxProblemDisplay.prototype.getCurrentSubmission = function() {
-      return this.newSubmission;
-    };
-
-    return MinimaxProblemDisplay;
-
-  })(XProblemDisplay);
-
-  root = typeof exports !== "undefined" && exports !== null ? exports : this;
-
-  root.TestProblemDisplay = TestProblemDisplay;
-
-}).call(this);
-;

common/lib/capa/capa/tests/test_files/js/compiled/javascriptresponse.js

@@ -1,50 +0,0 @@
-// Generated by CoffeeScript 1.3.3
-(function() {
-  var MinimaxProblemDisplay, root,
-    __hasProp = {}.hasOwnProperty,
-    __extends = function(child, parent) { for (var key in parent) { if (__hasProp.call(parent, key)) child[key] = parent[key]; } function ctor() { this.constructor = child; } ctor.prototype = parent.prototype; child.prototype = new ctor(); child.__super__ = parent.prototype; return child; };
-
-  MinimaxProblemDisplay = (function(_super) {
-
-    __extends(MinimaxProblemDisplay, _super);
-
-    function MinimaxProblemDisplay(state, submission, evaluation, container, submissionField, parameters) {
-      this.state = state;
-      this.submission = submission;
-      this.evaluation = evaluation;
-      this.container = container;
-      this.submissionField = submissionField;
-      this.parameters = parameters != null ? parameters : {};
-      MinimaxProblemDisplay.__super__.constructor.call(this, this.state, this.submission, this.evaluation, this.container, this.submissionField, this.parameters);
-    }
-
-    MinimaxProblemDisplay.prototype.render = function() {};
-
-    MinimaxProblemDisplay.prototype.createSubmission = function() {
-      var id, value, _ref, _results;
-      this.newSubmission = {};
-      if (this.submission != null) {
-        _ref = this.submission;
-        _results = [];
-        for (id in _ref) {
-          value = _ref[id];
-          _results.push(this.newSubmission[id] = value);
-        }
-        return _results;
-      }
-    };
-
-    MinimaxProblemDisplay.prototype.getCurrentSubmission = function() {
-      return this.newSubmission;
-    };
-
-    return MinimaxProblemDisplay;
-
-  })(XProblemDisplay);
-
-  root = typeof exports !== "undefined" && exports !== null ? exports : this;
-
-  root.TestProblemDisplay = TestProblemDisplay;
-
-}).call(this);
-;

common/lib/capa/capa/tests/test_files/multi_bare.xml

@@ -1,21 +0,0 @@
-<problem>
-<multiplechoiceresponse>
-  <choicegroup>
-    <choice correct="false" >
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false" >
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true" >
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </choicegroup>
-</multiplechoiceresponse>
-</problem>

common/lib/capa/capa/tests/test_files/multichoice.xml

@@ -1,21 +0,0 @@
-<problem>
-<multiplechoiceresponse>
-  <choicegroup>
-    <choice correct="false" name="foil1">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice correct="false" name="foil2">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice correct="true" name="foil3">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice correct="false" name="foil4">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice correct="false" name="foil5">
-      <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </choicegroup>
-</multiplechoiceresponse>
-</problem>

common/lib/capa/capa/tests/test_files/optionresponse.xml

@@ -1,63 +0,0 @@
-  <problem>
-    <text>
-      <p>
-Why do bicycles benefit from having larger wheels when going up a bump as shown in the picture? <br/>
-Assume that for both bicycles:<br/>
-1.)	The tires have equal air pressure.<br/>
-2.)	The bicycles never leave the contact with the bump.<br/>
-3.)	The bicycles have the same mass. The bicycle tires (regardless of size) have the same mass.<br/>
-</p>
-    </text>
-    <optionresponse texlayout="horizontal" max="10" randomize="yes">
-      <ul>
-        <li>
-          <text>
-            <p>The bicycles with larger wheels have more time to go over the bump. This decreases the magnitude of the force needed to lift the bicycle.</p>
-          </text>
-          <optioninput name="Foil1" location="random" options="('True','False')" correct="True">
-</optioninput>
-        </li>
-        <li>
-          <text>
-            <p>The bicycles with larger wheels always have a smaller vertical displacement regardless of speed.</p>
-          </text>
-          <optioninput name="Foil2" location="random" options="('True','False')" correct="False">
-</optioninput>
-        </li>
-        <li>
-          <text>
-            <p>The bicycles with larger wheels experience a force backward with less magnitude for the same amount of time.</p>
-          </text>
-          <optioninput name="Foil3" location="random" options="('True','False')" correct="False">
-</optioninput>
-        </li>
-        <li>
-          <text>
-            <p>The bicycles with larger wheels experience a force backward with less magnitude for a greater amount of time.</p>
-          </text>
-          <optioninput name="Foil4" location="random" options="('True','False')" correct="True">
-</optioninput>
-        </li>
-        <li>
-          <text>
-            <p>The bicycles with larger wheels have more kinetic energy turned into gravitational potential energy.</p>
-          </text>
-          <optioninput name="Foil5" location="random" options="('True','False')" correct="False">
-</optioninput>
-        </li>
-        <li>
-          <text>
-            <p>The bicycles with larger wheels have more rotational kinetic energy, so the horizontal velocity of the biker changes less.</p>
-          </text>
-          <optioninput name="Foil6" location="random" options="('True','False')" correct="False">
-</optioninput>
-        </li>
-      </ul>
-      <hintgroup showoncorrect="no">
-        <text>
-          <br/>
-          <br/>
-        </text>
-      </hintgroup>
-    </optionresponse>
-  </problem>

common/lib/capa/capa/tests/test_files/stringresponse_with_hint.xml

@@ -1,25 +0,0 @@
-<problem >
- <text><h2>Example: String Response Problem</h2>
- <br/>
- </text>
- 
- <text>Which US state has Lansing as its capital?</text>
-  <stringresponse answer="Michigan" type="ci">
-  <textline size="20" />
-  <hintgroup>
-   <stringhint answer="wisconsin" type="cs" name="wisc">
-   </stringhint>
-   <stringhint answer="minnesota" type="cs" name="minn">
-   </stringhint>
-   <hintpart on="wisc">
-    <text>The state capital of Wisconsin is Madison.</text>
-   </hintpart>
-   <hintpart on="minn">
-    <text>The state capital of Minnesota is St. Paul.</text>
-   </hintpart>
-   <hintpart on="default">
-    <text>The state you are looking for is also known as the 'Great Lakes State'</text>
-   </hintpart>     
-  </hintgroup>
- </stringresponse>
-</problem>

common/lib/capa/capa/tests/test_files/symbolicresponse.xml

@@ -1,29 +0,0 @@
-<problem>
-<text>
-<h2>Example: Symbolic Math Response Problem</h2>
-
-<p>
-A symbolic math response problem presents one or more symbolic math
-input fields for input.  Correctness of input is evaluated based on
-the symbolic properties of the expression entered.  The student enters
-text, but sees a proper symbolic rendition of the entered formula, in
-real time, next to the input box.
-</p>
-
-<p>This is a correct answer which may be entered below: </p>
-<p><tt>cos(theta)*[[1,0],[0,1]] + i*sin(theta)*[[0,1],[1,0]]</tt></p>
-
-<script>
-from symmath import *
-</script>
-  <text>Compute [mathjax] U = \exp\left( i \theta \left[ \begin{matrix} 0 &amp; 1 \\ 1 &amp; 0 \end{matrix} \right] \right) [/mathjax]
-      and give the resulting \(2 \times 2\) matrix. <br/>
-      Your input should be typed in as a list of lists, eg <tt>[[1,2],[3,4]]</tt>. <br/>
-       [mathjax]U=[/mathjax] <symbolicresponse cfn="symmath_check" answer="[[cos(theta),I*sin(theta)],[I*sin(theta),cos(theta)]]" options="matrix,imaginaryi" id="filenamedogi0VpEBOWedxsymmathresponse_1" state="unsubmitted">
-          <textline size="80" math="1" response_id="2" answer_id="1" id="filenamedogi0VpEBOWedxsymmathresponse_2_1"/>
-        </symbolicresponse>
-      <br/>
-  </text>
-
-</text>
-</problem>

common/lib/capa/capa/tests/test_files/truefalse.xml

@@ -1,21 +0,0 @@
-<problem>
-  <truefalseresponse max="10" randomize="yes">
-  <choicegroup>
-    <choice location="random" correct="true" name="foil1">
-      <startouttext />This is foil One.<endouttext />
-    </choice>
-    <choice location="random" correct="true" name="foil2">
-      <startouttext />This is foil Two.<endouttext />
-    </choice>
-    <choice location="random" correct="false" name="foil3">
-      <startouttext />This is foil Three.<endouttext />
-    </choice>
-    <choice location="random" correct="false" name="foil4">
-      <startouttext />This is foil Four.<endouttext />
-    </choice>
-    <choice location="random" correct="false" name="foil5">
-            <startouttext />This is foil Five.<endouttext />
-    </choice>
-  </choicegroup>
-</truefalseresponse>
-</problem>

common/lib/capa/capa/tests/test_responsetypes.py

@@ -16,93 +16,151 @@ from capa.correctmap import CorrectMap
 from capa.util import convert_files_to_filenames
 from capa.xqueue_interface import dateformat
 
+class ResponseTest(unittest.TestCase):
+    """ Base class for tests of capa responses."""
+    
+    xml_factory_class = None
 
-class MultiChoiceTest(unittest.TestCase):
-    def test_MC_grade(self):
-        multichoice_file = os.path.dirname(__file__) + "/test_files/multichoice.xml"
-        test_lcp = lcp.LoncapaProblem(open(multichoice_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': 'choice_foil3'}
-        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
-        false_answers = {'1_2_1': 'choice_foil2'}
-        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
+    def setUp(self):
+        if self.xml_factory_class:
+            self.xml_factory = self.xml_factory_class()
 
-    def test_MC_bare_grades(self):
-        multichoice_file = os.path.dirname(__file__) + "/test_files/multi_bare.xml"
-        test_lcp = lcp.LoncapaProblem(open(multichoice_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': 'choice_2'}
-        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
-        false_answers = {'1_2_1': 'choice_1'}
-        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
+    def build_problem(self, **kwargs):
+        xml = self.xml_factory.build_xml(**kwargs)
+        return lcp.LoncapaProblem(xml, '1', system=test_system)
 
-    def test_TF_grade(self):
-        truefalse_file = os.path.dirname(__file__) + "/test_files/truefalse.xml"
-        test_lcp = lcp.LoncapaProblem(open(truefalse_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': ['choice_foil2', 'choice_foil1']}
-        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
-        false_answers = {'1_2_1': ['choice_foil1']}
-        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
-        false_answers = {'1_2_1': ['choice_foil1', 'choice_foil3']}
-        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
-        false_answers = {'1_2_1': ['choice_foil3']}
-        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
-        false_answers = {'1_2_1': ['choice_foil1', 'choice_foil2', 'choice_foil3']}
-        self.assertEquals(test_lcp.grade_answers(false_answers).get_correctness('1_2_1'), 'incorrect')
+    def assert_grade(self, problem, submission, expected_correctness):
+        input_dict = {'1_2_1': submission}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEquals(correct_map.get_correctness('1_2_1'), expected_correctness)
+
+    def assert_multiple_grade(self, problem, correct_answers, incorrect_answers):
+        for input_str in correct_answers:
+            result = problem.grade_answers({'1_2_1': input_str}).get_correctness('1_2_1')
+            self.assertEqual(result, 'correct',
+                        msg="%s should be marked correct" % str(input_str))
+
+        for input_str in incorrect_answers:
+            result = problem.grade_answers({'1_2_1': input_str}).get_correctness('1_2_1')
+            self.assertEqual(result, 'incorrect', 
+                            msg="%s should be marked incorrect" % str(input_str))
+
+class MultiChoiceResponseTest(ResponseTest):
+    from response_xml_factory import MultipleChoiceResponseXMLFactory
+    xml_factory_class = MultipleChoiceResponseXMLFactory
+
+    def test_multiple_choice_grade(self):
+        problem = self.build_problem(choices=[False, True, False])
+
+        # Ensure that we get the expected grades
+        self.assert_grade(problem, 'choice_0', 'incorrect')
+        self.assert_grade(problem, 'choice_1', 'correct')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+
+    def test_named_multiple_choice_grade(self):
+        problem = self.build_problem(choices=[False, True, False],
+                                    choice_names=["foil_1", "foil_2", "foil_3"])
+        
+        # Ensure that we get the expected grades
+        self.assert_grade(problem, 'choice_foil_1', 'incorrect')
+        self.assert_grade(problem, 'choice_foil_2', 'correct')
+        self.assert_grade(problem, 'choice_foil_3', 'incorrect')
 
 
-class ImageResponseTest(unittest.TestCase):
-    def test_ir_grade(self):
-        imageresponse_file = os.path.dirname(__file__) + "/test_files/imageresponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(imageresponse_file).read(), '1', system=test_system)
-        # testing regions only
-        correct_answers = {
-           #regions
-           '1_2_1': '(490,11)-(556,98)',
-           '1_2_2': '(242,202)-(296,276)',
-           '1_2_3': '(490,11)-(556,98);(242,202)-(296,276)',
-           '1_2_4': '(490,11)-(556,98);(242,202)-(296,276)',
-           '1_2_5': '(490,11)-(556,98);(242,202)-(296,276)',
-           #testing regions and rectanges
-           '1_3_1': 'rectangle="(490,11)-(556,98)" \
-           regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
-           '1_3_2': 'rectangle="(490,11)-(556,98)" \
-           regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
-           '1_3_3': 'regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
-           '1_3_4': 'regions="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"',
-           '1_3_5': 'regions="[[[10,10], [20,10], [20, 30]]]"',
-           '1_3_6': 'regions="[[10,10], [30,30], [15, 15]]"',
-           '1_3_7': 'regions="[[10,10], [30,30], [10, 30], [30, 10]]"',
-                          }
-        test_answers = {
-            '1_2_1': '[500,20]',
-            '1_2_2': '[250,300]',
-            '1_2_3': '[500,20]',
-            '1_2_4': '[250,250]',
-            '1_2_5': '[10,10]',
+class TrueFalseResponseTest(ResponseTest):
+    from response_xml_factory import TrueFalseResponseXMLFactory
+    xml_factory_class = TrueFalseResponseXMLFactory
 
-            '1_3_1': '[500,20]',
-            '1_3_2': '[15,15]',
-            '1_3_3': '[500,20]',
-            '1_3_4': '[115,115]',
-            '1_3_5': '[15,15]',
-            '1_3_6': '[20,20]',
-            '1_3_7': '[20,15]',
-                        }
+    def test_true_false_grade(self):
+        problem = self.build_problem(choices=[False, True, True])
 
-        # regions
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_2'), 'incorrect')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_3'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_4'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_5'), 'incorrect')
+        # Check the results
+        # Mark correct if and only if ALL (and only) correct choices selected
+        self.assert_grade(problem, 'choice_0', 'incorrect')
+        self.assert_grade(problem, 'choice_1', 'incorrect')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_1', 'choice_2'], 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_2'], 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_1'], 'incorrect')
+        self.assert_grade(problem, ['choice_1', 'choice_2'], 'correct')
 
-        # regions and rectangles
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_1'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_2'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_3'), 'incorrect')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_4'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_5'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_6'), 'incorrect')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_7'), 'correct')
+        # Invalid choices should be marked incorrect (we have no choice 3)
+        self.assert_grade(problem, 'choice_3', 'incorrect')
+        self.assert_grade(problem, 'not_a_choice', 'incorrect')
+
+    def test_named_true_false_grade(self):
+        problem = self.build_problem(choices=[False, True, True],
+                                    choice_names=['foil_1','foil_2','foil_3'])
+
+        # Check the results
+        # Mark correct if and only if ALL (and only) correct chocies selected
+        self.assert_grade(problem, 'choice_foil_1', 'incorrect')
+        self.assert_grade(problem, 'choice_foil_2', 'incorrect')
+        self.assert_grade(problem, 'choice_foil_3', 'incorrect')
+        self.assert_grade(problem, ['choice_foil_1', 'choice_foil_2', 'choice_foil_3'], 'incorrect')
+        self.assert_grade(problem, ['choice_foil_1', 'choice_foil_3'], 'incorrect')
+        self.assert_grade(problem, ['choice_foil_1', 'choice_foil_2'], 'incorrect')
+        self.assert_grade(problem, ['choice_foil_2', 'choice_foil_3'], 'correct')
+
+        # Invalid choices should be marked incorrect
+        self.assert_grade(problem, 'choice_foil_4', 'incorrect')
+        self.assert_grade(problem, 'not_a_choice', 'incorrect')
+
+class ImageResponseTest(ResponseTest):
+    from response_xml_factory import ImageResponseXMLFactory
+    xml_factory_class = ImageResponseXMLFactory
+
+    def test_rectangle_grade(self):
+        # Define a rectangle with corners (10,10) and (20,20)
+        problem = self.build_problem(rectangle="(10,10)-(20,20)")
+
+        # Anything inside the rectangle (and along the borders) is correct
+        # Everything else is incorrect
+        correct_inputs = ["[12,19]", "[10,10]", "[20,20]", 
+                            "[10,15]", "[20,15]", "[15,10]", "[15,20]"]
+        incorrect_inputs = ["[4,6]", "[25,15]", "[15,40]", "[15,4]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_multiple_rectangles_grade(self):
+        # Define two rectangles
+        rectangle_str = "(10,10)-(20,20);(100,100)-(200,200)"
+
+        # Expect that only points inside the rectangles are marked correct
+        problem = self.build_problem(rectangle=rectangle_str)
+        correct_inputs = ["[12,19]", "[120, 130]"]
+        incorrect_inputs = ["[4,6]", "[25,15]", "[15,40]", "[15,4]",
+                            "[50,55]", "[300, 14]", "[120, 400]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_region_grade(self):
+        # Define a triangular region with corners (0,0), (5,10), and (0, 10)
+        region_str = "[ [1,1], [5,10], [0,10] ]"
+
+        # Expect that only points inside the triangle are marked correct
+        problem = self.build_problem(regions=region_str)
+        correct_inputs = ["[2,4]", "[1,3]"]
+        incorrect_inputs = ["[0,0]", "[3,5]", "[5,15]", "[30, 12]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_multiple_regions_grade(self):
+        # Define multiple regions that the user can select
+        region_str="[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"
+
+        # Expect that only points inside the regions are marked correct
+        problem = self.build_problem(regions=region_str)
+        correct_inputs = ["[15,12]", "[110,112]"]
+        incorrect_inputs = ["[0,0]", "[600,300]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_region_and_rectangle_grade(self):
+        rectangle_str = "(100,100)-(200,200)"
+        region_str="[[10,10], [20,10], [20, 30]]"
+
+        # Expect that only points inside the rectangle or region are marked correct
+        problem = self.build_problem(regions=region_str, rectangle=rectangle_str)
+        correct_inputs = ["[13,12]", "[110,112]"]
+        incorrect_inputs = ["[0,0]", "[600,300]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
 
 
 class SymbolicResponseTest(unittest.TestCase):
@@ -195,60 +253,165 @@ class SymbolicResponseTest(unittest.TestCase):
         self.assertEquals(test_lcp.grade_answers(wrong_answers).get_correctness('1_2_1'), 'incorrect')
 
 
-class OptionResponseTest(unittest.TestCase):
-    '''
-    Run this with
+class OptionResponseTest(ResponseTest):
+    from response_xml_factory import OptionResponseXMLFactory
+    xml_factory_class = OptionResponseXMLFactory
 
-    python manage.py test courseware.OptionResponseTest
-    '''
-    def test_or_grade(self):
-        optionresponse_file = os.path.dirname(__file__) + "/test_files/optionresponse.xml"
-        test_lcp = lcp.LoncapaProblem(open(optionresponse_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': 'True',
-                           '1_2_2': 'False'}
-        test_answers = {'1_2_1': 'True',
-                        '1_2_2': 'True',
-                        }
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_2'), 'incorrect')
+    def test_grade(self):
+        problem = self.build_problem(options=["first", "second", "third"], 
+                                    correct_option="second")
+
+        # Assert that we get the expected grades
+        self.assert_grade(problem, "first", "incorrect")
+        self.assert_grade(problem, "second", "correct")
+        self.assert_grade(problem, "third", "incorrect")
+
+        # Options not in the list should be marked incorrect
+        self.assert_grade(problem, "invalid_option", "incorrect")
 
 
-class FormulaResponseWithHintTest(unittest.TestCase):
-    '''
-    Test Formula response problem with a hint
-    This problem also uses calc.
-    '''
-    def test_or_grade(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/formularesponse_with_hint.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': '2.5*x-5.0'}
-        test_answers = {'1_2_1': '0.4*x-5.0'}
-        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
-        cmap = test_lcp.grade_answers(test_answers)
-        self.assertEquals(cmap.get_correctness('1_2_1'), 'incorrect')
-        self.assertTrue('You have inverted' in cmap.get_hint('1_2_1'))
+class FormulaResponseTest(ResponseTest):
+    from response_xml_factory import FormulaResponseXMLFactory
+    xml_factory_class = FormulaResponseXMLFactory
+
+    def test_grade(self):
+        # Sample variables x and y in the range [-10, 10]
+        sample_dict = {'x': (-10, 10), 'y': (-10, 10)}
+
+        # The expected solution is numerically equivalent to x+2y
+        problem = self.build_problem(sample_dict=sample_dict,
+                                    num_samples=10,
+                                    tolerance=0.01,
+                                    answer="x+2*y")
+
+        # Expect an equivalent formula to be marked correct
+        # 2x - x + y + y = x + 2y
+        input_formula = "2*x - x + y + y"
+        self.assert_grade(problem, input_formula, "correct")
+
+        # Expect an incorrect formula to be marked incorrect
+        # x + y != x + 2y
+        input_formula = "x + y"
+        self.assert_grade(problem, input_formula, "incorrect")
+
+    def test_hint(self):
+        # Sample variables x and y in the range [-10, 10]
+        sample_dict = {'x': (-10, 10), 'y': (-10,10) }
+
+        # Give a hint if the user leaves off the coefficient
+        # or leaves out x
+        hints = [('x + 3*y', 'y_coefficient', 'Check the coefficient of y'),
+                ('2*y', 'missing_x', 'Try including the variable x')]
 
 
-class StringResponseWithHintTest(unittest.TestCase):
-    '''
-    Test String response problem with a hint
-    '''
-    def test_or_grade(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/stringresponse_with_hint.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': 'Michigan'}
-        test_answers = {'1_2_1': 'Minnesota'}
-        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
-        cmap = test_lcp.grade_answers(test_answers)
-        self.assertEquals(cmap.get_correctness('1_2_1'), 'incorrect')
-        self.assertTrue('St. Paul' in cmap.get_hint('1_2_1'))
+        # The expected solution is numerically equivalent to x+2y
+        problem = self.build_problem(sample_dict=sample_dict,
+                                    num_samples=10,
+                                    tolerance=0.01,
+                                    answer="x+2*y",
+                                    hints=hints)
+
+        # Expect to receive a hint  if we add an extra y
+        input_dict = {'1_2_1': "x + 2*y + y"}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEquals(correct_map.get_hint('1_2_1'),
+                'Check the coefficient of y')
+
+        # Expect to receive a hint if we leave out x
+        input_dict = {'1_2_1': "2*y"}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEquals(correct_map.get_hint('1_2_1'),
+                'Try including the variable x')
 
 
-class CodeResponseTest(unittest.TestCase):
-    '''
-    Test CodeResponse
-    TODO: Add tests for external grader messages
-    '''
+    def test_script(self):
+        # Calculate the answer using a script
+        script = "calculated_ans = 'x+x'"
+
+        # Sample x in the range [-10,10]
+        sample_dict = {'x': (-10, 10)}
+
+        # The expected solution is numerically equivalent to 2*x
+        problem = self.build_problem(sample_dict=sample_dict,
+                                    num_samples=10,
+                                    tolerance=0.01,
+                                    answer="$calculated_ans",
+                                    script=script)
+
+        # Expect that the inputs are graded correctly
+        self.assert_grade(problem, '2*x', 'correct')
+        self.assert_grade(problem, '3*x', 'incorrect')
+
+
+class StringResponseTest(ResponseTest):
+    from response_xml_factory import StringResponseXMLFactory
+    xml_factory_class = StringResponseXMLFactory
+
+
+    def test_case_sensitive(self):
+        problem = self.build_problem(answer="Second", case_sensitive=True)
+
+        # Exact string should be correct
+        self.assert_grade(problem, "Second", "correct")
+
+        # Other strings and the lowercase version of the string are incorrect
+        self.assert_grade(problem, "Other String", "incorrect")
+        self.assert_grade(problem, "second", "incorrect")
+
+    def test_case_insensitive(self):
+        problem = self.build_problem(answer="Second", case_sensitive=False)
+
+        # Both versions of the string should be allowed, regardless
+        # of capitalization
+        self.assert_grade(problem, "Second", "correct")
+        self.assert_grade(problem, "second", "correct")
+
+        # Other strings are not allowed
+        self.assert_grade(problem, "Other String", "incorrect")
+
+    def test_hints(self):
+        hints = [("wisconsin", "wisc", "The state capital of Wisconsin is Madison"),
+                ("minnesota", "minn", "The state capital of Minnesota is St. Paul")]
+
+        problem = self.build_problem(answer="Michigan", 
+                                    case_sensitive=False, 
+                                    hints=hints)
+
+        # We should get a hint for Wisconsin
+        input_dict = {'1_2_1': 'Wisconsin'}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEquals(correct_map.get_hint('1_2_1'),
+                        "The state capital of Wisconsin is Madison")
+
+        # We should get a hint for Minnesota
+        input_dict = {'1_2_1': 'Minnesota'}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEquals(correct_map.get_hint('1_2_1'),
+                        "The state capital of Minnesota is St. Paul")
+
+        # We should NOT get a hint for Michigan (the correct answer)
+        input_dict = {'1_2_1': 'Michigan'}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEquals(correct_map.get_hint('1_2_1'), "")
+
+        # We should NOT get a hint for any other string
+        input_dict = {'1_2_1': 'California'}
+        correct_map = problem.grade_answers(input_dict)
+        self.assertEquals(correct_map.get_hint('1_2_1'), "")
+
+class CodeResponseTest(ResponseTest):
+    from response_xml_factory import CodeResponseXMLFactory
+    xml_factory_class = CodeResponseXMLFactory
+
+    def setUp(self):
+        super(CodeResponseTest, self).setUp()
+
+        grader_payload = json.dumps({"grader": "ps04/grade_square.py"})
+        self.problem = self.build_problem(initial_display="def square(x):",
+                                        answer_display="answer",
+                                        grader_payload=grader_payload,
+                                        num_responses=2)
+
     @staticmethod
     def make_queuestate(key, time):
         timestr = datetime.strftime(time, dateformat)
@@ -258,171 +421,354 @@ class CodeResponseTest(unittest.TestCase):
         """
         Simple test of whether LoncapaProblem knows when it's been queued
         """
-        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-        with open(problem_file) as input_file:
-            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=test_system)
 
-            answer_ids = sorted(test_lcp.get_question_answers())
+        answer_ids = sorted(self.problem.get_question_answers())
 
-            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
-            cmap = CorrectMap()
-            for answer_id in answer_ids:
-                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
-            test_lcp.correct_map.update(cmap)
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for answer_id in answer_ids:
+            cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+        self.problem.correct_map.update(cmap)
 
-            self.assertEquals(test_lcp.is_queued(), False)
+        self.assertEquals(self.problem.is_queued(), False)
 
-            # Now we queue the LCP
-            cmap = CorrectMap()
-            for i, answer_id in enumerate(answer_ids):
-                queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
-                cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
-            test_lcp.correct_map.update(cmap)
+        # Now we queue the LCP
+        cmap = CorrectMap()
+        for i, answer_id in enumerate(answer_ids):
+            queuestate = CodeResponseTest.make_queuestate(i, datetime.now())
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+        self.problem.correct_map.update(cmap)
 
-            self.assertEquals(test_lcp.is_queued(), True)
+        self.assertEquals(self.problem.is_queued(), True)
 
 
     def test_update_score(self):
         '''
         Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
         '''
-        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-        with open(problem_file) as input_file:
-            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=test_system)
+        answer_ids = sorted(self.problem.get_question_answers())
 
-            answer_ids = sorted(test_lcp.get_question_answers())
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        old_cmap = CorrectMap()
+        for i, answer_id in enumerate(answer_ids):
+            queuekey = 1000 + i
+            queuestate = CodeResponseTest.make_queuestate(1000 + i, datetime.now())
+            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
 
-            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
-            old_cmap = CorrectMap()
+        # Message format common to external graders
+        grader_msg = '<span>MESSAGE</span>'   # Must be valid XML
+        correct_score_msg = json.dumps({'correct': True, 'score': 1, 'msg': grader_msg})
+        incorrect_score_msg = json.dumps({'correct': False, 'score': 0, 'msg': grader_msg})
+
+        xserver_msgs = {'correct': correct_score_msg,
+                        'incorrect': incorrect_score_msg, }
+
+        # Incorrect queuekey, state should not be updated
+        for correctness in ['correct', 'incorrect']:
+            self.problem.correct_map = CorrectMap()
+            self.problem.correct_map.update(old_cmap)  # Deep copy
+
+            self.problem.update_score(xserver_msgs[correctness], queuekey=0)
+            self.assertEquals(self.problem.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
+
+            for answer_id in answer_ids:
+                self.assertTrue(self.problem.correct_map.is_queued(answer_id))  # Should be still queued, since message undelivered
+
+        # Correct queuekey, state should be updated
+        for correctness in ['correct', 'incorrect']:
             for i, answer_id in enumerate(answer_ids):
-                queuekey = 1000 + i
-                queuestate = CodeResponseTest.make_queuestate(1000 + i, datetime.now())
-                old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+                self.problem.correct_map = CorrectMap()
+                self.problem.correct_map.update(old_cmap)
 
-            # Message format common to external graders
-            grader_msg = '<span>MESSAGE</span>'   # Must be valid XML
-            correct_score_msg = json.dumps({'correct': True, 'score': 1, 'msg': grader_msg})
-            incorrect_score_msg = json.dumps({'correct': False, 'score': 0, 'msg': grader_msg})
+                new_cmap = CorrectMap()
+                new_cmap.update(old_cmap)
+                npoints = 1 if correctness == 'correct' else 0
+                new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg=grader_msg, queuestate=None)
 
-            xserver_msgs = {'correct': correct_score_msg,
-                            'incorrect': incorrect_score_msg, }
+                self.problem.update_score(xserver_msgs[correctness], queuekey=1000 + i)
+                self.assertEquals(self.problem.correct_map.get_dict(), new_cmap.get_dict())
 
-            # Incorrect queuekey, state should not be updated
-            for correctness in ['correct', 'incorrect']:
-                test_lcp.correct_map = CorrectMap()
-                test_lcp.correct_map.update(old_cmap)  # Deep copy
-
-                test_lcp.update_score(xserver_msgs[correctness], queuekey=0)
-                self.assertEquals(test_lcp.correct_map.get_dict(), old_cmap.get_dict())  # Deep comparison
-
-                for answer_id in answer_ids:
-                    self.assertTrue(test_lcp.correct_map.is_queued(answer_id))  # Should be still queued, since message undelivered
-
-            # Correct queuekey, state should be updated
-            for correctness in ['correct', 'incorrect']:
-                for i, answer_id in enumerate(answer_ids):
-                    test_lcp.correct_map = CorrectMap()
-                    test_lcp.correct_map.update(old_cmap)
-
-                    new_cmap = CorrectMap()
-                    new_cmap.update(old_cmap)
-                    npoints = 1 if correctness == 'correct' else 0
-                    new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg=grader_msg, queuestate=None)
-
-                    test_lcp.update_score(xserver_msgs[correctness], queuekey=1000 + i)
-                    self.assertEquals(test_lcp.correct_map.get_dict(), new_cmap.get_dict())
-
-                    for j, test_id in enumerate(answer_ids):
-                        if j == i:
-                            self.assertFalse(test_lcp.correct_map.is_queued(test_id))  # Should be dequeued, message delivered
-                        else:
-                            self.assertTrue(test_lcp.correct_map.is_queued(test_id))  # Should be queued, message undelivered
+                for j, test_id in enumerate(answer_ids):
+                    if j == i:
+                        self.assertFalse(self.problem.correct_map.is_queued(test_id))  # Should be dequeued, message delivered
+                    else:
+                        self.assertTrue(self.problem.correct_map.is_queued(test_id))  # Should be queued, message undelivered
 
 
     def test_recentmost_queuetime(self):
         '''
         Test whether the LoncapaProblem knows about the time of queue requests
         '''
-        problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-        with open(problem_file) as input_file:
-            test_lcp = lcp.LoncapaProblem(input_file.read(), '1', system=test_system)
+        answer_ids = sorted(self.problem.get_question_answers())
 
-            answer_ids = sorted(test_lcp.get_question_answers())
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for answer_id in answer_ids:
+            cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+        self.problem.correct_map.update(cmap)
 
-            # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
-            cmap = CorrectMap()
-            for answer_id in answer_ids:
-                cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
-            test_lcp.correct_map.update(cmap)
+        self.assertEquals(self.problem.get_recentmost_queuetime(), None)
 
-            self.assertEquals(test_lcp.get_recentmost_queuetime(), None)
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        cmap = CorrectMap()
+        for i, answer_id in enumerate(answer_ids):
+            queuekey = 1000 + i
+            latest_timestamp = datetime.now()
+            queuestate = CodeResponseTest.make_queuestate(1000 + i, latest_timestamp)
+            cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
+        self.problem.correct_map.update(cmap)
 
-            # CodeResponse requires internal CorrectMap state. Build it now in the queued state
-            cmap = CorrectMap()
-            for i, answer_id in enumerate(answer_ids):
-                queuekey = 1000 + i
-                latest_timestamp = datetime.now()
-                queuestate = CodeResponseTest.make_queuestate(1000 + i, latest_timestamp)
-                cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
-            test_lcp.correct_map.update(cmap)
+        # Queue state only tracks up to second
+        latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp, dateformat), dateformat)
 
-            # Queue state only tracks up to second
-            latest_timestamp = datetime.strptime(datetime.strftime(latest_timestamp, dateformat), dateformat)
+        self.assertEquals(self.problem.get_recentmost_queuetime(), latest_timestamp)
 
-            self.assertEquals(test_lcp.get_recentmost_queuetime(), latest_timestamp)
+    def test_convert_files_to_filenames(self):
+        '''
+        Test whether file objects are converted to filenames without altering other structures
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/filename_convert_test.txt")
+        with open(problem_file) as fp:
+            answers_with_file = {'1_2_1': 'String-based answer',
+                                 '1_3_1': ['answer1', 'answer2', 'answer3'],
+                                 '1_4_1': [fp, fp]}
+            answers_converted = convert_files_to_filenames(answers_with_file)
+            self.assertEquals(answers_converted['1_2_1'], 'String-based answer')
+            self.assertEquals(answers_converted['1_3_1'], ['answer1', 'answer2', 'answer3'])
+            self.assertEquals(answers_converted['1_4_1'], [fp.name, fp.name])
 
-        def test_convert_files_to_filenames(self):
-            '''
-            Test whether file objects are converted to filenames without altering other structures
-            '''
-            problem_file = os.path.join(os.path.dirname(__file__), "test_files/coderesponse.xml")
-            with open(problem_file) as fp:
-                answers_with_file = {'1_2_1': 'String-based answer',
-                                     '1_3_1': ['answer1', 'answer2', 'answer3'],
-                                     '1_4_1': [fp, fp]}
-                answers_converted = convert_files_to_filenames(answers_with_file)
-                self.assertEquals(answers_converted['1_2_1'], 'String-based answer')
-                self.assertEquals(answers_converted['1_3_1'], ['answer1', 'answer2', 'answer3'])
-                self.assertEquals(answers_converted['1_4_1'], [fp.name, fp.name])
+class ChoiceResponseTest(ResponseTest):
+    from response_xml_factory import ChoiceResponseXMLFactory
+    xml_factory_class = ChoiceResponseXMLFactory
+
+    def test_radio_group_grade(self):
+        problem = self.build_problem(choice_type='radio', 
+                                        choices=[False, True, False])
+
+        # Check that we get the expected results
+        self.assert_grade(problem, 'choice_0', 'incorrect')
+        self.assert_grade(problem, 'choice_1', 'correct')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+
+        # No choice 3 exists --> mark incorrect
+        self.assert_grade(problem, 'choice_3', 'incorrect')
 
 
-class ChoiceResponseTest(unittest.TestCase):
+    def test_checkbox_group_grade(self):
+        problem = self.build_problem(choice_type='checkbox',
+                                        choices=[False, True, True])
 
-    def test_cr_rb_grade(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/choiceresponse_radio.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': 'choice_2',
-                           '1_3_1': ['choice_2', 'choice_3']}
-        test_answers = {'1_2_1': 'choice_2',
-                        '1_3_1': 'choice_2',
-                        }
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_1'), 'incorrect')
+        # Check that we get the expected results
+        # (correct if and only if BOTH correct choices chosen)
+        self.assert_grade(problem, ['choice_1', 'choice_2'], 'correct')
+        self.assert_grade(problem, 'choice_1', 'incorrect')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_1'], 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_2'], 'incorrect')
 
-    def test_cr_cb_grade(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/choiceresponse_checkbox.xml"
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': 'choice_2',
-                           '1_3_1': ['choice_2', 'choice_3'],
-                           '1_4_1': ['choice_2', 'choice_3']}
-        test_answers = {'1_2_1': 'choice_2',
-                        '1_3_1': 'choice_2',
-                        '1_4_1': ['choice_2', 'choice_3'],
-                        }
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_2_1'), 'correct')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_3_1'), 'incorrect')
-        self.assertEquals(test_lcp.grade_answers(test_answers).get_correctness('1_4_1'), 'correct')
+        # No choice 3 exists --> mark incorrect
+        self.assert_grade(problem, 'choice_3', 'incorrect')
 
 
-class JavascriptResponseTest(unittest.TestCase):
+class JavascriptResponseTest(ResponseTest):
+    from response_xml_factory import JavascriptResponseXMLFactory
+    xml_factory_class = JavascriptResponseXMLFactory
 
-    def test_jr_grade(self):
-        problem_file = os.path.dirname(__file__) + "/test_files/javascriptresponse.xml"
+    def test_grade(self):
+        # Compile coffee files into javascript used by the response
         coffee_file_path = os.path.dirname(__file__) + "/test_files/js/*.coffee"
         os.system("coffee -c %s" % (coffee_file_path))
-        test_lcp = lcp.LoncapaProblem(open(problem_file).read(), '1', system=test_system)
-        correct_answers = {'1_2_1': json.dumps({0: 4})}
-        incorrect_answers = {'1_2_1': json.dumps({0: 5})}
 
-        self.assertEquals(test_lcp.grade_answers(incorrect_answers).get_correctness('1_2_1'), 'incorrect')
-        self.assertEquals(test_lcp.grade_answers(correct_answers).get_correctness('1_2_1'), 'correct')
+        problem = self.build_problem(generator_src="test_problem_generator.js",
+                                    grader_src="test_problem_grader.js",
+                                    display_class="TestProblemDisplay",
+                                    display_src="test_problem_display.js",
+                                    param_dict={'value': '4'})
+
+        # Test that we get graded correctly
+        self.assert_grade(problem, json.dumps({0:4}), "correct")
+        self.assert_grade(problem, json.dumps({0:5}), "incorrect")
+
+class NumericalResponseTest(ResponseTest):
+    from response_xml_factory import NumericalResponseXMLFactory
+    xml_factory_class = NumericalResponseXMLFactory
+
+    def test_grade_exact(self):
+        problem = self.build_problem(question_text="What is 2 + 2?",
+                                        explanation="The answer is 4",
+                                        answer=4)
+        correct_responses = ["4", "4.0", "4.00"]
+        incorrect_responses = ["", "3.9", "4.1", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+        
+
+    def test_grade_decimal_tolerance(self):
+        problem = self.build_problem(question_text="What is 2 + 2 approximately?",
+                                        explanation="The answer is 4",
+                                        answer=4,
+                                        tolerance=0.1)
+        correct_responses = ["4.0", "4.00", "4.09", "3.91"] 
+        incorrect_responses = ["", "4.11", "3.89", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+                        
+    def test_grade_percent_tolerance(self):
+        problem = self.build_problem(question_text="What is 2 + 2 approximately?",
+                                        explanation="The answer is 4",
+                                        answer=4,
+                                        tolerance="10%")
+        correct_responses = ["4.0", "4.3", "3.7", "4.30", "3.70"]
+        incorrect_responses = ["", "4.5", "3.5", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_grade_with_script(self):
+        script_text = "computed_response = math.sqrt(4)"
+        problem = self.build_problem(question_text="What is sqrt(4)?",
+                                        explanation="The answer is 2",
+                                        answer="$computed_response",
+                                        script=script_text)
+        correct_responses = ["2", "2.0"]
+        incorrect_responses = ["", "2.01", "1.99", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_grade_with_script_and_tolerance(self):
+        script_text = "computed_response = math.sqrt(4)"
+        problem = self.build_problem(question_text="What is sqrt(4)?",
+                                        explanation="The answer is 2",
+                                        answer="$computed_response",
+                                        tolerance="0.1",
+                                        script=script_text)
+        correct_responses = ["2", "2.0", "2.05", "1.95"]
+        incorrect_responses = ["", "2.11", "1.89", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+
+class CustomResponseTest(ResponseTest):
+    from response_xml_factory import CustomResponseXMLFactory
+    xml_factory_class = CustomResponseXMLFactory
+
+    def test_inline_code(self):
+
+        # For inline code, we directly modify global context variables
+        # 'answers' is a list of answers provided to us
+        # 'correct' is a list we fill in with True/False
+        # 'expect' is given to us (if provided in the XML)
+        inline_script = """correct[0] = 'correct' if (answers['1_2_1'] == expect) else 'incorrect'"""
+        problem = self.build_problem(answer=inline_script, expect="42")
+
+        # Check results
+        self.assert_grade(problem, '42', 'correct')
+        self.assert_grade(problem, '0', 'incorrect')
+
+    def test_inline_message(self):
+
+        # Inline code can update the global messages list
+        # to pass messages to the CorrectMap for a particular input
+        inline_script = """messages[0] = "Test Message" """
+        problem = self.build_problem(answer=inline_script)
+
+        input_dict = {'1_2_1': '0'}
+        msg = problem.grade_answers(input_dict).get_msg('1_2_1')
+        self.assertEqual(msg, "Test Message")
+
+    def test_function_code(self):
+
+        # For function code, we pass in three arguments:
+        # 
+        #   'expect' is the expect attribute of the <customresponse>
+        #
+        #   'answer_given' is the answer the student gave (if there is just one input)
+        #       or an ordered list of answers (if there are multiple inputs)
+        #   
+        #   'student_answers' is a dictionary of answers by input ID
+        #
+        #
+        # The function should return a dict of the form 
+        # { 'ok': BOOL, 'msg': STRING }
+        #
+        script = """def check_func(expect, answer_given, student_answers):
+    return {'ok': answer_given == expect, 'msg': 'Message text'}"""
+
+        problem = self.build_problem(script=script, cfn="check_func", expect="42")
+
+        # Correct answer
+        input_dict = {'1_2_1': '42'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+
+        self.assertEqual(correctness, 'correct')
+        self.assertEqual(msg, "Message text\n")
+
+        # Incorrect answer
+        input_dict = {'1_2_1': '0'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+
+        self.assertEqual(correctness, 'incorrect')
+        self.assertEqual(msg, "Message text\n")
+
+    def test_multiple_inputs(self):
+        # When given multiple inputs, the 'answer_given' argument
+        # to the check_func() is a list of inputs
+        # The sample script below marks the problem as correct
+        # if and only if it receives answer_given=[1,2,3]
+        # (or string values ['1','2','3'])
+        script = """def check_func(expect, answer_given, student_answers):
+    check1 = (int(answer_given[0]) == 1)
+    check2 = (int(answer_given[1]) == 2)
+    check3 = (int(answer_given[2]) == 3)
+    return {'ok': (check1 and check2 and check3),  'msg': 'Message text'}"""
+
+        problem = self.build_problem(script=script, 
+                                    cfn="check_func", num_inputs=3)
+
+        # Grade the inputs (one input incorrect)
+        input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3' }
+        correct_map = problem.grade_answers(input_dict)
+
+        # Everything marked incorrect
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'incorrect')
+        self.assertEqual(correct_map.get_correctness('1_2_2'), 'incorrect')
+        self.assertEqual(correct_map.get_correctness('1_2_3'), 'incorrect')
+
+        # Grade the inputs (everything correct)
+        input_dict = {'1_2_1': '1', '1_2_2': '2', '1_2_3': '3' }
+        correct_map = problem.grade_answers(input_dict)
+
+        # Everything marked incorrect
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct')
+        self.assertEqual(correct_map.get_correctness('1_2_2'), 'correct')
+        self.assertEqual(correct_map.get_correctness('1_2_3'), 'correct')
+
+
+class SchematicResponseTest(ResponseTest):
+    from response_xml_factory import SchematicResponseXMLFactory
+    xml_factory_class = SchematicResponseXMLFactory
+
+    def test_grade(self):
+
+        # Most of the schematic-specific work is handled elsewhere
+        # (in client-side JavaScript)
+        # The <schematicresponse> is responsible only for executing the
+        # Python code in <answer> with *submission* (list)
+        # in the global context.
+
+        # To test that the context is set up correctly,
+        # we create a script that sets *correct* to true
+        # if and only if we find the *submission* (list)
+        script="correct = ['correct' if 'test' in submission[0] else 'incorrect']"
+        problem = self.build_problem(answer=script)
+
+        # The actual dictionary would contain schematic information
+        # sent from the JavaScript simulation
+        submission_dict = {'test': 'test'}
+        input_dict = { '1_2_1': json.dumps(submission_dict) }
+        correct_map = problem.grade_answers(input_dict)
+
+        # Expect that the problem is graded as true
+        # (That is, our script verifies that the context
+        # is what we expect)
+        self.assertEqual(correct_map.get_correctness('1_2_1'), 'correct')