Revert "Revert "refactor: move common/lib/capa/capa to xmodule/capa" (#30762)"

This reverts commit 4463ee751d.

xmodule/capa/capa_problem.py

@@ -0,0 +1,1229 @@
+#
+# File:   capa/capa_problem.py
+#
+# Nomenclature:
+#
+# A capa Problem is a collection of text and capa Response questions.
+# Each Response may have one or more Input entry fields.
+# The capa problem may include a solution.
+#
+"""
+Main module which shows problems (of "capa" type).
+
+This is used by capa_module.
+"""
+
+
+import logging
+import os.path
+import re
+from collections import OrderedDict
+from copy import deepcopy
+from datetime import datetime
+from xml.sax.saxutils import unescape
+
+import six
+
+from lxml import etree
+from pytz import UTC
+
+import xmodule.capa.customrender as customrender
+import xmodule.capa.inputtypes as inputtypes
+import xmodule.capa.responsetypes as responsetypes
+import xmodule.capa.xqueue_interface as xqueue_interface
+from xmodule.capa.correctmap import CorrectMap
+from xmodule.capa.safe_exec import safe_exec
+from xmodule.capa.util import contextualize_text, convert_files_to_filenames, get_course_id_from_capa_module
+from openedx.core.djangolib.markup import HTML, Text
+from openedx.core.lib.edx_six import get_gettext
+from xmodule.stringify import stringify_children
+
+# extra things displayed after "show answers" is pressed
+solution_tags = ['solution']
+
+# fully accessible capa input types
+ACCESSIBLE_CAPA_INPUT_TYPES = [
+    'checkboxgroup',
+    'radiogroup',
+    'choicegroup',
+    'optioninput',
+    'textline',
+    'formulaequationinput',
+    'textbox',
+]
+
+# these get captured as student responses
+response_properties = ["codeparam", "responseparam", "answer", "openendedparam"]
+
+# special problem tags which should be turned into innocuous HTML
+html_transforms = {
+    'problem': {'tag': 'div'},
+    'text': {'tag': 'span'},
+    'math': {'tag': 'span'},
+}
+
+# These should be removed from HTML output, including all subelements
+html_problem_semantics = [
+    "additional_answer",
+    "codeparam",
+    "responseparam",
+    "answer",
+    "script",
+    "hintgroup",
+    "openendedparam",
+    "openendedrubric",
+]
+
+log = logging.getLogger(__name__)
+
+#-----------------------------------------------------------------------------
+# main class for this module
+
+
+class LoncapaSystem(object):
+    """
+    An encapsulation of resources needed from the outside.
+
+    These interfaces are collected here so that a caller of LoncapaProblem
+    can provide these resources however make sense for their environment, and
+    this code can remain independent.
+
+    Attributes:
+        i18n: an object implementing the `gettext.Translations` interface so
+            that we can use `.ugettext` to localize strings.
+
+    See :class:`ModuleSystem` for documentation of other attributes.
+
+    """
+    def __init__(
+        self,
+        ajax_url,
+        anonymous_student_id,
+        cache,
+        can_execute_unsafe_code,
+        get_python_lib_zip,
+        DEBUG,
+        i18n,
+        render_template,
+        resources_fs,
+        seed,      # Why do we do this if we have self.seed?
+        STATIC_URL,
+        xqueue,
+        matlab_api_key=None
+    ):
+        self.ajax_url = ajax_url
+        self.anonymous_student_id = anonymous_student_id
+        self.cache = cache
+        self.can_execute_unsafe_code = can_execute_unsafe_code
+        self.get_python_lib_zip = get_python_lib_zip
+        self.DEBUG = DEBUG                              # pylint: disable=invalid-name
+        self.i18n = i18n
+        self.render_template = render_template
+        self.resources_fs = resources_fs
+        self.seed = seed                     # Why do we do this if we have self.seed?
+        self.STATIC_URL = STATIC_URL                    # pylint: disable=invalid-name
+        self.xqueue = xqueue
+        self.matlab_api_key = matlab_api_key
+
+
+class LoncapaProblem(object):
+    """
+    Main class for capa Problems.
+    """
+    def __init__(self, problem_text, id, capa_system, capa_module,  # pylint: disable=redefined-builtin
+                 state=None, seed=None, minimal_init=False, extract_tree=True):
+        """
+        Initializes capa Problem.
+
+        Arguments:
+
+            problem_text (string): xml defining the problem.
+            id (string): identifier for this problem, often a filename (no spaces).
+            capa_system (LoncapaSystem): LoncapaSystem instance which provides OS,
+                rendering, user context, and other resources.
+            capa_module: instance needed to access runtime/logging
+            state (dict): containing the following keys:
+                - `seed` (int) random number generator seed
+                - `student_answers` (dict) maps input id to the stored answer for that input
+                - 'has_saved_answers' (Boolean) True if the answer has been saved since last submit.
+                - `correct_map` (CorrectMap) a map of each input to their 'correctness'
+                - `done` (bool) indicates whether or not this problem is considered done
+                - `input_state` (dict) maps input_id to a dictionary that holds the state for that input
+            seed (int): random number generator seed.
+            minimal_init (bool): whether to skip pre-processing student answers
+            extract_tree (bool): whether to parse the problem XML and store the HTML
+
+        """
+
+        ## Initialize class variables from state
+        self.do_reset()
+        self.problem_id = id
+        self.capa_system = capa_system
+        self.capa_module = capa_module
+
+        state = state or {}
+
+        # Set seed according to the following priority:
+        #       1. Contained in problem's state
+        #       2. Passed into capa_problem via constructor
+        self.seed = state.get('seed', seed)
+        assert self.seed is not None, "Seed must be provided for LoncapaProblem."
+
+        self.student_answers = state.get('student_answers', {})
+        self.has_saved_answers = state.get('has_saved_answers', False)
+        if 'correct_map' in state:
+            self.correct_map.set_dict(state['correct_map'])
+        self.done = state.get('done', False)
+        self.input_state = state.get('input_state', {})
+
+        # Convert startouttext and endouttext to proper <text></text>
+        problem_text = re.sub(r"startouttext\s*/", "text", problem_text)
+        problem_text = re.sub(r"endouttext\s*/", "/text", problem_text)
+        self.problem_text = problem_text
+
+        # parse problem XML file into an element tree
+        if isinstance(problem_text, six.text_type):
+            # etree chokes on Unicode XML with an encoding declaration
+            problem_text = problem_text.encode('utf-8')
+        self.tree = etree.XML(problem_text)
+
+        try:
+            self.make_xml_compatible(self.tree)
+        except Exception:
+            capa_module = self.capa_module
+            log.exception(
+                "CAPAProblemError: %s, id:%s, data: %s",
+                capa_module.display_name,
+                self.problem_id,
+                capa_module.data
+            )
+            raise
+
+        # handle any <include file="foo"> tags
+        self._process_includes()
+
+        # construct script processor context (eg for customresponse problems)
+        if minimal_init:
+            self.context = {}
+        else:
+            self.context = self._extract_context(self.tree)
+
+        # Pre-parse the XML tree: modifies it to add ID's and perform some in-place
+        # transformations.  This also creates the dict (self.responders) of Response
+        # instances for each question in the problem. The dict has keys = xml subtree of
+        # Response, values = Response instance
+        self.problem_data = self._preprocess_problem(self.tree, minimal_init)
+
+        if not minimal_init:
+            if not self.student_answers:  # True when student_answers is an empty dict
+                self.set_initial_display()
+
+            # dictionary of InputType objects associated with this problem
+            #   input_id string -> InputType object
+            self.inputs = {}
+
+            # Run response late_transforms last (see MultipleChoiceResponse)
+            # Sort the responses to be in *_1 *_2 ... order.
+            responses = list(self.responders.values())
+            responses = sorted(responses, key=lambda resp: int(resp.id[resp.id.rindex('_') + 1:]))
+            for response in responses:
+                if hasattr(response, 'late_transforms'):
+                    response.late_transforms(self)
+
+            if extract_tree:
+                self.extracted_tree = self._extract_html(self.tree)
+
+    def make_xml_compatible(self, tree):
+        """
+        Adjust tree xml in-place for compatibility before creating
+        a problem from it.
+        The idea here is to provide a central point for XML translation,
+        for example, supporting an old XML format. At present, there just two translations.
+
+        1. <additional_answer> compatibility translation:
+        old:    <additional_answer>ANSWER</additional_answer>
+        convert to
+        new:    <additional_answer answer="ANSWER">OPTIONAL-HINT</addional_answer>
+
+        2. <optioninput> compatibility translation:
+        optioninput works like this internally:
+            <optioninput options="('yellow','blue','green')" correct="blue" />
+        With extended hints there is a new <option> tag, like this
+            <option correct="True">blue <optionhint>sky color</optionhint> </option>
+        This translation takes in the new format and synthesizes the old option= attribute
+        so all downstream logic works unchanged with the new <option> tag format.
+        """
+        def is_optioninput_valid(optioninput):
+            """
+            Verifies if a given optioninput xml is valid or not.
+
+            A given optioninput(Dropdown) problem is invalid if it has more than one correct answer.
+
+            Argument:
+                optioninput: dropdown specification tree
+            Returns:
+                boolean: signifying if the optioninput is valid or not.
+            """
+            correct_options = [
+                option.get('correct').upper() == 'TRUE' for option in optioninput.findall('./option')
+            ]
+            return correct_options.count(True) in (0, 1)
+
+        additionals = tree.xpath('//stringresponse/additional_answer')
+        for additional in additionals:
+            answer = additional.get('answer')
+            text = additional.text
+            if not answer and text:  # trigger of old->new conversion
+                additional.set('answer', text)
+                additional.text = ''
+        for optioninput in tree.xpath('//optioninput'):
+            if not is_optioninput_valid(optioninput):
+                raise responsetypes.LoncapaProblemError("Dropdown questions can only have one correct answer.")
+            correct_option = None
+            child_options = []
+            for option_element in optioninput.findall('./option'):
+                text = option_element.text
+                text = text or ''
+                option_name = text.strip()
+                if option_element.get('correct').upper() == 'TRUE':
+                    correct_option = option_name
+                child_options.append("'" + option_name + "'")
+
+            if len(child_options) > 0:
+                options_string = '(' + ','.join(child_options) + ')'
+                optioninput.attrib.update({'options': options_string})
+                if correct_option:
+                    optioninput.attrib.update({'correct': correct_option})
+
+    def do_reset(self):
+        """
+        Reset internal state to unfinished, with no answers
+        """
+        self.student_answers = {}
+        self.has_saved_answers = False
+        self.correct_map = CorrectMap()
+        self.done = False
+
+    def set_initial_display(self):
+        """
+        Set the student's answers to the responders' initial displays, if specified.
+        """
+        initial_answers = {}
+        for responder in self.responders.values():
+            if hasattr(responder, 'get_initial_display'):
+                initial_answers.update(responder.get_initial_display())
+
+        self.student_answers = initial_answers
+
+    def __str__(self):
+        return "LoncapaProblem ({0})".format(self.problem_id)
+
+    def get_state(self):
+        """
+        Stored per-user session data neeeded to:
+            1) Recreate the problem
+            2) Populate any student answers.
+        """
+
+        return {'seed': self.seed,
+                'student_answers': self.student_answers,
+                'has_saved_answers': self.has_saved_answers,
+                'correct_map': self.correct_map.get_dict(),
+                'input_state': self.input_state,
+                'done': self.done}
+
+    def get_max_score(self):
+        """
+        Return the maximum score for this problem.
+        """
+        maxscore = 0
+        for responder in self.responders.values():
+            maxscore += responder.get_max_score()
+        return maxscore
+
+    def calculate_score(self, correct_map=None):
+        """
+        Compute score for this problem.  The score is the number of points awarded.
+        Returns a dictionary {'score': integer, from 0 to get_max_score(),
+                              'total': get_max_score()}.
+
+        Takes an optional correctness map for use in the rescore workflow.
+        """
+        if correct_map is None:
+            correct_map = self.correct_map
+        correct = 0
+        for key in correct_map:
+            try:
+                correct += correct_map.get_npoints(key)
+            except Exception:
+                log.error('key=%s, correct_map = %s', key, correct_map)
+                raise
+
+        return {'score': correct, 'total': self.get_max_score()}
+
+    def update_score(self, score_msg, queuekey):
+        """
+        Deliver grading response (e.g. from async code checking) to
+            the specific ResponseType that requested grading
+
+        Returns an updated CorrectMap
+        """
+        cmap = CorrectMap()
+        cmap.update(self.correct_map)
+        for responder in self.responders.values():
+            if hasattr(responder, 'update_score'):
+                # Each LoncapaResponse will update its specific entries in cmap
+                #   cmap is passed by reference
+                responder.update_score(score_msg, cmap, queuekey)
+        self.correct_map.set_dict(cmap.get_dict())
+        return cmap
+
+    def ungraded_response(self, xqueue_msg, queuekey):
+        """
+        Handle any responses from the xqueue that do not contain grades
+        Will try to pass the queue message to all inputtypes that can handle ungraded responses
+
+        Does not return any value
+        """
+        # check against each inputtype
+        for the_input in self.inputs.values():
+            # if the input type has an ungraded function, pass in the values
+            if hasattr(the_input, 'ungraded_response'):
+                the_input.ungraded_response(xqueue_msg, queuekey)
+
+    def is_queued(self):
+        """
+        Returns True if any part of the problem has been submitted to an external queue
+        (e.g. for grading.)
+        """
+        return any(self.correct_map.is_queued(answer_id) for answer_id in self.correct_map)
+
+    def get_recentmost_queuetime(self):
+        """
+        Returns a DateTime object that represents the timestamp of the most recent
+        queueing request, or None if not queued
+        """
+        if not self.is_queued():
+            return None
+
+        # Get a list of timestamps of all queueing requests, then convert it to a DateTime object
+        queuetime_strs = [
+            self.correct_map.get_queuetime_str(answer_id)
+            for answer_id in self.correct_map
+            if self.correct_map.is_queued(answer_id)
+        ]
+        queuetimes = [
+            datetime.strptime(qt_str, xqueue_interface.dateformat).replace(tzinfo=UTC)
+            for qt_str in queuetime_strs
+        ]
+
+        return max(queuetimes)
+
+    def grade_answers(self, answers):
+        """
+        Grade student responses.  Called by capa_module.submit_problem.
+
+        `answers` is a dict of all the entries from request.POST, but with the first part
+        of each key removed (the string before the first "_").
+
+        Thus, for example, input_ID123 -> ID123, and input_fromjs_ID123 -> fromjs_ID123
+
+        Calls the Response for each question in this problem, to do the actual grading.
+        """
+
+        # if answers include File objects, convert them to filenames.
+        self.student_answers = convert_files_to_filenames(answers)
+        new_cmap = self.get_grade_from_current_answers(answers)
+        self.correct_map = new_cmap  # lint-amnesty, pylint: disable=attribute-defined-outside-init
+        return self.correct_map
+
+    def supports_rescoring(self):
+        """
+        Checks that the current problem definition permits rescoring.
+
+        More precisely, it checks that there are no response types in
+        the current problem that are not fully supported (yet) for rescoring.
+
+        This includes responsetypes for which the student's answer
+        is not properly stored in state, i.e. file submissions.  At present,
+        we have no way to know if an existing response was actually a real
+        answer or merely the filename of a file submitted as an answer.
+
+        It turns out that because rescoring is a background task, limiting
+        it to responsetypes that don't support file submissions also means
+        that the responsetypes are synchronous.  This is convenient as it
+        permits rescoring to be complete when the rescoring call returns.
+        """
+        return all('filesubmission' not in responder.allowed_inputfields for responder in self.responders.values())
+
+    def get_grade_from_current_answers(self, student_answers):
+        """
+        Gets the grade for the currently-saved problem state, but does not save it
+        to the block.
+
+        For new student_answers being graded, `student_answers` is a dict of all the
+        entries from request.POST, but with the first part of each key removed
+        (the string before the first "_").  Thus, for example,
+        input_ID123 -> ID123, and input_fromjs_ID123 -> fromjs_ID123.
+
+        For rescoring, `student_answers` is None.
+
+        Calls the Response for each question in this problem, to do the actual grading.
+        """
+        # old CorrectMap
+        oldcmap = self.correct_map
+
+        # start new with empty CorrectMap
+        newcmap = CorrectMap()
+        # Call each responsetype instance to do actual grading
+        for responder in self.responders.values():
+            # File objects are passed only if responsetype explicitly allows
+            # for file submissions.  But we have no way of knowing if
+            # student_answers contains a proper answer or the filename of
+            # an earlier submission, so for now skip these entirely.
+            # TODO: figure out where to get file submissions when rescoring.
+            if 'filesubmission' in responder.allowed_inputfields and student_answers is None:
+                _ = get_gettext(self.capa_system.i18n)
+                raise Exception(_("Cannot rescore problems with possible file submissions"))
+
+            # use 'student_answers' only if it is provided, and if it might contain a file
+            # submission that would not exist in the persisted "student_answers".
+            if 'filesubmission' in responder.allowed_inputfields and student_answers is not None:
+                results = responder.evaluate_answers(student_answers, oldcmap)
+            else:
+                results = responder.evaluate_answers(self.student_answers, oldcmap)
+            newcmap.update(results)
+
+        return newcmap
+
+    def get_question_answers(self):
+        """
+        Returns a dict of answer_ids to answer values. If we cannot generate
+        an answer (this sometimes happens in customresponses), that answer_id is
+        not included. Called by "show answers" button JSON request
+        (see capa_module)
+        """
+        # dict of (id, correct_answer)
+        answer_map = {}
+        for response in self.responders.keys():  # lint-amnesty, pylint: disable=consider-iterating-dictionary
+            results = self.responder_answers[response]
+            answer_map.update(results)
+
+        # include solutions from <solution>...</solution> stanzas
+        for entry in self.tree.xpath("//" + "|//".join(solution_tags)):
+            answer = etree.tostring(entry).decode('utf-8')
+            if answer:
+                answer_map[entry.get('id')] = contextualize_text(answer, self.context)
+
+        log.debug('answer_map = %s', answer_map)
+        return answer_map
+
+    def get_answer_ids(self):
+        """
+        Return the IDs of all the responses -- these are the keys used for
+        the dicts returned by grade_answers and get_question_answers. (Though
+        get_question_answers may only return a subset of these.
+        """
+        answer_ids = []
+        for response in self.responders.keys():  # lint-amnesty, pylint: disable=consider-iterating-dictionary
+            results = self.responder_answers[response]
+            answer_ids.append(list(results.keys()))
+        return answer_ids
+
+    def find_correct_answer_text(self, answer_id):
+        """
+        Returns the correct answer(s) for the provided answer_id as a single string.
+
+        Arguments::
+            answer_id (str): a string like "98e6a8e915904d5389821a94e48babcf_13_1"
+
+        Returns:
+            str: A string containing the answer or multiple answers separated by commas.
+        """
+        xml_elements = self.tree.xpath('//*[@id="' + answer_id + '"]')
+        if not xml_elements:
+            return
+        xml_element = xml_elements[0]
+        answer_text = xml_element.xpath('@answer')
+        if answer_text:
+            return answer_id[0]
+        if xml_element.tag == 'optioninput':
+            return xml_element.xpath('@correct')[0]
+        return ', '.join(xml_element.xpath('*[@correct="true"]/text()'))
+
+    def find_question_label(self, answer_id):
+        """
+        Obtain the most relevant question text for a particular answer.
+
+        E.g. in a problem like "How much is 2+2?" "Two"/"Three"/"More than three",
+        this function returns the "How much is 2+2?" text.
+
+        It uses, in order:
+        - the question prompt, if the question has one
+        - the <p> or <label> element which precedes the choices (skipping descriptive elements)
+        - a text like "Question 5" if no other name could be found
+
+        Arguments::
+            answer_id: a string like "98e6a8e915904d5389821a94e48babcf_13_1"
+
+        Returns:
+            a string with the question text
+        """
+
+        def generate_default_question_label():
+            """
+            To create question string like "Question 2" by adding "Question" and its position number.
+            For instance 'd2e35c1d294b4ba0b3b1048615605d2a_2_1' contains 2,
+            which is used in question number 1 (see example XML in comment above)
+            There's no question 0 (question IDs start at 1, answer IDs at 2)
+            """
+            question_nr = int(answer_id.split('_')[-2]) - 1
+            return _("Question {}").format(question_nr)
+
+        _ = get_gettext(self.capa_system.i18n)
+        # Some questions define a prompt with this format:   >>This is a prompt<<
+        try:
+            prompt = self.problem_data[answer_id].get('label')
+        except KeyError:
+            prompt = None
+
+        if prompt:
+            question_text = prompt.striptags()
+        else:
+            # If no prompt, then we must look for something resembling a question ourselves
+            #
+            # We have a structure like:
+            #
+            # <p />
+            # <optionresponse id="a0effb954cca4759994f1ac9e9434bf4_2">
+            #   <optioninput id="a0effb954cca4759994f1ac9e9434bf4_3_1" />
+            # <optionresponse>
+            #
+            # Starting from  answer (the optioninput in this example) we go up and backwards
+            xml_elems = self.tree.xpath('//*[@id="' + answer_id + '"]')
+            if len(xml_elems) != 1:
+                return generate_default_question_label()
+
+            xml_elem = xml_elems[0].getparent()
+
+            # Get the element that probably contains the question text
+            questiontext_elem = xml_elem.getprevious()
+
+            # Go backwards looking for a <p> or <label>, but skip <description> because it doesn't
+            # contain the question text.
+            #
+            # E.g if we have this:
+            #   <p /> <description /> <optionresponse /> <optionresponse />
+            #
+            # then from the first optionresponse we'll end with the <p>.
+            # If we start in the second optionresponse, we'll find another response in the way,
+            # stop early, and instead of a question we'll report "Question 2".
+            SKIP_ELEMS = ['description']
+            LABEL_ELEMS = ['p', 'label']
+            while questiontext_elem is not None and questiontext_elem.tag in SKIP_ELEMS:
+                questiontext_elem = questiontext_elem.getprevious()
+
+            if questiontext_elem is not None and questiontext_elem.tag in LABEL_ELEMS:
+                question_text = questiontext_elem.text
+            else:
+                question_text = generate_default_question_label()
+
+        return question_text
+
+    def find_answer_text(self, answer_id, current_answer):
+        """
+        Process a raw answer text to make it more meaningful.
+
+        E.g. in a choice problem like "How much is 2+2?" "Two"/"Three"/"More than three",
+        this function will transform "choice_1" (which is the internal response given by
+        many capa methods) to the human version, e.g. "More than three".
+
+        If the answers are multiple (e.g. because they're from a multiple choice problem),
+        this will join them with a comma.
+
+        If passed a normal string which is already the answer, it doesn't change it.
+
+        TODO merge with response_a11y_data?
+
+        Arguments:
+            answer_id: a string like "98e6a8e915904d5389821a94e48babcf_13_1"
+            current_answer: a data structure as found in `LoncapaProblem.student_answers`
+                which represents the best response we have until now
+
+        Returns:
+            a string with the human version of the response
+        """
+        if isinstance(current_answer, list):
+            # Multiple answers. This case happens e.g. in multiple choice problems
+            answer_text = ", ".join(
+                self.find_answer_text(answer_id, answer) for answer in current_answer
+            )
+
+        elif isinstance(current_answer, six.string_types) and current_answer.startswith('choice_'):
+            # Many problem (e.g. checkbox) report "choice_0" "choice_1" etc.
+            # Here we transform it
+            elems = self.tree.xpath('//*[@id="{answer_id}"]//*[@name="{choice_number}"]'.format(
+                answer_id=answer_id,
+                choice_number=current_answer
+            ))
+            if len(elems) == 0:
+                log.warning("Answer Text Missing for answer id: %s and choice number: %s", answer_id, current_answer)
+                answer_text = "Answer Text Missing"
+            elif len(elems) == 1:
+                choicegroup = elems[0].getparent()
+                input_cls = inputtypes.registry.get_class_for_tag(choicegroup.tag)
+                choices_map = dict(input_cls.extract_choices(choicegroup, self.capa_system.i18n, text_only=True))
+                answer_text = choices_map.get(current_answer, "Answer Text Missing")
+            else:
+                log.warning("Multiple answers found for answer id: %s and choice number: %s", answer_id, current_answer)
+                answer_text = "Multiple answers found"
+
+        elif isinstance(current_answer, six.string_types):
+            # Already a string with the answer
+            answer_text = current_answer
+
+        else:
+            raise NotImplementedError()
+
+        return answer_text or "Answer Text Missing"
+
+    def do_targeted_feedback(self, tree):
+        """
+        Implements targeted-feedback in-place on  <multiplechoiceresponse> --
+        choice-level explanations shown to a student after submission.
+        Does nothing if there is no targeted-feedback attribute.
+        """
+        _ = get_gettext(self.capa_system.i18n)
+        # Note that the modifications has been done, avoiding problems if called twice.
+        if hasattr(self, 'has_targeted'):
+            return
+        self.has_targeted = True  # pylint: disable=attribute-defined-outside-init
+
+        for mult_choice_response in tree.xpath('//multiplechoiceresponse[@targeted-feedback]'):
+            show_explanation = mult_choice_response.get('targeted-feedback') == 'alwaysShowCorrectChoiceExplanation'
+
+            # Grab the first choicegroup (there should only be one within each <multiplechoiceresponse> tag)
+            choicegroup = mult_choice_response.xpath('./choicegroup[@type="MultipleChoice"]')[0]
+            choices_list = list(choicegroup.iter('choice'))
+
+            # Find the student answer key that matches our <choicegroup> id
+            student_answer = self.student_answers.get(choicegroup.get('id'))
+            expl_id_for_student_answer = None
+
+            # Keep track of the explanation-id that corresponds to the student's answer
+            # Also, keep track of the solution-id
+            solution_id = None
+            choice_correctness_for_student_answer = _('Incorrect')
+            for choice in choices_list:
+                if choice.get('name') == student_answer:
+                    expl_id_for_student_answer = choice.get('explanation-id')
+                    if choice.get('correct') == 'true':
+                        choice_correctness_for_student_answer = _('Correct')
+                if choice.get('correct') == 'true':
+                    solution_id = choice.get('explanation-id')
+
+            # Filter out targetedfeedback that doesn't correspond to the answer the student selected
+            # Note: following-sibling will grab all following siblings, so we just want the first in the list
+            targetedfeedbackset = mult_choice_response.xpath('./following-sibling::targetedfeedbackset')
+            if len(targetedfeedbackset) != 0:
+                targetedfeedbackset = targetedfeedbackset[0]
+                targetedfeedbacks = targetedfeedbackset.xpath('./targetedfeedback')
+                # find the legend by id in choicegroup.html for aria-describedby
+                problem_legend_id = str(choicegroup.get('id')) + '-legend'
+                for targetedfeedback in targetedfeedbacks:
+                    screenreadertext = etree.Element("span")
+                    targetedfeedback.insert(0, screenreadertext)
+                    screenreadertext.set('class', 'sr')
+                    screenreadertext.text = choice_correctness_for_student_answer
+                    targetedfeedback.set('role', 'group')
+                    targetedfeedback.set('aria-describedby', problem_legend_id)
+                    # Don't show targeted feedback if the student hasn't answer the problem
+                    # or if the target feedback doesn't match the student's (incorrect) answer
+                    if not self.done or targetedfeedback.get('explanation-id') != expl_id_for_student_answer:
+                        targetedfeedbackset.remove(targetedfeedback)
+
+            # Do not displace the solution under these circumstances
+            if not show_explanation or not self.done:
+                continue
+
+            # The next element should either be <solution> or <solutionset>
+            next_element = targetedfeedbackset.getnext()
+            parent_element = tree
+            solution_element = None
+            if next_element is not None and next_element.tag == 'solution':
+                solution_element = next_element
+            elif next_element is not None and next_element.tag == 'solutionset':
+                solutions = next_element.xpath('./solution')
+                for solution in solutions:
+                    if solution.get('explanation-id') == solution_id:
+                        parent_element = next_element
+                        solution_element = solution
+
+            # If could not find the solution element, then skip the remaining steps below
+            if solution_element is None:
+                continue
+
+            # Change our correct-choice explanation from a "solution explanation" to within
+            # the set of targeted feedback, which means the explanation will render on the page
+            # without the student clicking "Show Answer" or seeing a checkmark next to the correct choice
+            parent_element.remove(solution_element)
+
+            # Add our solution instead to the targetedfeedbackset and change its tag name
+            solution_element.tag = 'targetedfeedback'
+
+            targetedfeedbackset.append(solution_element)
+
+    def get_html(self):
+        """
+        Main method called externally to get the HTML to be rendered for this capa Problem.
+        """
+        self.do_targeted_feedback(self.tree)
+        html = contextualize_text(
+            etree.tostring(self._extract_html(self.tree)).decode('utf-8'),
+            self.context
+        )
+        return html
+
+    def handle_input_ajax(self, data):
+        """
+        InputTypes can support specialized AJAX calls. Find the correct input and pass along the correct data
+
+        Also, parse out the dispatch from the get so that it can be passed onto the input type nicely
+        """
+
+        # pull out the id
+        input_id = data['input_id']
+        if self.inputs[input_id]:
+            dispatch = data['dispatch']
+            return self.inputs[input_id].handle_ajax(dispatch, data)
+        else:
+            log.warning("Could not find matching input for id: %s", input_id)
+            return {}
+
+    # ======= Private Methods Below ========
+
+    def _process_includes(self):
+        """
+        Handle any <include file="foo"> tags by reading in the specified file and inserting it
+        into our XML tree.  Fail gracefully if debugging.
+        """
+        includes = self.tree.findall('.//include')
+        for inc in includes:
+            filename = inc.get('file') if six.PY3 else inc.get('file').decode('utf-8')
+            if filename is not None:
+                try:
+                    # open using LoncapaSystem OSFS filesystem
+                    ifp = self.capa_system.resources_fs.open(filename)
+                except Exception as err:  # lint-amnesty, pylint: disable=broad-except
+                    log.warning(
+                        'Error %s in problem xml include: %s',
+                        err,
+                        etree.tostring(inc, pretty_print=True)
+                    )
+                    log.warning(
+                        'Cannot find file %s in %s', filename, self.capa_system.resources_fs
+                    )
+                    # if debugging, don't fail - just log error
+                    # TODO (vshnayder): need real error handling, display to users
+                    if not self.capa_system.DEBUG:  # lint-amnesty, pylint: disable=no-else-raise
+                        raise
+                    else:
+                        continue
+                try:
+                    # read in and convert to XML
+                    incxml = etree.XML(ifp.read())
+                except Exception as err:  # lint-amnesty, pylint: disable=broad-except
+                    log.warning(
+                        'Error %s in problem xml include: %s',
+                        err,
+                        etree.tostring(inc, pretty_print=True)
+                    )
+                    log.warning('Cannot parse XML in %s', (filename))
+                    # if debugging, don't fail - just log error
+                    # TODO (vshnayder): same as above
+                    if not self.capa_system.DEBUG:  # lint-amnesty, pylint: disable=no-else-raise
+                        raise
+                    else:
+                        continue
+
+                # insert new XML into tree in place of include
+                parent = inc.getparent()
+                parent.insert(parent.index(inc), incxml)
+                parent.remove(inc)
+                log.debug('Included %s into %s', filename, self.problem_id)
+
+    def _extract_system_path(self, script):
+        """
+        Extracts and normalizes additional paths for code execution.
+        For now, there's a default path of data/course/code; this may be removed
+        at some point.
+
+        script : ?? (TODO)
+        """
+
+        DEFAULT_PATH = ['code']
+
+        # Separate paths by :, like the system path.
+        raw_path = script.get('system_path', '').split(":") + DEFAULT_PATH
+
+        # find additional comma-separated modules search path
+        path = []
+
+        for dir in raw_path:  # lint-amnesty, pylint: disable=redefined-builtin
+            if not dir:
+                continue
+
+            # path is an absolute path or a path relative to the data dir
+            dir = os.path.join(self.capa_system.resources_fs.root_path, dir)
+            # Check that we are within the resources_fs tree.
+            reldir = os.path.relpath(dir, self.capa_system.resources_fs.root_path)
+            if ".." in reldir:
+                log.warning("Ignoring Python directory outside of course: %r", dir)
+                continue
+
+            abs_dir = os.path.normpath(dir)
+            path.append(abs_dir)
+
+        return path
+
+    def _extract_context(self, tree):
+        """
+        Extract content of <script>...</script> from the problem.xml file, and exec it in the
+        context of this problem.  Provides ability to randomize problems, and also set
+        variables for problem answer checking.
+
+        Problem XML goes to Python execution context. Runs everything in script tags.
+        """
+        context = {}
+        context['seed'] = self.seed
+        context['anonymous_student_id'] = self.capa_system.anonymous_student_id
+        all_code = ''
+
+        python_path = []
+
+        for script in tree.findall('.//script'):
+
+            stype = script.get('type')
+            if stype:
+                if 'javascript' in stype:
+                    continue    # skip javascript
+                if 'perl' in stype:
+                    continue        # skip perl
+            # TODO: evaluate only python
+
+            for d in self._extract_system_path(script):
+                if d not in python_path and os.path.exists(d):
+                    python_path.append(d)
+
+            XMLESC = {"&apos;": "'", "&quot;": '"'}
+            code = unescape(script.text, XMLESC)
+            all_code += code
+
+        extra_files = []
+        if all_code:
+            # An asset named python_lib.zip can be imported by Python code.
+            zip_lib = self.capa_system.get_python_lib_zip()
+            if zip_lib is not None:
+                extra_files.append(("python_lib.zip", zip_lib))
+                python_path.append("python_lib.zip")
+
+            try:
+                safe_exec(
+                    all_code,
+                    context,
+                    random_seed=self.seed,
+                    python_path=python_path,
+                    extra_files=extra_files,
+                    cache=self.capa_system.cache,
+                    limit_overrides_context=get_course_id_from_capa_module(
+                        self.capa_module
+                    ),
+                    slug=self.problem_id,
+                    unsafely=self.capa_system.can_execute_unsafe_code(),
+                )
+            except Exception as err:
+                log.exception("Error while execing script code: " + all_code)  # lint-amnesty, pylint: disable=logging-not-lazy
+                msg = Text("Error while executing script code: %s" % str(err))
+                raise responsetypes.LoncapaProblemError(msg)
+
+        # Store code source in context, along with the Python path needed to run it correctly.
+        context['script_code'] = all_code
+        context['python_path'] = python_path
+        context['extra_files'] = extra_files or None
+        return context
+
+    def _extract_html(self, problemtree):  # private
+        """
+        Main (private) function which converts Problem XML tree to HTML.
+        Calls itself recursively.
+
+        Returns Element tree of XHTML representation of problemtree.
+        Calls render_html of Response instances to render responses into XHTML.
+
+        Used by get_html.
+        """
+        if not isinstance(problemtree.tag, six.string_types):
+            # Comment and ProcessingInstruction nodes are not Elements,
+            # and we're ok leaving those behind.
+            # BTW: etree gives us no good way to distinguish these things
+            # other than to examine .tag to see if it's a string. :(
+            return
+
+        if (problemtree.tag == 'script' and problemtree.get('type')
+                and 'javascript' in problemtree.get('type')):
+            # leave javascript intact.
+            return deepcopy(problemtree)
+
+        if problemtree.tag in html_problem_semantics:
+            return
+
+        problemid = problemtree.get('id')    # my ID
+
+        if problemtree.tag in inputtypes.registry.registered_tags():
+            # If this is an inputtype subtree, let it render itself.
+            response_data = self.problem_data[problemid]
+
+            status = 'unsubmitted'
+            msg = ''
+            hint = ''
+            hintmode = None
+            input_id = problemtree.get('id')
+            answervariable = None
+            if problemid in self.correct_map:
+                pid = input_id
+
+                # If we're withholding correctness, don't show adaptive hints either.
+                # Note that regular, "demand" hints will be shown, if the course author has added them to the problem.
+                if not self.capa_module.correctness_available():
+                    status = 'submitted'
+                else:
+                    # If the the problem has not been saved since the last submit set the status to the
+                    # current correctness value and set the message as expected. Otherwise we do not want to
+                    # display correctness because the answer may have changed since the problem was graded.
+                    if not self.has_saved_answers:
+                        status = self.correct_map.get_correctness(pid)
+                        msg = self.correct_map.get_msg(pid)
+
+                    hint = self.correct_map.get_hint(pid)
+                    hintmode = self.correct_map.get_hintmode(pid)
+                    answervariable = self.correct_map.get_property(pid, 'answervariable')
+
+            value = ''
+            if self.student_answers and problemid in self.student_answers:
+                value = self.student_answers[problemid]
+
+            if input_id not in self.input_state:
+                self.input_state[input_id] = {}
+
+            # do the rendering
+            state = {
+                'value': value,
+                'status': status,
+                'id': input_id,
+                'input_state': self.input_state[input_id],
+                'answervariable': answervariable,
+                'response_data': response_data,
+                'has_saved_answers': self.has_saved_answers,
+                'feedback': {
+                    'message': msg,
+                    'hint': hint,
+                    'hintmode': hintmode,
+                }
+            }
+
+            input_type_cls = inputtypes.registry.get_class_for_tag(problemtree.tag)
+            # save the input type so that we can make ajax calls on it if we need to
+            self.inputs[input_id] = input_type_cls(self.capa_system, problemtree, state)
+            return self.inputs[input_id].get_html()
+
+        # let each Response render itself
+        if problemtree in self.responders:
+            overall_msg = self.correct_map.get_overall_message()
+            return self.responders[problemtree].render_html(
+                self._extract_html, response_msg=overall_msg
+            )
+
+        # let each custom renderer render itself:
+        if problemtree.tag in customrender.registry.registered_tags():
+            renderer_class = customrender.registry.get_class_for_tag(problemtree.tag)
+            renderer = renderer_class(self.capa_system, problemtree)
+            return renderer.get_html()
+
+        # otherwise, render children recursively, and copy over attributes
+        tree = etree.Element(problemtree.tag)
+        for item in problemtree:
+            item_xhtml = self._extract_html(item)
+            if item_xhtml is not None:
+                tree.append(item_xhtml)
+
+        if tree.tag in html_transforms:
+            tree.tag = html_transforms[problemtree.tag]['tag']
+        else:
+            # copy attributes over if not innocufying
+            for (key, value) in problemtree.items():
+                tree.set(key, value)
+
+        tree.text = problemtree.text
+        tree.tail = problemtree.tail
+
+        return tree
+
+    def _preprocess_problem(self, tree, minimal_init):  # private
+        """
+        Assign IDs to all the responses
+        Assign sub-IDs to all entries (textline, schematic, etc.)
+        Annoted correctness and value
+        In-place transformation
+
+        Also create capa Response instances for each responsetype and save as self.responders
+
+        Obtain all responder answers and save as self.responder_answers dict (key = response)
+        """
+        response_id = 1
+        problem_data = {}
+        self.responders = {}
+        for response in tree.xpath('//' + "|//".join(responsetypes.registry.registered_tags())):
+            responsetype_id = self.problem_id + "_" + str(response_id)
+            # create and save ID for this response
+            response.set('id', responsetype_id)
+            response_id += 1
+
+            answer_id = 1
+            input_tags = inputtypes.registry.registered_tags()
+            inputfields = tree.xpath(
+                "|".join(['//' + response.tag + '[@id=$id]//' + x for x in input_tags]),
+                id=responsetype_id
+            )
+
+            # assign one answer_id for each input type
+            for entry in inputfields:
+                entry.attrib['response_id'] = str(response_id)
+                entry.attrib['answer_id'] = str(answer_id)
+                entry.attrib['id'] = "%s_%i_%i" % (self.problem_id, response_id, answer_id)
+                answer_id = answer_id + 1
+
+            self.response_a11y_data(response, inputfields, responsetype_id, problem_data)
+
+            # instantiate capa Response
+            responsetype_cls = responsetypes.registry.get_class_for_tag(response.tag)
+            responder = responsetype_cls(
+                response, inputfields, self.context, self.capa_system, self.capa_module, minimal_init
+            )
+            # save in list in self
+            self.responders[response] = responder
+
+        if not minimal_init:
+            # get responder answers (do this only once, since there may be a performance cost,
+            # eg with externalresponse)
+            self.responder_answers = {}
+            for response in self.responders.keys():  # lint-amnesty, pylint: disable=consider-iterating-dictionary
+                try:
+                    self.responder_answers[response] = self.responders[response].get_answers()
+                except:
+                    log.debug('responder %s failed to properly return get_answers()',
+                              self.responders[response])  # FIXME
+                    raise
+
+            # <solution>...</solution> may not be associated with any specific response; give
+            # IDs for those separately
+            # TODO: We should make the namespaces consistent and unique (e.g. %s_problem_%i).
+            solution_id = 1
+            for solution in tree.findall('.//solution'):
+                solution.attrib['id'] = "%s_solution_%i" % (self.problem_id, solution_id)
+                solution_id += 1
+
+        return problem_data
+
+    def response_a11y_data(self, response, inputfields, responsetype_id, problem_data):
+        """
+        Construct data to be used for a11y.
+
+        Arguments:
+            response (object): xml response object
+            inputfields (list): list of inputfields in a responsetype
+            responsetype_id (str): responsetype id
+            problem_data (dict): dict to be filled with response data
+        """
+        # if there are no inputtypes then don't do anything
+        if not inputfields:
+            return
+
+        element_to_be_deleted = None
+        label = ''
+
+        if len(inputfields) > 1:
+            response.set('multiple_inputtypes', 'true')
+            group_label_tag = response.find('label')
+            group_description_tags = response.findall('description')
+            group_label_tag_id = 'multiinput-group-label-{}'.format(responsetype_id)
+            group_label_tag_text = ''
+            if group_label_tag is not None:
+                group_label_tag.tag = 'p'
+                group_label_tag.set('id', group_label_tag_id)
+                group_label_tag.set('class', 'multi-inputs-group-label')
+                group_label_tag_text = stringify_children(group_label_tag)
+                response.set('multiinput-group-label-id', group_label_tag_id)
+
+            group_description_ids = []
+            for index, group_description_tag in enumerate(group_description_tags):
+                group_description_tag_id = 'multiinput-group-description-{}-{}'.format(responsetype_id, index)
+                group_description_tag.tag = 'p'
+                group_description_tag.set('id', group_description_tag_id)
+                group_description_tag.set('class', 'multi-inputs-group-description question-description')
+                group_description_ids.append(group_description_tag_id)
+
+            if group_description_ids:
+                response.set('multiinput-group_description_ids', ' '.join(group_description_ids))
+
+            for inputfield in inputfields:
+                problem_data[inputfield.get('id')] = {
+                    'group_label': group_label_tag_text,
+                    'label': HTML(inputfield.attrib.get('label', '')),
+                    'descriptions': {}
+                }
+        else:
+            # Extract label value from <label> tag or label attribute from inside the responsetype
+            responsetype_label_tag = response.find('label')
+            if responsetype_label_tag is not None:
+                label = stringify_children(responsetype_label_tag)
+                # store <label> tag containing question text to delete
+                # it later otherwise question will be rendered twice
+                element_to_be_deleted = responsetype_label_tag
+            elif 'label' in inputfields[0].attrib:
+                # in this case we have old problems with label attribute and p tag having question in it
+                # we will pick the first sibling of responsetype if its a p tag and match the text with
+                # the label attribute text. if they are equal then we will use this text as question.
+                # Get first <p> tag before responsetype, this <p> may contains the question text.
+                p_tag = response.xpath('preceding-sibling::*[1][self::p]')
+
+                if p_tag and p_tag[0].text == inputfields[0].attrib['label']:
+                    label = stringify_children(p_tag[0])
+                    element_to_be_deleted = p_tag[0]
+            else:
+                # In this case the problems don't have tag or label attribute inside the responsetype
+                # so we will get the first preceding label tag w.r.t to this responsetype.
+                # This will take care of those multi-question problems that are not using --- in their markdown.
+                label_tag = response.xpath('preceding-sibling::*[1][self::label]')
+                if label_tag:
+                    label = stringify_children(label_tag[0])
+                    element_to_be_deleted = label_tag[0]
+
+            # delete label or p element only if inputtype is fully accessible
+            if inputfields[0].tag in ACCESSIBLE_CAPA_INPUT_TYPES and element_to_be_deleted is not None:
+                element_to_be_deleted.getparent().remove(element_to_be_deleted)
+
+            # Extract descriptions and set unique id on each description tag
+            description_tags = response.findall('description')
+            description_id = 1
+            descriptions = OrderedDict()
+            for description in description_tags:
+                descriptions[
+                    "description_%s_%i" % (responsetype_id, description_id)
+                ] = HTML(stringify_children(description))
+                response.remove(description)
+                description_id += 1
+
+            problem_data[inputfields[0].get('id')] = {
+                'label': HTML(label.strip()) if label else '',
+                'descriptions': descriptions
+            }

xmodule/capa/checker.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+"""
+Commandline tool for doing operations on Problems
+"""
+
+
+import argparse
+import logging
+import sys
+from io import BytesIO
+
+from calc import UndefinedVariable
+from mako.lookup import TemplateLookup
+from path import Path as path
+
+from xmodule.capa.capa_problem import LoncapaProblem
+
+logging.basicConfig(format="%(levelname)s %(message)s")
+log = logging.getLogger('capa.checker')
+
+
+class DemoSystem(object):  # lint-amnesty, pylint: disable=missing-class-docstring
+    def __init__(self):
+        self.lookup = TemplateLookup(directories=[path(__file__).dirname() / 'templates'])
+        self.DEBUG = True
+
+    def render_template(self, template_filename, dictionary):
+        """
+        Render the specified template with the given dictionary of context data.
+        """
+        return self.lookup.get_template(template_filename).render(**dictionary)
+
+
+def main():  # lint-amnesty, pylint: disable=missing-function-docstring
+    parser = argparse.ArgumentParser(description='Check Problem Files')
+    parser.add_argument("command", choices=['test', 'show'])  # Watch? Render? Open?
+    parser.add_argument("files", nargs="+", type=argparse.FileType('r'))
+    parser.add_argument("--seed", required=False, type=int)
+    parser.add_argument("--log-level", required=False, default="INFO",
+                        choices=['info', 'debug', 'warn', 'error',
+                                 'INFO', 'DEBUG', 'WARN', 'ERROR'])
+
+    args = parser.parse_args()
+    log.setLevel(args.log_level.upper())
+
+    system = DemoSystem()
+
+    for problem_file in args.files:
+        log.info("Opening {0}".format(problem_file.name))
+
+        try:
+            problem = LoncapaProblem(problem_file, "fakeid", seed=args.seed, system=system)  # lint-amnesty, pylint: disable=no-value-for-parameter, unexpected-keyword-arg
+        except Exception as ex:  # lint-amnesty, pylint: disable=broad-except
+            log.error("Could not parse file {0}".format(problem_file.name))
+            log.exception(ex)
+            continue
+
+        if args.command == 'test':
+            command_test(problem)
+        elif args.command == 'show':
+            command_show(problem)
+
+        problem_file.close()
+
+    # In case we want to do anything else here.
+
+
+def command_show(problem):
+    """Display the text for this problem"""
+    print(problem.get_html())
+
+
+def command_test(problem):  # lint-amnesty, pylint: disable=missing-function-docstring
+    # We're going to trap stdout/stderr from the problems (yes, some print)
+    old_stdout, old_stderr = sys.stdout, sys.stderr
+    try:
+        sys.stdout = BytesIO()
+        sys.stderr = BytesIO()
+
+        check_that_suggested_answers_work(problem)
+        check_that_blanks_fail(problem)
+
+        log_captured_output(sys.stdout,
+                            "captured stdout from {0}".format(problem))
+        log_captured_output(sys.stderr,
+                            "captured stderr from {0}".format(problem))
+    except Exception as e:  # lint-amnesty, pylint: disable=broad-except
+        log.exception(e)
+    finally:
+        sys.stdout, sys.stderr = old_stdout, old_stderr
+
+
+def check_that_blanks_fail(problem):
+    """Leaving it blank should never work. Neither should a space."""
+    blank_answers = dict((answer_id, "")
+                         for answer_id in problem.get_question_answers())
+    grading_results = problem.grade_answers(blank_answers)
+    try:
+        assert all(result == 'incorrect' for result in grading_results.values())
+    except AssertionError:
+        log.error("Blank accepted as correct answer in {0} for {1}"
+                  .format(problem,
+                          [answer_id for answer_id, result
+                           in sorted(grading_results.items())
+                           if result != 'incorrect']))
+
+
+def check_that_suggested_answers_work(problem):
+    """Split this up so that we're only used for formula/numeric answers.
+
+    Examples of where this fails:
+    * Displayed answers use units but acceptable ones do not.
+      - L1e0.xml
+      - Presents itself as UndefinedVariable (when it tries to pass to calc)
+    * "a or d" is what's displayed, but only "a" or "d" is accepted, not the
+      string "a or d".
+      - L1-e00.xml
+    """
+    # These are actual answers we get from the responsetypes
+    real_answers = problem.get_question_answers()
+
+    # all_answers is real_answers + blanks for other answer_ids for which the
+    # responsetypes can't provide us pre-canned answers (customresponse)
+    all_answer_ids = problem.get_answer_ids()
+    all_answers = dict((answer_id, real_answers.get(answer_id, ""))
+                       for answer_id in all_answer_ids)
+
+    log.debug("Real answers: {0}".format(real_answers))
+    if real_answers:
+        try:
+            real_results = dict((answer_id, result) for answer_id, result
+                                in problem.grade_answers(all_answers).items()
+                                if answer_id in real_answers)
+            log.debug(real_results)
+            assert(all(result == 'correct'
+                       for answer_id, result in real_results.items()))
+        except UndefinedVariable as uv_exc:
+            log.error("The variable \"{0}\" specified in the ".format(uv_exc) +  # lint-amnesty, pylint: disable=logging-not-lazy
+                      "solution isn't recognized (is it a units measure?).")
+        except AssertionError:
+            log.error("The following generated answers were not accepted for {0}:"
+                      .format(problem))
+            for question_id, result in sorted(real_results.items()):
+                if result != 'correct':
+                    log.error("  {0} = {1}".format(question_id, real_answers[question_id]))
+        except Exception as ex:  # lint-amnesty, pylint: disable=broad-except
+            log.error("Uncaught error in {0}".format(problem))
+            log.exception(ex)
+
+
+def log_captured_output(output_stream, stream_name):  # lint-amnesty, pylint: disable=missing-function-docstring
+    output_stream.seek(0)
+    output_text = output_stream.read()
+    if output_text:
+        log.info("##### Begin {0} #####\n".format(stream_name) + output_text)  # lint-amnesty, pylint: disable=logging-not-lazy
+        log.info("##### End {0} #####".format(stream_name))
+
+
+if __name__ == '__main__':
+    sys.exit(main())

xmodule/capa/correctmap.py

@@ -0,0 +1,196 @@
+# lint-amnesty, pylint: disable=missing-module-docstring
+# -----------------------------------------------------------------------------
+# class used to store graded responses to CAPA questions
+#
+# Used by responsetypes and capa_problem
+
+
+class CorrectMap(object):
+    """
+    Stores map between answer_id and response evaluation result for each question
+    in a capa problem.  The response evaluation result for each answer_id includes
+    (correctness, npoints, msg, hint, hintmode).
+
+    - correctness : 'correct', 'incorrect', 'partially-correct', or 'incomplete'
+    - npoints     : None, or integer specifying number of points awarded for this answer_id
+    - msg         : string (may have HTML) giving extra message response
+                    (displayed below textline or textbox)
+    - hint        : string (may have HTML) giving optional hint
+                    (displayed below textline or textbox, above msg)
+    - hintmode    : one of (None,'on_request','always') criteria for displaying hint
+    - queuestate  : Dict {key:'', time:''} where key is a secret string, and time is a string dump
+                    of a DateTime object in the format '%Y%m%d%H%M%S'. Is None when not queued
+
+    Behaves as a dict.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # start with empty dict
+        self.cmap = {}
+        self.items = self.cmap.items
+        self.keys = self.cmap.keys
+        self.overall_message = ""
+        self.set(*args, **kwargs)
+
+    def __getitem__(self, *args, **kwargs):
+        return self.cmap.__getitem__(*args, **kwargs)
+
+    def __iter__(self):
+        return self.cmap.__iter__()
+
+    # See the documentation for 'set_dict' for the use of kwargs
+    def set(  # lint-amnesty, pylint: disable=missing-function-docstring
+        self,  # lint-amnesty, pylint: disable=unused-argument
+        answer_id=None,
+        correctness=None,
+        npoints=None,
+        msg='',
+        hint='',
+        hintmode=None,
+        queuestate=None,
+        answervariable=None,
+        **kwargs
+    ):
+
+        if answer_id is not None:
+            self.cmap[answer_id] = {
+                'correctness': correctness,
+                'npoints': npoints,
+                'msg': msg,
+                'hint': hint,
+                'hintmode': hintmode,
+                'queuestate': queuestate,
+                'answervariable': answervariable,
+            }
+
+    def __repr__(self):
+        return repr(self.cmap)
+
+    def get_dict(self):
+        """
+        return dict version of self
+        """
+        return self.cmap
+
+    def set_dict(self, correct_map):
+        """
+        Set internal dict of CorrectMap to provided correct_map dict
+
+        correct_map is saved by LMS as a plaintext JSON dump of the correctmap dict. This
+        means that when the definition of CorrectMap (e.g. its properties) are altered,
+        an existing correct_map dict will not coincide with the newest CorrectMap format as
+        defined by self.set.
+
+        For graceful migration, feed the contents of each correct map to self.set, rather than
+        making a direct copy of the given correct_map dict. This way, the common keys between
+        the incoming correct_map dict and the new CorrectMap instance will be written, while
+        mismatched keys will be gracefully ignored.
+
+        Special migration case:
+            If correct_map is a one-level dict, then convert it to the new dict of dicts format.
+
+        """
+        # empty current dict
+        self.__init__()
+
+        if not correct_map:
+            return
+
+        # create new dict entries
+        if not isinstance(list(correct_map.values())[0], dict):
+            # special migration
+            for k in correct_map:
+                self.set(k, correctness=correct_map[k])
+        else:
+            for k in correct_map:
+                self.set(k, **correct_map[k])
+
+    def is_correct(self, answer_id):
+        """
+        Takes an answer_id
+        Returns true if the problem is correct OR partially correct.
+        """
+        if answer_id in self.cmap:
+            return self.cmap[answer_id]['correctness'] in ['correct', 'partially-correct']
+        return None
+
+    def is_partially_correct(self, answer_id):
+        """
+        Takes an answer_id
+        Returns true if the problem is partially correct.
+        """
+        if answer_id in self.cmap:
+            return self.cmap[answer_id]['correctness'] == 'partially-correct'
+        return None
+
+    def is_queued(self, answer_id):
+        return answer_id in self.cmap and self.cmap[answer_id]['queuestate'] is not None
+
+    def is_right_queuekey(self, answer_id, test_key):
+        return self.is_queued(answer_id) and self.cmap[answer_id]['queuestate']['key'] == test_key
+
+    def get_queuetime_str(self, answer_id):
+        if self.cmap[answer_id]['queuestate']:
+            return self.cmap[answer_id]['queuestate']['time']
+        else:
+            return None
+
+    def get_npoints(self, answer_id):
+        """Return the number of points for an answer, used for partial credit."""
+        npoints = self.get_property(answer_id, 'npoints')
+        if npoints is not None:
+            return npoints
+        elif self.is_correct(answer_id):
+            return 1
+        # if not correct and no points have been assigned, return 0
+        return 0
+
+    def set_property(self, answer_id, property, value):  # lint-amnesty, pylint: disable=redefined-builtin
+        if answer_id in self.cmap:
+            self.cmap[answer_id][property] = value
+        else:
+            self.cmap[answer_id] = {property: value}
+
+    def get_property(self, answer_id, property, default=None):  # lint-amnesty, pylint: disable=redefined-builtin
+        if answer_id in self.cmap:
+            return self.cmap[answer_id].get(property, default)
+        return default
+
+    def get_correctness(self, answer_id):
+        return self.get_property(answer_id, 'correctness')
+
+    def get_msg(self, answer_id):
+        return self.get_property(answer_id, 'msg', '')
+
+    def get_hint(self, answer_id):
+        return self.get_property(answer_id, 'hint', '')
+
+    def get_hintmode(self, answer_id):
+        return self.get_property(answer_id, 'hintmode', None)
+
+    def set_hint_and_mode(self, answer_id, hint, hintmode):
+        """
+          - hint     : (string) HTML text for hint
+          - hintmode : (string) mode for hint display ('always' or 'on_request')
+        """
+        self.set_property(answer_id, 'hint', hint)
+        self.set_property(answer_id, 'hintmode', hintmode)
+
+    def update(self, other_cmap):
+        """
+        Update this CorrectMap with the contents of another CorrectMap
+        """
+        if not isinstance(other_cmap, CorrectMap):
+            raise Exception('CorrectMap.update called with invalid argument %s' % other_cmap)
+        self.cmap.update(other_cmap.get_dict())
+        self.set_overall_message(other_cmap.get_overall_message())
+
+    def set_overall_message(self, message_str):
+        """ Set a message that applies to the question as a whole,
+            rather than to individual inputs. """
+        self.overall_message = str(message_str) if message_str else ""
+
+    def get_overall_message(self):
+        """ Retrieve a message that applies to the question as a whole.
+        If no message is available, returns the empty string """
+        return self.overall_message

xmodule/capa/customrender.py

@@ -0,0 +1,185 @@
+"""
+This has custom renderers: classes that know how to render certain problem tags (e.g. <math> and
+<solution>) to html.
+
+These tags do not have state, so they just get passed the system (for access to render_template),
+and the xml element.
+"""
+
+
+import logging
+import re
+import xml.sax.saxutils as saxutils
+
+from django.utils import html
+from lxml import etree
+
+from .registry import TagRegistry
+
+log = logging.getLogger(__name__)
+
+registry = TagRegistry()
+
+
+# -----------------------------------------------------------------------------
+
+
+class MathRenderer(object):  # lint-amnesty, pylint: disable=missing-class-docstring
+    tags = ['math']
+
+    def __init__(self, system, xml):
+        r"""
+        Render math using latex-like formatting.
+
+        Examples:
+
+        <math>$\displaystyle U(r)=4 U_0 $</math>
+        <math>$r_0$</math>
+
+        We convert these to [mathjax]...[/mathjax] and [mathjaxinline]...[/mathjaxinline]
+
+        TODO: use shorter tags (but this will require converting problem XML files!)
+        """
+        self.system = system
+        self.xml = xml
+
+        mathstr = re.sub(r'\$(.*)\$', r'[mathjaxinline]\1[/mathjaxinline]', xml.text)
+        mtag = 'mathjax'
+        if r'\displaystyle' not in mathstr:
+            mtag += 'inline'
+        else:
+            mathstr = mathstr.replace(r'\displaystyle', '')
+        self.mathstr = mathstr.replace('mathjaxinline]', '%s]' % mtag)
+
+    def get_html(self):
+        """
+        Return the contents of this tag, rendered to html, as an etree element.
+        """
+        # TODO: why are there nested html tags here??  Why are there html tags at all, in fact?
+        # xss-lint: disable=python-interpolate-html
+        html = '<html><html>%s</html><html>%s</html></html>' % (  # lint-amnesty, pylint: disable=redefined-outer-name
+            self.mathstr, saxutils.escape(self.xml.tail))
+        try:
+            xhtml = etree.XML(html)
+        except Exception as err:  # lint-amnesty, pylint: disable=broad-except
+            if self.system.DEBUG:
+                # xss-lint: disable=python-interpolate-html
+                msg = '<html><div class="inline-error"><p>Error %s</p>' % (
+                    str(err).replace('<', '&lt;'))  # xss-lint: disable=python-custom-escape
+                # xss-lint: disable=python-interpolate-html
+                msg += ('<p>Failed to construct math expression from <pre>%s</pre></p>' %
+                        html.replace('<', '&lt;'))  # xss-lint: disable=python-custom-escape
+                msg += "</div></html>"
+                log.error(msg)
+                return etree.XML(msg)
+            else:
+                raise
+        return xhtml
+
+
+registry.register(MathRenderer)
+
+
+# -----------------------------------------------------------------------------
+
+
+class SolutionRenderer(object):
+    """
+    A solution is just a <span>...</span> which is given an ID, that is used for displaying an
+    extended answer (a problem "solution") after "show answers" is pressed.
+
+    Note that the solution content is NOT rendered and returned in the HTML. It is obtained by an
+    ajax call.
+    """
+    tags = ['solution']
+
+    def __init__(self, system, xml):
+        self.system = system
+        self.id = xml.get('id')
+
+    def get_html(self):
+        context = {'id': self.id}
+        html = self.system.render_template("solutionspan.html", context)  # lint-amnesty, pylint: disable=redefined-outer-name
+        return etree.XML(html)
+
+
+registry.register(SolutionRenderer)
+
+
+# -----------------------------------------------------------------------------
+
+
+class TargetedFeedbackRenderer(object):
+    """
+    A targeted feedback is just a <span>...</span> that is used for displaying an
+    extended piece of feedback to students if they incorrectly answered a question.
+    """
+    tags = ['targetedfeedback']
+
+    def __init__(self, system, xml):
+        self.system = system
+        self.xml = xml
+
+    def get_html(self):
+        """
+        Return the contents of this tag, rendered to html, as an etree element.
+        """
+        # xss-lint: disable=python-wrap-html
+        html_str = '<section class="targeted-feedback-span"><span>{}</span></section>'.format(
+            etree.tostring(self.xml, encoding='unicode'))
+        try:
+            xhtml = etree.XML(html_str)
+
+        except Exception as err:  # pylint: disable=broad-except
+            if self.system.DEBUG:
+                # xss-lint: disable=python-wrap-html
+                msg = """
+                    <html>
+                      <div class="inline-error">
+                        <p>Error {err}</p>
+                        <p>Failed to construct targeted feedback from <pre>{html}</pre></p>
+                      </div>
+                    </html>
+                """.format(err=html.escape(err), html=html.escape(html_str))
+                log.error(msg)
+                return etree.XML(msg)
+            else:
+                raise
+        return xhtml
+
+
+registry.register(TargetedFeedbackRenderer)
+
+
+# -----------------------------------------------------------------------------
+
+
+class ClarificationRenderer(object):
+    """
+    A clarification appears as an inline icon which reveals more information when the user
+    hovers over it.
+
+    e.g. <p>Enter the ROA <clarification>Return on Assets</clarification> for 2015:</p>
+    """
+    tags = ['clarification']
+
+    def __init__(self, system, xml):
+        self.system = system
+        # Get any text content found inside this tag prior to the first child tag. It may be a string or None type.
+        initial_text = xml.text if xml.text else ''
+        self.inner_html = initial_text + ''.join(etree.tostring(element, encoding='unicode') for element in xml)
+        self.tail = xml.tail
+
+    def get_html(self):
+        """
+        Return the contents of this tag, rendered to html, as an etree element.
+        """
+        context = {'clarification': self.inner_html}
+        html = self.system.render_template("clarification.html", context)  # lint-amnesty, pylint: disable=redefined-outer-name
+        xml = etree.XML(html)
+        # We must include any text that was following our original <clarification>...</clarification> XML node.:
+        xml.tail = self.tail
+        return xml
+
+
+registry.register(ClarificationRenderer)

xmodule/capa/inputtypes.py

@@ -0,0 +1,1820 @@
+#
+# File:   courseware/capa/inputtypes.py
+#
+
+"""
+Module containing the problem elements which render into input objects
+
+- textline
+- textbox (aka codeinput)
+- schematic
+- choicegroup (aka radiogroup, checkboxgroup)
+- imageinput  (for clickable image)
+- optioninput (for option list)
+- filesubmission (upload a file)
+- crystallography
+- vsepr_input
+- drag_and_drop
+- formulaequationinput
+- chemicalequationinput
+
+These are matched by *.html files templates/*.html which are mako templates with the
+actual html.
+
+Each input type takes the xml tree as 'element', the previous answer as 'value', and the
+graded status as'status'
+"""
+
+# TODO: make hints do something
+
+# TODO: make all inputtypes actually render msg
+
+# TODO: remove unused fields (e.g. 'hidden' in a few places)
+
+# TODO: add validators so that content folks get better error messages.
+
+
+# Possible todo: make inline the default for textlines and other "one-line" inputs.  It probably
+# makes sense, but a bunch of problems have markup that assumes block.  Bigger TODO: figure out a
+# general css and layout strategy for capa, document it, then implement it.
+
+
+import json
+import logging
+import re
+import shlex  # for splitting quoted strings
+import sys
+import time
+from datetime import datetime
+
+import bleach
+import html5lib
+import pyparsing
+import six
+from calc.preview import latex_preview
+from chem import chemcalc
+
+from lxml import etree
+from six import text_type
+
+from xmodule.capa.xqueue_interface import XQUEUE_TIMEOUT
+from openedx.core.djangolib.markup import HTML, Text
+from openedx.core.lib import edx_six
+from xmodule.stringify import stringify_children
+
+from . import xqueue_interface
+from .registry import TagRegistry
+from .util import sanitize_html
+
+log = logging.getLogger(__name__)
+
+#########################################################################
+
+registry = TagRegistry()  # pylint: disable=invalid-name
+
+
+class Status(object):
+    """
+    Problem status
+    attributes: classname, display_name, display_tooltip
+    """
+    css_classes = {
+        # status: css class
+        'unsubmitted': 'unanswered',
+        'incomplete': 'incorrect',
+        'queued': 'processing',
+    }
+    __slots__ = ('classname', '_status', 'display_name', 'display_tooltip')
+
+    def __init__(self, status, gettext_func=six.text_type):
+        self.classname = self.css_classes.get(status, status)
+        _ = gettext_func
+        names = {
+            'correct': _('correct'),
+            'incorrect': _('incorrect'),
+            'partially-correct': _('partially correct'),
+            'incomplete': _('incomplete'),
+            'unanswered': _('unanswered'),
+            'unsubmitted': _('unanswered'),
+            'submitted': _('submitted'),
+            'queued': _('processing'),
+        }
+        tooltips = {
+            # Translators: these are tooltips that indicate the state of an assessment question
+            'correct': _('This answer is correct.'),
+            'incorrect': _('This answer is incorrect.'),
+            'partially-correct': _('This answer is partially correct.'),
+            'queued': _('This answer is being processed.'),
+        }
+        tooltips.update(
+            dict.fromkeys(
+                ['incomplete', 'unanswered', 'unsubmitted'], _('Not yet answered.')
+            )
+        )
+        self.display_name = names.get(status, six.text_type(status))
+        self.display_tooltip = tooltips.get(status, '')
+        self._status = status or ''
+
+    def __str__(self):
+        return self._status
+
+    def __repr__(self):
+        return 'Status(%r)' % self._status
+
+    def __eq__(self, other):
+        return self._status == str(other)
+
+    def __hash__(self):
+        return hash(str(self))
+
+
+class Attribute(object):
+    """
+    Allows specifying required and optional attributes for input types.
+    """
+
+    # want to allow default to be None, but also allow required objects
+    _sentinel = object()
+
+    def __init__(self, name, default=_sentinel, transform=None, validate=None, render=True):
+        """
+        Define an attribute
+
+        name (str): then name of the attribute--should be alphanumeric (valid for an XML attribute)
+
+        default (any type): If not specified, this attribute is required.  If specified, use this as the default value
+                        if the attribute is not specified.  Note that this value will not be transformed or validated.
+
+        transform (function str -> any type): If not None, will be called to transform the parsed value into an internal
+                        representation.
+
+        validate (function str-or-return-type-of-tranform -> unit or exception): If not None, called to validate the
+                       (possibly transformed) value of the attribute.  Should raise ValueError with a helpful message if
+                       the value is invalid.
+
+        render (bool): if False, don't include this attribute in the template context.
+        """
+        self.name = name
+        self.default = default
+        self.validate = validate
+        self.transform = transform
+        self.render = render
+
+    def parse_from_xml(self, element):
+        """
+        Given an etree xml element that should have this attribute, do the obvious thing:
+          - look for it.  raise ValueError if not found and required.
+          - transform and validate.  pass through any exceptions from transform or validate.
+        """
+        val = element.get(self.name)
+        if self.default == self._sentinel and val is None:
+            raise ValueError(
+                'Missing required attribute {0}.'.format(self.name)
+            )
+
+        if val is None:
+            # not required, so return default
+            return self.default
+
+        if self.transform is not None:
+            val = self.transform(val)
+
+        if self.validate is not None:
+            self.validate(val)
+
+        return val
+
+
+class InputTypeBase(object):
+    """
+    Abstract base class for input types.
+    """
+
+    template = None
+
+    def __init__(self, system, xml, state):
+        """
+        Instantiate an InputType class.  Arguments:
+
+        - system    : LoncapaModule instance which provides OS, rendering, and user context.
+                      Specifically, must have a render_template function.
+        - xml       : Element tree of this Input element
+        - state     : a dictionary with optional keys:
+                      * 'value'  -- the current value of this input
+                                    (what the student entered last time)
+                      * 'id' -- the id of this input, typically
+                                "{problem-location}_{response-num}_{input-num}"
+                      * 'status' (submitted, unanswered, unsubmitted)
+                      * 'input_state' -- dictionary containing any inputtype-specific state
+                                        that has been preserved
+                      * 'feedback' (dictionary containing keys for hints, errors, or other
+                         feedback from previous attempt.  Specifically 'message', 'hint',
+                         'hintmode'.  If 'hintmode' is 'always', the hint is always displayed.)
+        """
+
+        self.xml = xml
+        self.tag = xml.tag
+        self.capa_system = system
+
+        # NOTE: ID should only come from one place.  If it comes from multiple,
+        # we use state first, XML second (in case the xml changed, but we have
+        # existing state with an old id). Since we don't make this guarantee,
+        # we can swap this around in the future if there's a more logical
+        # order.
+
+        self.input_id = state.get('id', xml.get('id'))
+        if self.input_id is None:
+            raise ValueError(
+                "input id state is None. xml is {0}".format(etree.tostring(xml))
+            )
+
+        self.value = state.get('value', '')
+
+        feedback = state.get('feedback', {})
+        self.msg = feedback.get('message', '')
+        self.hint = feedback.get('hint', '')
+        self.hintmode = feedback.get('hintmode', None)
+        self.input_state = state.get('input_state', {})
+        self.answervariable = state.get('answervariable', None)
+        self.response_data = state.get('response_data')
+
+        # put hint above msg if it should be displayed
+        if self.hintmode == 'always':
+            self.msg = HTML('{hint}<br/>{msg}' if self.msg else '{hint}').format(hint=HTML(self.hint),
+                                                                                 msg=HTML(self.msg))
+
+        self.status = state.get('status', 'unanswered')
+
+        try:
+            # Pre-parse and process all the declared requirements.
+            self.process_requirements()
+
+            # Call subclass "constructor" -- means they don't have to worry about calling
+            # super().__init__, and are isolated from changes to the input
+            # constructor interface.
+            self.setup()
+        except Exception as err:  # lint-amnesty, pylint: disable=broad-except
+            # Something went wrong: add xml to message, but keep the traceback
+            msg = "Error in xml '{x}': {err} ".format(
+                x=etree.tostring(xml), err=text_type(err))
+            six.reraise(Exception, Exception(msg), sys.exc_info()[2])
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Should return a list of Attribute objects (see docstring there for details). Subclasses should override.  e.g.
+
+        return [Attribute('unicorn', True), Attribute('num_dragons', 12, transform=int), ...]
+        """
+        return []
+
+    def process_requirements(self):
+        """
+        Subclasses can declare lists of required and optional attributes.  This
+        function parses the input xml and pulls out those attributes.  This
+        isolates most simple input types from needing to deal with xml parsing at all.
+
+        Processes attributes, putting the results in the self.loaded_attributes dictionary.  Also creates a set
+        self.to_render, containing the names of attributes that should be included in the context by default.
+        """
+        # Use local dicts and sets so that if there are exceptions, we don't
+        # end up in a partially-initialized state.
+        loaded = {}
+        to_render = set()
+        for attribute in self.get_attributes():
+            loaded[attribute.name] = attribute.parse_from_xml(self.xml)
+            if attribute.render:
+                to_render.add(attribute.name)
+
+        self.loaded_attributes = loaded
+        self.to_render = to_render
+
+    def setup(self):
+        """
+        InputTypes should override this to do any needed initialization.  It is called after the
+        constructor, so all base attributes will be set.
+
+        If this method raises an exception, it will be wrapped with a message that includes the
+        problem xml.
+        """
+        pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+    def handle_ajax(self, dispatch, data):
+        """
+        InputTypes that need to handle specialized AJAX should override this.
+
+        Input:
+            dispatch: a string that can be used to determine how to handle the data passed in
+            data: a dictionary containing the data that was sent with the ajax call
+
+        Output:
+            a dictionary object that can be serialized into JSON. This will be sent back to the Javascript.
+        """
+        pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+    def _get_render_context(self):
+        """
+        Should return a dictionary of keys needed to render the template for the input type.
+
+        (Separate from get_html to faciliate testing of logic separately from the rendering)
+
+        The default implementation gets the following rendering context: basic things like value, id, status, and msg,
+        as well as everything in self.loaded_attributes, and everything returned by self._extra_context().
+
+        This means that input types that only parse attributes and pass them to the template get everything they need,
+        and don't need to override this method.
+        """
+        context = {
+            'id': self.input_id,
+            'value': self.value,
+            'status': Status(self.status, edx_six.get_gettext(self.capa_system.i18n)),
+            'msg': self.msg,
+            'response_data': self.response_data,
+            'STATIC_URL': self.capa_system.STATIC_URL,
+            'describedby_html': HTML(''),
+        }
+
+        # Generate the list of ids to be used with the aria-describedby field.
+        descriptions = []
+
+        # If there is trailing text, add the id as the first element to the list before adding the status id
+        if 'trailing_text' in self.loaded_attributes and self.loaded_attributes['trailing_text']:
+            trailing_text_id = 'trailing_text_' + self.input_id
+            descriptions.append(trailing_text_id)
+
+        # Every list should contain the status id
+        status_id = 'status_' + self.input_id
+        descriptions.append(status_id)
+        descriptions.extend(list(self.response_data.get('descriptions', {}).keys()))
+        description_ids = ' '.join(descriptions)
+        context.update(
+            {'describedby_html': HTML('aria-describedby="{}"').format(description_ids)}
+        )
+
+        context.update(
+            (a, v) for (a, v) in six.iteritems(self.loaded_attributes) if a in self.to_render
+        )
+        context.update(self._extra_context())
+        if self.answervariable:
+            context.update({'answervariable': self.answervariable})
+        return context
+
+    def _extra_context(self):
+        """
+        Subclasses can override this to return extra context that should be passed to their templates for rendering.
+
+        This is useful when the input type requires computing new template variables from the parsed attributes.
+        """
+        return {}
+
+    def get_html(self):
+        """
+        Return the html for this input, as an etree element.
+        """
+        if self.template is None:
+            raise NotImplementedError("no rendering template specified for class {0}"
+                                      .format(self.__class__))
+
+        context = self._get_render_context()
+
+        html = self.capa_system.render_template(self.template, context).strip()
+
+        try:
+            output = etree.XML(html)
+        except etree.XMLSyntaxError as ex:
+            # If `html` contains attrs with no values, like `controls` in <audio controls src='smth'/>,
+            # XML parser will raise exception, so wee fallback to html5parser, which will set empty "" values for such attrs.  # lint-amnesty, pylint: disable=line-too-long
+            try:
+                output = html5lib.parseFragment(html, treebuilder='lxml', namespaceHTMLElements=False)[0]
+            except IndexError:
+                raise ex  # lint-amnesty, pylint: disable=raise-missing-from
+
+        return output
+
+    def get_user_visible_answer(self, internal_answer):
+        """
+        Given the internal representation of the answer provided by the user, return the representation of the answer
+        as the user saw it.  Subclasses should override this method if and only if the internal represenation of the
+        answer is different from the answer that is displayed to the user.
+        """
+        return internal_answer
+
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class OptionInput(InputTypeBase):
+    """
+    Input type for selecting and Select option input type.
+
+    Example:
+
+    <optioninput options="('Up','Down')" correct="Up"/><text>The location of the sky</text>
+
+    # TODO: allow ordering to be randomized
+    """
+
+    template = "optioninput.html"
+    tags = ['optioninput']
+
+    @staticmethod
+    def parse_options(options):
+        """
+        Given options string, convert it into an ordered list of (option_id, option_description) tuples, where
+        id==description for now.  TODO: make it possible to specify different id and descriptions.
+        """
+        # convert single quotes inside option values to html encoded string
+        options = re.sub(r"([a-zA-Z])('|\\')([a-zA-Z])", r"\1&#39;\3", options)
+        options = re.sub(r"\\'", r"&#39;", options)  # replace already escaped single quotes
+        # parse the set of possible options
+        if six.PY3:
+            lexer = shlex.shlex(options[1:-1])
+        else:
+            lexer = shlex.shlex(options[1:-1].encode('utf-8'))
+
+        lexer.quotes = "'"
+        # Allow options to be separated by whitespace as well as commas
+        lexer.whitespace = ", "
+
+        # remove quotes
+        # convert escaped single quotes (html encoded string) back to single quotes
+        if six.PY3:
+            tokens = [x[1:-1].replace("&#39;", "'") for x in lexer]
+        else:
+            tokens = [x[1:-1].decode('utf-8').replace("&#39;", "'") for x in lexer]
+
+        # make list of (option_id, option_description), with description=id
+        return [(t, t) for t in tokens]
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Convert options to a convenient format.
+        """
+        return [Attribute('options', transform=cls.parse_options),
+                Attribute('inline', False)]
+
+    def _extra_context(self):
+        """
+        Return extra context.
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        return {'default_option_text': _('Select an option')}
+
+#-----------------------------------------------------------------------------
+
+
+# TODO: consolidate choicegroup, radiogroup, checkboxgroup after discussion of
+# desired semantics.
+
+@registry.register
+class ChoiceGroup(InputTypeBase):
+    """
+    Radio button or checkbox inputs: multiple choice or true/false
+
+    TODO: allow order of choices to be randomized, following lon-capa spec.  Use
+    "location" attribute, ie random, top, bottom.
+
+    Example:
+
+    <choicegroup>
+      <choice correct="false" name="foil1">
+        <text>This is foil One.</text>
+      </choice>
+      <choice correct="false" name="foil2">
+        <text>This is foil Two.</text>
+      </choice>
+      <choice correct="true" name="foil3">
+        <text>This is foil Three.</text>
+      </choice>
+    </choicegroup>
+    """
+    template = "choicegroup.html"
+    tags = ['choicegroup', 'radiogroup', 'checkboxgroup']
+
+    def setup(self):
+        i18n = self.capa_system.i18n
+        # suffix is '' or [] to change the way the input is handled in --as a scalar or vector
+        # value.  (VS: would be nice to make this less hackish).
+        if self.tag == 'choicegroup':
+            self.suffix = ''
+            self.html_input_type = "radio"
+        elif self.tag == 'radiogroup':
+            self.html_input_type = "radio"
+            self.suffix = '[]'
+        elif self.tag == 'checkboxgroup':
+            self.html_input_type = "checkbox"
+            self.suffix = '[]'
+        else:
+            _ = edx_six.get_gettext(i18n)
+            # Translators: 'ChoiceGroup' is an input type and should not be translated.
+            msg = _("ChoiceGroup: unexpected tag {tag_name}").format(tag_name=self.tag)
+            raise Exception(msg)
+
+        self.choices = self.extract_choices(self.xml, i18n)
+        self._choices_map = dict(self.choices,)
+
+    @classmethod
+    def get_attributes(cls):
+        # Make '_' a no-op so we can scrape strings. Using lambda instead of
+        #  `django.utils.translation.ugettext_noop` because Django cannot be imported in this file
+        _ = lambda text: text
+        return [Attribute("show_correctness", "always"),
+                Attribute("submitted_message", _("Answer received."))]
+
+    def _extra_context(self):
+        return {'input_type': self.html_input_type,
+                'choices': self.choices,
+                'name_array_suffix': self.suffix}
+
+    @staticmethod
+    def extract_choices(element, i18n, text_only=False):
+        """
+        Extracts choices for a few input types, such as ChoiceGroup, RadioGroup and
+        CheckboxGroup.
+
+        returns list of (choice_name, choice_text) tuples
+
+        By default it will return any XML tag in the choice (e.g. <choicehint>) unless text_only=True is passed.
+
+        TODO: allow order of choices to be randomized, following lon-capa spec.  Use
+        "location" attribute, ie random, top, bottom.
+        """
+
+        choices = []
+        _ = edx_six.get_gettext(i18n)
+
+        for choice in element:
+            if choice.tag == 'choice':
+                if not text_only:
+                    text = stringify_children(choice)
+                else:
+                    text = choice.text
+                choices.append((choice.get("name"), text))
+            else:
+                if choice.tag != 'compoundhint':
+                    msg = Text('[capa.inputtypes.extract_choices] {error_message}').format(
+                        error_message=Text(
+                            # Translators: '<choice>' and '<compoundhint>' are tag names and should not be translated.
+                            _('Expected a <choice> or <compoundhint> tag; got {given_tag} instead')).format(
+                            given_tag=choice.tag
+                        )
+                    )
+                    raise Exception(msg)
+        return choices
+
+    def get_user_visible_answer(self, internal_answer):
+        if isinstance(internal_answer, six.string_types):
+            return self._choices_map[internal_answer]
+
+        return [self._choices_map[i] for i in internal_answer]
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class JSInput(InputTypeBase):
+    """
+    Inputtype for general javascript inputs. Intended to be used with
+    customresponse.
+    Loads in a sandboxed iframe to help prevent css and js conflicts between
+    frame and top-level window.
+
+    iframe sandbox whitelist:
+        - allow-scripts
+        - allow-popups
+        - allow-forms
+        - allow-pointer-lock
+        - allow-downloads
+
+    This in turn means that the iframe cannot directly access the top-level
+    window elements.
+      Example:
+
+        <jsinput html_file="/static/test.html"
+                 gradefn="grade"
+                 height="500"
+                 width="400"/>
+
+    See the documentation in docs/data/source/course_data_formats/jsinput.rst
+    for more information.
+    """
+
+    template = "jsinput.html"
+    tags = ['jsinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Register the attributes.
+        """
+        return [
+            Attribute('params', None),       # extra iframe params
+            Attribute('html_file', None),
+            Attribute('gradefn', "gradefn"),
+            Attribute('get_statefn', None),  # Function to call in iframe
+                                             #   to get current state.
+            Attribute('initial_state', None),  # JSON string to be used as initial state
+            Attribute('set_statefn', None),  # Function to call iframe to
+                                             #   set state
+            Attribute('width', "400"),       # iframe width
+            Attribute('height', "300"),      # iframe height
+            # Title for the iframe, which should be supplied by the author of the problem. Not translated
+            # because we are in a class method and therefore do not have access to capa_system.i18n.
+            # Note that the default "display name" for the problem is also not translated.
+            Attribute('title', "Problem Remote Content"),
+            # SOP will be relaxed only if this attribute is set to false.
+            Attribute('sop', None)
+        ]
+
+    def _extra_context(self):
+        context = {
+            'jschannel_loader': '{static_url}js/capa/src/jschannel.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+            'jsinput_loader': '{static_url}js/capa/src/jsinput.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+            'saved_state': self.value
+        }
+
+        return context
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class TextLine(InputTypeBase):
+    """
+    A text line input.  Can do math preview if "math"="1" is specified.
+
+    If "trailing_text" is set to a value, then the textline will be shown with
+    the value after the text input, and before the checkmark or any input-specific
+    feedback. HTML will not work, but properly escaped HTML characters will. This
+    feature is useful if you would like to specify a specific type of units for the
+    text input.
+
+    If the hidden attribute is specified, the textline is hidden and the input id
+    is stored in a div with name equal to the value of the hidden attribute.  This
+    is used e.g. for embedding simulations turned into questions.
+
+    Example:
+        <textline math="1" trailing_text="m/s"/>
+
+    This example will render out a text line with a math preview and the text 'm/s'
+    after the end of the text line.
+    """
+
+    template = "textline.html"
+    tags = ['textline']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Register the attributes.
+        """
+        return [
+            Attribute('size', None),
+
+            Attribute('hidden', False),
+            Attribute('inline', False),
+
+            # Attributes below used in setup(), not rendered directly.
+            Attribute('math', None, render=False),
+            # TODO: 'dojs' flag is temporary, for backwards compatibility with
+            # 8.02x
+            Attribute('dojs', None, render=False),
+            Attribute('preprocessorClassName', None, render=False),
+            Attribute('preprocessorSrc', None, render=False),
+            Attribute('trailing_text', ''),
+        ]
+
+    def setup(self):
+        self.do_math = bool(self.loaded_attributes['math'] or
+                            self.loaded_attributes['dojs'])
+
+        # TODO: do math checking using ajax instead of using js, so
+        # that we only have one math parser.
+        self.preprocessor = None
+        if self.do_math:
+            # Preprocessor to insert between raw input and Mathjax
+            self.preprocessor = {
+                'class_name': self.loaded_attributes['preprocessorClassName'],
+                'script_src': self.loaded_attributes['preprocessorSrc'],
+            }
+            if None in list(self.preprocessor.values()):
+                self.preprocessor = None
+
+    def _extra_context(self):
+        return {'do_math': self.do_math,
+                'preprocessor': self.preprocessor, }
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class FileSubmission(InputTypeBase):
+    """
+    Upload some files (e.g. for programming assignments)
+    """
+
+    template = "filesubmission.html"
+    tags = ['filesubmission']
+
+    @staticmethod
+    def parse_files(files):
+        """
+        Given a string like 'a.py b.py c.out', split on whitespace and return as a json list.
+        """
+        return json.dumps(files.split())
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Convert the list of allowed files to a convenient format.
+        """
+        return [Attribute('allowed_files', '[]', transform=cls.parse_files),
+                Attribute('required_files', '[]', transform=cls.parse_files), ]
+
+    def setup(self):
+        """
+        Do some magic to handle queueing status (render as "queued" instead of "incomplete"),
+        pull queue_len from the msg field.  (TODO: get rid of the queue_len hack).
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        submitted_msg = _("Your files have been submitted. As soon as your submission is"
+                          " graded, this message will be replaced with the grader's feedback.")
+        self.submitted_msg = submitted_msg
+
+        # Check if problem has been queued
+        self.queue_len = 0
+        # Flag indicating that the problem has been queued, 'msg' is length of
+        # queue
+        if self.status == 'incomplete':
+            self.status = 'queued'
+            self.queue_len = self.msg
+            self.msg = self.submitted_msg
+
+    def _extra_context(self):
+        return {'queue_len': self.queue_len, }
+
+
+#-----------------------------------------------------------------------------
+
+@registry.register
+class CodeInput(InputTypeBase):
+    """
+    A text area input for code--uses codemirror, does syntax highlighting, special tab handling,
+    etc.
+    """
+
+    template = "codeinput.html"
+    tags = [
+        'codeinput',
+        'textbox',
+        # Another (older) name--at some point we may want to make it use a
+        # non-codemirror editor.
+    ]
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Convert options to a convenient format.
+        """
+        return [
+            Attribute('rows', '30'),
+            Attribute('cols', '80'),
+            Attribute('hidden', ''),
+
+            # For CodeMirror
+            Attribute('mode', 'python'),
+            Attribute('linenumbers', 'true'),
+            # Template expects tabsize to be an int it can do math with
+            Attribute('tabsize', 4, transform=int),
+        ]
+
+    def setup_code_response_rendering(self):
+        """
+        Implement special logic: handle queueing state, and default input.
+        """
+        # if no student input yet, then use the default input given by the
+        # problem
+        if not self.value and self.xml.text:
+            self.value = self.xml.text.strip()
+
+        # Check if problem has been queued
+        self.queue_len = 0  # lint-amnesty, pylint: disable=attribute-defined-outside-init
+        # Flag indicating that the problem has been queued, 'msg' is length of
+        # queue
+        if self.status == 'incomplete':
+            self.status = 'queued'
+            self.queue_len = self.msg  # lint-amnesty, pylint: disable=attribute-defined-outside-init
+            self.msg = bleach.clean(self.submitted_msg)
+
+    def setup(self):
+        """ setup this input type """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        submitted_msg = _("Your answer has been submitted. As soon as your submission is"
+                          " graded, this message will be replaced with the grader's feedback.")
+        self.submitted_msg = submitted_msg
+
+        self.setup_code_response_rendering()
+
+    def _extra_context(self):
+        """
+        Define queue_len, arial_label and code mirror exit message context variables
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        return {
+            'queue_len': self.queue_len,
+            'aria_label': _('{programming_language} editor').format(
+                programming_language=self.loaded_attributes.get('mode')
+            ),
+            'code_mirror_exit_message': _('Press ESC then TAB or click outside of the code editor to exit')
+        }
+
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class MatlabInput(CodeInput):
+    """
+    InputType for handling Matlab code input
+
+    Example:
+     <matlabinput rows="10" cols="80" tabsize="4">
+        Initial Text
+     </matlabinput>
+    """
+    template = "matlabinput.html"
+    tags = ['matlabinput']
+
+    def setup(self):
+        """
+        Handle matlab-specific parsing
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+
+        submitted_msg = _("Submitted. As soon as a response is returned, "
+                          "this message will be replaced by that feedback.")
+        self.submitted_msg = submitted_msg
+
+        self.setup_code_response_rendering()
+
+        xml = self.xml
+
+        self.plot_payload = xml.findtext('./plot_payload')
+        # Check if problem has been queued
+        self.queuename = 'matlab'
+        self.queue_msg = ''
+        # this is only set if we don't have a graded response
+        # the graded response takes precedence
+        if 'queue_msg' in self.input_state and self.status in ['queued', 'incomplete', 'unsubmitted']:
+            self.queue_msg = sanitize_html(self.input_state['queue_msg'])
+
+        if 'queuestate' in self.input_state and self.input_state['queuestate'] == 'queued':
+            self.status = 'queued'
+            self.queue_len = 1
+            self.msg = self.submitted_msg
+            # Handle situation if no response from xqueue arrived during specified time.
+            if ('queuetime' not in self.input_state or
+                    time.time() - self.input_state['queuetime'] > XQUEUE_TIMEOUT):
+                self.queue_len = 0
+                self.status = 'unsubmitted'
+                self.msg = _(
+                    'No response from Xqueue within {xqueue_timeout} seconds. Aborted.'
+                ).format(xqueue_timeout=XQUEUE_TIMEOUT)
+
+    def handle_ajax(self, dispatch, data):
+        """
+        Handle AJAX calls directed to this input
+
+        Args:
+            - dispatch (str) - indicates how we want this ajax call to be handled
+            - data (dict) - dictionary of key-value pairs that contain useful data
+        Returns:
+            dict - 'success' - whether or not we successfully queued this submission
+                 - 'message' - message to be rendered in case of error
+        """
+
+        if dispatch == 'plot':
+            return self._plot_data(data)
+        return {}
+
+    def ungraded_response(self, queue_msg, queuekey):
+        """
+        Handle the response from the XQueue
+        Stores the response in the input_state so it can be rendered later
+
+        Args:
+            - queue_msg (str) - message returned from the queue. The message to be rendered
+            - queuekey (str) - a key passed to the queue. Will be matched up to verify that this is the response we're waiting for  # lint-amnesty, pylint: disable=line-too-long
+
+        Returns:
+            nothing
+        """
+        # check the queuekey against the saved queuekey
+        if('queuestate' in self.input_state and self.input_state['queuestate'] == 'queued'
+                and self.input_state['queuekey'] == queuekey):
+            msg = self._parse_data(queue_msg)
+            # save the queue message so that it can be rendered later
+            self.input_state['queue_msg'] = msg
+            self.input_state['queuestate'] = None
+            self.input_state['queuekey'] = None
+
+    def button_enabled(self):
+        """ Return whether or not we want the 'Test Code' button visible
+
+        Right now, we only want this button to show up when a problem has not been
+        checked.
+        """
+        if self.status in ['correct', 'incorrect', 'partially-correct']:
+            return False
+        else:
+            return True
+
+    def _extra_context(self):
+        """ Set up additional context variables"""
+
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+
+        queue_msg = self.queue_msg
+        if len(self.queue_msg) > 0:  # An empty string cannot be parsed as XML but is okay to include in the template.
+            try:
+                etree.XML(HTML('<div>{0}</div>').format(HTML(self.queue_msg)))
+            except etree.XMLSyntaxError:
+                try:
+                    html5lib.parseFragment(self.queue_msg, treebuilder='lxml', namespaceHTMLElements=False)[0]
+                except (IndexError, ValueError):
+                    # If neither can parse queue_msg, it contains invalid xml.
+                    queue_msg = HTML("<span>{0}</span>").format(_("Error running code."))
+
+        extra_context = {
+            'queue_len': str(self.queue_len),
+            'queue_msg': queue_msg,
+            'button_enabled': self.button_enabled(),
+            'matlab_editor_js': '{static_url}js/vendor/CodeMirror/octave.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+            'msg': sanitize_html(self.msg)  # sanitize msg before rendering into template
+        }
+        return extra_context
+
+    def _parse_data(self, queue_msg):
+        """
+        Parses the message out of the queue message
+        Args:
+            queue_msg (str) - a JSON encoded string
+        Returns:
+            returns the value for the the key 'msg' in queue_msg
+        """
+        try:
+            result = json.loads(queue_msg)
+        except (TypeError, ValueError):
+            log.error("External message should be a JSON serialized dict."
+                      " Received queue_msg = %s", queue_msg)
+            raise
+        msg = result['msg']
+        return msg
+
+    def _plot_data(self, data):
+        """
+        AJAX handler for the plot button
+        Args:
+            get (dict) - should have key 'submission' which contains the student submission
+        Returns:
+            dict - 'success' - whether or not we successfully queued this submission
+                 - 'message' - message to be rendered in case of error
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        # only send data if xqueue exists
+        if self.capa_system.xqueue is None:
+            return {'success': False, 'message': _('Cannot connect to the queue')}
+
+        # pull relevant info out of get
+        response = data['submission']
+
+        # construct xqueue headers
+        qinterface = self.capa_system.xqueue.interface
+        qtime = datetime.utcnow().strftime(xqueue_interface.dateformat)
+        callback_url = self.capa_system.xqueue.construct_callback('ungraded_response')
+        anonymous_student_id = self.capa_system.anonymous_student_id
+        # TODO: Why is this using self.capa_system.seed when we have self.seed???
+        queuekey = xqueue_interface.make_hashkey(str(self.capa_system.seed) + qtime +
+                                                 anonymous_student_id +
+                                                 self.input_id)
+        xheader = xqueue_interface.make_xheader(
+            lms_callback_url=callback_url,
+            lms_key=queuekey,
+            queue_name=self.queuename)
+
+        # construct xqueue body
+        student_info = {
+            'anonymous_student_id': anonymous_student_id,
+            'submission_time': qtime
+        }
+        contents = {
+            'grader_payload': self.plot_payload,
+            'student_info': json.dumps(student_info),
+            'student_response': response,
+            'token': getattr(self.capa_system, 'matlab_api_key', None),
+            'endpoint_version': "2",
+            'requestor_id': anonymous_student_id,
+        }
+
+        (error, msg) = qinterface.send_to_queue(header=xheader,
+                                                body=json.dumps(contents))
+        # save the input state if successful
+        if error == 0:
+            self.input_state['queuekey'] = queuekey
+            self.input_state['queuestate'] = 'queued'
+            self.input_state['queuetime'] = time.time()
+
+        return {'success': error == 0, 'message': msg}
+
+
+#-----------------------------------------------------------------------------
+
+@registry.register
+class Schematic(InputTypeBase):
+    """
+    InputType for the schematic editor
+    """
+
+    template = "schematicinput.html"
+    tags = ['schematic']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Convert options to a convenient format.
+        """
+        return [
+            Attribute('height', None),
+            Attribute('width', None),
+            Attribute('parts', None),
+            Attribute('analyses', None),
+            Attribute('initial_value', None),
+            Attribute('submit_analyses', None),
+        ]
+
+    def _extra_context(self):
+        context = {
+            'setup_script': '{static_url}js/capa/schematicinput.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+        }
+
+        return context
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class ImageInput(InputTypeBase):
+    """
+    Clickable image as an input field.  Element should specify the image source, height,
+    and width, e.g.
+
+    <imageinput src="/static/Figures/Skier-conservation-of-energy.jpg" width="388" height="560" />
+
+    TODO: showanswer for imageimput does not work yet - need javascript to put rectangle
+    over acceptable area of image.
+    """
+
+    template = "imageinput.html"
+    tags = ['imageinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Note: src, height, and width are all required.
+        """
+        return [Attribute('src'),
+                Attribute('height'),
+                Attribute('width'), ]
+
+    def setup(self):
+        """
+        if value is of the form [x,y] then parse it and send along coordinates of previous answer
+        """
+        m = re.match(r'\[([0-9]+),([0-9]+)]',
+                     self.value.strip().replace(' ', ''))
+        if m:
+            # Note: we subtract 15 to compensate for the size of the dot on the screen.
+            # (is a 30x30 image--lms/static/images/green-pointer.png).
+            (self.gx, self.gy) = [int(x) - 15 for x in m.groups()]
+        else:
+            (self.gx, self.gy) = (0, 0)
+
+    def _extra_context(self):
+
+        return {'gx': self.gx,
+                'gy': self.gy}
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class Crystallography(InputTypeBase):
+    """
+    An input for crystallography -- user selects 3 points on the axes, and we get a plane.
+
+    TODO: what's the actual value format?
+    """
+
+    template = "crystallography.html"
+    tags = ['crystallography']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Note: height, width are required.
+        """
+        return [Attribute('height'),
+                Attribute('width'),
+                ]
+
+# -------------------------------------------------------------------------
+
+
+@registry.register
+class VseprInput(InputTypeBase):
+    """
+    Input for molecular geometry--show possible structures, let student
+    pick structure and label positions with atoms or electron pairs.
+    """
+
+    template = 'vsepr_input.html'
+    tags = ['vsepr_input']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Note: height, width, molecules and geometries are required.
+        """
+        return [Attribute('height'),
+                Attribute('width'),
+                Attribute('molecules'),
+                Attribute('geometries'),
+                ]
+
+#-------------------------------------------------------------------------
+
+
+@registry.register
+class ChemicalEquationInput(InputTypeBase):
+    """
+    An input type for entering chemical equations.  Supports live preview.
+
+    Example:
+
+    <chemicalequationinput size="50"/>
+
+    options: size -- width of the textbox.
+    """
+
+    template = "chemicalequationinput.html"
+    tags = ['chemicalequationinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Can set size of text field.
+        """
+        return [Attribute('size', '20'), ]
+
+    def _extra_context(self):
+        """
+        TODO (vshnayder): Get rid of this once we have a standard way of requiring js to be loaded.
+        """
+        return {
+            'previewer': '{static_url}js/capa/chemical_equation_preview.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+        }
+
+    def handle_ajax(self, dispatch, data):
+        """
+        Since we only have chemcalc preview this input, check to see if it
+        matches the corresponding dispatch and send it through if it does
+        """
+        if dispatch == 'preview_chemcalc':
+            return self.preview_chemcalc(data)
+        return {}
+
+    def preview_chemcalc(self, data):
+        """
+        Render an html preview of a chemical formula or equation.  get should
+        contain a key 'formula' and value 'some formula string'.
+
+        Returns a json dictionary:
+        {
+           'preview' : 'the-preview-html' or ''
+           'error' : 'the-error' or ''
+        }
+        """
+
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        result = {'preview': '',
+                  'error': ''}
+        try:
+            formula = data['formula']
+        except KeyError:
+            result['error'] = _("No formula specified.")
+            return result
+
+        try:
+            result['preview'] = chemcalc.render_to_html(formula)
+        except pyparsing.ParseException as err:
+            result['error'] = _("Couldn't parse formula: {error_msg}").format(error_msg=err.msg)
+        except Exception:  # lint-amnesty, pylint: disable=broad-except
+            # this is unexpected, so log
+            log.warning(
+                "Error while previewing chemical formula", exc_info=True)
+            result['error'] = _("Error while rendering preview")
+
+        return result
+
+#-------------------------------------------------------------------------
+
+
+@registry.register
+class FormulaEquationInput(InputTypeBase):
+    """
+    An input type for entering formula equations.  Supports live preview.
+
+    Example:
+
+    <formulaequationinput size="50"/>
+
+    options: size -- width of the textbox.
+             trailing_text -- text to show after the input textbox when
+                              rendered, same as textline (useful for units)
+    """
+
+    template = "formulaequationinput.html"
+    tags = ['formulaequationinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Can set size of text field.
+        """
+        return [
+            Attribute('size', '20'),
+            Attribute('inline', False),
+            Attribute('trailing_text', ''),
+        ]
+
+    def _extra_context(self):
+        """
+        TODO (vshnayder): Get rid of 'previewer' once we have a standard way of requiring js to be loaded.
+        """
+        # `reported_status` is basically `status`, except we say 'unanswered'
+
+        return {
+            'previewer': '{static_url}js/capa/src/formula_equation_preview.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+        }
+
+    def handle_ajax(self, dispatch, get):  # lint-amnesty, pylint: disable=arguments-differ
+        """
+        Since we only have formcalc preview this input, check to see if it
+        matches the corresponding dispatch and send it through if it does
+        """
+        if dispatch == 'preview_formcalc':
+            return self.preview_formcalc(get)
+        return {}
+
+    def preview_formcalc(self, get):
+        """
+        Render an preview of a formula or equation. `get` should
+        contain a key 'formula' with a math expression.
+
+        Returns a json dictionary:
+        {
+           'preview' : '<some latex>' or ''
+           'error' : 'the-error' or ''
+           'request_start' : <time sent with request>
+        }
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        result = {'preview': '',
+                  'error': ''}
+
+        try:
+            formula = get['formula']
+        except KeyError:
+            result['error'] = _("No formula specified.")
+            return result
+
+        result['request_start'] = int(get.get('request_start', 0))
+
+        try:
+            # TODO add references to valid variables and functions
+            # At some point, we might want to mark invalid variables as red
+            # or something, and this is where we would need to pass those in.
+            result['preview'] = latex_preview(formula)
+        except pyparsing.ParseException:
+            result['error'] = _("Sorry, couldn't parse formula")
+            result['formula'] = formula
+        except Exception:  # lint-amnesty, pylint: disable=broad-except
+            # this is unexpected, so log
+            log.warning(
+                "Error while previewing formula", exc_info=True
+            )
+            result['error'] = _("Error while rendering preview")
+
+        return result
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class DragAndDropInput(InputTypeBase):
+    """
+    Input for drag and drop problems. Allows student to drag and drop images and
+    labels to base image.
+    """
+
+    template = 'drag_and_drop_input.html'
+    tags = ['drag_and_drop_input']
+
+    def setup(self):
+
+        def parse(tag, tag_type):
+            """Parses <tag ... /> xml element to dictionary. Stores
+                'draggable' and 'target' tags with attributes to dictionary and
+                returns last.
+
+                Args:
+                    tag: xml etree element <tag...> with attributes
+
+                    tag_type: 'draggable' or 'target'.
+
+                    If tag_type is 'draggable' : all attributes except id
+                    (name or label or icon or can_reuse) are optional
+
+                    If tag_type is 'target' all attributes (name, x, y, w, h)
+                    are required. (x, y) - coordinates of center of target,
+                    w, h - weight and height of target.
+
+                Returns:
+                    Dictionary of vaues of attributes:
+                    dict{'name': smth, 'label': smth, 'icon': smth,
+                    'can_reuse': smth}.
+            """
+            tag_attrs = {}
+            tag_attrs['draggable'] = {
+                'id': Attribute._sentinel,  # lint-amnesty, pylint: disable=protected-access
+                'label': "", 'icon': "",
+                'can_reuse': ""
+            }
+
+            tag_attrs['target'] = {
+                'id': Attribute._sentinel,  # lint-amnesty, pylint: disable=protected-access
+                'x': Attribute._sentinel,  # lint-amnesty, pylint: disable=protected-access
+                'y': Attribute._sentinel,  # lint-amnesty, pylint: disable=protected-access
+                'w': Attribute._sentinel,  # lint-amnesty, pylint: disable=protected-access
+                'h': Attribute._sentinel  # lint-amnesty, pylint: disable=protected-access
+            }
+
+            dic = {}
+
+            for attr_name in tag_attrs[tag_type].keys():
+                dic[attr_name] = Attribute(attr_name,
+                                           default=tag_attrs[tag_type][attr_name]).parse_from_xml(tag)
+
+            if tag_type == 'draggable' and not self.no_labels:
+                dic['label'] = dic['label'] or dic['id']
+
+            if tag_type == 'draggable':
+                dic['target_fields'] = [parse(target, 'target') for target in
+                                        tag.iterchildren('target')]
+
+            return dic
+
+        # add labels to images?:
+        self.no_labels = Attribute('no_labels',
+                                   default="False").parse_from_xml(self.xml)
+
+        to_js = {}
+
+        # image drag and drop onto
+        to_js['base_image'] = Attribute('img').parse_from_xml(self.xml)
+
+        # outline places on image where to drag adn drop
+        to_js['target_outline'] = Attribute('target_outline',
+                                            default="False").parse_from_xml(self.xml)
+        # one draggable per target?
+        to_js['one_per_target'] = Attribute('one_per_target',
+                                            default="True").parse_from_xml(self.xml)
+        # list of draggables
+        to_js['draggables'] = [parse(draggable, 'draggable') for draggable in
+                               self.xml.iterchildren('draggable')]
+        # list of targets
+        to_js['targets'] = [parse(target, 'target') for target in
+                            self.xml.iterchildren('target')]
+
+        # custom background color for labels:
+        label_bg_color = Attribute('label_bg_color',
+                                   default=None).parse_from_xml(self.xml)
+        if label_bg_color:
+            to_js['label_bg_color'] = label_bg_color
+
+        self.loaded_attributes['drag_and_drop_json'] = json.dumps(to_js)
+        self.to_render.add('drag_and_drop_json')
+
+#-------------------------------------------------------------------------
+
+
+@registry.register
+class DesignProtein2dInput(InputTypeBase):
+    """
+    An input type for design of a protein in 2D. Integrates with the Protex java applet.
+
+    Example:
+
+    <designprotein2d width="800" hight="500" target_shape="E;NE;NW;W;SW;E;none" />
+    """
+
+    template = "designprotein2dinput.html"
+    tags = ['designprotein2dinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Note: width, hight, and target_shape are required.
+        """
+        return [Attribute('width'),
+                Attribute('height'),
+                Attribute('target_shape')
+                ]
+
+    def _extra_context(self):
+        context = {
+            'applet_loader': '{static_url}js/capa/design-protein-2d.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+        }
+
+        return context
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class EditAGeneInput(InputTypeBase):
+    """
+        An input type for editing a gene.
+        Integrates with the genex GWT application.
+
+        Example:
+
+        <editagene genex_dna_sequence="CGAT" genex_problem_number="1"/>
+    """
+
+    template = "editageneinput.html"
+    tags = ['editageneinput']
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Note: width, height, and dna_sequencee are required.
+        """
+        return [Attribute('genex_dna_sequence'),
+                Attribute('genex_problem_number')
+                ]
+
+    def _extra_context(self):
+        context = {
+            'applet_loader': '{static_url}js/capa/edit-a-gene.js'.format(
+                static_url=self.capa_system.STATIC_URL),
+        }
+
+        return context
+
+#---------------------------------------------------------------------
+
+
+@registry.register
+class AnnotationInput(InputTypeBase):
+    """
+    Input type for annotations: students can enter some notes or other text
+    (currently ungraded), and then choose from a set of tags/optoins, which are graded.
+
+    Example:
+
+        <annotationinput>
+            <title>Annotation Exercise</title>
+            <text>
+                They are the ones who, at the public assembly, had put savage derangement [ate] into my thinking
+                [phrenes] |89 on that day when I myself deprived Achilles of his honorific portion [geras]
+            </text>
+            <comment>Agamemnon says that ate or 'derangement' was the cause of his actions: why could Zeus say the same thing?</comment>  # lint-amnesty, pylint: disable=line-too-long
+            <comment_prompt>Type a commentary below:</comment_prompt>
+            <tag_prompt>Select one tag:</tag_prompt>
+            <options>
+                <option choice="correct">ate - both a cause and an effect</option>
+                <option choice="incorrect">ate - a cause</option>
+                <option choice="partially-correct">ate - an effect</option>
+            </options>
+        </annotationinput>
+
+    # TODO: allow ordering to be randomized
+    """
+
+    template = "annotationinput.html"
+    tags = ['annotationinput']
+
+    def setup(self):
+        xml = self.xml
+
+        self.debug = False  # set to True to display extra debug info with input
+        self.return_to_annotation = True  # return only works in conjunction with annotatable xmodule
+
+        self.title = xml.findtext('./title', 'Annotation Exercise')
+        self.text = xml.findtext('./text')
+        self.comment = xml.findtext('./comment')
+        self.comment_prompt = xml.findtext(
+            './comment_prompt', 'Type a commentary below:')
+        self.tag_prompt = xml.findtext('./tag_prompt', 'Select one tag:')
+        self.options = self._find_options()
+
+        # Need to provide a value that JSON can parse if there is no
+        # student-supplied value yet.
+        if self.value == '':
+            self.value = 'null'
+
+        self._validate_options()
+
+    def _find_options(self):
+        """ Returns an array of dicts where each dict represents an option. """
+        elements = self.xml.findall('./options/option')
+        return [{
+                'id': index,
+                'description': option.text,
+                'choice': option.get('choice')
+                } for (index, option) in enumerate(elements)]
+
+    def _validate_options(self):
+        """ Raises a ValueError if the choice attribute is missing or invalid. """
+        valid_choices = ('correct', 'partially-correct', 'incorrect')
+        for option in self.options:
+            choice = option['choice']
+            if choice is None:  # lint-amnesty, pylint: disable=no-else-raise
+                raise ValueError('Missing required choice attribute.')
+            elif choice not in valid_choices:
+                raise ValueError('Invalid choice attribute: {0}. Must be one of: {1}'.format(
+                    choice, ', '.join(valid_choices)))
+
+    def _unpack(self, json_value):
+        """ Unpacks the json input state into a dict. """
+        d = json.loads(json_value)
+        if not isinstance(d, dict):
+            d = {}
+
+        comment_value = d.get('comment', '')
+        if not isinstance(comment_value, six.string_types):
+            comment_value = ''
+
+        options_value = d.get('options', [])
+        if not isinstance(options_value, list):
+            options_value = []
+
+        return {
+            'options_value': options_value,
+            'has_options_value': len(options_value) > 0,  # for convenience
+            'comment_value': comment_value,
+        }
+
+    def _extra_context(self):
+        extra_context = {
+            'title': self.title,
+            'text': self.text,
+            'comment': self.comment,
+            'comment_prompt': self.comment_prompt,
+            'tag_prompt': self.tag_prompt,
+            'options': self.options,
+            'return_to_annotation': self.return_to_annotation,
+            'debug': self.debug
+        }
+
+        extra_context.update(self._unpack(self.value))
+
+        return extra_context
+
+
+@registry.register
+class ChoiceTextGroup(InputTypeBase):
+    r"""
+    Groups of radiobutton/checkboxes with text inputs.
+
+    Examples:
+    RadioButton problem
+    <problem>
+      <startouttext/>
+        A person rolls a standard die 100 times and records the results.
+        On the first roll they received a "1". Given this information
+        select the correct choice and fill in numbers to make it accurate.
+      <endouttext/>
+      <choicetextresponse>
+        <radiotextgroup>
+          <choice correct="false">The lowest number rolled was:
+            <decoy_input/> and the highest number rolled was:
+            <decoy_input/> .</choice>
+          <choice correct="true">The lowest number rolled was <numtolerance_input answer="1"/>
+            and there is not enough information to determine the highest number rolled.
+          </choice>
+          <choice correct="false">There is not enough information to determine the lowest
+          number rolled, and the highest number rolled was:
+          <decoy_input/> .
+          </choice>
+        </radiotextgroup>
+      </choicetextresponse>
+    </problem>
+
+    CheckboxProblem:
+    <problem>
+      <startouttext/>
+        A person randomly selects 100 times, with replacement, from the list of numbers \(\sqrt{2}\) , 2, 3, 4 ,5 ,6
+        and records the results. The first number they pick is \(\sqrt{2}\) Given this information
+        select the correct choices and fill in numbers to make them accurate.
+      <endouttext/>
+      <choicetextresponse>
+        <checkboxtextgroup>
+             <choice correct="true">
+                The lowest number selected was <numtolerance_input answer="1.4142" tolerance="0.01"/>
+             </choice>
+             <choice correct="false">
+                The highest number selected was <decoy_input/> .
+            </choice>
+            <choice correct="true">There is not enough information given to determine the highest number
+                which was selected.
+            </choice>
+            <choice correct="false">There is not enough information given to determine the lowest number
+                selected.
+            </choice>
+        </checkboxtextgroup>
+      </choicetextresponse>
+    </problem>
+
+    In the preceding examples the <decoy_input/> is used to generate a textinput html element
+    in the problem's display. Since it is inside of an incorrect choice, no answer given
+    for it will be correct, and thus specifying an answer for it is not needed.
+    """
+    template = "choicetext.html"
+    tags = ['radiotextgroup', 'checkboxtextgroup']
+
+    def setup(self):
+        """
+        Performs setup for the initial rendering of the problem.
+        `self.html_input_type` determines whether this problem is displayed
+        with radiobuttons or checkboxes
+
+        If the initial value of `self.value` is '' change it to {} so that
+        the template has an empty dictionary to work with.
+
+        sets the value of self.choices to be equal to the return value of
+        `self.extract_choices`
+        """
+        self.text_input_values = {}
+        if self.tag == 'radiotextgroup':
+            self.html_input_type = "radio"
+        elif self.tag == 'checkboxtextgroup':
+            self.html_input_type = "checkbox"
+        else:
+            _ = edx_six.get_gettext(self.capa_system.i18n)
+            msg = _("{input_type}: unexpected tag {tag_name}").format(
+                input_type="ChoiceTextGroup", tag_name=self.tag
+            )
+            raise Exception(msg)
+
+        if self.value == '':
+            # Make `value` an empty dictionary, if it currently has an empty
+            # value. This is necessary because the template expects a
+            # dictionary.
+            self.value = {}
+        self.choices = self.extract_choices(self.xml, self.capa_system.i18n)
+
+    @classmethod
+    def get_attributes(cls):
+        """
+        Returns a list of `Attribute` for this problem type
+        """
+        # Make '_' a no-op so we can scrape strings. Using lambda instead of
+        #  `django.utils.translation.ugettext_noop` because Django cannot be imported in this file
+        _ = lambda text: text
+        return [
+            Attribute("show_correctness", "always"),
+            Attribute("submitted_message", _("Answer received.")),
+        ]
+
+    def _extra_context(self):
+        """
+        Returns a dictionary of extra content necessary for rendering this InputType.
+
+        `input_type` is either 'radio' or 'checkbox' indicating whether the choices for
+        this problem will have radiobuttons or checkboxes.
+        """
+        return {
+            'input_type': self.html_input_type,
+            'choices': self.choices
+        }
+
+    @staticmethod
+    def extract_choices(element, i18n):
+        """
+        Extracts choices from the xml for this problem type.
+        If we have xml that is as follows(choice names will have been assigned
+        by now)
+        <radiotextgroup>
+        <choice correct = "true" name ="1_2_1_choiceinput_0bc">
+            The number
+                <numtolerance_input name = "1_2_1_choiceinput0_numtolerance_input_0" answer="5"/>
+            Is the mean of the list.
+        </choice>
+        <choice correct = "false" name = "1_2_1_choiceinput_1bc>
+            False demonstration choice
+        </choice>
+        </radiotextgroup>
+
+        Choices are used for rendering the problem properly
+        The function will setup choices as follows:
+        choices =[
+            ("1_2_1_choiceinput_0bc",
+                [{'type': 'text', 'contents': "The number", 'tail_text': '',
+                  'value': ''
+                  },
+                  {'type': 'textinput',
+                   'contents': "1_2_1_choiceinput0_numtolerance_input_0",
+                   'tail_text': 'Is the mean of the list',
+                   'value': ''
+                   }
+                ]
+             ),
+            ("1_2_1_choiceinput_1bc",
+                [{'type': 'text', 'contents': "False demonstration choice",
+                 'tail_text': '',
+                  'value': ''
+                  }
+                ]
+            )
+        ]
+        """
+
+        _ = edx_six.get_gettext(i18n)
+        choices = []
+
+        for choice in element:
+            if choice.tag != 'choice':
+                msg = Text("[capa.inputtypes.extract_choices] {0}").format(
+                    # Translators: a "tag" is an XML element, such as "<b>" in HTML
+                    Text(_("Expected a {expected_tag} tag; got {given_tag} instead")).format(
+                        # xss-lint: disable=python-wrap-html
+                        expected_tag="<choice>",
+                        given_tag=choice.tag,
+                    )
+                )
+                raise Exception(msg)
+
+            components = []
+            choice_text = ''
+            if choice.text is not None:
+                choice_text += choice.text
+            # Initialize our dict for the next content
+            adder = {
+                'type': 'text',
+                'contents': choice_text,
+                'tail_text': '',
+                'value': ''
+            }
+            components.append(adder)
+
+            for elt in choice:
+                # for elements in the choice e.g. <text> <numtolerance_input>
+                adder = {
+                    'type': 'text',
+                    'contents': '',
+                    'tail_text': '',
+                    'value': ''
+                }
+                tag_type = elt.tag
+                # If the current `elt` is a <numtolerance_input> set the
+                # `adder`type to 'numtolerance_input', and 'contents' to
+                # the `elt`'s name.
+                # Treat decoy_inputs and numtolerance_inputs the same in order
+                # to prevent students from reading the Html and figuring out
+                # which inputs are valid
+                if tag_type in ('numtolerance_input', 'decoy_input'):
+                    # We set this to textinput, so that we get a textinput html
+                    # element.
+                    adder['type'] = 'textinput'
+                    adder['contents'] = elt.get('name')
+                else:
+                    adder['contents'] = elt.text
+
+                # Add any tail text("is the mean" in the example)
+                adder['tail_text'] = elt.tail if elt.tail else ''
+                components.append(adder)
+
+            # Add the tuple for the current choice to the list of choices
+            choices.append((choice.get("name"), components))
+        return choices

xmodule/capa/registry.py

@@ -0,0 +1,60 @@
+"""A registry for finding classes based on tags in the class."""
+
+
+class TagRegistry(object):
+    """
+    A registry mapping tags to handlers.
+
+    (A dictionary with some extra error checking.)
+    """
+    def __init__(self):
+        self._mapping = {}
+
+    def register(self, cls):
+        """
+        Register cls as a supported tag type.  It is expected to define cls.tags as a list of tags
+        that it implements.
+
+        If an already-registered type has registered one of those tags, will raise ValueError.
+
+        If there are no tags in cls.tags, will also raise ValueError.
+        """
+
+        # Do all checks and complain before changing any state.
+        if len(cls.tags) == 0:
+            raise ValueError("No tags specified for class {0}".format(cls.__name__))
+
+        for tag in cls.tags:
+            if tag in self._mapping:
+                other_cls = self._mapping[tag]
+                if cls == other_cls:
+                    # registering the same class multiple times seems silly, but ok
+                    continue
+                raise ValueError(
+                    "Tag {0} already registered by class {1}."
+                    " Can't register for class {2}".format(
+                        tag,
+                        other_cls.__name__,
+                        cls.__name__,
+                    )
+                )
+
+        # Ok, should be good to change state now.
+        for t in cls.tags:
+            self._mapping[t] = cls
+
+        # Returning the cls means we can use this as a decorator.
+        return cls
+
+    def registered_tags(self):
+        """
+        Get a list of all the tags that have been registered.
+        """
+        return list(self._mapping.keys())
+
+    def get_class_for_tag(self, tag):
+        """
+        For any tag in registered_tags(), returns the corresponding class.  Otherwise, will raise
+        KeyError.
+        """
+        return self._mapping[tag]

xmodule/capa/responsetypes.py

@@ -0,0 +1,3930 @@
+#
+# File:   courseware/capa/responsetypes.py
+#
+"""
+Problem response evaluation.  Handles checking of student responses,
+of a variety of types.
+
+Used by capa_problem.py
+"""
+
+# standard library imports
+
+
+import abc
+# TODO: Refactor this code and fix this issue.
+import inspect
+import json
+import logging
+import numbers
+import re
+import sys
+import textwrap
+import traceback
+from cmath import isnan
+from collections import namedtuple
+from datetime import datetime
+from sys import float_info
+
+import html5lib
+import numpy
+import random2 as random
+import requests
+import six
+# specific library imports
+from calc import UndefinedVariable, UnmatchedParenthesis, evaluator
+from django.utils import html
+
+from lxml import etree
+from lxml.html.soupparser import fromstring as fromstring_bs  # uses Beautiful Soup!!! FIXME?
+from pyparsing import ParseException
+from pytz import UTC
+from shapely.geometry import MultiPoint, Point
+from six import text_type
+from six.moves import map, range, zip
+
+import xmodule.capa.safe_exec as safe_exec
+import xmodule.capa.xqueue_interface as xqueue_interface
+from openedx.core.djangolib.markup import HTML, Text
+from openedx.core.lib import edx_six
+from openedx.core.lib.grade_utils import round_away_from_zero
+
+from . import correctmap
+from .registry import TagRegistry
+from .util import (
+    compare_with_tolerance,
+    contextualize_text,
+    convert_files_to_filenames,
+    default_tolerance,
+    find_with_default,
+    get_course_id_from_capa_module,
+    get_inner_html_from_xpath,
+    is_list_of_files
+)
+
+log = logging.getLogger(__name__)
+
+registry = TagRegistry()
+
+CorrectMap = correctmap.CorrectMap
+CORRECTMAP_PY = None
+
+# Make '_' a no-op so we can scrape strings. Using lambda instead of
+#  `django.utils.translation.ugettext_noop` because Django cannot be imported in this file
+_ = lambda text: text
+
+QUESTION_HINT_CORRECT_STYLE = 'feedback-hint-correct'
+QUESTION_HINT_INCORRECT_STYLE = 'feedback-hint-incorrect'
+QUESTION_HINT_LABEL_STYLE = 'hint-label'
+QUESTION_HINT_TEXT_STYLE = 'hint-text'
+QUESTION_HINT_MULTILINE = 'feedback-hint-multi'
+
+#-----------------------------------------------------------------------------
+# Exceptions
+
+
+class LoncapaProblemError(Exception):
+    """
+    Error in specification of a problem
+    """
+    pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+
+class ResponseError(Exception):
+    """
+    Error for failure in processing a response, including
+    exceptions that occur when executing a custom script.
+    """
+    pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+
+class StudentInputError(Exception):
+    """
+    Error for an invalid student input.
+    For example, submitting a string when the problem expects a number
+    """
+    pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+#-----------------------------------------------------------------------------
+#
+# Main base class for CAPA responsetypes
+
+
+class LoncapaResponse(six.with_metaclass(abc.ABCMeta, object)):
+    """
+    Base class for CAPA responsetypes.  Each response type (ie a capa question,
+    which is part of a capa problem) is represented as a subclass,
+    which should provide the following methods:
+
+      - get_score           : evaluate the given student answers, and return a CorrectMap
+      - get_answers         : provide a dict of the expected answers for this problem
+
+    Each subclass must also define the following attributes:
+
+      - tags                : xhtml tags identifying this response (used in auto-registering)
+
+    In addition, these methods are optional:
+
+      - setup_response : find and note the answer input field IDs for the response; called
+                         by __init__
+
+      - check_hint_condition : check to see if the student's answers satisfy a particular
+                               condition for a hint to be displayed
+
+      - render_html          : render this Response as HTML (must return XHTML-compliant string)
+      - __str__              : unicode representation of this Response
+
+    Each response type may also specify the following attributes:
+
+      - max_inputfields      : (int) maximum number of answer input fields (checked in __init__
+                               if not None)
+
+      - allowed_inputfields  : list of allowed input fields (each a string) for this Response
+
+      - required_attributes  : list of required attributes (each a string) on the main
+                               response XML stanza
+
+      - hint_tag             : xhtml tag identifying hint associated with this response inside
+                               hintgroup
+    """
+
+    tags = None
+    hint_tag = None
+    has_partial_credit = False
+    credit_type = []
+
+    max_inputfields = None
+    allowed_inputfields = []
+    required_attributes = []
+
+    # Overridable field that specifies whether this capa response type has support for
+    # for rendering on devices of different sizes and shapes.
+    # By default, we set this to False, allowing subclasses to override as appropriate.
+    multi_device_support = False
+
+    def __init__(self, xml, inputfields, context, system, capa_module, minimal_init):
+        """
+        Init is passed the following arguments:
+
+          - xml         : ElementTree of this Response
+          - inputfields : ordered list of ElementTrees for each input entry field in this Response
+          - context     : script processor context
+          - system      : LoncapaSystem instance which provides OS, rendering, and user context
+          - capa_module : Capa module, to access runtime
+        """
+        self.xml = xml
+        self.inputfields = inputfields
+        self.context = context
+        self.capa_system = system
+        self.capa_module = capa_module  # njp, note None
+
+        self.id = xml.get('id')
+
+        # The LoncapaProblemError messages here do not need to be translated as they are
+        # only displayed to the user when settings.DEBUG is True
+        for abox in inputfields:
+            if abox.tag not in self.allowed_inputfields:
+                msg = "%s: cannot have input field %s" % (
+                    six.text_type(self), abox.tag)
+                msg += "\nSee XML source line %s" % getattr(
+                    xml, 'sourceline', '[unavailable]')
+                raise LoncapaProblemError(msg)
+
+        if self.max_inputfields and len(inputfields) > self.max_inputfields:
+            msg = "%s: cannot have more than %s input fields" % (
+                six.text_type(self), self.max_inputfields)
+            msg += "\nSee XML source line %s" % getattr(
+                xml, 'sourceline', '[unavailable]')
+            raise LoncapaProblemError(msg)
+
+        for prop in self.required_attributes:
+            if not xml.get(prop):
+                msg = "Error in problem specification: %s missing required attribute %s" % (
+                    six.text_type(self), prop)
+                msg += "\nSee XML source line %s" % getattr(
+                    xml, 'sourceline', '[unavailable]')
+                raise LoncapaProblemError(msg)
+
+        # ordered list of answer_id values for this response
+        self.answer_ids = [x.get('id') for x in self.inputfields]
+        if self.max_inputfields == 1:
+            # for convenience
+            self.answer_id = self.answer_ids[0]
+
+        # map input_id -> maxpoints
+        self.maxpoints = {}
+        for inputfield in self.inputfields:
+            # By default, each answerfield is worth 1 point
+            maxpoints = inputfield.get('points', '1')
+            self.maxpoints.update({inputfield.get('id'): int(maxpoints)})
+
+        if not minimal_init:
+            # dict for default answer map (provided in input elements)
+            self.default_answer_map = {}
+            for entry in self.inputfields:
+                answer = entry.get('correct_answer')
+                if answer:
+                    self.default_answer_map[entry.get(
+                        'id')] = contextualize_text(answer, self.context)
+
+            # Does this problem have partial credit?
+            # If so, what kind? Get it as a list of strings.
+            partial_credit = xml.xpath('.')[0].get('partial_credit', default=False)
+
+            if str(partial_credit).lower().strip() == 'false':
+                self.has_partial_credit = False
+                self.credit_type = []
+            else:
+                self.has_partial_credit = True
+                self.credit_type = partial_credit.split(',')
+                self.credit_type = [word.strip().lower() for word in self.credit_type]
+
+            if hasattr(self, 'setup_response'):
+                self.setup_response()
+
+    def get_max_score(self):
+        """
+        Return the total maximum points of all answer fields under this Response
+        """
+        return sum(self.maxpoints.values())
+
+    def render_html(self, renderer, response_msg=''):
+        """
+        Return XHTML Element tree representation of this Response.
+
+        Arguments:
+
+          - renderer : procedure which produces HTML given an ElementTree
+          - response_msg: a message displayed at the end of the Response
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+
+        # response_id = problem_id + response index
+        response_id = self.xml.attrib['id']
+
+        response_index = response_id.split('_')[-1]
+        # Translators: index here could be 1,2,3 and so on
+        response_label = _('Question {index}').format(index=response_index)
+
+        # wrap the content inside a section
+        tree = etree.Element('div')
+        tree.set('class', 'wrapper-problem-response')
+        tree.set('tabindex', '-1')
+        tree.set('aria-label', response_label)
+        tree.set('role', 'group')
+
+        if self.xml.get('multiple_inputtypes'):
+            # add <div> to wrap all inputtypes
+            content = etree.SubElement(tree, 'div')
+            content.set('class', 'multi-inputs-group')
+            content.set('role', 'group')
+
+            if self.xml.get('multiinput-group-label-id'):
+                content.set('aria-labelledby', self.xml.get('multiinput-group-label-id'))
+
+            if self.xml.get('multiinput-group_description_ids'):
+                content.set('aria-describedby', self.xml.get('multiinput-group_description_ids'))
+        else:
+            content = tree
+
+        # problem author can make this span display:inline
+        if self.xml.get('inline', ''):
+            tree.set('class', 'inline')
+
+        for item in self.xml:
+            # call provided procedure to do the rendering
+            item_xhtml = renderer(item)
+            if item_xhtml is not None:
+                content.append(item_xhtml)
+        tree.tail = self.xml.tail
+
+        # Add a <div> for the message at the end of the response
+        if response_msg:
+            content.append(self._render_response_msg_html(response_msg))
+
+        return tree
+
+    def evaluate_answers(self, student_answers, old_cmap):
+        """
+        Called by capa_problem.LoncapaProblem to evaluate student answers, and to
+        generate hints (if any).
+
+        Returns the new CorrectMap, with (correctness,msg,hint,hintmode) for each answer_id.
+        """
+        new_cmap = self.get_score(student_answers)
+        self.get_hints(convert_files_to_filenames(
+            student_answers), new_cmap, old_cmap)
+        return new_cmap
+
+    def make_hint_div(self, hint_node, correct, student_answer, question_tag,
+                      label=None, hint_log=None, multiline_mode=False, log_extra=None):
+        """
+        Returns the extended hint div based on the student_answer
+        or the empty string if, after processing all the arguments, there is no hint.
+        As a side effect, logs a tracking log event detailing the hint.
+
+        Keyword args:
+            * hint_node: xml node such as <optionhint>, holding extended hint text. May be passed in as None.
+            * correct: bool indication if the student answer is correct
+            * student_answer: list length 1 or more of string answers
+              (only checkboxes make multiple answers)
+            * question_tag: string name of enclosing question, e.g. 'choiceresponse'
+            * label: (optional) if None (the default), extracts the label from the node,
+              otherwise using this value. The value '' inhibits labeling of the hint.
+            * hint_log: (optional) hints to be used, passed in as list-of-dict format (below)
+            * multiline_mode: (optional) bool, default False, hints should be shown one-per line
+            * log_extra: (optional) dict items to be injected in the tracking log
+
+        There are many parameters to this method because a variety of extended hint contexts
+        all bottleneck through here. In addition, the caller must provide detailed background
+        information about the hint-trigger to go in the tracking log.
+
+        hint_log format: list of dicts with each hint as a 'text' key. Each dict has extra
+        information for logging, essentially recording the logic which triggered the feedback.
+        Case 1: records which choices triggered
+        e.g. [{'text': 'feedback 1', 'trigger': [{'choice': 'choice_0', 'selected': True}]},...
+        Case 2: a compound hint, the trigger list has 1 or more choices
+        e.g. [{'text': 'a hint', 'trigger':[{'choice': 'choice_0', 'selected': True},
+              {'choice': 'choice_1', 'selected':True}]}]
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        # 1. Establish the hint_texts
+        # This can lead to early-exit if the hint is blank.
+        if not hint_log:
+            # .text can be None when node has immediate children nodes
+            if hint_node is None or (hint_node.text is None and len(hint_node.getchildren()) == 0):
+                return ''
+            hint_text = get_inner_html_from_xpath(hint_node)
+            if not hint_text:
+                return ''
+            hint_log = [{'text': hint_text}]
+        # invariant: xxxx
+
+        # 2. Establish the label:
+        # Passed in, or from the node, or the default
+        if not label and hint_node is not None:
+            label = hint_node.get('label', None)
+        # Tricky: label None means output defaults, while '' means output empty label
+        if label is None:
+            if correct:
+                label = _('Correct:')
+            else:
+                label = _('Incorrect:')
+
+        # self.runtime.track_function('get_demand_hint', event_info)
+        # This this "feedback hint" event
+        event_info = {}
+        event_info['module_id'] = text_type(self.capa_module.location)
+        event_info['problem_part_id'] = self.id
+        event_info['trigger_type'] = 'single'  # maybe be overwritten by log_extra
+        event_info['hint_label'] = label
+        event_info['hints'] = hint_log
+        event_info['correctness'] = correct
+        event_info['student_answer'] = student_answer
+        event_info['question_type'] = question_tag
+        if log_extra:
+            event_info.update(log_extra)
+        self.capa_module.runtime.track_function('edx.problem.hint.feedback_displayed', event_info)
+
+        # Form the div-wrapped hint texts
+        hints_wrap = HTML('').join(
+            [HTML('<div class="{question_hint_text_style}">{hint_content}</div>').format(
+                question_hint_text_style=QUESTION_HINT_TEXT_STYLE,
+                hint_content=HTML(dct.get('text'))
+            ) for dct in hint_log]
+        )
+        if multiline_mode:
+            hints_wrap = HTML('<div class="{question_hint_multiline}">{hints_wrap}</div>').format(
+                question_hint_multiline=QUESTION_HINT_MULTILINE,
+                hints_wrap=hints_wrap
+            )
+        label_wrap = ''
+        if label:
+            label_wrap = HTML('<span class="{question_hint_label_style}">{label} </span>').format(
+                question_hint_label_style=QUESTION_HINT_LABEL_STYLE,
+                label=Text(label)
+            )
+
+        # Establish the outer style
+        if correct:
+            style = QUESTION_HINT_CORRECT_STYLE
+        else:
+            style = QUESTION_HINT_INCORRECT_STYLE
+
+        # Ready to go
+        return HTML('<div class="{st}"><div class="explanation-title">{text}</div>{lwrp}{hintswrap}</div>').format(
+            st=style,
+            text=Text(_("Answer")),
+            lwrp=label_wrap,
+            hintswrap=hints_wrap
+        )
+
+    def get_extended_hints(self, student_answers, new_cmap):
+        """
+        Pull "extended hint" information out the xml based on the student answers,
+        installing it in the new_map for display.
+        Implemented by subclasses that have extended hints.
+        """
+        pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+    def get_hints(self, student_answers, new_cmap, old_cmap):
+        """
+        Generate adaptive hints for this problem based on student answers, the old CorrectMap,
+        and the new CorrectMap produced by get_score.
+
+        Does not return anything.
+
+        Modifies new_cmap, by adding hints to answer_id entries as appropriate.
+        """
+
+        hintfn = None
+        hint_function_provided = False
+        hintgroup = self.xml.find('hintgroup')
+        if hintgroup is not None:
+            hintfn = hintgroup.get('hintfn')
+            if hintfn is not None:
+                hint_function_provided = True
+
+        if hint_function_provided:
+            # if a hint function has been supplied, it will take precedence
+            # Hint is determined by a function defined in the <script> context; evaluate
+            # that function to obtain list of hint, hintmode for each answer_id.
+
+            # The function should take arguments (answer_ids, student_answers, new_cmap, old_cmap)
+            # and it should modify new_cmap as appropriate.
+
+            # We may extend this in the future to add another argument which provides a
+            # callback procedure to a social hint generation system.
+
+            global CORRECTMAP_PY
+            if CORRECTMAP_PY is None:
+                # We need the CorrectMap code for hint functions. No, this is not great.
+                CORRECTMAP_PY = inspect.getsource(correctmap)
+
+            code = (
+                CORRECTMAP_PY + "\n" +
+                self.context['script_code'] + "\n" +
+                textwrap.dedent("""
+                    new_cmap = CorrectMap()
+                    new_cmap.set_dict(new_cmap_dict)
+                    old_cmap = CorrectMap()
+                    old_cmap.set_dict(old_cmap_dict)
+                    {hintfn}(answer_ids, student_answers, new_cmap, old_cmap)
+                    new_cmap_dict.update(new_cmap.get_dict())
+                    old_cmap_dict.update(old_cmap.get_dict())
+                    """).format(hintfn=hintfn)
+            )
+            globals_dict = {
+                'answer_ids': self.answer_ids,
+                'student_answers': student_answers,
+                'new_cmap_dict': new_cmap.get_dict(),
+                'old_cmap_dict': old_cmap.get_dict(),
+            }
+
+            try:
+                safe_exec.safe_exec(
+                    code,
+                    globals_dict,
+                    python_path=self.context['python_path'],
+                    extra_files=self.context['extra_files'],
+                    limit_overrides_context=get_course_id_from_capa_module(
+                        self.capa_module
+                    ),
+                    slug=self.id,
+                    random_seed=self.context['seed'],
+                    unsafely=self.capa_system.can_execute_unsafe_code(),
+                )
+            except Exception as err:
+                _ = edx_six.get_gettext(self.capa_system.i18n)
+                msg = _('Error {err} in evaluating hint function {hintfn}.').format(err=err, hintfn=hintfn)
+                sourcenum = getattr(self.xml, 'sourceline', _('(Source code line unavailable)'))
+                msg += "\n" + _("See XML source line {sourcenum}.").format(sourcenum=sourcenum)
+                raise ResponseError(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+
+            new_cmap.set_dict(globals_dict['new_cmap_dict'])
+            return
+
+        # no hint function provided
+        # hint specified by conditions and text dependent on conditions (a-la Loncapa design)
+        # see http://help.loncapa.org/cgi-bin/fom?file=291
+        #
+        # Example:
+        #
+        # <formularesponse samples="x@-5:5#11" id="11" answer="$answer">
+        #   <textline size="25" />
+        #   <hintgroup>
+        #     <formulahint samples="x@-5:5#11" answer="$wrongans" name="inversegrad"></formulahint>
+        #     <hintpart on="inversegrad">
+        #       <text>You have inverted the slope in the question.  The slope is
+        #             (y2-y1)/(x2 - x1) you have the slope as (x2-x1)/(y2-y1).</text>
+        #     </hintpart>
+        #   </hintgroup>
+        # </formularesponse>
+
+        if (self.hint_tag is not None
+                and hintgroup is not None
+                and hintgroup.find(self.hint_tag) is not None
+                and hasattr(self, 'check_hint_condition')):
+
+            rephints = hintgroup.findall(self.hint_tag)
+            hints_to_show = self.check_hint_condition(  # lint-amnesty, pylint: disable=assignment-from-no-return
+                rephints, student_answers)
+            # can be 'on_request' or 'always' (default)
+
+            hintmode = hintgroup.get('mode', 'always')
+            for hintpart in hintgroup.findall('hintpart'):
+                if hintpart.get('on') in hints_to_show:
+                    hint_text = hintpart.find('text').text
+                    # make the hint appear after the last answer box in this
+                    # response
+                    aid = self.answer_ids[-1]
+                    new_cmap.set_hint_and_mode(aid, hint_text, hintmode)
+            log.debug('after hint: new_cmap = %s', new_cmap)
+        else:
+            # If no other hint form matches, try extended hints.
+            self.get_extended_hints(student_answers, new_cmap)
+
+    @abc.abstractmethod
+    def get_score(self, student_answers):
+        """
+        Return a CorrectMap for the answers expected vs given.  This includes
+        (correctness, npoints, msg) for each answer_id.
+
+        Arguments:
+         - student_answers : dict of (answer_id, answer) where answer = student input (string)
+        """
+        pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+    @abc.abstractmethod
+    def get_answers(self):
+        """
+        Return a dict of (answer_id, answer_text) for each answer for this question.
+        """
+        pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+    def check_hint_condition(self, hxml_set, student_answers):
+        """
+        Return a list of hints to show.
+
+          - hxml_set        : list of Element trees, each specifying a condition to be
+                              satisfied for a named hint condition
+
+          - student_answers : dict of student answers
+
+        Returns a list of names of hint conditions which were satisfied.  Those are used
+        to determine which hints are displayed.
+        """
+        pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+    def setup_response(self):
+        pass
+
+    def __str__(self):
+        return 'LoncapaProblem Response %s' % self.xml.tag
+
+    def _render_response_msg_html(self, response_msg):
+        """ Render a <div> for a message that applies to the entire response.
+
+        *response_msg* is a string, which may contain XHTML markup
+
+        Returns an etree element representing the response message <div> """
+        # First try wrapping the text in a <div> and parsing
+        # it as an XHTML tree
+        try:
+            response_msg_div = etree.XML(HTML('<div>{}</div>').format(HTML(str(response_msg))))
+
+        # If we can't do that, create the <div> and set the message
+        # as the text of the <div>
+        except Exception:  # pylint: disable=broad-except
+            response_msg_div = etree.Element('div')
+            response_msg_div.text = str(response_msg)
+
+        # Set the css class of the message <div>
+        response_msg_div.set("class", "response_message")
+
+        return response_msg_div
+
+    # These accessor functions allow polymorphic checking of response
+    # objects without having to call hasattr() directly.
+    def has_mask(self):
+        """True if the response has masking."""
+        return hasattr(self, '_has_mask')
+
+    def has_shuffle(self):
+        """True if the response has a shuffle transformation."""
+        return hasattr(self, '_has_shuffle')
+
+    def has_answerpool(self):
+        """True if the response has an answer-pool transformation."""
+        return hasattr(self, '_has_answerpool')
+
+
+#-----------------------------------------------------------------------------
+@registry.register
+class ChoiceResponse(LoncapaResponse):
+    """
+    This response type is used when the student chooses from a discrete set of
+    choices. Currently, to be marked correct, all "correct" choices must be
+    supplied by the student, and no extraneous choices may be included.
+
+    This response type allows for two inputtypes: radiogroups and checkbox
+    groups. radiogroups are used when the student should select a single answer,
+    and checkbox groups are used when the student may supply 0+ answers.
+    Note: it is suggested to include a "None of the above" choice when no
+    answer is correct for a checkboxgroup inputtype; this ensures that a student
+    must actively mark something to get credit.
+
+    If two choices are marked as correct with a radiogroup, the student will
+    have no way to get the answer right.
+
+    TODO: Allow for marking choices as 'optional' and 'required', which would
+    not penalize a student for including optional answers and would also allow
+    for questions in which the student can supply one out of a set of correct
+    answers.This would also allow for survey-style questions in which all
+    answers are correct.
+
+    Example:
+
+    <choiceresponse>
+        <radiogroup>
+            <choice correct="false">
+                <text>This is a wrong answer.</text>
+            </choice>
+            <choice correct="true">
+                <text>This is the right answer.</text>
+            </choice>
+            <choice correct="false">
+                <text>This is another wrong answer.</text>
+            </choice>
+        </radiogroup>
+    </choiceresponse>
+
+    In the above example, radiogroup can be replaced with checkboxgroup to allow
+    the student to select more than one choice.
+
+    TODO: In order for the inputtypes to render properly, this response type
+    must run setup_response prior to the input type rendering. Specifically, the
+    choices must be given names. This behavior seems like a leaky abstraction,
+    and it'd be nice to change this at some point.
+
+    """
+    human_name = _('Checkboxes')
+    tags = ['choiceresponse']
+    max_inputfields = 1
+    allowed_inputfields = ['checkboxgroup', 'radiogroup']
+    correct_choices = None
+    multi_device_support = True
+
+    def setup_response(self):
+        self.assign_choice_names()
+
+        self.correct_choices = set()
+        self.incorrect_choices = set()
+        for choice in self.get_choices():
+
+            # contextualize the name and correct attributes
+            name = contextualize_text(choice.get('name'), self.context)
+            correct = contextualize_text(choice.get('correct'), self.context).upper()
+
+            # divide choices into correct and incorrect
+            if correct == 'TRUE':
+                self.correct_choices.add(name)
+            elif correct == 'FALSE':
+                self.incorrect_choices.add(name)
+
+    def get_choices(self):
+        """Returns this response's XML choice elements."""
+        return self.xml.xpath('//*[@id=$id]//choice', id=self.xml.get('id'))
+
+    def assign_choice_names(self):
+        """
+        Initialize name attributes in <choice> tags for this response.
+        """
+
+        for index, choice in enumerate(self.get_choices()):
+            choice.set("name", "choice_" + str(index))
+            # If a choice does not have an id, assign 'A' 'B', .. used by CompoundHint
+            if not choice.get('id'):
+                choice.set("id", chr(ord("A") + index))
+
+    def grade_via_every_decision_counts(self, **kwargs):
+        """
+        Calculates partial credit on the Every Decision Counts scheme.
+        For each correctly selected or correctly blank choice, score 1 point.
+        Divide by total number of choices.
+        Arguments:
+            all_choices, the full set of checkboxes
+            student_answer, what the student actually chose
+            student_non_answers, what the student didn't choose
+        Returns a CorrectMap.
+        """
+
+        all_choices = kwargs['all_choices']
+        student_answer = kwargs['student_answer']
+        student_non_answers = kwargs['student_non_answers']
+
+        edc_max_grade = len(all_choices)
+        edc_current_grade = 0
+
+        good_answers = sum([1 for answer in student_answer if answer in self.correct_choices])
+        good_non_answers = sum([1 for blank in student_non_answers if blank in self.incorrect_choices])
+        edc_current_grade = good_answers + good_non_answers
+
+        return_grade = round_away_from_zero(self.get_max_score() * float(edc_current_grade) / float(edc_max_grade), 2)
+
+        if edc_current_grade == edc_max_grade:
+            return CorrectMap(self.answer_id, correctness='correct')
+        elif edc_current_grade > 0:
+            return CorrectMap(self.answer_id, correctness='partially-correct', npoints=return_grade)
+        else:
+            return CorrectMap(self.answer_id, correctness='incorrect', npoints=0)
+
+    def grade_via_halves(self, **kwargs):
+        """
+        Calculates partial credit on the Halves scheme.
+        If no errors, full credit.
+        If one error, half credit as long as there are 3+ choices
+        If two errors, 1/4 credit as long as there are 5+ choices
+        (If not enough choices, no credit.)
+        Arguments:
+            all_choices, the full set of checkboxes
+            student_answer, what the student actually chose
+            student_non_answers, what the student didn't choose
+        Returns a CorrectMap
+        """
+
+        all_choices = kwargs['all_choices']
+        student_answer = kwargs['student_answer']
+        student_non_answers = kwargs['student_non_answers']
+
+        halves_error_count = 0
+
+        incorrect_answers = sum([1 for answer in student_answer if answer in self.incorrect_choices])
+        missed_answers = sum([1 for blank in student_non_answers if blank in self.correct_choices])
+        halves_error_count = incorrect_answers + missed_answers
+
+        if halves_error_count == 0:
+            return_grade = self.get_max_score()
+            return CorrectMap(self.answer_id, correctness='correct', npoints=return_grade)
+        elif halves_error_count == 1 and len(all_choices) > 2:
+            return_grade = round_away_from_zero(self.get_max_score() / 2.0, 2)
+            return CorrectMap(self.answer_id, correctness='partially-correct', npoints=return_grade)
+        elif halves_error_count == 2 and len(all_choices) > 4:
+            return_grade = round_away_from_zero(self.get_max_score() / 4.0, 2)
+            return CorrectMap(self.answer_id, correctness='partially-correct', npoints=return_grade)
+        else:
+            return CorrectMap(self.answer_id, 'incorrect')
+
+    def grade_without_partial_credit(self, **kwargs):
+        """
+        Standard grading for checkbox problems.
+        100% credit if all choices are correct; 0% otherwise
+        Arguments: student_answer, which is the items the student actually chose
+        """
+
+        student_answer = kwargs['student_answer']
+
+        required_selected = len(self.correct_choices - student_answer) == 0
+        no_extra_selected = len(student_answer - self.correct_choices) == 0
+
+        correct = required_selected & no_extra_selected
+
+        if correct:
+            return CorrectMap(self.answer_id, 'correct')
+        else:
+            return CorrectMap(self.answer_id, 'incorrect')
+
+    def get_score(self, student_answers):
+
+        # Setting up answer sets:
+        #  all_choices: the full set of checkboxes
+        #  student_answer: what the student actually chose (note no "s")
+        #  student_non_answers: what they didn't choose
+        #  self.correct_choices: boxes that should be checked
+        #  self.incorrect_choices: boxes that should NOT be checked
+
+        all_choices = self.correct_choices.union(self.incorrect_choices)
+
+        student_answer = student_answers.get(self.answer_id, [])
+
+        if not isinstance(student_answer, list):
+            student_answer = [student_answer]
+
+        # When a student leaves all the boxes unmarked, edX throws an error.
+        # This line checks for blank answers so that we can throw "false".
+        # This is not ideal. "None apply" should be a valid choice.
+        # Sadly, this is not the place where we can fix that problem.
+        empty_answer = student_answer == []
+
+        if empty_answer:
+            return CorrectMap(self.answer_id, 'incorrect')
+
+        student_answer = set(student_answer)
+
+        student_non_answers = all_choices - student_answer
+
+        # No partial credit? Get grade right now.
+        if not self.has_partial_credit:
+            return self.grade_without_partial_credit(student_answer=student_answer)
+
+        # This below checks to see whether we're using an alternate grading scheme.
+        #  Set partial_credit="false" (or remove it) to require an exact answer for any credit.
+        #  Set partial_credit="EDC" to count each choice for equal points (Every Decision Counts).
+        #  Set partial_credit="halves" to take half credit off for each error.
+
+        # Translators: 'partial_credit' and the items in the 'graders' object
+        # are attribute names or values and should not be translated.
+        graders = {
+            'edc': self.grade_via_every_decision_counts,
+            'halves': self.grade_via_halves,
+            'false': self.grade_without_partial_credit
+        }
+
+        # Only one type of credit at a time.
+        if len(self.credit_type) > 1:
+            raise LoncapaProblemError('Only one type of partial credit is allowed for Checkbox problems.')
+
+        # Make sure we're using an approved style.
+        if self.credit_type[0] not in graders:
+            raise LoncapaProblemError('partial_credit attribute should be one of: ' + ','.join(graders))
+
+        # Run the appropriate grader.
+        return graders[self.credit_type[0]](
+            all_choices=all_choices,
+            student_answer=student_answer,
+            student_non_answers=student_non_answers
+        )
+
+    def get_answers(self):
+        return {self.answer_id: list(self.correct_choices)}
+
+    def get_extended_hints(self, student_answers, new_cmap):
+        """
+        Extract compound and extended hint information from the xml based on the student_answers.
+        The hint information goes into the msg= in new_cmap for display.
+        Each choice in the checkboxgroup can have 2 extended hints, matching the
+        case that the student has or has not selected that choice:
+          <checkboxgroup label="Select the best snack">
+             <choice correct="true">Donut
+               <choicehint selected="tRuE">A Hint!</choicehint>
+               <choicehint selected="false">Another hint!</choicehint>
+             </choice>
+        """
+        # Tricky: student_answers may be *empty* here. That is the representation that
+        # no checkboxes were selected. For typical responsetypes, you look at
+        # student_answers[self.answer_id], but that does not work here.
+
+        # Compound hints are a special thing just for checkboxgroup, trying
+        # them first before the regular extended hints.
+        if self.get_compound_hints(new_cmap, student_answers):
+            return
+
+        # Look at all the choices - each can generate some hint text
+        choices = self.xml.xpath('//checkboxgroup[@id=$id]/choice', id=self.answer_id)
+        hint_log = []
+        label = None
+        label_count = 0
+        choice_all = []
+        # Tricky: in the case that the student selects nothing, there is simply
+        # no entry in student_answers, rather than an entry with the empty list value.
+        # That explains the following line.
+        student_choice_list = student_answers.get(self.answer_id, [])
+        # We build up several hints in hint_divs, then wrap it once at the end.
+        for choice in choices:
+            name = choice.get('name')  # generated name, e.g. choice_2
+            choice_all.append(name)
+            selected = name in student_choice_list  # looking for 'true' vs. 'false'
+            if selected:
+                selector = 'true'
+            else:
+                selector = 'false'
+            # We find the matching <choicehint> in python vs xpath so we can be case-insensitive
+            hint_nodes = choice.findall('./choicehint')
+            for hint_node in hint_nodes:
+                if hint_node.get('selected', '').lower() == selector:
+                    text = get_inner_html_from_xpath(hint_node)
+                    if hint_node.get('label') is not None:  # tricky: label '' vs None is significant
+                        label = hint_node.get('label')
+                        label_count += 1
+                    if text:
+                        hint_log.append({'text': text, 'trigger': [{'choice': name, 'selected': selected}]})
+
+        if hint_log:
+            # Complication: if there is only a single label specified, we use it.
+            # However if there are multiple, we use none.
+            if label_count > 1:
+                label = None
+            new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                None,
+                new_cmap[self.answer_id]['correctness'] == 'correct',
+                student_choice_list,
+                self.tags[0],
+                label,
+                hint_log,
+                multiline_mode=True,  # the one case where we do this
+                log_extra={'choice_all': choice_all}  # checkbox specific logging
+            )
+
+    def get_compound_hints(self, new_cmap, student_answers):
+        """
+        Compound hints are a type of extended hint specific to checkboxgroup with the
+        <compoundhint value="A C"> meaning choices A and C were selected.
+        Checks for a matching compound hint, installing it in new_cmap.
+        Returns True if compound condition hints were matched.
+        """
+        compound_hint_matched = False
+        if self.answer_id in student_answers:
+            # First create a set of the student's selected ids
+            student_set = set()
+            names = []
+            for student_answer in student_answers[self.answer_id]:
+                choice_list = self.xml.xpath('//checkboxgroup[@id=$id]/choice[@name=$name]',
+                                             id=self.answer_id, name=student_answer)
+                if choice_list:
+                    choice = choice_list[0]
+                    student_set.add(choice.get('id').upper())
+                    names.append(student_answer)
+
+            for compound_hint in self.xml.xpath('//checkboxgroup[@id=$id]/compoundhint', id=self.answer_id):
+                # Selector words are space separated and not case-sensitive
+                selectors = compound_hint.get('value').upper().split()
+                selector_set = set(selectors)
+
+                if selector_set == student_set:
+                    # This is the atypical case where the hint text is in an inner div with its own style.
+                    hint_text = compound_hint.text.strip()
+                    # Compute the choice names just for logging
+                    choices = self.xml.xpath('//checkboxgroup[@id=$id]/choice', id=self.answer_id)
+                    choice_all = [choice.get('name') for choice in choices]
+                    hint_log = [{'text': hint_text, 'trigger': [{'choice': name, 'selected': True} for name in names]}]
+                    new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                        compound_hint,
+                        new_cmap[self.answer_id]['correctness'] == 'correct',
+                        student_answers[self.answer_id],
+                        self.tags[0],
+                        hint_log=hint_log,
+                        log_extra={'trigger_type': 'compound', 'choice_all': choice_all}
+                    )
+                    compound_hint_matched = True
+                    break
+        return compound_hint_matched
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class MultipleChoiceResponse(LoncapaResponse):
+    """
+    Multiple Choice Response
+    The shuffle and answer-pool features on this class enable permuting and
+    subsetting the choices shown to the student.
+    Both features enable name "masking":
+    With masking, the regular names of multiplechoice choices
+    choice_0 choice_1 ... are not used. Instead we use random masked names
+    mask_2 mask_0 ... so that a view-source of the names reveals nothing about
+    the original order. We introduce the masked names right at init time, so the
+    whole software stack works with just the one system of naming.
+    The .has_mask() test on a response checks for masking, implemented by a
+    ._has_mask attribute on the response object.
+    The logging functionality in capa_module calls the unmask functions here
+    to translate back to choice_0 name style for recording in the logs, so
+    the logging is in terms of the regular names.
+    """
+    # TODO: randomize
+
+    human_name = _('Multiple Choice')
+    tags = ['multiplechoiceresponse']
+    max_inputfields = 1
+    allowed_inputfields = ['choicegroup']
+    correct_choices = None
+    multi_device_support = True
+
+    def setup_response(self):
+        """
+        Collects information from the XML for later use.
+
+        correct_choices is a list of the correct choices.
+        partial_choices is a list of the partially-correct choices.
+        partial_values is a list of the scores that go with those
+          choices, defaulting to 0.5 if no value is specified.
+        """
+        # call secondary setup for MultipleChoice questions, to set name
+        # attributes
+        self.mc_setup_response()
+
+        # define correct choices (after calling secondary setup)
+        xml = self.xml
+        cxml = xml.xpath('//*[@id=$id]//choice', id=xml.get('id'))
+
+        # contextualize correct attribute and then select ones for which
+        # correct = "true"
+        self.correct_choices = [
+            contextualize_text(choice.get('name'), self.context)
+            for choice in cxml
+            if contextualize_text(choice.get('correct'), self.context).upper() == "TRUE"
+        ]
+
+        if self.has_partial_credit:
+            self.partial_choices = [
+                contextualize_text(choice.get('name'), self.context)
+                for choice in cxml
+                if contextualize_text(choice.get('correct'), self.context).lower() == 'partial'
+            ]
+            self.partial_values = [
+                float(choice.get('point_value', default='0.5'))    # Default partial credit: 50%
+                for choice in cxml
+                if contextualize_text(choice.get('correct'), self.context).lower() == 'partial'
+            ]
+
+    def get_extended_hints(self, student_answer_dict, new_cmap):  # lint-amnesty, pylint: disable=arguments-differ
+        """
+        Extract any hints in a <choicegroup> matching the student's answers
+        <choicegroup label="What is your favorite color?" type="MultipleChoice">
+          <choice correct="false">Red
+            <choicehint>No, Blue!</choicehint>
+          </choice>
+          ...
+        Any hint text is installed in the new_cmap.
+        """
+        if self.answer_id in student_answer_dict:
+            student_answer = student_answer_dict[self.answer_id]
+
+            # Warning: mostly student_answer is a string, but sometimes it is a list of strings.
+            if isinstance(student_answer, list):
+                student_answer = student_answer[0]
+
+            # Find the named choice used by the student. Silently ignore a non-matching
+            # choice name.
+            choice = self.xml.find('./choicegroup[@id="{0}"]/choice[@name="{1}"]'.format(self.answer_id,
+                                                                                         student_answer))
+            if choice is not None:
+                hint_node = choice.find('./choicehint')
+                new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                    hint_node,
+                    choice.get('correct').upper() == 'TRUE',
+                    [student_answer],
+                    self.tags[0]
+                )
+
+    def mc_setup_response(self):
+        """
+        Initialize name attributes in <choice> stanzas in the <choicegroup> in this response.
+        Masks the choice names if applicable.
+        """
+        i = 0
+        for response in self.xml.xpath("choicegroup"):
+            # Is Masking enabled? -- check for shuffle or answer-pool features
+            # Masking (self._has_mask) is off, to be re-enabled with a future PR.
+            rtype = response.get('type')
+            if rtype not in ["MultipleChoice"]:
+                # force choicegroup to be MultipleChoice if not valid
+                response.set("type", "MultipleChoice")
+            for choice in list(response):
+                # The regular, non-masked name:
+                if choice.get("name") is not None:
+                    name = "choice_" + choice.get("name")
+                else:
+                    name = "choice_" + str(i)
+                    i += 1
+                # If using the masked name, e.g. mask_0, save the regular name
+                # to support unmasking later (for the logs).
+                # Masking is currently disabled so this code is commented, as
+                # the variable `mask_ids` is not defined. (the feature appears to not be fully implemented)
+                # The original work for masking was done by Nick Parlante as part of the OLI Hinting feature.
+                # if self.has_mask():
+                #     mask_name = "mask_" + str(mask_ids.pop())
+                #     self._mask_dict[mask_name] = name
+                #     choice.set("name", mask_name)
+                # else:
+                choice.set("name", name)
+
+    def late_transforms(self, problem):
+        """
+        Rearrangements run late in the __init__ process.
+        Cannot do these at response init time, as not enough
+        other stuff exists at that time.
+        """
+        self.do_shuffle(self.xml, problem)
+        self.do_answer_pool(self.xml, problem)
+
+    def grade_via_points(self, **kwargs):
+        """
+        Calculates partial credit based on the Points scheme.
+        Answer choices marked "partial" are given partial credit.
+        Default is 50%; other amounts may be set in point_value attributes.
+        Arguments: student_answers
+        Returns: a CorrectMap
+        """
+
+        student_answers = kwargs['student_answers']
+
+        if (self.answer_id in student_answers
+                and student_answers[self.answer_id] in self.correct_choices):
+            return CorrectMap(self.answer_id, correctness='correct')
+
+        elif (
+                self.answer_id in student_answers
+                and student_answers[self.answer_id] in self.partial_choices
+        ):
+            choice_index = self.partial_choices.index(student_answers[self.answer_id])
+            credit_amount = self.partial_values[choice_index]
+            return CorrectMap(self.answer_id, correctness='partially-correct', npoints=credit_amount)
+        else:
+            return CorrectMap(self.answer_id, 'incorrect')
+
+    def grade_without_partial_credit(self, **kwargs):
+        """
+        Standard grading for multiple-choice problems.
+        100% credit if choices are correct; 0% otherwise
+        Arguments: student_answers
+        Returns: a CorrectMap
+        """
+
+        student_answers = kwargs['student_answers']
+
+        if (self.answer_id in student_answers
+                and student_answers[self.answer_id] in self.correct_choices):
+            return CorrectMap(self.answer_id, correctness='correct')
+        else:
+            return CorrectMap(self.answer_id, 'incorrect')
+
+    def get_score(self, student_answers):
+        """
+        grade student response.
+        """
+
+        # No partial credit? Grade it right away.
+        if not self.has_partial_credit:
+            return self.grade_without_partial_credit(student_answers=student_answers)
+
+        # This below checks to see whether we're using an alternate grading scheme.
+        #  Set partial_credit="false" (or remove it) to require an exact answer for any credit.
+        #  Set partial_credit="points" to set specific point values for specific choices.
+
+        # Translators: 'partial_credit' and the items in the 'graders' object
+        # are attribute names or values and should not be translated.
+        graders = {
+            'points': self.grade_via_points,
+            'false': self.grade_without_partial_credit
+        }
+
+        # Only one type of credit at a time.
+        if len(self.credit_type) > 1:
+            raise LoncapaProblemError('Only one type of partial credit is allowed for Multiple Choice problems.')
+
+        # Make sure we're using an approved style.
+        if self.credit_type[0] not in graders:
+            raise LoncapaProblemError('partial_credit attribute should be one of: ' + ','.join(graders))
+
+        # Run the appropriate grader.
+        return graders[self.credit_type[0]](
+            student_answers=student_answers
+        )
+
+    def get_answers(self):
+        return {self.answer_id: self.correct_choices}
+
+    def unmask_name(self, name):
+        """
+        Given a masked name, e.g. mask_2, returns the regular name, e.g. choice_0.
+        Fails with LoncapaProblemError if called on a response that is not masking.
+        """
+        # if not self.has_mask():
+        #     _ = edx_six.get_gettext(self.capa_system.i18n)
+        #     # Translators: 'unmask_name' is a method name and should not be translated.
+        #     msg = "unmask_name called on response that is not masked"
+        #     raise LoncapaProblemError(msg)
+        # return self._mask_dict[name]  # TODO: this is not defined
+        raise NotImplementedError()
+
+    def unmask_order(self):
+        """
+        Returns a list of the choice names in the order displayed to the user,
+        using the regular (non-masked) names.
+        """
+        # With masking disabled, this computation remains interesting to see
+        # the displayed order, even though there is no unmasking.
+        choices = self.xml.xpath('choicegroup/choice')
+        return [choice.get("name") for choice in choices]
+
+    def do_shuffle(self, tree, problem):
+        """
+        For a choicegroup with shuffle="true", shuffles the choices in-place in the given tree
+        based on the seed. Otherwise does nothing.
+        Raises LoncapaProblemError if both shuffle and answer-pool are active:
+        a problem should use one or the other but not both.
+        Does nothing if the tree has already been processed.
+        """
+        # The tree is already pared down to this <multichoiceresponse> so this query just
+        # gets the child choicegroup (i.e. no leading //)
+        choicegroups = tree.xpath('choicegroup[@shuffle="true"]')
+        if choicegroups:
+            choicegroup = choicegroups[0]
+            if choicegroup.get('answer-pool') is not None:
+                _ = edx_six.get_gettext(self.capa_system.i18n)
+                # Translators: 'shuffle' and 'answer-pool' are attribute names and should not be translated.
+                msg = _("Do not use shuffle and answer-pool at the same time")
+                raise LoncapaProblemError(msg)
+            # Note in the response that shuffling is done.
+            # Both to avoid double-processing, and to feed the logs.
+            if self.has_shuffle():
+                return
+            self._has_shuffle = True  # pylint: disable=attribute-defined-outside-init
+            # Move elements from tree to list for shuffling, then put them back.
+            ordering = list(choicegroup.getchildren())
+            for choice in ordering:
+                choicegroup.remove(choice)
+            ordering = self.shuffle_choices(ordering, self.get_rng(problem))
+            for choice in ordering:
+                choicegroup.append(choice)
+
+    def shuffle_choices(self, choices, rng):
+        """
+        Returns a list of choice nodes with the shuffling done,
+        using the provided random number generator.
+        Choices with 'fixed'='true' are held back from the shuffle.
+        """
+        # Separate out a list of the stuff to be shuffled
+        # vs. the head/tail of fixed==true choices to be held back from the shuffle.
+        # Rare corner case: A fixed==true choice "island" in the middle is lumped in
+        # with the tail group of fixed choices.
+        # Slightly tricky one-pass implementation using a state machine
+        head = []
+        middle = []  # only this one gets shuffled
+        tail = []
+        at_head = True
+        for choice in choices:
+            if at_head and choice.get('fixed') == 'true':
+                head.append(choice)
+                continue
+            at_head = False
+            if choice.get('fixed') == 'true':
+                tail.append(choice)
+            else:
+                middle.append(choice)
+        rng.shuffle(middle)
+        return head + middle + tail
+
+    def get_rng(self, problem):
+        """
+        Get the random number generator to be shared by responses
+        of the problem, creating it on the problem if needed.
+        """
+        # Multiple questions in a problem share one random number generator (rng) object
+        # stored on the problem. If each question got its own rng, the structure of multiple
+        # questions within a problem could appear predictable to the student,
+        # e.g. (c) keeps being the correct choice. This is due to the seed being
+        # defined at the problem level, so the multiple rng's would be seeded the same.
+        # The name _shared_rng begins with an _ to suggest that it is not a facility
+        # for general use.
+        # pylint: disable=protected-access
+        if not hasattr(problem, '_shared_rng'):
+            problem._shared_rng = random.Random(self.context['seed'])
+        return problem._shared_rng
+
+    def do_answer_pool(self, tree, problem):
+        """
+        Implements the answer-pool subsetting operation in-place on the tree.
+        Allows for problem questions with a pool of answers, from which answer options shown to the student
+        and randomly selected so that there is always 1 correct answer and n-1 incorrect answers,
+        where the author specifies n as the value of the attribute "answer-pool" within <choicegroup>
+
+        The <choicegroup> tag must have an attribute 'answer-pool' giving the desired
+        pool size. If that attribute is zero or not present, no operation is performed.
+        Calling this a second time does nothing.
+        Raises LoncapaProblemError if the answer-pool value is not an integer,
+        or if the number of correct or incorrect choices available is zero.
+        """
+        choicegroups = tree.xpath("choicegroup[@answer-pool]")
+        if choicegroups:
+            choicegroup = choicegroups[0]
+            num_str = choicegroup.get('answer-pool')
+            if num_str == '0':
+                return
+            try:
+                num_choices = int(num_str)
+            except ValueError:
+                _ = edx_six.get_gettext(self.capa_system.i18n)
+                # Translators: 'answer-pool' is an attribute name and should not be translated.
+                msg = _("answer-pool value should be an integer")
+                raise LoncapaProblemError(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+
+            # Note in the response that answerpool is done.
+            # Both to avoid double-processing, and to feed the logs.
+            if self.has_answerpool():
+                return
+            self._has_answerpool = True  # pylint: disable=attribute-defined-outside-init
+
+            choices_list = list(choicegroup.getchildren())
+
+            # Remove all choices in the choices_list (we will add some back in later)
+            for choice in choices_list:
+                choicegroup.remove(choice)
+
+            rng = self.get_rng(problem)  # random number generator to use
+            # Sample from the answer pool to get the subset choices and solution id
+            (solution_id, subset_choices) = self.sample_from_answer_pool(choices_list, rng, num_choices)
+
+            # Add back in randomly selected choices
+            for choice in subset_choices:
+                choicegroup.append(choice)
+
+            # Filter out solutions that don't correspond to the correct answer we selected to show
+            # Note that this means that if the user simply provides a <solution> tag, nothing is filtered
+            solutionset = choicegroup.xpath('../following-sibling::solutionset')
+            if len(solutionset) != 0:
+                solutionset = solutionset[0]
+                solutions = solutionset.xpath('./solution')
+                for solution in solutions:
+                    if solution.get('explanation-id') != solution_id:
+                        solutionset.remove(solution)
+
+    def sample_from_answer_pool(self, choices, rng, num_pool):
+        """
+        Takes in:
+            1. list of choices
+            2. random number generator
+            3. the requested size "answer-pool" number, in effect a max
+
+        Returns a tuple with 2 items:
+            1. the solution_id corresponding with the chosen correct answer
+            2. (subset) list of choice nodes with num-1 incorrect and 1 correct
+
+        Raises an error if the number of correct or incorrect choices is 0.
+        """
+
+        correct_choices = []
+        incorrect_choices = []
+
+        for choice in choices:
+            if choice.get('correct').upper() == 'TRUE':
+                correct_choices.append(choice)
+            else:
+                incorrect_choices.append(choice)
+                # In my small test, capa seems to treat the absence of any correct=
+                # attribute as equivalent to ="false", so that's what we do here.
+
+        # We raise an error if the problem is highly ill-formed.
+        # There must be at least one correct and one incorrect choice.
+        # IDEA: perhaps this sort semantic-lint constraint should be generalized to all multichoice
+        # not just down in this corner when answer-pool is used.
+        # Or perhaps in the overall author workflow, these errors are unhelpful and
+        # should all be removed.
+        if len(correct_choices) < 1 or len(incorrect_choices) < 1:
+            _ = edx_six.get_gettext(self.capa_system.i18n)
+            # Translators: 'Choicegroup' is an input type and should not be translated.
+            msg = _("Choicegroup must include at least 1 correct and 1 incorrect choice")
+            raise LoncapaProblemError(msg)
+
+        # Limit the number of incorrect choices to what we actually have
+        num_incorrect = num_pool - 1
+        num_incorrect = min(num_incorrect, len(incorrect_choices))
+
+        # Select the one correct choice
+        index = rng.randint(0, len(correct_choices) - 1)
+        correct_choice = correct_choices[index]
+        solution_id = correct_choice.get('explanation-id')
+
+        # Put together the result, pushing most of the work onto rng.shuffle()
+        subset_choices = [correct_choice]
+        rng.shuffle(incorrect_choices)
+        subset_choices += incorrect_choices[:num_incorrect]
+        rng.shuffle(subset_choices)
+
+        return (solution_id, subset_choices)
+
+
+@registry.register
+class TrueFalseResponse(MultipleChoiceResponse):  # lint-amnesty, pylint: disable=abstract-method, missing-class-docstring
+
+    human_name = _('True/False Choice')
+    tags = ['truefalseresponse']
+
+    def mc_setup_response(self):
+        i = 0
+        for response in self.xml.xpath("choicegroup"):
+            response.set("type", "TrueFalse")
+            for choice in list(response):
+                if choice.get("name") is None:
+                    choice.set("name", "choice_" + str(i))
+                    i += 1
+                else:
+                    choice.set("name", "choice_" + choice.get("name"))
+
+    def get_score(self, student_answers):
+        correct = set(self.correct_choices)
+        answers = student_answers.get(self.answer_id, [])
+        if not isinstance(answers, list):
+            answers = [answers]
+
+        if correct == set(answers):
+            return CorrectMap(self.answer_id, 'correct')
+
+        return CorrectMap(self.answer_id, 'incorrect')
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class OptionResponse(LoncapaResponse):
+    """
+    TODO: handle randomize
+    """
+
+    human_name = _('Dropdown')
+    tags = ['optionresponse']
+    hint_tag = 'optionhint'
+    allowed_inputfields = ['optioninput']
+    answer_fields = None
+    multi_device_support = True
+
+    def setup_response(self):
+        self.answer_fields = self.inputfields
+
+    def get_score(self, student_answers):
+        cmap = CorrectMap()
+        amap = self.get_answers()
+        for aid in amap:
+            if aid in student_answers and student_answers[aid] == amap[aid]:
+                cmap.set(aid, 'correct')
+            else:
+                cmap.set(aid, 'incorrect')
+            answer_variable = self.get_student_answer_variable_name(student_answers, aid)
+            if answer_variable:
+                cmap.set_property(aid, 'answervariable', answer_variable)
+        return cmap
+
+    def get_answers(self):
+        amap = dict([(af.get('id'), contextualize_text(af.get(  # lint-amnesty, pylint: disable=consider-using-dict-comprehension
+            'correct'), self.context)) for af in self.answer_fields])
+        return amap
+
+    def get_student_answer_variable_name(self, student_answers, aid):
+        """
+        Return student answers variable name if exist in context else None.
+        """
+        if aid in student_answers:
+            for key, val in six.iteritems(self.context):
+                # convert val into unicode because student answer always be a unicode string
+                # even it is a list, dict etc.
+                if six.text_type(val) == student_answers[aid]:
+                    return '$' + key
+        return None
+
+    def get_extended_hints(self, student_answers, new_cmap):
+        """
+        Extract optioninput extended hint, e.g.
+        <optioninput>
+          <option correct="True">Donut <optionhint>Of course</optionhint> </option>
+        """
+        answer_id = self.answer_ids[0]  # Note *not* self.answer_id
+        if answer_id in student_answers:
+            student_answer = student_answers[answer_id]
+            # If we run into an old-style optioninput, there is no <option> tag, so this safely does nothing
+            options = self.xml.xpath('//optioninput[@id=$id]/option', id=answer_id)
+            # Extra pass here to ignore whitespace around the answer in the matching
+            options = [option for option in options if option.text.strip() == student_answer]
+            if options:
+                option = options[0]
+                hint_node = option.find('./optionhint')
+                if hint_node is not None:
+                    new_cmap[answer_id]['msg'] += self.make_hint_div(
+                        hint_node,
+                        option.get('correct').upper() == 'TRUE',
+                        [student_answer],
+                        self.tags[0]
+                    )
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class NumericalResponse(LoncapaResponse):
+    """
+    This response type expects a number or formulaic expression that evaluates
+    to a number (e.g. `4+5/2^2`), and accepts with a tolerance.
+    """
+
+    human_name = _('Numerical Input')
+    tags = ['numericalresponse']
+    hint_tag = 'numericalhint'
+    allowed_inputfields = ['textline', 'formulaequationinput']
+    required_attributes = ['answer']
+    max_inputfields = 1
+    multi_device_support = True
+
+    def __init__(self, *args, **kwargs):
+        self.correct_answer = ''
+        self.additional_answers = []
+        self.additional_answer_index = -1
+        self.tolerance = default_tolerance
+        self.range_tolerance = False
+        self.answer_range = self.inclusion = None
+        super(NumericalResponse, self).__init__(*args, **kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def setup_response(self):
+        xml = self.xml
+        context = self.context
+        answer = xml.get('answer')
+
+        self.additional_answers = (
+            [element.get('answer') for element in xml.findall('additional_answer')]
+        )
+
+        if answer.startswith(('[', '(')) and answer.endswith((']', ')')):  # range tolerance case
+            self.range_tolerance = True
+            self.inclusion = (
+                True if answer.startswith('[') else False, True if answer.endswith(']') else False  # lint-amnesty, pylint: disable=simplifiable-if-expression
+            )
+            try:
+                self.answer_range = [contextualize_text(x, context) for x in answer[1:-1].split(',')]
+                self.correct_answer = answer[0] + self.answer_range[0] + ', ' + self.answer_range[1] + answer[-1]
+            except Exception:
+                log.debug("Content error--answer '%s' is not a valid range tolerance answer", answer)
+                _ = edx_six.get_gettext(self.capa_system.i18n)
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    _("There was a problem with the staff answer to this problem.")
+                )
+        else:
+            self.correct_answer = contextualize_text(answer, context)
+
+            # Find the tolerance
+            tolerance_xml = xml.xpath(
+                '//*[@id=$id]//responseparam[@type="tolerance"]/@default',
+                id=xml.get('id')
+            )
+            if tolerance_xml:  # If it isn't an empty list...
+                self.tolerance = contextualize_text(tolerance_xml[0].strip(), context)
+
+    def get_staff_ans(self, answer):
+        """
+        Given the staff answer as a string, find its float value.
+
+        Use `evaluator` for this, but for backward compatability, try the
+        built-in method `complex` (which used to be the standard).
+        """
+        try:
+            correct_ans = complex(answer)
+        except ValueError:
+            # When `correct_answer` is not of the form X+Yj, it raises a
+            # `ValueError`. Then test if instead it is a math expression.
+            # `complex` seems to only generate `ValueErrors`, only catch these.
+            try:
+                correct_ans = evaluator({}, {}, answer)
+            except Exception:
+                log.debug("Content error--answer '%s' is not a valid number", answer)
+                _ = edx_six.get_gettext(self.capa_system.i18n)
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    _("There was a problem with the staff answer to this problem.")
+                )
+
+        return correct_ans
+
+    def get_score(self, student_answers):  # lint-amnesty, pylint: disable=too-many-statements
+        """
+        Grade a numeric response.
+        """
+        if self.answer_id not in student_answers:
+            return CorrectMap(self.answer_id, 'incorrect')
+
+        # Make sure we're using an approved partial credit style.
+        # Currently implemented: 'close' and 'list'
+        if self.has_partial_credit:
+            graders = ['list', 'close']
+            for style in self.credit_type:
+                if style not in graders:
+                    raise LoncapaProblemError('partial_credit attribute should be one of: ' + ','.join(graders))
+
+        student_answer = student_answers[self.answer_id]
+
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        general_exception = StudentInputError(
+            _("Could not interpret '{student_answer}' as a number.").format(student_answer=html.escape(student_answer))
+        )
+
+        # Begin `evaluator` block
+        # Catch a bunch of exceptions and give nicer messages to the student.
+        try:
+            student_float = evaluator({}, {}, student_answer)
+        except UndefinedVariable as err:
+            raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                err.args[0]
+            )
+        except UnmatchedParenthesis as err:
+            raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                err.args[0]
+            )
+        except ValueError as val_err:
+            if 'factorial' in text_type(val_err):  # lint-amnesty, pylint: disable=no-else-raise
+                # This is thrown when fact() or factorial() is used in an answer
+                #   that evaluates on negative and/or non-integer inputs
+                # text_type(ve) will be: `factorial() only accepts integral values` or
+                # `factorial() not defined for negative values`
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    _("Factorial function evaluated outside its domain:"
+                      "'{student_answer}'").format(student_answer=html.escape(student_answer))
+                )
+            else:
+                raise general_exception  # lint-amnesty, pylint: disable=raise-missing-from
+        except ParseException:
+            raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                _("Invalid math syntax: '{student_answer}'").format(student_answer=html.escape(student_answer))
+            )
+        except Exception:
+            raise general_exception  # lint-amnesty, pylint: disable=raise-missing-from
+        # End `evaluator` block -- we figured out the student's answer!
+
+        tree = self.xml
+
+        # What multiple of the tolerance is worth partial credit?
+        has_partial_range = tree.xpath('responseparam[@partial_range]')
+        if has_partial_range:
+            partial_range = float(has_partial_range[0].get('partial_range', default='2'))
+        else:
+            partial_range = 2
+
+        # Take in alternative answers that are worth partial credit.
+        has_partial_answers = tree.xpath('responseparam[@partial_answers]')
+        if has_partial_answers:
+            partial_answers = has_partial_answers[0].get('partial_answers').split(',')
+            for index, word in enumerate(partial_answers):
+                partial_answers[index] = word.strip()
+                partial_answers[index] = self.get_staff_ans(partial_answers[index])
+
+        else:
+            partial_answers = False
+
+        partial_score = 0.5
+        is_correct = 'incorrect'
+
+        if self.range_tolerance:
+            if isinstance(student_float, complex):
+                raise StudentInputError(_("You may not use complex numbers in range tolerance problems"))
+            boundaries = []
+            for inclusion, answer in zip(self.inclusion, self.answer_range):
+                boundary = self.get_staff_ans(answer)
+                if boundary.imag != 0:
+                    raise StudentInputError(
+                        # Translators: This is an error message for a math problem. If the instructor provided a
+                        # boundary (end limit) for a variable that is a complex number (a + bi), this message displays.
+                        _("There was a problem with the staff answer to this problem: complex boundary.")
+                    )
+                if isnan(boundary):
+                    raise StudentInputError(
+                        # Translators: This is an error message for a math problem. If the instructor did not
+                        # provide a boundary (end limit) for a variable, this message displays.
+                        _("There was a problem with the staff answer to this problem: empty boundary.")
+                    )
+                boundaries.append(boundary.real)
+                if compare_with_tolerance(
+                        student_float,
+                        boundary,
+                        tolerance=float_info.epsilon,
+                        relative_tolerance=True
+                ):
+                    is_correct = 'correct' if inclusion else 'incorrect'
+                    break
+            else:
+                if boundaries[0] < student_float < boundaries[1]:
+                    is_correct = 'correct'
+                else:
+                    if self.has_partial_credit is False:
+                        pass
+                    elif 'close' in self.credit_type:
+                        # Partial credit: 50% if the student is outside the specified boundaries,
+                        # but within an extended set of boundaries.
+
+                        extended_boundaries = []
+                        boundary_range = boundaries[1] - boundaries[0]
+                        extended_boundaries.append(boundaries[0] - partial_range * boundary_range)
+                        extended_boundaries.append(boundaries[1] + partial_range * boundary_range)
+                        if extended_boundaries[0] < student_float < extended_boundaries[1]:
+                            is_correct = 'partially-correct'
+
+        else:
+            correct_float = self.get_staff_ans(self.correct_answer)
+
+            # Partial credit is available in three cases:
+            #  If the student answer is within expanded tolerance of the actual answer,
+            #  the student gets 50% credit. (Currently set as the default.)
+            #  Set via partial_credit="close" in the numericalresponse tag.
+            #
+            #  If the student answer is within regular tolerance of an alternative answer,
+            #  the student gets 50% credit. (Same default.)
+            #  Set via partial_credit="list"
+            #
+            #  If the student answer is within expanded tolerance of an alternative answer,
+            #  the student gets 25%. (We take the 50% and square it, at the moment.)
+            #  Set via partial_credit="list,close" or "close, list" or the like.
+
+            if str(self.tolerance).endswith('%'):
+                expanded_tolerance = str(partial_range * float(str(self.tolerance)[:-1])) + '%'
+            else:
+                expanded_tolerance = partial_range * float(self.tolerance)
+
+            if compare_with_tolerance(student_float, correct_float, self.tolerance):
+                is_correct = 'correct'
+            elif self.has_partial_credit is False:
+                pass
+            elif 'list' in self.credit_type:
+                for value in partial_answers:
+                    if compare_with_tolerance(student_float, value, self.tolerance):
+                        is_correct = 'partially-correct'
+                        break
+                    elif 'close' in self.credit_type:
+                        if compare_with_tolerance(student_float, correct_float, expanded_tolerance):
+                            is_correct = 'partially-correct'
+                            break
+                        elif compare_with_tolerance(student_float, value, expanded_tolerance):
+                            is_correct = 'partially-correct'
+                            partial_score = partial_score * partial_score
+                            break
+            elif 'close' in self.credit_type:
+                if compare_with_tolerance(student_float, correct_float, expanded_tolerance):
+                    is_correct = 'partially-correct'
+
+        # Reset self.additional_answer_index to -1 so that we always have a fresh index to look up.
+        self.additional_answer_index = -1
+
+        # Compare with additional answers.
+        if is_correct == 'incorrect':
+            temp_additional_answer_idx = 0
+            for additional_answer in self.additional_answers:
+                staff_answer = self.get_staff_ans(additional_answer)
+                if complex(student_float) == staff_answer:
+                    is_correct = 'correct'
+                    self.additional_answer_index = temp_additional_answer_idx
+                    break
+                temp_additional_answer_idx += 1
+
+        if is_correct == 'partially-correct':
+            return CorrectMap(self.answer_id, is_correct, npoints=partial_score)
+        else:
+            return CorrectMap(self.answer_id, is_correct)
+
+    def compare_answer(self, ans1, ans2):
+        """
+        Outside-facing function that lets us compare two numerical answers,
+        with this problem's tolerance.
+        """
+        return compare_with_tolerance(
+            evaluator({}, {}, ans1),
+            evaluator({}, {}, ans2),
+            self.tolerance
+        )
+
+    def validate_answer(self, answer):
+        """
+        Returns whether this answer is in a valid form.
+        """
+        try:
+            evaluator({}, {}, answer)
+            return True
+        except (StudentInputError, UndefinedVariable, UnmatchedParenthesis):
+            return False
+
+    def get_answers(self):
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        # Example: "Answer: Answer_1 or Answer_2 or Answer_3".
+        separator = Text(' {b_start}{or_separator}{b_end} ').format(
+            # Translators: Separator used in NumericalResponse to display multiple answers.
+            or_separator=_('or'),
+            b_start=HTML('<b>'),
+            b_end=HTML('</b>'),
+        )
+        return {self.answer_id: separator.join([self.correct_answer] + self.additional_answers)}
+
+    def set_cmap_msg(self, student_answers, new_cmap, hint_type, hint_index):
+        """
+        Sets feedback to correct hint node in correct map.
+
+        Arguments:
+            student_answers (dict): Dict containing student input.
+            new_cmap (dict): Dict containing correct map properties.
+            hint_type (str): Hint type, either `correcthint` or `additional_answer`
+            hint_index (int): Index of the hint node
+        """
+        # Note: using self.id here, not the more typical self.answer_id
+        hint_nodes = self.xml.xpath('//numericalresponse[@id=$id]/' + hint_type, id=self.id)
+        if hint_nodes:
+            hint_node = hint_nodes[hint_index]
+            if hint_type == 'additional_answer':
+                hint_node = hint_nodes[hint_index].find('./correcthint')
+            new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                hint_node,
+                True,
+                [student_answers[self.answer_id]],
+                self.tags[0]
+            )
+
+    def get_extended_hints(self, student_answers, new_cmap):
+        """
+        Extract numericalresponse extended hint, e.g.
+          <correcthint>Yes, 1+1 IS 2<correcthint>
+        """
+        if self.answer_id in student_answers:
+            if new_cmap.cmap[self.answer_id]['correctness'] == 'correct':  # if the grader liked the student's answer
+                # Answer is not an additional answer.
+                if self.additional_answer_index == -1:
+                    self.set_cmap_msg(student_answers, new_cmap, 'correcthint', 0)
+                else:
+                    self.set_cmap_msg(student_answers, new_cmap, 'additional_answer', self.additional_answer_index)
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class StringResponse(LoncapaResponse):
+    r"""
+    This response type allows one or more answers.
+
+    Additional answers are added by `additional_answer` tag.
+    If `regexp` is in `type` attribute, then answers and hints are treated as regular expressions.
+
+    Examples:
+        <stringresponse answer="Michigan">
+            <textline size="20" />
+        </stringresponse >
+
+        <stringresponse answer="a1" type="ci regexp">
+            <additional_answer>d5</additional_answer>
+            <additional_answer answer="a3"><correcthint>a hint - new format</correcthint></additional_answer>
+            <textline size="20"/>
+            <hintgroup>
+                <stringhint answer="a0" type="ci" name="ha0" />
+                <stringhint answer="a4" type="ci" name="ha4" />
+                <stringhint answer="^\d" type="ci" name="re1" />
+                <hintpart on="ha0">
+                    <startouttext />+1<endouttext />
+                </hintpart >
+                <hintpart on="ha4">
+                    <startouttext />-1<endouttext />
+                </hintpart >
+                <hintpart on="re1">
+                    <startouttext />Any number+5<endouttext />
+                </hintpart >
+            </hintgroup>
+        </stringresponse>
+    """
+    human_name = _('Text Input')
+    tags = ['stringresponse']
+    hint_tag = 'stringhint'
+    allowed_inputfields = ['textline']
+    required_attributes = ['answer']
+    max_inputfields = 1
+    correct_answer = []
+    multi_device_support = True
+
+    def setup_response_backward(self):
+        self.correct_answer = [
+            contextualize_text(answer, self.context).strip() for answer in self.xml.get('answer').split('_or_')
+        ]
+
+    def setup_response(self):
+        self.backward = '_or_' in self.xml.get('answer').lower()
+        self.regexp = False
+        self.case_insensitive = False
+        if self.xml.get('type') is not None:
+            self.regexp = 'regexp' in self.xml.get('type').lower().split(' ')
+            self.case_insensitive = 'ci' in self.xml.get('type').lower().split(' ')
+
+        # backward compatibility, can be removed in future, it is up to @Lyla Fisher.
+        if self.backward:
+            self.setup_response_backward()
+            return
+        # end of backward compatibility
+
+        # XML compatibility note: in 2015, additional_answer switched to having a 'answer' attribute.
+        # See make_xml_compatible in capa_problem which translates the old format.
+        correct_answers = (
+            [self.xml.get('answer')] +
+            [element.get('answer') for element in self.xml.findall('additional_answer')]
+        )
+        self.correct_answer = [contextualize_text(answer, self.context).strip() for answer in correct_answers]
+
+    def get_score(self, student_answers):
+        """Grade a string response """
+        if self.answer_id not in student_answers:
+            correct = False
+        else:
+            student_answer = student_answers[self.answer_id].strip()
+            correct = self.check_string(self.correct_answer, student_answer)
+        return CorrectMap(self.answer_id, 'correct' if correct else 'incorrect')
+
+    def check_string_backward(self, expected, given):
+        if self.case_insensitive:
+            return given.lower() in [i.lower() for i in expected]
+        return given in expected
+
+    def get_extended_hints(self, student_answers, new_cmap):
+        """
+        Find and install extended hints in new_cmap depending on the student answers.
+        StringResponse is probably the most complicated form we have.
+        The forms show below match in the order given, and the first matching one stops the matching.
+        <stringresponse answer="A" type="ci">
+          <correcthint>hint1</correcthint>                         <!-- hint for correct answer -->
+          <additional_answer answer="B">hint2</additional_answer>  <!-- additional_answer with its own hint -->
+          <stringequalhint answer="C">hint3</stringequalhint>      <!-- string matcher/hint for an incorrect answer -->
+          <regexphint answer="FG+">hint4</regexphint>              <!-- regex matcher/hint for an incorrect answer -->
+          <textline size="20"/>
+        </stringresponse>
+        The "ci" and "regexp" options are inherited from the parent stringresponse as appropriate.
+        """
+        if self.answer_id in student_answers:
+            student_answer = student_answers[self.answer_id]
+            # Note the atypical case of using self.id instead of self.answer_id
+            responses = self.xml.xpath('//stringresponse[@id=$id]', id=self.id)
+            if responses:
+                response = responses[0]
+
+                # First call the existing check_string to see if this is a right answer by that test.
+                # It handles the various "ci" "regexp" cases internally.
+                expected = response.get('answer').strip()
+                if self.check_string([expected], student_answer):
+                    hint_node = response.find('./correcthint')
+                    if hint_node is not None:
+                        new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                            hint_node,
+                            True,
+                            [student_answer],
+                            self.tags[0]
+                        )
+                    return
+
+                # Then look for additional answer with an answer= attribute
+                for node in response.findall('./additional_answer'):
+                    if self.match_hint_node(node, student_answer, self.regexp, self.case_insensitive):
+                        hint_node = node.find('./correcthint')
+                        new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                            hint_node,
+                            True,
+                            [student_answer],
+                            self.tags[0]
+                        )
+                        return
+
+                # stringequalhint and regexphint represent wrong answers
+                for hint_node in response.findall('./stringequalhint'):
+                    if self.match_hint_node(hint_node, student_answer, False, self.case_insensitive):
+                        new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                            hint_node,
+                            False,
+                            [student_answer],
+                            self.tags[0]
+                        )
+                        return
+
+                for hint_node in response.findall('./regexphint'):
+                    if self.match_hint_node(hint_node, student_answer, True, self.case_insensitive):
+                        new_cmap[self.answer_id]['msg'] += self.make_hint_div(
+                            hint_node,
+                            False,
+                            [student_answer],
+                            self.tags[0]
+                        )
+                        return
+
+    def match_hint_node(self, node, given, regex_mode, ci_mode):
+        """
+        Given an xml extended hint node such as additional_answer or regexphint,
+        which contain an answer= attribute, returns True if the given student answer is a match.
+        The boolean arguments regex_mode and ci_mode control how the answer stored in
+        the question is treated for the comparison (analogously to check_string).
+        """
+        answer = node.get('answer', '').strip()
+        if not answer:
+            return False
+
+        if regex_mode:
+            flags = 0
+            if ci_mode:
+                flags = re.IGNORECASE
+            try:
+                # We follow the check_string convention/exception, adding ^ and $
+                regex = re.compile('^' + answer + '$', flags=flags | re.UNICODE)
+                return re.search(regex, given)
+            except Exception:  # pylint: disable=broad-except
+                return False
+
+        if ci_mode:
+            return answer.lower() == given.lower()
+        else:
+            return answer == given
+
+    def check_string(self, expected, given):
+        """
+        Find given in expected.
+
+        If self.regexp is true, regular expression search is used.
+        if self.case_insensitive is true, case insensitive search is used, otherwise case sensitive search is used.
+        Spaces around values of attributes are stripped in XML parsing step.
+
+        Args:
+            expected: list.
+            given: str.
+
+        Returns: bool
+
+        Raises: `ResponseError` if it fails to compile regular expression.
+
+        Note: for old code, which supports _or_ separator, we add some  backward compatibility handling.
+        Should be removed soon. When to remove it, is up to Lyla Fisher.
+        """
+        # if given answer is empty.
+        if not given:
+            return False
+
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        # backward compatibility, should be removed in future.
+        if self.backward:
+            return self.check_string_backward(expected, given)
+        # end of backward compatibility
+
+        if self.regexp:  # regexp match
+            flags = re.IGNORECASE if self.case_insensitive else 0
+            try:
+                regexp = re.compile('^' + '|'.join(expected) + '$', flags=flags | re.UNICODE)
+                result = re.search(regexp, given)
+            except Exception as err:
+                msg = '[courseware.capa.responsetypes.stringresponse] {error}: {message}'.format(
+                    error=_('error'),
+                    message=text_type(err)
+                )
+                log.error(msg, exc_info=True)
+                raise ResponseError(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+            return bool(result)
+        else:  # string match
+            if self.case_insensitive:
+                return given.lower() in [i.lower() for i in expected]
+            else:
+                return given in expected
+
+    def check_hint_condition(self, hxml_set, student_answers):
+        given = student_answers[self.answer_id].strip()
+        hints_to_show = []
+        for hxml in hxml_set:
+            name = hxml.get('name')
+
+            hinted_answer = contextualize_text(hxml.get('answer'), self.context).strip()
+
+            if self.check_string([hinted_answer], given):
+                hints_to_show.append(name)
+        log.debug('hints_to_show = %s', hints_to_show)
+        return hints_to_show
+
+    def get_answers(self):
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        # Translators: Separator used in StringResponse to display multiple answers.
+        # Example: "Answer: Answer_1 or Answer_2 or Answer_3".
+        separator = HTML(' <b>{}</b> ').format(_('or'))
+        return {self.answer_id: separator.join(self.correct_answer)}
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class CustomResponse(LoncapaResponse):
+    """
+    Custom response.  The python code to be run should be in <answer>...</answer>
+    or in a <script>...</script>
+    """
+
+    human_name = _('Custom Evaluated Script')
+    tags = ['customresponse']
+
+    allowed_inputfields = ['textline', 'textbox', 'crystallography',
+                           'chemicalequationinput', 'vsepr_input',
+                           'drag_and_drop_input', 'designprotein2dinput',
+                           'editageneinput', 'annotationinput',
+                           'jsinput', 'formulaequationinput']
+    code = None
+    expect = None
+
+    # Standard amount for partial credit if not otherwise specified:
+    default_pc = 0.5
+
+    def setup_response(self):
+        xml = self.xml
+
+        # if <customresponse> has an "expect" (or "answer") attribute then save
+        # that
+        self.expect = contextualize_text(xml.get('expect') or xml.get('answer'), self.context)
+
+        log.debug('answer_ids=%s', self.answer_ids)
+
+        # the <answer>...</answer> stanza should be local to the current <customresponse>.
+        # So try looking there first.
+        self.code = None
+        answer = None
+        try:
+            answer = xml.xpath('//*[@id=$id]//answer', id=xml.get('id'))[0]
+        except IndexError:
+            # print "xml = ",etree.tostring(xml,pretty_print=True)
+
+            # if we have a "cfn" attribute then look for the function specified by cfn, in
+            # the problem context ie the comparison function is defined in the
+            # <script>...</script> stanza instead
+            cfn = xml.get('cfn')
+            if cfn:
+                log.debug("cfn = %s", cfn)
+
+                # This is a bit twisty.  We used to grab the cfn function from
+                # the context, but now that we sandbox Python execution, we
+                # can't get functions from previous executions.  So we make an
+                # actual function that will re-execute the original script,
+                # and invoke the function with the data needed.
+                def make_check_function(script_code, cfn):
+                    def check_function(expect, ans, **kwargs):
+                        extra_args = "".join(", {0}={0}".format(k) for k in kwargs)
+                        code = (
+                            script_code + "\n" +
+                            "cfn_return = %s(expect, ans%s)\n" % (cfn, extra_args)
+                        )
+                        globals_dict = {
+                            'expect': expect,
+                            'ans': ans,
+                        }
+                        globals_dict.update(kwargs)
+                        safe_exec.safe_exec(
+                            code,
+                            globals_dict,
+                            python_path=self.context['python_path'],
+                            extra_files=self.context['extra_files'],
+                            slug=self.id,
+                            random_seed=self.context['seed'],
+                            unsafely=self.capa_system.can_execute_unsafe_code(),
+                        )
+                        return globals_dict['cfn_return']
+                    return check_function
+
+                self.code = make_check_function(self.context['script_code'], cfn)
+
+        if not self.code:
+            if answer is None:
+                log.error("[courseware.capa.responsetypes.customresponse] missing"
+                          " code checking script! id=%s", self.id)
+                self.code = ''
+            else:
+                answer_src = answer.get('src')
+                if answer_src is not None:
+                    # TODO: this code seems not to be used any more since self.capa_system.filesystem doesn't exist.
+                    self.code = self.capa_system.filesystem.open('src/' + answer_src).read()
+                else:
+                    self.code = answer.text
+
+    def get_score(self, student_answers):
+        """
+        student_answers is a dict with everything from request.POST, but with the first part
+        of each key removed (the string before the first "_").
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+
+        log.debug('%s: student_answers=%s', six.text_type(self), student_answers)
+
+        # ordered list of answer id's
+        # sort the responses on the bases of the problem's position number
+        # which can be found in the last place in the problem id. Then convert
+        # this number into an int, so that we sort on ints instead of strings
+        idset = sorted(self.answer_ids, key=lambda x: int(x.split("_")[-1]))
+        try:
+            # ordered list of answers
+            submission = [student_answers[k] for k in idset]
+        except Exception as err:
+            msg = "[courseware.capa.responsetypes.customresponse] {message}\n idset = {idset}, error = {err}".format(
+                message=_("error getting student answer from {student_answers}").format(
+                    student_answers=student_answers,
+                ),
+                idset=idset,
+                err=err
+            )
+
+            log.error(
+                "[courseware.capa.responsetypes.customresponse] error getting"
+                " student answer from %s"
+                "\n idset = %s, error = %s",
+                student_answers, idset, err
+            )
+            raise Exception(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+
+        # global variable in context which holds the Presentation MathML from dynamic math input
+        # ordered list of dynamath responses
+        dynamath = [student_answers.get(k + '_dynamath', None) for k in idset]
+
+        # if there is only one box, and it's empty, then don't evaluate
+        if len(idset) == 1 and not submission[0]:
+            # default to no error message on empty answer (to be consistent with other
+            # responsetypes) but allow author to still have the old behavior by setting
+            # empty_answer_err attribute
+            msg = (HTML('<span class="inline-error">{0}</span>').format(_('No answer entered!'))
+                   if self.xml.get('empty_answer_err') else '')
+            return CorrectMap(idset[0], 'incorrect', msg=msg)
+
+        # NOTE: correct = 'unknown' could be dangerous. Inputtypes such as textline are
+        # not expecting 'unknown's
+        correct = ['unknown'] * len(idset)
+        messages = [''] * len(idset)
+        overall_message = ""
+
+        # put these in the context of the check function evaluator
+        # note that this doesn't help the "cfn" version - only the exec version
+        self.context.update({
+            # my ID
+            'response_id': self.id,
+
+            # expected answer (if given as attribute)
+            'expect': self.expect,
+
+            # ordered list of student answers from entry boxes in our subtree
+            'submission': submission,
+
+            # ordered list of ID's of all entry boxes in our subtree
+            'idset': idset,
+
+            # ordered list of all javascript inputs in our subtree
+            'dynamath': dynamath,
+
+            # dict of student's responses, with keys being entry box IDs
+            'answers': student_answers,
+
+            # the list to be filled in by the check function
+            'correct': correct,
+
+            # the list of messages to be filled in by the check function
+            'messages': messages,
+
+            # a message that applies to the entire response
+            # instead of a particular input
+            'overall_message': overall_message,
+
+            # any options to be passed to the cfn
+            'options': self.xml.get('options'),
+            'testdat': 'hello world',
+        })
+
+        # Pass DEBUG to the check function.
+        self.context['debug'] = self.capa_system.DEBUG
+
+        # Run the check function
+        self.execute_check_function(idset, submission)
+
+        # build map giving "correct"ness of the answer(s)
+        correct = self.context['correct']
+        messages = self.context['messages']
+        overall_message = self.clean_message_html(self.context['overall_message'])
+        grade_decimals = self.context.get('grade_decimals')
+
+        correct_map = CorrectMap()
+        correct_map.set_overall_message(overall_message)
+
+        for k in range(len(idset)):  # lint-amnesty, pylint: disable=consider-using-enumerate
+            max_points = self.maxpoints[idset[k]]
+            if grade_decimals:
+                npoints = max_points * grade_decimals[k]
+            else:
+                if correct[k] == 'correct':
+                    npoints = max_points
+                elif correct[k] == 'partially-correct':
+                    npoints = max_points * self.default_pc
+                else:
+                    npoints = 0
+            correct_map.set(idset[k], correct[k], msg=messages[k],
+                            npoints=npoints)
+        return correct_map
+
+    def execute_check_function(self, idset, submission):  # lint-amnesty, pylint: disable=missing-function-docstring, too-many-statements
+        # exec the check function
+        if isinstance(self.code, six.string_types):  # lint-amnesty, pylint: disable=too-many-nested-blocks
+            try:
+                safe_exec.safe_exec(
+                    self.code,
+                    self.context,
+                    cache=self.capa_system.cache,
+                    python_path=self.context['python_path'],
+                    extra_files=self.context['extra_files'],
+                    slug=self.id,
+                    random_seed=self.context['seed'],
+                    unsafely=self.capa_system.can_execute_unsafe_code(),
+                )
+            except Exception as err:  # pylint: disable=broad-except
+                self._handle_exec_exception(err)
+
+        else:
+            # self.code is not a string; it's a function we created earlier.
+
+            # this is an interface to the Tutor2 check functions
+            tutor_cfn = self.code
+            answer_given = submission[0] if (len(idset) == 1) else submission
+            kwnames = self.xml.get("cfn_extra_args", "").split()
+            kwargs = {n: self.context.get(n) for n in kwnames}
+            log.debug(" submission = %s", submission)
+            try:
+                ret = tutor_cfn(self.expect, answer_given, **kwargs)
+            except Exception as err:  # pylint: disable=broad-except
+                self._handle_exec_exception(err)
+            log.debug(
+                "[courseware.capa.responsetypes.customresponse.get_score] ret = %s",
+                ret
+            )
+            if isinstance(ret, dict):
+                # One kind of dictionary the check function can return has the
+                # form {'ok': BOOLEAN or STRING, 'msg': STRING, 'grade_decimal' (optional): FLOAT (between 0.0 and 1.0)}
+                # 'ok' will control the checkmark, while grade_decimal, if present, will scale
+                # the score the student receives on the response.
+                # If there are multiple inputs, they all get marked
+                # to the same correct/incorrect value
+                if 'ok' in ret:
+
+                    # Returning any falsy value or the "false" string for "ok" gives incorrect.
+                    # Returning any string that includes "partial" for "ok" gives partial credit.
+                    # Returning any other truthy value for "ok" gives correct
+
+                    ok_val = str(ret['ok']).lower().strip() if bool(ret['ok']) else 'false'
+
+                    if ok_val == 'false':
+                        correct = 'incorrect'
+                    elif 'partial' in ok_val:
+                        correct = 'partially-correct'
+                    else:
+                        correct = 'correct'
+                    correct = [correct] * len(idset)   # All inputs share the same mark.
+
+                    # old version, no partial credit:
+                    # correct = ['correct' if ret['ok'] else 'incorrect'] * len(idset)
+
+                    msg = ret.get('msg', None)
+                    msg = self.clean_message_html(msg)
+
+                    # If there is only one input, apply the message to that input
+                    # Otherwise, apply the message to the whole problem
+                    if len(idset) > 1:
+                        self.context['overall_message'] = msg
+                    else:
+                        self.context['messages'][0] = msg
+
+                    if 'grade_decimal' in ret:
+                        decimal = float(ret['grade_decimal'])
+                    else:
+                        if correct[0] == 'correct':
+                            decimal = 1.0
+                        elif correct[0] == 'partially-correct':
+                            decimal = self.default_pc
+                        else:
+                            decimal = 0.0
+                    grade_decimals = [decimal] * len(idset)
+                    self.context['grade_decimals'] = grade_decimals
+
+                # Another kind of dictionary the check function can return has
+                # the form:
+                # { 'overall_message': STRING,
+                #   'input_list': [
+                #     {
+                #         'ok': BOOLEAN or STRING,
+                #         'msg': STRING,
+                #         'grade_decimal' (optional): FLOAT (between 0.0 and 1.0)
+                #     },
+                #   ...
+                #   ]
+                # }
+                # 'ok' will control the checkmark, while grade_decimal, if present, will scale
+                # the score the student receives on the response.
+                #
+                # This allows the function to return an 'overall message'
+                # that applies to the entire problem, as well as correct/incorrect
+                # status, scaled grades, and messages for individual inputs
+                elif 'input_list' in ret:
+                    overall_message = ret.get('overall_message', '')
+                    input_list = ret['input_list']
+
+                    correct = []
+                    messages = []
+                    grade_decimals = []
+
+                    # Returning any falsy value or the "false" string for "ok" gives incorrect.
+                    # Returning any string that includes "partial" for "ok" gives partial credit.
+                    # Returning any other truthy value for "ok" gives correct
+
+                    for input_dict in input_list:
+                        if str(input_dict['ok']).lower().strip() == "false" or not input_dict['ok']:
+                            correct.append('incorrect')
+                        elif 'partial' in str(input_dict['ok']).lower().strip():
+                            correct.append('partially-correct')
+                        else:
+                            correct.append('correct')
+
+                        # old version, no partial credit
+                        # correct.append('correct'
+                        #                if input_dict['ok'] else 'incorrect')
+
+                        msg = (self.clean_message_html(input_dict['msg'])
+                               if 'msg' in input_dict else None)
+                        messages.append(msg)
+                        if 'grade_decimal' in input_dict:
+                            decimal = input_dict['grade_decimal']
+                        else:
+                            if str(input_dict['ok']).lower().strip() == 'true':
+                                decimal = 1.0
+                            elif 'partial' in str(input_dict['ok']).lower().strip():
+                                decimal = self.default_pc
+                            else:
+                                decimal = 0.0
+                        grade_decimals.append(decimal)
+
+                    self.context['messages'] = messages
+                    self.context['overall_message'] = overall_message
+                    self.context['grade_decimals'] = grade_decimals
+
+                # Otherwise, we do not recognize the dictionary
+                # Raise an exception
+                else:
+                    log.error(traceback.format_exc())
+                    _ = edx_six.get_gettext(self.capa_system.i18n)
+                    raise ResponseError(
+                        _("CustomResponse: check function returned an invalid dictionary!")
+                    )
+
+            else:
+
+                # Returning any falsy value or the "false" string for "ok" gives incorrect.
+                # Returning any string that includes "partial" for "ok" gives partial credit.
+                # Returning any other truthy value for "ok" gives correct
+
+                if str(ret).lower().strip() == "false" or not bool(ret):
+                    correct = 'incorrect'
+                elif 'partial' in str(ret).lower().strip():
+                    correct = 'partially-correct'
+                else:
+                    correct = 'correct'
+                correct = [correct] * len(idset)
+
+                # old version, no partial credit:
+                # correct = ['correct' if ret else 'incorrect'] * len(idset)
+
+            self.context['correct'] = correct
+
+    def clean_message_html(self, msg):  # lint-amnesty, pylint: disable=missing-function-docstring
+
+        # If *msg* is an empty string, then the code below
+        # will return "</html>".  To avoid this, we first check
+        # that *msg* is a non-empty string.
+        if msg:
+
+            # When we parse *msg* using etree, there needs to be a root
+            # element, so we wrap the *msg* text in <html> tags
+            msg = HTML('<html>{msg}</html>').format(msg=HTML(msg))
+
+            # Replace < characters
+            msg = msg.replace('&#60;', '&lt;')
+
+            # Use etree to prettify the HTML
+            msg = etree.tostring(fromstring_bs(msg), pretty_print=True).decode('utf-8')
+
+            msg = msg.replace('&#13;', '')
+
+            # Remove the <html> tags we introduced earlier, so we're
+            # left with just the prettified message markup
+            msg = re.sub('(?ms)<html>(.*)</html>', '\\1', msg)
+
+            # Strip leading and trailing whitespace
+            return msg.strip()
+
+        # If we start with an empty string, then return an empty string
+        else:
+            return ""
+
+    def get_answers(self):
+        """
+        Give correct answer expected for this response.
+
+        use default_answer_map from entry elements (eg textline),
+        when this response has multiple entry objects.
+
+        but for simplicity, if an "expect" attribute was given by the content author
+        ie <customresponse expect="foo" ...> then that.
+        """
+        if len(self.answer_ids) > 1:
+            return self.default_answer_map
+        if self.expect:
+            return {self.answer_ids[0]: self.expect}
+        return self.default_answer_map
+
+    def _handle_exec_exception(self, err):
+        """
+        Handle an exception raised during the execution of
+        custom Python code.
+
+        Raises a ResponseError
+        """
+
+        # Log the error if we are debugging
+        msg = 'Error occurred while evaluating CustomResponse'
+        log.warning(msg, exc_info=True)
+
+        # Notify student with a student input error
+        _, _, traceback_obj = sys.exc_info()
+        raise ResponseError(text_type(err), traceback_obj)
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class SymbolicResponse(CustomResponse):
+    """
+    Symbolic math response checking, using symmath library.
+    """
+
+    human_name = _('Symbolic Math Input')
+    tags = ['symbolicresponse']
+    max_inputfields = 1
+
+    def setup_response(self):
+        # Symbolic response always uses symmath_check()
+        # If the XML did not specify this, then set it now
+        # Otherwise, we get an error from the superclass
+        self.xml.set('cfn', 'symmath_check')
+
+        # Let CustomResponse do its setup
+        super(SymbolicResponse, self).setup_response()  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def execute_check_function(self, idset, submission):
+        from symmath import symmath_check
+        try:
+            # Since we have limited max_inputfields to 1,
+            # we can assume that there is only one submission
+            answer_given = submission[0]
+
+            ret = symmath_check(
+                self.expect, answer_given,
+                dynamath=self.context.get('dynamath'),
+                options=self.context.get('options'),
+                debug=self.context.get('debug'),
+            )
+        except Exception as err:
+            log.error("oops in SymbolicResponse (cfn) error %s", err)
+            log.error(traceback.format_exc())
+            _ = edx_six.get_gettext(self.capa_system.i18n)
+            # Translators: 'SymbolicResponse' is a problem type and should not be translated.
+            msg = _("An error occurred with SymbolicResponse. The error was: {error_msg}").format(
+                error_msg=err,
+            )
+            raise Exception(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+        self.context['messages'][0] = self.clean_message_html(ret['msg'])
+        self.context['correct'] = ['correct' if ret['ok'] else 'incorrect'] * len(idset)
+
+#-----------------------------------------------------------------------------
+
+## ScoreMessage named tuple ##
+## valid:       Flag indicating valid score_msg format (Boolean)
+## correct:     Correctness of submission (Boolean)
+## score:       Points to be assigned (numeric, can be float)
+## msg:         Message from grader to display to student (string)
+
+ScoreMessage = namedtuple('ScoreMessage', ['valid', 'correct', 'points', 'msg'])
+
+
+@registry.register
+class CodeResponse(LoncapaResponse):
+    """
+    Grade student code using an external queueing server, called 'xqueue'.
+
+    Expects 'xqueue' dict in LoncapaSystem with the following properties that are
+    needed by CodeResponse::
+
+        capa_system.xqueue = object with properties:
+            interface: XQueueInterface object.
+            construct_callback: Per-StudentModule callback URL constructor,
+                defaults to using 'score_update' as the correct dispatch (function).
+            default_queuename: Default queue name to submit request (string).
+        }
+
+    External requests are only submitted for student submission grading, not
+    for getting reference answers.
+
+    """
+
+    human_name = _('Code Input')
+    tags = ['coderesponse']
+    allowed_inputfields = ['textbox', 'filesubmission', 'matlabinput']
+    max_inputfields = 1
+    payload = None
+    initial_display = None
+    url = None
+    answer = None
+    queue_name = None
+
+    def setup_response(self):
+        """
+        Configure CodeResponse from XML. Supports both CodeResponse and ExternalResponse XML
+
+        TODO: Determines whether in synchronous or asynchronous (queued) mode
+        """
+        xml = self.xml
+        # TODO: XML can override external resource (grader/queue) URL
+        self.url = xml.get('url', None)
+
+        # We do not support xqueue within Studio.
+        if self.capa_system.xqueue is not None:
+            default_queuename = self.capa_system.xqueue.default_queuename
+        else:
+            default_queuename = None
+        self.queue_name = xml.get('queuename', default_queuename)
+
+        # VS[compat]:
+        # Check if XML uses the ExternalResponse format or the generic
+        # CodeResponse format
+        codeparam = self.xml.find('codeparam')
+        assert codeparam is not None, "Unsupported old format! <coderesponse> without <codeparam>"
+        self._parse_coderesponse_xml(codeparam)
+
+    def _parse_coderesponse_xml(self, codeparam):
+        """
+        Parse the new CodeResponse XML format. When successful, sets:
+            self.initial_display
+            self.answer (an answer to display to the student in the LMS)
+            self.payload
+        """
+        grader_payload = codeparam.find('grader_payload')
+        grader_payload = grader_payload.text if grader_payload is not None else ''
+        self.payload = {
+            'grader_payload': grader_payload,
+        }
+
+        # matlab api key can be defined in course settings. if so, add it to the grader payload
+        api_key = getattr(self.capa_system, 'matlab_api_key', None)
+        if api_key and self.xml.find('matlabinput') is not None:
+            self.payload['token'] = api_key
+            self.payload['endpoint_version'] = "2"
+            self.payload['requestor_id'] = self.capa_system.anonymous_student_id
+
+        self.initial_display = find_with_default(
+            codeparam, 'initial_display', '')
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        self.answer = find_with_default(codeparam, 'answer_display',
+                                        _('No answer provided.'))
+
+    def get_score(self, student_answers):
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        try:
+            # Note that submission can be a file
+            submission = student_answers[self.answer_id]
+        except Exception as err:
+            log.error(
+                'Error in CodeResponse %s: cannot get student answer for %s;'
+                ' student_answers=%s',
+                err, self.answer_id, convert_files_to_filenames(student_answers)
+            )
+            raise Exception(err)  # lint-amnesty, pylint: disable=raise-missing-from
+
+        # We do not support xqueue within Studio.
+        if self.capa_system.xqueue is None:
+            cmap = CorrectMap()
+            cmap.set(self.answer_id, queuestate=None,
+                     msg=_('Error: No grader has been set up for this problem.'))
+            return cmap
+
+        # Prepare xqueue request
+        #------------------------------------------------------------
+
+        qinterface = self.capa_system.xqueue.interface
+        qtime = datetime.strftime(datetime.now(UTC), xqueue_interface.dateformat)
+
+        anonymous_student_id = self.capa_system.anonymous_student_id
+
+        # Generate header
+        queuekey = xqueue_interface.make_hashkey(
+            str(self.capa_system.seed) + qtime + anonymous_student_id + self.answer_id
+        )
+        callback_url = self.capa_system.xqueue.construct_callback()
+        xheader = xqueue_interface.make_xheader(
+            lms_callback_url=callback_url,
+            lms_key=queuekey,
+            queue_name=self.queue_name
+        )
+
+        # Generate body
+        if is_list_of_files(submission):
+            # TODO: Get S3 pointer from the Queue
+            self.context.update({'submission': ''})
+        else:
+            self.context.update({'submission': submission})
+
+        contents = self.payload.copy()
+
+        # Metadata related to the student submission revealed to the external
+        # grader
+        student_info = {
+            'anonymous_student_id': anonymous_student_id,
+            'submission_time': qtime,
+            'random_seed': self.context['seed'],
+        }
+        contents.update({'student_info': json.dumps(student_info)})
+
+        # Submit request. When successful, 'msg' is the prior length of the
+        # queue
+
+        if is_list_of_files(submission):
+            # TODO: Is there any information we want to send here?
+            contents.update({'student_response': ''})
+            (error, msg) = qinterface.send_to_queue(header=xheader,
+                                                    body=json.dumps(contents),
+                                                    files_to_upload=submission)
+        else:
+            contents.update({'student_response': submission})
+            (error, msg) = qinterface.send_to_queue(header=xheader,
+                                                    body=json.dumps(contents))
+
+        # State associated with the queueing request
+        queuestate = {'key': queuekey,
+                      'time': qtime, }
+
+        cmap = CorrectMap()
+        if error:
+            _ = edx_six.get_gettext(self.capa_system.i18n)
+            error_msg = _('Unable to deliver your submission to grader (Reason: {error_msg}).'
+                          ' Please try again later.').format(error_msg=msg)
+            cmap.set(self.answer_id, queuestate=None, msg=error_msg)
+        else:
+            # Queueing mechanism flags:
+            #   1) Backend: Non-null CorrectMap['queuestate'] indicates that
+            #      the problem has been queued
+            #   2) Frontend: correctness='incomplete' eventually trickles down
+            #      through inputtypes.textbox and .filesubmission to inform the
+            #      browser to poll the LMS
+            cmap.set(self.answer_id, queuestate=queuestate,
+                     correctness='incomplete', msg=msg)
+
+        return cmap
+
+    def update_score(self, score_msg, oldcmap, queuekey):
+        """Updates the user's score based on the returned message from the grader."""
+        (valid_score_msg, correct, points, msg) = self._parse_score_msg(score_msg)
+
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+
+        if not valid_score_msg:
+            # Translators: 'grader' refers to the edX automatic code grader.
+            error_msg = _('Invalid grader reply. Please contact the course staff.')
+            oldcmap.set(self.answer_id, msg=error_msg)
+            return oldcmap
+
+        correctness = 'correct' if correct else 'incorrect'
+
+        # TODO: Find out how this is used elsewhere, if any
+        self.context['correct'] = correctness
+
+        # Replace 'oldcmap' with new grading results if queuekey matches.  If queuekey
+        # does not match, we keep waiting for the score_msg whose key actually
+        # matches
+        if oldcmap.is_right_queuekey(self.answer_id, queuekey):
+            # Sanity check on returned points
+            if points < 0:  # lint-amnesty, pylint: disable=consider-using-max-builtin
+                points = 0
+            # Queuestate is consumed
+            oldcmap.set(
+                self.answer_id, npoints=points, correctness=correctness,
+                msg=msg.replace('&nbsp;', '&#160;'), queuestate=None)
+        else:
+            log.debug(
+                'CodeResponse: queuekey %s does not match for answer_id=%s.',
+                queuekey,
+                self.answer_id
+            )
+
+        return oldcmap
+
+    def get_answers(self):
+        anshtml = HTML('<span class="code-answer"><pre><code>{}</code></pre></span>').format(self.answer)
+        return {self.answer_id: anshtml}
+
+    def get_initial_display(self):
+        """
+        The course author can specify an initial display
+        to be displayed the code response box.
+        """
+        return {self.answer_id: self.initial_display}
+
+    def _parse_score_msg(self, score_msg):
+        """
+         Grader reply is a JSON-dump of the following dict
+           { 'correct': True/False,
+             'score': Numeric value (floating point is okay) to assign to answer
+             'msg': grader_msg }
+
+        Returns (valid_score_msg, correct, score, msg):
+            valid_score_msg: Flag indicating valid score_msg format (Boolean)
+            correct:         Correctness of submission (Boolean)
+            score:           Points to be assigned (numeric, can be float)
+            msg:             Message from grader to display to student (string)
+        """
+        fail = (False, False, 0, '')
+        try:
+            score_result = json.loads(score_msg)
+        except (TypeError, ValueError):
+            log.error("External grader message should be a JSON-serialized dict."
+                      " Received score_msg = %s", score_msg)
+            return fail
+        if not isinstance(score_result, dict):
+            log.error("External grader message should be a JSON-serialized dict."
+                      " Received score_result = %s", score_result)
+            return fail
+        for tag in ['correct', 'score', 'msg']:
+            if tag not in score_result:
+                log.error("External grader message is missing one or more required"
+                          " tags: 'correct', 'score', 'msg'")
+                return fail
+
+        # Next, we need to check that the contents of the external grader message is safe for the LMS.
+        # 1) Make sure that the message is valid XML (proper opening/closing tags)
+        # 2) If it is not valid XML, make sure it is valid HTML.
+        #    Note: html5lib parser will try to repair any broken HTML
+        #    For example: <aaa></bbb> will become <aaa/>.
+        msg = score_result['msg']
+
+        try:
+            etree.fromstring(msg)
+        except etree.XMLSyntaxError as _err:
+            # If `html` contains attrs with no values, like `controls` in <audio controls src='smth'/>,
+            # XML parser will raise exception, so wee fallback to html5parser,
+            # which will set empty "" values for such attrs.
+            try:
+                parsed = html5lib.parseFragment(msg, treebuilder='lxml', namespaceHTMLElements=False)
+            except ValueError:
+                # the parsed message might contain strings that are not
+                # xml compatible, in which case, throw the error message
+                parsed = False
+
+            if not parsed:
+                log.error(
+                    "Unable to parse external grader message as valid"
+                    " XML: score_msg['msg']=%s",
+                    msg,
+                )
+                return fail
+
+        return (True, score_result['correct'], score_result['score'], msg)
+
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class ExternalResponse(LoncapaResponse):
+    """
+    Grade the students input using an external server.
+
+    Typically used by coding problems.
+
+    """
+
+    human_name = _('External Grader')
+    tags = ['externalresponse']
+    allowed_inputfields = ['textline', 'textbox']
+    awdmap = {
+        'EXACT_ANS': 'correct',         # TODO: handle other loncapa responses
+        'WRONG_FORMAT': 'incorrect',
+    }
+
+    def __init__(self, *args, **kwargs):
+        self.url = ''
+        self.tests = []
+        self.code = ''
+        super(ExternalResponse, self).__init__(*args, **kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def setup_response(self):
+        xml = self.xml
+        # FIXME - hardcoded URL
+        self.url = xml.get('url') or "http://qisx.mit.edu:8889/pyloncapa"
+
+        answer = xml.find('answer')
+        if answer is not None:
+            answer_src = answer.get('src')
+            if answer_src is not None:
+                # TODO: this code seems not to be used any more since self.capa_system.filesystem doesn't exist.
+                self.code = self.capa_system.filesystem.open('src/' + answer_src).read()
+            else:
+                self.code = answer.text
+        else:
+            # no <answer> stanza; get code from <script>
+            self.code = self.context['script_code']
+            if not self.code:
+                msg = '%s: Missing answer script code for externalresponse' % six.text_type(
+                    self)
+                msg += "\nSee XML source line %s" % getattr(
+                    self.xml, 'sourceline', '[unavailable]')
+                raise LoncapaProblemError(msg)
+
+        self.tests = xml.get('tests')
+
+    def do_external_request(self, cmd, extra_payload):
+        """
+        Perform HTTP request / post to external server.
+
+        cmd = remote command to perform (str)
+        extra_payload = dict of extra stuff to post.
+
+        Return XML tree of response (from response body)
+        """
+        xmlstr = etree.tostring(self.xml, pretty_print=True)
+        payload = {
+            'xml': xmlstr,
+            'edX_cmd': cmd,
+            'edX_tests': self.tests,
+            'processor': self.code,
+        }
+        payload.update(extra_payload)
+
+        try:
+            # call external server. TODO: synchronous call, can block for a
+            # long time
+            req = requests.post(self.url, data=payload)
+        except Exception as err:
+            msg = 'Error {0} - cannot connect to external server url={1}'.format(err, self.url)
+            log.error(msg)
+            raise Exception(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+
+        if self.capa_system.DEBUG:
+            log.info('response = %s', req.text)
+
+        if (not req.text) or (not req.text.strip()):
+            raise Exception(
+                'Error: no response from external server url=%s' % self.url)
+
+        try:
+            # response is XML; parse it
+            rxml = etree.fromstring(req.text)
+        except Exception as err:
+            msg = 'Error {0} - cannot parse response from external server req.text={1}'.format(err, req.text)
+            log.error(msg)
+            raise Exception(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+
+        return rxml
+
+    def get_score(self, student_answers):
+        idset = sorted(self.answer_ids)
+        cmap = CorrectMap()
+        try:
+            submission = [student_answers[k] for k in idset]
+        except Exception as err:
+            log.error(
+                'Error %s: cannot get student answer for %s; student_answers=%s',
+                err,
+                self.answer_ids,
+                student_answers
+            )
+            raise Exception(err)  # lint-amnesty, pylint: disable=raise-missing-from
+
+        self.context.update({'submission': submission})
+
+        extra_payload = {'edX_student_response': json.dumps(submission)}
+
+        try:
+            rxml = self.do_external_request('get_score', extra_payload)
+        except Exception as err:  # pylint: disable=broad-except
+            log.error('Error %s', err)
+            if self.capa_system.DEBUG:
+                cmap.set_dict(dict(list(zip(sorted(
+                    self.answer_ids), ['incorrect'] * len(idset)))))
+                cmap.set_property(
+                    self.answer_ids[0], 'msg',
+                    Text('<span class="inline-error">{}</span>').format(str(err))
+                )
+                return cmap
+
+        awd = rxml.find('awarddetail').text
+
+        self.context['correct'] = ['correct']
+        if awd in self.awdmap:
+            self.context['correct'][0] = self.awdmap[awd]
+
+        # create CorrectMap
+        for key in idset:
+            idx = idset.index(key)
+            msg = rxml.find('message').text.replace(
+                '&nbsp;', '&#160;') if idx == 0 else None
+            cmap.set(key, self.context['correct'][idx], msg=msg)
+
+        return cmap
+
+    def get_answers(self):
+        """
+        Use external server to get expected answers
+        """
+        try:
+            rxml = self.do_external_request('get_answers', {})
+            exans = json.loads(rxml.find('expected').text)
+        except Exception as err:  # pylint: disable=broad-except
+            log.error('Error %s', err)
+            if self.capa_system.DEBUG:
+                msg = HTML('<span class="inline-error">{}</span>').format(err)
+                exans = [''] * len(self.answer_ids)
+                exans[0] = msg
+
+        if not len(exans) == len(self.answer_ids):
+            log.error('Expected %s answers from external server, only got %s!',
+                      len(self.answer_ids), len(exans))
+            raise Exception('Short response from external server')
+        return dict(list(zip(self.answer_ids, exans)))
+
+
+#-----------------------------------------------------------------------------
+
+@registry.register
+class FormulaResponse(LoncapaResponse):
+    """
+    Checking of symbolic math response using numerical sampling.
+    """
+
+    human_name = _('Math Expression Input')
+    tags = ['formularesponse']
+    hint_tag = 'formulahint'
+    allowed_inputfields = ['textline', 'formulaequationinput']
+    required_attributes = ['answer', 'samples']
+    max_inputfields = 1
+    multi_device_support = True
+
+    def __init__(self, *args, **kwargs):
+        self.correct_answer = ''
+        self.samples = ''
+        self.tolerance = default_tolerance
+        self.case_sensitive = False
+        super(FormulaResponse, self).__init__(*args, **kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def setup_response(self):
+        xml = self.xml
+        context = self.context
+        self.correct_answer = contextualize_text(xml.get('answer'), context)
+        self.samples = contextualize_text(xml.get('samples'), context)
+
+        # Find the tolerance
+        tolerance_xml = xml.xpath(
+            '//*[@id=$id]//responseparam[@type="tolerance"]/@default',
+            id=xml.get('id')
+        )
+        if tolerance_xml:  # If it isn't an empty list...
+            self.tolerance = contextualize_text(tolerance_xml[0].strip(), context)
+
+        types = xml.get('type')
+        if types is None:
+            typeslist = []
+        else:
+            typeslist = types.split(',')
+        if 'ci' in typeslist:
+            # Case insensitive
+            self.case_sensitive = False
+        elif 'cs' in typeslist:
+            # Case sensitive
+            self.case_sensitive = True
+        else:
+            # Default
+            self.case_sensitive = False
+
+    def get_score(self, student_answers):
+        given = student_answers[self.answer_id]
+        correctness = self.check_formula(
+            self.correct_answer,
+            given,
+            self.samples
+        )
+        return CorrectMap(self.answer_id, correctness)
+
+    def tupleize_answers(self, answer, var_dict_list):
+        """
+        Takes in an answer and a list of dictionaries mapping variables to values.
+        Each dictionary represents a test case for the answer.
+        Returns a tuple of formula evaluation results.
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+
+        out = []
+        for var_dict in var_dict_list:
+            try:
+                out.append(evaluator(
+                    var_dict,
+                    {},
+                    answer,
+                    case_sensitive=self.case_sensitive,
+                ))
+            except UndefinedVariable as err:
+                log.debug(
+                    'formularesponse: undefined variable in formula=%s',
+                    html.escape(answer)
+                )
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    err.args[0]
+                )
+            except UnmatchedParenthesis as err:
+                log.debug(
+                    'formularesponse: unmatched parenthesis in formula=%s',
+                    html.escape(answer)
+                )
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    err.args[0]
+                )
+            except ValueError as err:
+                if 'factorial' in text_type(err):
+                    # This is thrown when fact() or factorial() is used in a formularesponse answer
+                    #   that tests on negative and/or non-integer inputs
+                    # text_type(err) will be: `factorial() only accepts integral values` or
+                    # `factorial() not defined for negative values`
+                    log.debug(
+                        ('formularesponse: factorial function used in response '
+                         'that tests negative and/or non-integer inputs. '
+                         'Provided answer was: %s'),
+                        html.escape(answer)
+                    )
+                    raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                        _("Factorial function not permitted in answer "
+                          "for this problem. Provided answer was: "
+                          "{bad_input}").format(bad_input=html.escape(answer))
+                    )
+                # If non-factorial related ValueError thrown, handle it the same as any other Exception
+                log.debug('formularesponse: error %s in formula', err)
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    _("Invalid input: Could not parse '{bad_input}' as a formula.").format(
+                        bad_input=html.escape(answer)
+                    )
+                )
+            except Exception as err:
+                # traceback.print_exc()
+                log.debug('formularesponse: error %s in formula', err)
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    _("Invalid input: Could not parse '{bad_input}' as a formula").format(
+                        bad_input=html.escape(answer)
+                    )
+                )
+        return out
+
+    def randomize_variables(self, samples):
+        """
+        Returns a list of dictionaries mapping variables to random values in range,
+        as expected by tupleize_answers.
+        """
+        variables = samples.split('@')[0].split(',')
+        numsamples = int(samples.split('@')[1].split('#')[1])
+        sranges = list(zip(*[list(map(float, x.split(","))) for x in samples.split('@')[1].split('#')[0].split(':')]))
+        ranges = dict(list(zip(variables, sranges)))
+
+        out = []
+        for _ in range(numsamples):
+            var_dict = {}
+            # ranges give numerical ranges for testing
+            for var in ranges:
+                # TODO: allow specified ranges (i.e. integers and complex numbers) for random variables
+                value = random.uniform(*ranges[var])
+                var_dict[str(var)] = value
+            out.append(var_dict)
+        return out
+
+    def check_formula(self, expected, given, samples):
+        """
+        Given an expected answer string, a given (student-produced) answer
+        string, and a samples string, return whether the given answer is
+        "correct" or "incorrect".
+        """
+        var_dict_list = self.randomize_variables(samples)
+        student_result = self.tupleize_answers(given, var_dict_list)
+        instructor_result = self.tupleize_answers(expected, var_dict_list)
+
+        correct = all(compare_with_tolerance(student, instructor, self.tolerance)
+                      for student, instructor in zip(student_result, instructor_result))
+        if correct:
+            return "correct"
+        else:
+            return "incorrect"
+
+    def compare_answer(self, ans1, ans2):
+        """
+        An external interface for comparing whether a and b are equal.
+        """
+        internal_result = self.check_formula(ans1, ans2, self.samples)
+        return internal_result == "correct"
+
+    def validate_answer(self, answer):
+        """
+        Returns whether this answer is in a valid form.
+        """
+        var_dict_list = self.randomize_variables(self.samples)
+        try:
+            self.tupleize_answers(answer, var_dict_list)
+            return True
+        except StudentInputError:
+            return False
+
+    def strip_dict(self, inp_d):
+        """
+        Takes a dict. Returns an identical dict, with all non-word
+        keys and all non-numeric values stripped out. All values also
+        converted to float. Used so we can safely use Python contexts.
+        """
+        inp_d = dict([(k, numpy.complex(inp_d[k]))  # lint-amnesty, pylint: disable=consider-using-dict-comprehension
+                      for k in inp_d if isinstance(k, str) and
+                      k.isalnum() and
+                      isinstance(inp_d[k], numbers.Number)])
+        return inp_d
+
+    def check_hint_condition(self, hxml_set, student_answers):
+        given = student_answers[self.answer_id]
+        hints_to_show = []
+        for hxml in hxml_set:
+            samples = hxml.get('samples')
+            name = hxml.get('name')
+            correct_answer = contextualize_text(
+                hxml.get('answer'), self.context)
+            # pylint: disable=broad-except
+            try:
+                correctness = self.check_formula(
+                    correct_answer,
+                    given,
+                    samples
+                )
+            except Exception:
+                correctness = 'incorrect'
+            if correctness == 'correct':
+                hints_to_show.append(name)
+        log.debug('hints_to_show = %s', hints_to_show)
+        return hints_to_show
+
+    def get_answers(self):
+        return {self.answer_id: self.correct_answer}
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class SchematicResponse(LoncapaResponse):
+    """
+    Circuit schematic response type.
+    """
+    human_name = _('Circuit Schematic Builder')
+    tags = ['schematicresponse']
+    allowed_inputfields = ['schematic']
+
+    def __init__(self, *args, **kwargs):
+        self.code = ''
+        super(SchematicResponse, self).__init__(*args, **kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def setup_response(self):
+        xml = self.xml
+        answer = xml.xpath('//*[@id=$id]//answer', id=xml.get('id'))[0]
+        answer_src = answer.get('src')
+        if answer_src is not None:
+            # Untested; never used
+            self.code = self.capa_system.resources_fs.open('src/' + answer_src).read()
+        else:
+            self.code = answer.text
+
+    def get_score(self, student_answers):
+        submission = [
+            json.loads(student_answers[k]) for k in sorted(self.answer_ids)
+        ]
+        self.context.update({'submission': submission})
+        try:
+            safe_exec.safe_exec(
+                self.code,
+                self.context,
+                cache=self.capa_system.cache,
+                python_path=self.context['python_path'],
+                extra_files=self.context['extra_files'],
+                slug=self.id,
+                random_seed=self.context['seed'],
+                unsafely=self.capa_system.can_execute_unsafe_code(),
+            )
+        except Exception as err:
+            _ = edx_six.get_gettext(self.capa_system.i18n)
+            # Translators: 'SchematicResponse' is a problem type and should not be translated.
+            msg = _('Error in evaluating SchematicResponse. The error was: {error_msg}').format(error_msg=err)
+            raise ResponseError(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+        cmap = CorrectMap()
+        cmap.set_dict(dict(list(zip(sorted(self.answer_ids), self.context['correct']))))
+        return cmap
+
+    def get_answers(self):
+        # use answers provided in input elements
+        return self.default_answer_map
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class ImageResponse(LoncapaResponse):
+    """
+    Handle student response for image input: the input is a click on an image,
+    which produces an [x,y] coordinate pair.  The click is correct if it falls
+    within a region specified.  This region is a union of rectangles.
+
+    Lon-CAPA requires that each <imageresponse> has a <foilgroup> inside it.
+    That doesn't make sense to me (Ike).  Instead, let's have it such that
+    <imageresponse> should contain one or more <imageinput> stanzas.
+    Each <imageinput> should specify a rectangle(s) or region(s), given as an
+    attribute, defining the correct answer.
+
+    <imageinput src="/static/images/Lecture2/S2_p04.png" width="811" height="610"
+    rectangle="(10,10)-(20,30);(12,12)-(40,60)"
+    regions="[[[10,10], [20,30], [40, 10]], [[100,100], [120,130], [110,150]]]"/>
+
+    Regions is list of lists [region1, region2, region3, ...] where regionN
+    is disordered list of points: [[1,1], [100,100], [50,50], [20, 70]].
+
+    If there is only one region in the list, simpler notation can be used:
+    regions="[[10,10], [30,30], [10, 30], [30, 10]]" (without explicitly
+        setting outer list)
+
+    Returns:
+        True, if click is inside any region or rectangle. Otherwise False.
+    """
+
+    human_name = _('Image Mapped Input')
+    tags = ['imageresponse']
+    allowed_inputfields = ['imageinput']
+
+    def __init__(self, *args, **kwargs):
+        self.ielements = []
+        super(ImageResponse, self).__init__(*args, **kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def setup_response(self):
+        self.ielements = self.inputfields
+        self.answer_ids = [ie.get('id') for ie in self.ielements]
+
+    def get_score(self, student_answers):
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        correct_map = CorrectMap()
+        expectedset = self.get_mapped_answers()
+        for aid in self.answer_ids:  # loop through IDs of <imageinput>
+            # Fields in our stanza
+            given = student_answers[aid]  # This should be a string of the form '[x,y]'
+            correct_map.set(aid, 'incorrect')
+            if not given:  # No answer to parse. Mark as incorrect and move on
+                continue
+            # Parse given answer
+            acoords = re.match(r'\[([0-9]+),([0-9]+)]', given.strip().replace(' ', ''))
+            if not acoords:
+                msg = _('error grading {image_input_id} (input={user_input})').format(
+                    image_input_id=aid,
+                    user_input=given
+                )
+                raise Exception('[capamodule.capa.responsetypes.imageinput] ' + msg)
+
+            (ans_x, ans_y) = [int(x) for x in acoords.groups()]
+
+            rectangles, regions = expectedset
+            if rectangles[aid]:  # Rectangles part - for backward compatibility
+                # Check whether given point lies in any of the solution
+                # rectangles
+                solution_rectangles = rectangles[aid].split(';')
+                for solution_rectangle in solution_rectangles:
+                    # parse expected answer
+                    # TODO: Compile regexp on file load
+                    sr_coords = re.match(
+                        r'[\(\[]([0-9]+),([0-9]+)[\)\]]-[\(\[]([0-9]+),([0-9]+)[\)\]]',
+                        solution_rectangle.strip().replace(' ', ''))
+                    if not sr_coords:
+                        # Translators: {sr_coords} are the coordinates of a rectangle
+                        msg = _('Error in problem specification! Cannot parse rectangle in {sr_coords}').format(
+                            sr_coords=etree.tostring(self.ielements[aid], pretty_print=True)
+                        )
+                        raise Exception('[capamodule.capa.responsetypes.imageinput] ' + msg)
+
+                    (llx, lly, urx, ury) = [int(x) for x in sr_coords.groups()]
+
+                    # answer is correct if (x,y) is within the specified
+                    # rectangle
+                    if (llx <= ans_x <= urx) and (lly <= ans_y <= ury):
+                        correct_map.set(aid, 'correct')
+                        break
+            if correct_map[aid]['correctness'] != 'correct' and regions[aid]:
+                parsed_region = json.loads(regions[aid])
+                if parsed_region:
+                    if not isinstance(parsed_region[0][0], list):
+                        # we have [[1,2],[3,4],[5,6]] - single region
+                        # instead of [[[1,2],[3,4],[5,6], [[1,2],[3,4],[5,6]]]
+                        # or [[[1,2],[3,4],[5,6]]] - multiple regions syntax
+                        parsed_region = [parsed_region]
+                    for region in parsed_region:
+                        polygon = MultiPoint(region).convex_hull
+                        if (polygon.type == 'Polygon' and
+                                polygon.contains(Point(ans_x, ans_y))):
+                            correct_map.set(aid, 'correct')
+                            break
+        return correct_map
+
+    def get_mapped_answers(self):
+        """
+        Returns the internal representation of the answers
+
+        Input:
+            None
+        Returns:
+            tuple (dict, dict) -
+                rectangles (dict) - a map of inputs to the defined rectangle for that input
+                regions (dict) - a map of inputs to the defined region for that input
+        """
+        answers = (
+            dict([(ie.get('id'), ie.get(  # lint-amnesty, pylint: disable=consider-using-dict-comprehension
+                'rectangle')) for ie in self.ielements]),
+            dict([(ie.get('id'), ie.get('regions')) for ie in self.ielements]))  # lint-amnesty, pylint: disable=consider-using-dict-comprehension
+        return answers
+
+    def get_answers(self):
+        """
+        Returns the external representation of the answers
+
+        Input:
+            None
+        Returns:
+            dict (str, (str, str)) - a map of inputs to a tuple of their rectangle
+                and their regions
+        """
+        answers = {}
+        for ielt in self.ielements:
+            ie_id = ielt.get('id')
+            answers[ie_id] = {'rectangle': ielt.get('rectangle'), 'regions': ielt.get('regions')}
+
+        return answers
+
+#-----------------------------------------------------------------------------
+
+
+@registry.register
+class AnnotationResponse(LoncapaResponse):
+    """
+    Checking of annotation responses.
+
+    The response contains both a comment (student commentary) and an option (student tag).
+    Only the tag is currently graded. Answers may be incorrect, partially correct, or correct.
+    """
+    human_name = _('Annotation Input')
+    tags = ['annotationresponse']
+    allowed_inputfields = ['annotationinput']
+    max_inputfields = 1
+    default_scoring = {'incorrect': 0, 'partially-correct': 1, 'correct': 2}
+
+    def __init__(self, *args, **kwargs):
+        self.scoring_map = {}
+        self.answer_map = {}
+        super(AnnotationResponse, self).__init__(*args, **kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def setup_response(self):
+        self.scoring_map = self._get_scoring_map()
+        self.answer_map = self._get_answer_map()
+        self.maxpoints = self._get_max_points()
+
+    def get_score(self, student_answers):
+        """
+        Returns a CorrectMap for the student answer, which may include
+        partially correct answers.
+        """
+        student_answer = student_answers[self.answer_id]
+        student_option = self._get_submitted_option_id(student_answer)
+
+        scoring = self.scoring_map[self.answer_id]
+        is_valid = student_option is not None and student_option in list(scoring.keys(
+        ))
+
+        (correctness, points) = ('incorrect', None)
+        if is_valid:
+            correctness = scoring[student_option]['correctness']
+            points = scoring[student_option]['points']
+
+        return CorrectMap(self.answer_id, correctness=correctness, npoints=points)
+
+    def get_answers(self):
+        return self.answer_map
+
+    def _get_scoring_map(self):
+        """Returns a dict of option->scoring for each input."""
+        scoring = self.default_scoring
+        choices = dict([(choice, choice) for choice in scoring])  # lint-amnesty, pylint: disable=consider-using-dict-comprehension
+        scoring_map = {}
+
+        for inputfield in self.inputfields:
+            option_scoring = dict([(  # lint-amnesty, pylint: disable=consider-using-dict-comprehension
+                option['id'],
+                {
+                    'correctness': choices.get(option['choice']),
+                    'points': scoring.get(option['choice'])
+                }
+            ) for option in self._find_options(inputfield)])
+
+            scoring_map[inputfield.get('id')] = option_scoring
+
+        return scoring_map
+
+    def _get_answer_map(self):
+        """Returns a dict of answers for each input."""
+        answer_map = {}
+        for inputfield in self.inputfields:
+            correct_option = self._find_option_with_choice(
+                inputfield, 'correct')
+            if correct_option is not None:
+                input_id = inputfield.get('id')
+                answer_map[input_id] = correct_option.get('description')
+        return answer_map
+
+    def _get_max_points(self):
+        """Returns a dict of the max points for each input: input id -> maxpoints."""
+        scoring = self.default_scoring
+        correct_points = scoring.get('correct')
+        return dict([(inputfield.get('id'), correct_points) for inputfield in self.inputfields])  # lint-amnesty, pylint: disable=consider-using-dict-comprehension
+
+    def _find_options(self, inputfield):
+        """Returns an array of dicts where each dict represents an option. """
+        elements = inputfield.findall('./options/option')
+        return [
+            {
+                'id': index,
+                'description': option.text,
+                'choice': option.get('choice')
+            } for (index, option) in enumerate(elements)
+        ]
+
+    def _find_option_with_choice(self, inputfield, choice):
+        """Returns the option with the given choice value, otherwise None. """
+        for option in self._find_options(inputfield):
+            if option['choice'] == choice:
+                return option
+
+    def _unpack(self, json_value):
+        """Unpacks a student response value submitted as JSON."""
+        json_d = json.loads(json_value)
+        if not isinstance(json_d, dict):
+            json_d = {}
+
+        comment_value = json_d.get('comment', '')
+        if not isinstance(json_d, six.string_types):
+            comment_value = ''
+
+        options_value = json_d.get('options', [])
+        if not isinstance(options_value, list):
+            options_value = []
+
+        return {
+            'options_value': options_value,
+            'comment_value': comment_value
+        }
+
+    def _get_submitted_option_id(self, student_answer):
+        """Return the single option that was selected, otherwise None."""
+        submitted = self._unpack(student_answer)
+        option_ids = submitted['options_value']
+        if len(option_ids) == 1:
+            return option_ids[0]
+        return None
+
+
+@registry.register
+class ChoiceTextResponse(LoncapaResponse):
+    """
+    Allows for multiple choice responses with text inputs
+    Desired semantics match those of NumericalResponse and
+    ChoiceResponse.
+    """
+
+    human_name = _('Checkboxes With Text Input')
+    tags = ['choicetextresponse']
+    max_inputfields = 1
+    allowed_inputfields = [
+        'choicetextgroup',
+        'checkboxtextgroup',
+        'radiotextgroup',
+    ]
+
+    def __init__(self, *args, **kwargs):
+        self.correct_inputs = {}
+        self.answer_values = {}
+        self.correct_choices = {}
+        super(ChoiceTextResponse, self).__init__(*args, **kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def setup_response(self):
+        """
+        Sets up three dictionaries for use later:
+        `correct_choices`: These are the correct binary choices(radio/checkbox)
+        `correct_inputs`: These are the numerical/string answers for required
+        inputs.
+        `answer_values`: This is a dict, keyed by the name of the binary choice
+            which contains the correct answers for the text inputs separated by
+            commas e.g. "1, 0.5"
+
+        `correct_choices` and `correct_inputs` are used for grading the problem
+        and `answer_values` is used for displaying correct answers.
+
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        context = self.context
+        self.answer_values = {self.answer_id: []}
+        self.assign_choice_names()
+        correct_xml = self.xml.xpath('//*[@id=$id]//choice[@correct="true"]',
+                                     id=self.xml.get('id'))
+
+        for node in correct_xml:
+            # For each correct choice, set the `parent_name` to the
+            # current choice's name
+            parent_name = node.get('name')
+            # Add the name of the correct binary choice to the
+            # correct choices list as a key. The value is not important.
+            self.correct_choices[parent_name] = {'answer': ''}
+            # Add the name of the parent to the list of correct answers
+            self.answer_values[self.answer_id].append(parent_name)
+            answer_list = []
+            # Loop over <numtolerance_input> elements inside of the correct choices
+            for child in node:
+                answer = child.get('answer', None)
+                if not answer:
+                    # If the question creator does not specify an answer for a
+                    # <numtolerance_input> inside of a correct choice, raise an error
+                    raise LoncapaProblemError(
+                        _("Answer not provided for {input_type}").format(input_type="numtolerance_input")
+                    )
+                # Contextualize the answer to allow script generated answers.
+                answer = contextualize_text(answer, context)
+                input_name = child.get('name')
+                # Contextualize the tolerance to value.
+                tolerance = contextualize_text(
+                    child.get('tolerance', default_tolerance),
+                    context
+                )
+                # Add the answer and tolerance information for the current
+                # numtolerance_input to `correct_inputs`
+                self.correct_inputs[input_name] = {
+                    'answer': answer,
+                    'tolerance': tolerance
+                }
+                # Add the correct answer for this input to the list for show
+                answer_list.append(answer)
+            # Turn the list of numtolerance_input answers into a comma separated string.
+            self.answer_values[parent_name] = ', '.join(answer_list)
+        # Turn correct choices into a set. Allows faster grading.
+        self.correct_choices = set(self.correct_choices.keys())
+
+    def assign_choice_names(self):
+        """
+        Initialize name attributes in <choice> and <numtolerance_input> tags
+        for this response.
+
+        Example:
+        Assuming for simplicity that `self.answer_id` = '1_2_1'
+
+        Before the function is called `self.xml` =
+        <radiotextgroup>
+            <choice correct = "true">
+                The number
+                    <numtolerance_input answer="5"/>
+                Is the mean of the list.
+            </choice>
+            <choice correct = "false">
+                False demonstration choice
+            </choice>
+        </radiotextgroup>
+
+        After this is called the choices and numtolerance_inputs will have a name
+        attribute initialized and self.xml will be:
+
+        <radiotextgroup>
+        <choice correct = "true" name ="1_2_1_choiceinput_0bc">
+            The number
+                <numtolerance_input name = "1_2_1_choiceinput0_numtolerance_input_0"
+                 answer="5"/>
+            Is the mean of the list.
+        </choice>
+        <choice correct = "false" name = "1_2_1_choiceinput_1bc>
+            False demonstration choice
+        </choice>
+        </radiotextgroup>
+        """
+
+        choices = self.xml.xpath('//*[@id=$id]//choice', id=self.xml.get('id'))
+        for index, choice in enumerate(choices):
+            # Set the name attribute for <choices>
+            # "bc" is appended at the end to indicate that this is a
+            # binary choice as opposed to a numtolerance_input, this convention
+            # is used when grading the problem
+            choice.set(
+                "name",
+                self.answer_id + "_choiceinput_" + str(index) + "bc"
+            )
+            # Set Name attributes for <numtolerance_input> elements
+            # Look for all <numtolerance_inputs> inside this choice.
+            numtolerance_inputs = choice.findall('numtolerance_input')
+            # Look for all <decoy_input> inside this choice
+            decoys = choice.findall('decoy_input')
+            # <decoy_input> would only be used in choices which do not contain
+            # <numtolerance_input>
+            inputs = numtolerance_inputs if numtolerance_inputs else decoys
+            # Give each input inside of the choice a name combining
+            # The ordinality of the choice, and the ordinality of the input
+            # within that choice e.g. 1_2_1_choiceinput_0_numtolerance_input_1
+            for ind, child in enumerate(inputs):
+                child.set(
+                    "name",
+                    self.answer_id + "_choiceinput_" + str(index) +
+                    "_numtolerance_input_" + str(ind)
+                )
+
+    def get_score(self, student_answers):
+        """
+        Returns a `CorrectMap` showing whether `student_answers` are correct.
+
+        `student_answers` contains keys for binary inputs(radiobutton,
+        checkbox) and numerical inputs. Keys ending with 'bc' are binary
+        choice inputs otherwise they are text fields.
+
+        This method first separates the two
+        types of answers and then grades them in separate methods.
+
+        The student is only correct if they have both the binary inputs and
+        numerical inputs correct.
+        """
+        answer_dict = student_answers.get(self.answer_id, "")
+        binary_choices, numtolerance_inputs = self._split_answers_dict(answer_dict)
+        # Check the binary choices first.
+        choices_correct = self._check_student_choices(binary_choices)
+        inputs_correct = self._check_student_inputs(numtolerance_inputs)
+        # Only return correct if the student got both the binary
+        # and numtolerance_inputs are correct
+        correct = choices_correct and inputs_correct
+
+        return CorrectMap(
+            self.answer_id,
+            'correct' if correct else 'incorrect'
+        )
+
+    def get_answers(self):
+        """
+        Returns a dictionary containing the names of binary choices as keys
+        and a string of answers to any numtolerance_inputs which they may have
+        e.g {choice_1bc : "answer1, answer2", choice_2bc : ""}
+        """
+        return self.answer_values
+
+    def _split_answers_dict(self, a_dict):
+        """
+        Returns two dicts:
+        `binary_choices` : dictionary {input_name: input_value} for
+        the binary choices which the student selected.
+        and
+        `numtolerance_choices` : a dictionary {input_name: input_value}
+        for the numtolerance_inputs inside of choices which were selected
+
+        Determines if an input is inside of a binary input by looking at
+        the beginning of it's name.
+
+        For example. If a binary_choice was named '1_2_1_choiceinput_0bc'
+        All of the numtolerance_inputs in it would have an idea that begins
+        with '1_2_1_choice_input_0_numtolerance_input'
+
+        Splits the name of the numtolerance_input at the occurence of
+        '_numtolerance_input_' and appends 'bc' to the end to get the name
+        of the choice it is contained in.
+
+        Example:
+        `a_dict` = {
+            '1_2_1_choiceinput_0bc': '1_2_1_choiceinput_0bc',
+            '1_2_1_choiceinput_0_numtolerance_input_0': '1',
+            '1_2_1_choiceinput_0_numtolerance_input_1': '2'
+            '1_2_1_choiceinput_1_numtolerance_input_0': '3'
+        }
+
+        In this case, the binary choice is '1_2_1_choiceinput_0bc', and
+        the numtolerance_inputs associated with it are
+        '1_2_1_choiceinput_0_numtolerance_input_0', and
+        '1_2_1_choiceinput_0_numtolerance_input_1'.
+
+        so the two return dictionaries would be
+        `binary_choices` = {'1_2_1_choiceinput_0bc': '1_2_1_choiceinput_0bc'}
+        and
+        `numtolerance_choices` ={
+            '1_2_1_choiceinput_0_numtolerance_input_0': '1',
+            '1_2_1_choiceinput_0_numtolerance_input_1': '2'
+        }
+
+        The entry '1_2_1_choiceinput_1_numtolerance_input_0': '3' is discarded
+        because it was not inside of a selected binary choice, and no validation
+        should be performed on numtolerance_inputs inside of non-selected choices.
+        """
+
+        # Initialize the two dictionaries that are returned
+        numtolerance_choices = {}
+        binary_choices = {}
+
+        # `selected_choices` is a list of binary choices which were "checked/selected"
+        # when the student submitted the problem.
+        # Keys in a_dict ending with 'bc' refer to binary choices.
+        selected_choices = [key for key in a_dict if key.endswith("bc")]
+        for key in selected_choices:
+            binary_choices[key] = a_dict[key]
+
+        # Convert the name of a numtolerance_input into the name of the binary
+        # choice that it is contained within, and append it to the list if
+        # the numtolerance_input's parent binary_choice is contained in
+        # `selected_choices`.
+        selected_numtolerance_inputs = [
+            key for key in a_dict if key.partition("_numtolerance_input_")[0] + "bc"
+            in selected_choices
+        ]
+
+        for key in selected_numtolerance_inputs:
+            numtolerance_choices[key] = a_dict[key]
+
+        return (binary_choices, numtolerance_choices)
+
+    def _check_student_choices(self, choices):
+        """
+        Compares student submitted checkbox/radiobutton answers against
+        the correct answers. Returns True or False.
+
+        True if all of the correct choices are selected and no incorrect
+        choices are selected.
+        """
+        student_choices = set(choices)
+        required_selected = len(self.correct_choices - student_choices) == 0
+        no_extra_selected = len(student_choices - self.correct_choices) == 0
+        correct = required_selected and no_extra_selected
+        return correct
+
+    def _check_student_inputs(self, numtolerance_inputs):
+        """
+        Compares student submitted numerical answers against the correct
+        answers and tolerances.
+
+        `numtolerance_inputs` is a dictionary {answer_name : answer_value}
+
+        Performs numerical validation by means of calling
+        `compare_with_tolerance()` on all of `numtolerance_inputs`
+
+        Performs a call to `compare_with_tolerance` even on values for
+        decoy_inputs. This is used to validate their numericality and
+        raise an error if the student entered a non numerical expression.
+
+        Returns True if and only if all student inputs are correct.
+        """
+        _ = edx_six.get_gettext(self.capa_system.i18n)
+        inputs_correct = True
+        for answer_name, answer_value in six.iteritems(numtolerance_inputs):
+            # If `self.corrrect_inputs` does not contain an entry for
+            # `answer_name`, this means that answer_name is a decoy
+            # input's value, and validation of its numericality is the
+            # only thing of interest from the later call to
+            # `compare_with_tolerance`.
+            params = self.correct_inputs.get(answer_name, {'answer': 0})
+
+            correct_ans = params['answer']
+            # Set the tolerance to '0' if it was not specified in the xml
+            tolerance = params.get('tolerance', default_tolerance)
+            # Make sure that the staff answer is a valid number
+            try:
+                correct_ans = complex(correct_ans)
+            except ValueError:
+                log.debug(
+                    "Content error--answer '%s' is not a valid complex number",
+                    correct_ans
+                )
+                raise StudentInputError(  # lint-amnesty, pylint: disable=raise-missing-from
+                    _("The Staff answer could not be interpreted as a number.")
+                )
+            # Compare the student answer to the staff answer/ or to 0
+            # if all that is important is verifying numericality
+            try:
+                partial_correct = compare_with_tolerance(
+                    evaluator({}, {}, answer_value),
+                    correct_ans,
+                    tolerance
+                )
+            except:
+                # Use the traceback-preserving version of re-raising with a
+                # different type
+                __, __, trace = sys.exc_info()
+                msg = _("Could not interpret '{given_answer}' as a number.").format(
+                    given_answer=html.escape(answer_value)
+                )
+                msg += " ({0})".format(trace)
+                raise StudentInputError(msg)  # lint-amnesty, pylint: disable=raise-missing-from
+
+            # Ignore the results of the comparisons which were just for
+            # Numerical Validation.
+            if answer_name in self.correct_inputs and not partial_correct:
+                # If any input is not correct, set the return value to False
+                inputs_correct = False
+        return inputs_correct
+
+#-----------------------------------------------------------------------------
+
+# TEMPORARY: List of all response subclasses
+# FIXME: To be replaced by auto-registration
+
+# pylint: disable=invalid-all-object
+__all__ = [
+    CodeResponse,
+    NumericalResponse,
+    FormulaResponse,
+    CustomResponse,
+    SchematicResponse,
+    ExternalResponse,
+    ImageResponse,
+    OptionResponse,
+    SymbolicResponse,
+    StringResponse,
+    ChoiceResponse,
+    MultipleChoiceResponse,
+    TrueFalseResponse,
+    AnnotationResponse,
+    ChoiceTextResponse,
+]
+# pylint: enable=invalid-all-object

xmodule/capa/safe_exec/README.rst

@@ -0,0 +1,43 @@
+Configuring Capa sandboxed execution
+====================================
+
+Capa problems can contain code authored by the course author.  We need to
+execute that code in a sandbox.  We use CodeJail as the sandboxing facility,
+but it needs to be configured specifically for Capa's use.
+
+As a developer, you don't have to do anything to configure sandboxing if you
+don't want to, and everything will operate properly, you just won't have
+protection on that code.
+
+If you want to configure sandboxing, you're going to use the `README from
+CodeJail`__, with a few customized tweaks.
+
+__ https://github.com/edx/codejail/blob/master/README.rst
+
+
+1. At the instruction to install packages into the sandboxed code, you'll
+   need to install the requirements from requirements/edx-sandbox::
+
+    $ pip install -r requirements/edx-sandbox/base.txt
+
+2. You can configure resource limits in settings.py.  A CODE_JAIL setting is
+   available, a dictionary.  The "limits" key lets you adjust the limits for
+   CPU time, real time, and memory use.  Setting any of them to zero disables
+   that limit::
+
+    # in settings.py...
+    CODE_JAIL = {
+        # Configurable limits.
+        'limits': {
+            # How many CPU seconds can jailed code use?
+            'CPU': 1,
+            # How many real-time seconds will a sandbox survive?
+            'REALTIME': 1,
+            # How much memory (in bytes) can a sandbox use?
+            'VMEM': 30000000,
+        },
+    }
+
+
+That's it.  Once you've finished the CodeJail configuration instructions,
+your course-hosted Python code should be run securely.

xmodule/capa/safe_exec/__init__.py

@@ -0,0 +1,3 @@
+"""Capa's specialized use of codejail.safe_exec."""
+
+from .safe_exec import safe_exec, update_hash

xmodule/capa/safe_exec/exceptions.py

@@ -0,0 +1,21 @@
+"""
+Exceptions related to safe exec.
+"""
+
+
+class CodejailServiceParseError(Exception):
+    """
+    An exception that is raised whenever we have issues with data parsing.
+    """
+
+
+class CodejailServiceStatusError(Exception):
+    """
+    An exception that is raised whenever Codejail service response status is different to 200.
+    """
+
+
+class CodejailServiceUnavailable(Exception):
+    """
+    An exception that is raised whenever Codejail service is unavailable.
+    """

xmodule/capa/safe_exec/lazymod.py

@@ -0,0 +1,43 @@
+"""A module proxy for delayed importing of modules.
+
+From http://barnesc.blogspot.com/2006/06/automatic-python-imports-with-autoimp.html,
+in the public domain.
+
+"""
+
+import sys
+
+
+class LazyModule(object):
+    """A lazy module proxy."""
+
+    def __init__(self, modname):
+        self.__dict__['__name__'] = modname
+        self._set_mod(None)
+
+    def _set_mod(self, mod):
+        if mod is not None:
+            self.__dict__ = mod.__dict__
+        self.__dict__['_lazymod_mod'] = mod
+
+    def _load_mod(self):
+        __import__(self.__name__)
+        self._set_mod(sys.modules[self.__name__])
+
+    def __getattr__(self, name):
+        if self.__dict__['_lazymod_mod'] is None:
+            self._load_mod()
+
+        mod = self.__dict__['_lazymod_mod']
+
+        if hasattr(mod, name):
+            return getattr(mod, name)
+        else:
+            try:
+                subname = '%s.%s' % (self.__name__, name)
+                __import__(subname)
+                submod = getattr(mod, name)  # lint-amnesty, pylint: disable=unused-variable
+            except ImportError:
+                raise AttributeError("'module' object has no attribute %r" % name)  # lint-amnesty, pylint: disable=raise-missing-from
+            self.__dict__[name] = LazyModule(subname)
+            return self.__dict__[name]

xmodule/capa/safe_exec/remote_exec.py

@@ -0,0 +1,107 @@
+"""
+Helper methods related to safe exec.
+"""
+
+import json
+import logging
+from importlib import import_module
+import requests
+
+from codejail.safe_exec import SafeExecException
+from django.conf import settings
+from edx_toggles.toggles import SettingToggle
+from requests.exceptions import RequestException, HTTPError
+from simplejson import JSONDecodeError
+
+from django.utils.translation import gettext as _
+from .exceptions import CodejailServiceParseError, CodejailServiceStatusError, CodejailServiceUnavailable
+
+log = logging.getLogger(__name__)
+
+# .. toggle_name: ENABLE_CODEJAIL_REST_SERVICE
+# .. toggle_implementation: SettingToggle
+# .. toggle_default: False
+# .. toggle_description: Set this to True if you want to run Codejail code using
+#   a separate VM or container and communicate with edx-platform using REST API.
+# .. toggle_use_cases: open_edx
+# .. toggle_creation_date: 2021-08-19
+ENABLE_CODEJAIL_REST_SERVICE = SettingToggle(
+    "ENABLE_CODEJAIL_REST_SERVICE", default=False, module_name=__name__
+)
+
+
+def is_codejail_rest_service_enabled():
+    return ENABLE_CODEJAIL_REST_SERVICE.is_enabled()
+
+
+def get_remote_exec(*args, **kwargs):
+    """Get remote exec function based on setting and executes it."""
+    remote_exec_function_name = settings.CODE_JAIL_REST_SERVICE_REMOTE_EXEC
+    try:
+        mod_name, func_name = remote_exec_function_name.rsplit('.', 1)
+        remote_exec_module = import_module(mod_name)
+        remote_exec_function = getattr(remote_exec_module, func_name)
+        if not remote_exec_function:
+            remote_exec_function = send_safe_exec_request_v0
+    except ModuleNotFoundError:
+        return send_safe_exec_request_v0(*args, **kwargs)
+    return remote_exec_function(*args, **kwargs)
+
+
+def get_codejail_rest_service_endpoint():
+    return f"{settings.CODE_JAIL_REST_SERVICE_HOST}/api/v0/code-exec"
+
+
+def send_safe_exec_request_v0(data):
+    """
+    Sends a request to a codejail api service forwarding required code and files.
+    Arguments:
+        data: Dict containing code and other parameters
+            required for jailed code execution.
+            It also includes extra_files (python_lib.zip) required by the codejail execution.
+    Returns:
+        Response received from codejail api service
+    """
+    globals_dict = data["globals_dict"]
+    extra_files = data.pop("extra_files")
+
+    codejail_service_endpoint = get_codejail_rest_service_endpoint()
+    payload = json.dumps(data)
+
+    try:
+        response = requests.post(
+            codejail_service_endpoint,
+            files=extra_files,
+            data={'payload': payload},
+            timeout=(settings.CODE_JAIL_REST_SERVICE_CONNECT_TIMEOUT, settings.CODE_JAIL_REST_SERVICE_READ_TIMEOUT)
+        )
+
+    except RequestException as err:
+        log.error("Failed to connect to codejail api service: url=%s, params=%s",
+                  codejail_service_endpoint, str(payload))
+        raise CodejailServiceUnavailable(_(
+            "Codejail API Service is unavailable. "
+            "Please try again in a few minutes."
+        )) from err
+
+    try:
+        response.raise_for_status()
+    except HTTPError as err:
+        raise CodejailServiceStatusError(_("Codejail API Service invalid response.")) from err
+
+    try:
+        response_json = response.json()
+    except JSONDecodeError as err:
+        log.error("Invalid JSON response received from codejail api service: Response_Content=%s", response.content)
+        raise CodejailServiceParseError(_("Invalid JSON response received from codejail api service.")) from err
+
+    emsg = response_json.get("emsg")
+    exception = None
+
+    if emsg:
+        exception_msg = f"{emsg}. For more information check Codejail Service logs."
+        exception = SafeExecException(exception_msg)
+
+    globals_dict.update(response_json.get("globals_dict"))
+
+    return emsg, exception

xmodule/capa/safe_exec/safe_exec.py

@@ -0,0 +1,192 @@
+"""Capa's specialized use of codejail.safe_exec."""
+
+
+import hashlib
+
+from codejail.safe_exec import SafeExecException, json_safe
+from codejail.safe_exec import not_safe_exec as codejail_not_safe_exec
+from codejail.safe_exec import safe_exec as codejail_safe_exec
+from edx_django_utils.monitoring import function_trace
+import six
+from six import text_type
+
+from . import lazymod
+from .remote_exec import is_codejail_rest_service_enabled, get_remote_exec
+
+# Establish the Python environment for Capa.
+# Capa assumes float-friendly division always.
+# The name "random" is a properly-seeded stand-in for the random module.
+CODE_PROLOG = """\
+from __future__ import absolute_import, division
+
+import os
+os.environ["OPENBLAS_NUM_THREADS"] = "1"    # See TNL-6456
+
+import random2 as random_module
+import sys
+from six.moves import xrange
+
+random = random_module.Random(%r)
+random.Random = random_module.Random
+sys.modules['random'] = random
+"""
+
+ASSUMED_IMPORTS = [
+    ("numpy", "numpy"),
+    ("math", "math"),
+    ("scipy", "scipy"),
+    ("calc", "calc"),
+    ("eia", "eia"),
+    ("chemcalc", "chem.chemcalc"),
+    ("chemtools", "chem.chemtools"),
+    ("miller", "chem.miller"),
+    ("draganddrop", "verifiers.draganddrop"),
+]
+
+# We'll need the code from lazymod.py for use in safe_exec, so read it now.
+lazymod_py_file = lazymod.__file__
+if lazymod_py_file.endswith("c"):
+    lazymod_py_file = lazymod_py_file[:-1]
+
+with open(lazymod_py_file) as f:
+    lazymod_py = f.read()
+
+LAZY_IMPORTS = [lazymod_py]
+for name, modname in ASSUMED_IMPORTS:
+    LAZY_IMPORTS.append("{} = LazyModule('{}')\n".format(name, modname))
+
+LAZY_IMPORTS = "".join(LAZY_IMPORTS)
+
+
+def update_hash(hasher, obj):
+    """
+    Update a `hashlib` hasher with a nested object.
+
+    To properly cache nested structures, we need to compute a hash from the
+    entire structure, canonicalizing at every level.
+
+    `hasher`'s `.update()` method is called a number of times, touching all of
+    `obj` in the process.  Only primitive JSON-safe types are supported.
+
+    """
+    hasher.update(six.b(str(type(obj))))
+    if isinstance(obj, (tuple, list)):
+        for e in obj:
+            update_hash(hasher, e)
+    elif isinstance(obj, dict):
+        for k in sorted(obj):
+            update_hash(hasher, k)
+            update_hash(hasher, obj[k])
+    else:
+        hasher.update(six.b(repr(obj)))
+
+
+@function_trace('safe_exec')
+def safe_exec(
+    code,
+    globals_dict,
+    random_seed=None,
+    python_path=None,
+    extra_files=None,
+    cache=None,
+    limit_overrides_context=None,
+    slug=None,
+    unsafely=False,
+):
+    """
+    Execute python code safely.
+
+    `code` is the Python code to execute.  It has access to the globals in `globals_dict`,
+    and any changes it makes to those globals are visible in `globals_dict` when this
+    function returns.
+
+    `random_seed` will be used to see the `random` module available to the code.
+
+    `python_path` is a list of filenames or directories to add to the Python
+    path before execution.  If the name is not in `extra_files`, then it will
+    also be copied into the sandbox.
+
+    `extra_files` is a list of (filename, contents) pairs.  These files are
+    created in the sandbox.
+
+    `cache` is an object with .get(key) and .set(key, value) methods.  It will be used
+    to cache the execution, taking into account the code, the values of the globals,
+    and the random seed.
+
+    `limit_overrides_context` is an optional string to be used as a key on
+    the `settings.CODE_JAIL['limit_overrides']` dictionary in order to apply
+    context-specific overrides to the codejail execution limits.
+    If `limit_overrides_context` is omitted or not present in limit_overrides,
+    then use the default limits specified insettings.CODE_JAIL['limits'].
+
+    `slug` is an arbitrary string, a description that's meaningful to the
+    caller, that will be used in log messages.
+
+    If `unsafely` is true, then the code will actually be executed without sandboxing.
+    """
+    # Check the cache for a previous result.
+    if cache:
+        safe_globals = json_safe(globals_dict)
+        md5er = hashlib.md5()
+        md5er.update(repr(code).encode('utf-8'))
+        update_hash(md5er, safe_globals)
+        key = "safe_exec.%r.%s" % (random_seed, md5er.hexdigest())
+        cached = cache.get(key)
+        if cached is not None:
+            # We have a cached result.  The result is a pair: the exception
+            # message, if any, else None; and the resulting globals dictionary.
+            emsg, cleaned_results = cached
+            globals_dict.update(cleaned_results)
+            if emsg:
+                raise SafeExecException(emsg)
+            return
+
+    # Create the complete code we'll run.
+    code_prolog = CODE_PROLOG % random_seed
+
+    if is_codejail_rest_service_enabled():
+        data = {
+            "code": code_prolog + LAZY_IMPORTS + code,
+            "globals_dict": globals_dict,
+            "python_path": python_path,
+            "limit_overrides_context": limit_overrides_context,
+            "slug": slug,
+            "unsafely": unsafely,
+            "extra_files": extra_files,
+        }
+
+        emsg, exception = get_remote_exec(data)
+
+    else:
+        # Decide which code executor to use.
+        if unsafely:
+            exec_fn = codejail_not_safe_exec
+        else:
+            exec_fn = codejail_safe_exec
+
+        # Run the code!  Results are side effects in globals_dict.
+        try:
+            exec_fn(
+                code_prolog + LAZY_IMPORTS + code,
+                globals_dict,
+                python_path=python_path,
+                extra_files=extra_files,
+                limit_overrides_context=limit_overrides_context,
+                slug=slug,
+            )
+        except SafeExecException as e:
+            # Saving SafeExecException e in exception to be used later.
+            exception = e
+            emsg = text_type(e)
+        else:
+            emsg = None
+
+    # Put the result back in the cache.  This is complicated by the fact that
+    # the globals dict might not be entirely serializable.
+    if cache:
+        cleaned_results = json_safe(globals_dict)
+        cache.set(key, (emsg, cleaned_results))
+
+    # If an exception happened, raise it now.
+    if emsg:
+        raise exception

xmodule/capa/safe_exec/tests/test_files/pylib/constant.py

@@ -0,0 +1 @@
+THE_CONST = 23

xmodule/capa/safe_exec/tests/test_lazymod.py

@@ -0,0 +1,57 @@
+"""Test lazymod.py"""
+
+
+import sys
+import unittest
+
+from xmodule.capa.safe_exec.lazymod import LazyModule
+
+
+class ModuleIsolation(object):
+    """
+    Manage changes to sys.modules so that we can roll back imported modules.
+
+    Create this object, it will snapshot the currently imported modules. When
+    you call `clean_up()`, it will delete any module imported since its creation.
+    """
+    def __init__(self):
+        # Save all the names of all the imported modules.
+        self.mods = set(sys.modules)
+
+    def clean_up(self):  # lint-amnesty, pylint: disable=missing-function-docstring
+        # Get a list of modules that didn't exist when we were created
+        new_mods = [m for m in sys.modules if m not in self.mods]
+        # and delete them all so another import will run code for real again.
+        for m in new_mods:
+            del sys.modules[m]
+
+
+class TestLazyMod(unittest.TestCase):  # lint-amnesty, pylint: disable=missing-class-docstring
+
+    def setUp(self):
+        super(TestLazyMod, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        # Each test will remove modules that it imported.
+        self.addCleanup(ModuleIsolation().clean_up)
+
+    def test_simple(self):
+        # Import some stdlib module that has not been imported before
+        module_name = 'colorsys'
+        if module_name in sys.modules:
+            # May have been imported during test discovery, remove it again
+            del sys.modules[module_name]
+        assert module_name not in sys.modules
+        colorsys = LazyModule(module_name)
+        hsv = colorsys.rgb_to_hsv(.3, .4, .2)
+        assert hsv[0] == 0.25
+
+    def test_dotted(self):
+        # wsgiref is a module with submodules that is not already imported.
+        # Any similar module would do. This test demonstrates that the module
+        # is not already imported
+        module_name = 'wsgiref.util'
+        if module_name in sys.modules:
+            # May have been imported during test discovery, remove it again
+            del sys.modules[module_name]
+        assert module_name not in sys.modules
+        wsgiref_util = LazyModule(module_name)
+        assert wsgiref_util.guess_scheme({}) == 'http'

xmodule/capa/safe_exec/tests/test_safe_exec.py

@@ -0,0 +1,398 @@
+"""Test safe_exec.py"""
+
+
+import hashlib
+import os
+import os.path
+import textwrap
+import unittest
+
+import pytest
+import random2 as random
+import six
+from codejail import jail_code
+from codejail.django_integration import ConfigureCodeJailMiddleware
+from codejail.safe_exec import SafeExecException
+from django.conf import settings
+from django.core.exceptions import MiddlewareNotUsed
+from django.test import override_settings
+from six import text_type, unichr
+from six.moves import range
+
+from xmodule.capa.safe_exec import safe_exec, update_hash
+
+
+class TestSafeExec(unittest.TestCase):  # lint-amnesty, pylint: disable=missing-class-docstring
+    def test_set_values(self):
+        g = {}
+        safe_exec("a = 17", g)
+        assert g['a'] == 17
+
+    def test_division(self):
+        g = {}
+        # Future division: 1/2 is 0.5.
+        safe_exec("a = 1/2", g)
+        assert g['a'] == 0.5
+
+    def test_assumed_imports(self):
+        g = {}
+        # Math is always available.
+        safe_exec("a = int(math.pi)", g)
+        assert g['a'] == 3
+
+    def test_random_seeding(self):
+        g = {}
+        r = random.Random(17)
+        rnums = [r.randint(0, 999) for _ in range(100)]
+
+        # Without a seed, the results are unpredictable
+        safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g)
+        assert g['rnums'] != rnums
+
+        # With a seed, the results are predictable
+        safe_exec("rnums = [random.randint(0, 999) for _ in xrange(100)]", g, random_seed=17)
+        assert g['rnums'] == rnums
+
+    def test_random_is_still_importable(self):
+        g = {}
+        r = random.Random(17)
+        rnums = [r.randint(0, 999) for _ in range(100)]
+
+        # With a seed, the results are predictable even from the random module
+        safe_exec(
+            "import random\n"
+            "rnums = [random.randint(0, 999) for _ in xrange(100)]\n",
+            g, random_seed=17)
+        assert g['rnums'] == rnums
+
+    def test_python_lib(self):
+        pylib = os.path.dirname(__file__) + "/test_files/pylib"
+        g = {}
+        safe_exec(
+            "import constant; a = constant.THE_CONST",
+            g, python_path=[pylib]
+        )
+
+    def test_raising_exceptions(self):
+        g = {}
+        with pytest.raises(SafeExecException) as cm:
+            safe_exec("1/0", g)
+        assert 'ZeroDivisionError' in text_type(cm.value)
+
+
+class TestSafeOrNot(unittest.TestCase):  # lint-amnesty, pylint: disable=missing-class-docstring
+    def test_cant_do_something_forbidden(self):
+        '''
+        Demonstrates that running unsafe code inside the code jail
+        throws SafeExecException, protecting the calling process.
+        '''
+        # Can't test for forbiddenness if CodeJail isn't configured for python.
+        if not jail_code.is_configured("python"):
+            pytest.skip()
+
+        g = {}
+        with pytest.raises(SafeExecException) as cm:
+            safe_exec('import sys; sys.exit(1)', g)
+        assert "SystemExit" not in text_type(cm)
+        assert "Couldn't execute jailed code" in text_type(cm)
+
+    def test_can_do_something_forbidden_if_run_unsafely(self):
+        '''
+        Demonstrates that running unsafe code outside the code jail
+        can cause issues directly in the calling process.
+        '''
+        g = {}
+        with pytest.raises(SystemExit) as cm:
+            safe_exec('import sys; sys.exit(1)', g, unsafely=True)
+        assert "SystemExit" in text_type(cm)
+
+
+class TestLimitConfiguration(unittest.TestCase):
+    """
+    Test that resource limits can be configured and overriden via Django settings.
+
+    We just test that the limits passed to `codejail` as we expect them to be.
+    Actual resource limiting tests are within the `codejail` package itself.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+
+        # Make a copy of codejail settings just for this test class.
+        # Set a global REALTIME limit of 100.
+        # Set a REALTIME limit override of 200 for a special course.
+        cls.test_codejail_settings = (getattr(settings, 'CODE_JAIL', None) or {}).copy()
+        cls.test_codejail_settings['limits'] = {
+            'REALTIME': 100,
+        }
+        cls.test_codejail_settings['limit_overrides'] = {
+            'course-v1:my+special+course': {'REALTIME': 200, 'NPROC': 30},
+        }
+        cls.configure_codejail(cls.test_codejail_settings)
+
+    @classmethod
+    def tearDownClass(cls):
+        super().tearDownClass()
+
+        # Re-apply original configuration.
+        cls.configure_codejail(getattr(settings, 'CODE_JAIL', None) or {})
+
+    @staticmethod
+    def configure_codejail(codejail_settings):
+        """
+        Given a `settings.CODE_JAIL` dictionary, apply it to the codejail package.
+
+        We use the `ConfigureCodeJailMiddleware` that comes with codejail.
+        """
+        with override_settings(CODE_JAIL=codejail_settings):
+            # To apply `settings.CODE_JAIL`, we just intialize an instance of the
+            # middleware class. We expect it to apply to changes, and then raise
+            # "MiddlewareNotUsed" to indicate that its work is done.
+            # This is exactly how the settings are applied in production (except the
+            # middleware is automatically initialized because it's an element of
+            # `settings.MIDDLEWARE`).
+            try:
+                ConfigureCodeJailMiddleware()
+            except MiddlewareNotUsed:
+                pass
+
+    def test_effective_limits_reflect_configuration(self):
+        """
+        Test that `get_effective_limits` returns configured limits with overrides
+        applied correctly.
+        """
+        # REALTIME has been configured with a global limit.
+        # Check it with no overrides context.
+        assert jail_code.get_effective_limits()['REALTIME'] == 100
+
+        # Now check REALTIME with an overrides context that we haven't configured.
+        # Should be the same.
+        assert jail_code.get_effective_limits('random-context-name')['REALTIME'] == 100
+
+        # Now check REALTIME limit for a special course.
+        # It should be overriden.
+        assert jail_code.get_effective_limits('course-v1:my+special+course')['REALTIME'] == 200
+
+        # We haven't configured a limit for NPROC.
+        # It should use the codejail default.
+        assert jail_code.get_effective_limits()['NPROC'] == 15
+
+        # But we have configured an NPROC limit override for a special course.
+        assert jail_code.get_effective_limits('course-v1:my+special+course')['NPROC'] == 30
+
+
+class DictCache(object):
+    """A cache implementation over a simple dict, for testing."""
+
+    def __init__(self, d):
+        self.cache = d
+
+    def get(self, key):
+        # Actual cache implementations have limits on key length
+        assert len(key) <= 250
+        return self.cache.get(key)
+
+    def set(self, key, value):
+        # Actual cache implementations have limits on key length
+        assert len(key) <= 250
+        self.cache[key] = value
+
+
+class TestSafeExecCaching(unittest.TestCase):
+    """Test that caching works on safe_exec."""
+
+    def test_cache_miss_then_hit(self):
+        g = {}
+        cache = {}
+
+        # Cache miss
+        safe_exec("a = int(math.pi)", g, cache=DictCache(cache))
+        assert g['a'] == 3
+        # A result has been cached
+        assert list(cache.values())[0] == (None, {'a': 3})
+
+        # Fiddle with the cache, then try it again.
+        cache[list(cache.keys())[0]] = (None, {'a': 17})
+
+        g = {}
+        safe_exec("a = int(math.pi)", g, cache=DictCache(cache))
+        assert g['a'] == 17
+
+    def test_cache_large_code_chunk(self):
+        # Caching used to die on memcache with more than 250 bytes of code.
+        # Check that it doesn't any more.
+        code = "a = 0\n" + ("a += 1\n" * 12345)
+
+        g = {}
+        cache = {}
+        safe_exec(code, g, cache=DictCache(cache))
+        assert g['a'] == 12345
+
+    def test_cache_exceptions(self):
+        # Used to be that running code that raised an exception didn't cache
+        # the result.  Check that now it does.
+        code = "1/0"
+        g = {}
+        cache = {}
+        with pytest.raises(SafeExecException):
+            safe_exec(code, g, cache=DictCache(cache))
+
+        # The exception should be in the cache now.
+        assert len(cache) == 1
+        cache_exc_msg, cache_globals = list(cache.values())[0]  # lint-amnesty, pylint: disable=unused-variable
+        assert 'ZeroDivisionError' in cache_exc_msg
+
+        # Change the value stored in the cache, the result should change.
+        cache[list(cache.keys())[0]] = ("Hey there!", {})
+
+        with pytest.raises(SafeExecException):
+            safe_exec(code, g, cache=DictCache(cache))
+
+        assert len(cache) == 1
+        cache_exc_msg, cache_globals = list(cache.values())[0]
+        assert 'Hey there!' == cache_exc_msg
+
+        # Change it again, now no exception!
+        cache[list(cache.keys())[0]] = (None, {'a': 17})
+        safe_exec(code, g, cache=DictCache(cache))
+        assert g['a'] == 17
+
+    def test_unicode_submission(self):
+        # Check that using non-ASCII unicode does not raise an encoding error.
+        # Try several non-ASCII unicode characters.
+        for code in [129, 500, 2 ** 8 - 1, 2 ** 16 - 1]:
+            code_with_unichr = six.text_type("# ") + unichr(code)
+            try:
+                safe_exec(code_with_unichr, {}, cache=DictCache({}))
+            except UnicodeEncodeError:
+                self.fail("Tried executing code with non-ASCII unicode: {0}".format(code))
+
+
+class TestUpdateHash(unittest.TestCase):
+    """Test the safe_exec.update_hash function to be sure it canonicalizes properly."""
+
+    def hash_obj(self, obj):
+        """Return the md5 hash that `update_hash` makes us."""
+        md5er = hashlib.md5()
+        update_hash(md5er, obj)
+        return md5er.hexdigest()
+
+    def equal_but_different_dicts(self):
+        """
+        Make two equal dicts with different key order.
+
+        Simple literals won't do it.  Filling one and then shrinking it will
+        make them different.
+
+        """
+        d1 = {k: 1 for k in "abcdefghijklmnopqrstuvwxyz"}
+        d2 = {k: 1 for k in "bcdefghijklmnopqrstuvwxyza"}
+        # TODO: remove the next lines once we are in python3.8
+        # since python3.8 dict preserve the order of insertion
+        # and therefore d2 and d1 keys are already in different order.
+        for i in range(10000):
+            d2[i] = 1
+        for i in range(10000):
+            del d2[i]
+
+        # Check that our dicts are equal, but with different key order.
+        assert d1 == d2
+        assert list(d1.keys()) != list(d2.keys())
+
+        return d1, d2
+
+    def test_simple_cases(self):
+        h1 = self.hash_obj(1)
+        h10 = self.hash_obj(10)
+        hs1 = self.hash_obj("1")
+
+        assert h1 != h10
+        assert h1 != hs1
+
+    def test_list_ordering(self):
+        h1 = self.hash_obj({'a': [1, 2, 3]})
+        h2 = self.hash_obj({'a': [3, 2, 1]})
+        assert h1 != h2
+
+    def test_dict_ordering(self):
+        d1, d2 = self.equal_but_different_dicts()
+        h1 = self.hash_obj(d1)
+        h2 = self.hash_obj(d2)
+        assert h1 == h2
+
+    def test_deep_ordering(self):
+        d1, d2 = self.equal_but_different_dicts()
+        o1 = {'a': [1, 2, [d1], 3, 4]}
+        o2 = {'a': [1, 2, [d2], 3, 4]}
+        h1 = self.hash_obj(o1)
+        h2 = self.hash_obj(o2)
+        assert h1 == h2
+
+
+class TestRealProblems(unittest.TestCase):  # lint-amnesty, pylint: disable=missing-class-docstring
+    def test_802x(self):
+        code = textwrap.dedent("""\
+            import math
+            import random
+            import numpy
+            e=1.602e-19 #C
+            me=9.1e-31  #kg
+            mp=1.672e-27 #kg
+            eps0=8.854e-12 #SI units
+            mu0=4e-7*math.pi #SI units
+
+            Rd1=random.randrange(1,30,1)
+            Rd2=random.randrange(30,50,1)
+            Rd3=random.randrange(50,70,1)
+            Rd4=random.randrange(70,100,1)
+            Rd5=random.randrange(100,120,1)
+
+            Vd1=random.randrange(1,20,1)
+            Vd2=random.randrange(20,40,1)
+            Vd3=random.randrange(40,60,1)
+
+            #R=[0,10,30,50,70,100] #Ohm
+            #V=[0,12,24,36] # Volt
+
+            R=[0,Rd1,Rd2,Rd3,Rd4,Rd5] #Ohms
+            V=[0,Vd1,Vd2,Vd3] #Volts
+            #here the currents IL and IR are defined as in figure ps3_p3_fig2
+            a=numpy.array([  [  R[1]+R[4]+R[5],R[4] ],[R[4], R[2]+R[3]+R[4] ] ])
+            b=numpy.array([V[1]-V[2],-V[3]-V[2]])
+            x=numpy.linalg.solve(a,b)
+            IL='%.2e' % x[0]
+            IR='%.2e' % x[1]
+            ILR='%.2e' % (x[0]+x[1])
+            def sign(x):
+                return abs(x)/x
+
+            RW="Rightwards"
+            LW="Leftwards"
+            UW="Upwards"
+            DW="Downwards"
+            I1='%.2e' % abs(x[0])
+            I1d=LW if sign(x[0])==1 else RW
+            I1not=LW if I1d==RW else RW
+            I2='%.2e' % abs(x[1])
+            I2d=RW if sign(x[1])==1 else LW
+            I2not=LW if I2d==RW else RW
+            I3='%.2e' % abs(x[1])
+            I3d=DW if sign(x[1])==1 else UW
+            I3not=DW if I3d==UW else UW
+            I4='%.2e' % abs(x[0]+x[1])
+            I4d=UW if sign(x[1]+x[0])==1 else DW
+            I4not=DW if I4d==UW else UW
+            I5='%.2e' % abs(x[0])
+            I5d=RW if sign(x[0])==1 else LW
+            I5not=LW if I5d==RW else RW
+            VAP=-x[0]*R[1]-(x[0]+x[1])*R[4]
+            VPN=-V[2]
+            VGD=+V[1]-x[0]*R[1]+V[3]+x[1]*R[2]
+            aVAP='%.2e' % VAP
+            aVPN='%.2e' % VPN
+            aVGD='%.2e' % VGD
+            """)
+        g = {}
+        safe_exec(code, g)
+        assert 'aVAP' in g

xmodule/capa/templates/annotationinput.html

@@ -0,0 +1,62 @@
+<%! from openedx.core.djangolib.markup import HTML %>
+<div class="annotation-input">
+<div class="script_placeholder" data-src="${STATIC_URL}js/capa/annotationinput.js"/>
+
+<div class="annotation-header">
+    ${title}
+
+    % if return_to_annotation:
+        <a class="annotation-return" href="javascript:void(0)">Return to Annotation</a><br/>
+    % endif
+</div>
+<div class="annotation-body">
+
+    <div class="block block-highlight">${text}</div>
+    <div class="block block-comment">${comment}</div>
+
+    <div class="block">${comment_prompt}</div>
+    <textarea class="comment" id="input_${id}_comment" name="input_${id}_comment" aria-describedby="answer_${id}">${comment_value|h}</textarea>
+
+    <div class="block" id="label_${id}">${tag_prompt}</div>
+    <ul class="tags">
+    % for option in options:
+        <li>
+            % if has_options_value:
+                % if all(c == status.classname for c in (option['choice'], status)):
+                <span class="tag-status ${status.classname}" aria-describedby="input_${id}_comment">
+                    <%include file="status_span.html" args="status=status"/>
+                </span>
+                % endif
+            % endif
+
+            <span class="tag
+            % if option['id'] in options_value:
+                selected
+            % endif
+            " data-id="${option['id']}">
+                ${option['description']}
+            </span>
+        </li>
+    % endfor
+    </ul>
+
+    % if debug:
+    <div class="debug-value">
+        Rendered with value:<br/>
+        <pre>${value|h}</pre>
+        Current input value:<br/>
+        <input type="text" class="value" name="input_${id}" id="input_${id}" value="${value|h}" />
+    </div>
+    % else:
+        <input type="hidden" class="value" name="input_${id}" id="input_${id}" value="${value|h}" />
+    % endif
+
+    <%include file="status_span.html" args="status=status, status_id=id"/>
+
+    <p id="answer_${id}" class="answer answer-annotation"></p>
+</div>
+</div>
+
+% if msg:
+<span class="message" aria-describedby="label_${id}" tabindex="-1">${HTML(msg)}</span>
+% endif

xmodule/capa/templates/chemicalequationinput.html

@@ -0,0 +1,22 @@
+<%! from xmodule.capa.util import remove_markup %>
+<div id="chemicalequationinput_${id}" class="chemicalequationinput">
+    <div class="script_placeholder" data-src="${previewer}"/>
+
+    <div class="${status.classname}">
+
+      <input type="text" name="input_${id}" id="input_${id}" aria-label="${remove_markup(response_data['label'])}"
+             aria-describedby="answer_${id}" data-input-id="${id}" value="${value|h}"
+             % if size:
+              size="${size}"
+             % endif
+      />
+
+      <p class="indicator-container">
+        ${value|h}
+        <%include file="status_span.html" args="status=status, status_id=id"/>
+      </p>
+
+      <div id="input_${id}_preview" class="equation"></div>
+      <p id="answer_${id}" class="answer"></p>
+  </div>
+</div>

xmodule/capa/templates/choicegroup.html

@@ -0,0 +1,65 @@
+<%page expression_filter="h"/>
+<%!
+from openedx.core.djangolib.markup import HTML
+import six
+%>
+<%
+  def is_radio_input(choice_id):
+    return input_type == 'radio' and ((isinstance(value, six.string_types) and (choice_id == value)) or (
+      not isinstance(value, six.string_types) and choice_id in value
+    ))
+%>
+<div class="choicegroup capa_inputtype" id="inputtype_${id}">
+    <fieldset ${describedby_html}>
+      % if response_data['label']:
+        <legend id="${id}-legend" class="response-fieldset-legend field-group-hd">${response_data['label']}</legend>
+      % endif
+      % for description_id, description_text in response_data['descriptions'].items():
+        <p class="question-description" id="${description_id}">${description_text}</p>
+      % endfor
+        % for choice_id, choice_label in choices:
+          <%
+            label_class = 'response-label field-label label-inline'
+            input_class = 'field-input input-' + input_type
+            input_checked = ''
+
+            if is_radio_input(choice_id) or (input_type != 'radio' and choice_id in value):
+              input_class += ' submitted'
+              if status.classname and not show_correctness == 'never':
+                label_class += ' choicegroup_' + status.classname
+          %>
+          <div class="field">
+            <input type="${input_type}" name="input_${id}${name_array_suffix}" id="input_${id}_${choice_id}"
+              class="${input_class}" value="${choice_id}"
+                ## If the student selected this choice...
+                % if is_radio_input(choice_id):
+                checked="true"
+                % elif input_type != 'radio' and choice_id in value:
+                checked="true"
+                % endif
+            /><label id="${id}-${choice_id}-label" for="input_${id}_${choice_id}"
+                class="${label_class}"
+                ${describedby_html}
+                >
+                <div>
+                    ${HTML(choice_label)}
+                </div>
+            </label>
+          </div>
+        % endfor
+        <span id="answer_${id}"></span>
+    </fieldset>
+    <div class="indicator-container">
+        % if show_correctness != 'never':
+          <%include file="status_span.html" args="status=status, status_id=id"/>
+        % else:
+          <%include file="status_span.html" args="status=status, status_id=id, hide_correctness=True"/>
+        % endif
+    </div>
+    % if show_correctness == "never" and (value or status not in ['unsubmitted']):
+    <div class="capa_alert">${submitted_message}</div>
+    %endif
+    % if msg:
+    <span class="message" aria-describedby="${id}-legend" tabindex="-1">${HTML(msg)}</span>
+    % endif
+</div>

xmodule/capa/templates/choicetext.html

@@ -0,0 +1,70 @@
+<%! from xmodule.capa.util import remove_markup
+from django.utils.translation import ugettext as _
+from openedx.core.djangolib.markup import HTML
+%>
+
+<% element_checked = False %>
+% for choice_id, _ in choices:
+    <% choice_id = choice_id %>
+    %if choice_id in value:
+        <% element_checked = True %>
+    %endif
+% endfor
+<section id="choicetextinput_${id}" class="choicetextinput">
+<div class="choicetextgroup capa_inputtype" id="inputtype_${id}">
+    <div class="script_placeholder" data-src="${STATIC_URL}js/capa/choicetextinput.js"/>
+
+    <fieldset aria-label="${remove_markup(response_data['label'])}">
+        % for choice_id, choice_description in choices:
+        <% choice_id = choice_id %>
+        <section id="forinput${choice_id}"
+            % if input_type == 'radio' and choice_id in value :
+                % if status.classname:
+                    class="choicetextgroup_${status.classname}"
+                % endif
+            % endif
+            >
+            <input class="ctinput" type="${input_type}" name="choiceinput_${id}" id="${choice_id}" value="${choice_id}"
+
+            % if choice_id in value:
+            checked="true"
+            % endif
+            />
+
+            % for content_node in choice_description:
+                % if content_node['type'] == 'text':
+                    <span class="mock_label">
+                         ${content_node['contents']}
+                     </span>
+                % else:
+                    <% my_id = content_node.get('contents','') %>
+                    <% my_val = value.get(my_id,'') %>
+                    <input class="ctinput" type="text" name="${content_node['contents']}" id="${content_node['contents']}" value="${my_val|h}"/>
+                %endif
+                <span class="mock_label">
+                    ${content_node['tail_text']}
+                </span>
+
+            % endfor
+            <p id="answer_${choice_id}" class="answer"></p>
+                </section>
+
+        % endfor
+        <span id="answer_${id}"></span>
+    </fieldset>
+        <input class= "choicetextvalue" type="hidden"  name="input_${id}{}" id="input_${id}" value="${value|h}" />
+
+    <div class="indicator-container">
+    % if input_type == 'checkbox' or not element_checked:
+        <%include file="status_span.html" args="status=status, status_id=id"/>
+    % endif
+    </div>
+
+    % if show_correctness == "never" and (value or status not in ['unsubmitted']):
+    <div class="capa_alert">${_(submitted_message)}</div>
+    %endif
+    % if msg:
+    <span class="message" tabindex="-1">${HTML(msg)}</span>
+    % endif
+</div>
+</section>

xmodule/capa/templates/clarification.html

@@ -0,0 +1,5 @@
+<span class="clarification" tabindex="0" role="note" aria-label="Clarification">
+    <span data-tooltip="${clarification | h}" data-tooltip-show-on-click="true"
+       class="fa fa-info-circle" aria-hidden="true"></span>
+    <span class="sr">(${clarification})</span>
+</span>

xmodule/capa/templates/codeinput.html

@@ -0,0 +1,50 @@
+<%page expression_filter="h"/>
+<%!
+from django.utils.translation import ugettext as _
+from openedx.core.djangolib.markup import HTML
+%>
+<div id="textbox_${id}" class="capa_inputtype textbox cminput">
+  % if response_data['label']:
+    <label class="problem-group-label" for="cm-textarea-${id}">${response_data['label']}</label>
+  % else:
+    <label class="sr problem-group-label" for="cm-textarea-${id}">${_('Code Editor')}</label>
+  % endif
+  <textarea rows="${rows}" cols="${cols}" name="input_${id}"
+            aria-label="${aria_label}"
+            aria-describedby="answer_${id}"
+            id="input_${id}"
+            tabindex="0"
+            data-mode="${mode}"
+            data-tabsize="${tabsize}"
+            % if linenumbers:
+            data-linenums="true"
+            % endif
+            % if hidden:
+      	    style="display:none;"
+            % endif
+  >${value}</textarea>
+  <span class="cm-editor-exit-message capa-message" id="cm-editor-exit-message-${id}">
+    ${code_mirror_exit_message}
+  </span>
+
+  <div class="grader-status" tabindex="-1">
+
+    <%include file="status_span.html" args="status=status, status_id=id"/>
+
+    % if status == 'queued':
+      <span style="display:none;" class="xqueue" id="${id}">${queue_len}</span>
+    % endif
+
+    % if hidden:
+      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+    % endif
+
+    <p class="debug">${status.display_name}</p>
+  </div>
+
+  <span id="answer_${id}"></span>
+
+  <div class="external-grader-message">
+    ${HTML(msg)}
+  </div>
+</div>

xmodule/capa/templates/crystallography.html

@@ -0,0 +1,30 @@
+<%! from openedx.core.djangolib.markup import HTML %>
+<section id="inputtype_${id}" class="capa_inputtype" >
+    <div class="crystalography_problem" style="width:${width};height:${height}"></div>
+
+    <div class="input_lattice">
+        Lattice: <select></select>
+    </div>
+
+    <div class="script_placeholder" data-src="/static/js/raphael.js"></div>
+    <div class="script_placeholder" data-src="/static/js/sylvester.js"></div>
+    <div class="script_placeholder" data-src="/static/js/crystallography.js"></div>
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+        <div class="status ${status.classname}" id="status_${id}">
+    % endif
+
+    <input type="text" name="input_${id}" aria-describedby="answer_${id}" id="input_${id}" value="${value|h}" style="display:none;"/>
+
+    <%include file="status_span.html" args="status=status, status_id=id"/>
+
+    <p id="answer_${id}" class="answer"></p>
+
+    % if msg:
+        <span class="message" tabindex="-1">${HTML(msg)}</span>
+    % endif
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+        </div>
+    % endif
+</section>

xmodule/capa/templates/designprotein2dinput.html

@@ -0,0 +1,19 @@
+<section id="designprotein2dinput_${id}" class="designprotein2dinput">
+    <div class="script_placeholder" data-src="/static/js/capa/protex/protex.nocache.js?raw"/>
+    <div class="script_placeholder" data-src="${applet_loader}"/>
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+    <div class="${status.classname}" id="status_${id}">
+    % endif
+
+    <div id="protex_container"></div>
+    <input type="hidden" name="target_shape" id="target_shape" value ="${target_shape}"></input>
+    <input type="hidden" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"/>
+
+    <%include file="status_span.html" args="status=status, status_id=id"/>
+
+    <p id="answer_${id}" class="answer"></p>
+  % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+  </div>
+% endif
+</section>

xmodule/capa/templates/drag_and_drop_input.html

@@ -0,0 +1,34 @@
+<%! from openedx.core.djangolib.markup import HTML %>
+<div id="inputtype_${id}" class="capa_inputtype">
+    <div class="drag_and_drop_problem_div" id="drag_and_drop_div_${id}"
+     data-plain-id="${id}">
+    </div>
+
+    <div class="drag_and_drop_problem_json" id="drag_and_drop_json_${id}"
+        style="display:none;">${drag_and_drop_json}</div>
+
+    <div class="script_placeholder" data-src="${STATIC_URL}js/capa/drag_and_drop.js"></div>
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+        <div class="${status.classname}" id="status_${id}">
+    % endif
+
+
+    <input type="text" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"
+    style="display:none;"/>
+
+
+    <p class="indicator-container drag-and-drop--status" aria-describedby="input_${id}">
+        <%include file="status_span.html" args="status=status, status_id=id"/>
+    </p>
+
+    <p id="answer_${id}" class="answer"></p>
+
+    % if msg:
+        <span class="message" tabindex="-1">${HTML(msg)}</span>
+    % endif
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+        </div>
+    % endif
+</div>

xmodule/capa/templates/editageneinput.html

@@ -0,0 +1,23 @@
+ <section id="editageneinput_${id}" class="editageneinput">
+    <div class="script_placeholder" data-src="/static/js/capa/genex/genex.nocache.js?raw"/>
+    <div class="script_placeholder" data-src="${applet_loader}"/>
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+    <div class="${status.classname}">
+    % endif
+
+    <div id="genex_container"></div>
+    <input type="hidden" name="genex_dna_sequence" id="genex_dna_sequence" value ="${genex_dna_sequence}"></input>
+    <input type="hidden" name="genex_problem_number" id="genex_problem_number" value ="${genex_problem_number}"></input>
+    <input type="hidden" name="input_${id}" aria-describedby="answer_${id}" id="input_${id}" value="${value|h}"/>
+
+    <p class="indicator-container" aria-describedby="input_${id}">
+        <%include file="status_span.html" args="status=status, status_id=id"/>
+    </p>
+
+    <p id="answer_${id}" class="answer"></p>
+  % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+  </div>
+% endif
+</section>
+

xmodule/capa/templates/editamolecule.html

@@ -0,0 +1,28 @@
+<section id="editamoleculeinput_${id}" class="editamoleculeinput">
+  <div class="script_placeholder" data-src="${applet_loader}"/>
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+    <div class="${status.classname}">
+    % endif
+
+    <div id="applet_${id}" class="applet" data-molfile-src="${file}" style="display:block;width:500px;height:400px">
+    </div>
+
+    <br/>
+
+    <input type="hidden" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"/>
+
+    <button id="reset_${id}" class="reset">Reset</button>
+
+    <p id="answer_${id}" class="answer"></p>
+
+    <p class="indicator-container" aria-describedby="input_${id}">
+      <%include file="status_span.html" args="status=status, status_id=id"/>
+    </p>
+
+    <div class="error_message" style="padding: 5px 5px 5px 5px; background-color:#FA6666; height:60px;width:400px; display: none"></div>
+
+  % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+  </div>
+  % endif
+</section>

xmodule/capa/templates/filesubmission.html

@@ -0,0 +1,14 @@
+<%! from openedx.core.djangolib.markup import HTML %>
+<section id="filesubmission_${id}" class="filesubmission">
+  <div class="grader-status file">
+
+    <span class="${status.classname}" id="status_${id}">${status.display_name}</span>
+    % if status == 'queued':
+    <span style="display:none;" class="xqueue" id="${id}">${queue_len}</span>
+    % endif
+    <p class="debug">${status}</p>
+
+    <input type="file" name="input_${id}" id="input_${id}" value="${value}" multiple="multiple" data-required_files="${required_files|h}" data-allowed_files="${allowed_files|h}" aria-label="${response_data['label']}"/>
+  </div>
+    <div class="message" tabindex="-1">${HTML(msg)}</div>
+</section>

xmodule/capa/templates/formulaequationinput.html

@@ -0,0 +1,36 @@
+<%page expression_filter="h"/>
+<%! from openedx.core.djangolib.markup import HTML %>
+<% doinline = 'style="display:inline-block;vertical-align:top"' if inline else "" %>
+<div id="formulaequationinput_${id}" class="inputtype formulaequationinput" ${doinline | n, decode.utf8}>
+    <div class="${status.classname}">
+      % if response_data['label']:
+          <label class="problem-group-label" for="input_${id}" id="label_${id}">${response_data['label']}</label>
+      % endif
+        % for description_id, description_text in response_data['descriptions'].items():
+            <p class="question-description" id="${description_id}">${description_text}</p>
+        % endfor
+        <input type="text" name="input_${id}" id="input_${id}"
+            data-input-id="${id}" value="${value}"
+            ${describedby_html}
+            % if size:
+            size="${size}"
+            % endif
+            />
+        <span class="trailing_text" id="trailing_text_${id}">${trailing_text}</span>
+
+        <%include file="status_span.html" args="status=status, status_id=id"/>
+
+        <p id="answer_${id}" class="answer"></p>
+
+        <div id="input_${id}_preview" class="equation">
+            \(\)
+            <img src="${STATIC_URL}images/spinner.gif" class="loading" alt="Loading"/>
+        </div>
+    </div>
+
+    <div class="script_placeholder" data-src="${previewer}"/>
+
+  % if msg:
+      <span class="message" aria-describedby="label_${id}" tabindex="-1">${HTML(msg)}</span>
+  % endif
+</div>

xmodule/capa/templates/imageinput.html

@@ -0,0 +1,45 @@
+<%page expression_filter="h"/>
+<div class="imageinput capa_inputtype" id="inputtype_${id}">
+  <input
+    type="hidden"
+    class="imageinput"
+    src="${src}"
+    name="input_${id}"
+    id="input_${id}"
+    value="${value}"
+  />
+  <div style="position:relative;">
+    <div
+      id="imageinput_${id}"
+      style="
+        background-image: url('${src}');
+        width: ${width}px;
+        height: ${height}px;
+        position: relative;
+        left: 0;
+        top: 0;"
+    >
+      <img
+        src="${STATIC_URL}images/green-pointer.png"
+        id="cross_${id}"
+        style="position: absolute; top: ${gy}px; left: ${gx}px;"
+        alt="Selection indicator"
+      />
+    </div>
+    <div
+      data-width="${width}"
+      data-height="${height}"
+      id="answer_${id}"
+      style="
+        position: absolute;
+        left: 0;
+        top: 0;"
+    ></div>
+  </div>
+
+  <script type="text/javascript" charset="utf-8">
+    (new ImageInput('${id | n, decode.utf8}'));
+  </script>
+
+  <%include file="status_span.html" args="status=status, status_id=id"/>
+</div>

xmodule/capa/templates/jsinput.html

@@ -0,0 +1,59 @@
+<%page expression_filter="h"/>
+<%! from openedx.core.djangolib.markup import HTML %>
+<div id="inputtype_${id}" class="jsinput"
+    data="${gradefn}"
+    % if saved_state:
+    data-stored="${saved_state}"
+    % endif
+    % if initial_state:
+    data-initial-state="${initial_state}"
+    % endif
+    % if get_statefn:
+    data-getstate="${get_statefn}"
+    % endif
+    % if set_statefn:
+    data-setstate="${set_statefn}"
+    % endif
+    % if sop:
+    data-sop="${sop}"
+    % endif
+    data-processed="false"
+    >
+
+  <div class="script_placeholder" data-src="${jschannel_loader}"/>
+  <div class="script_placeholder" data-src="${jsinput_loader}"/>
+  % if status in ['unsubmitted', 'submitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+  <div class="${status.classname}">
+  % endif
+
+  <iframe name="iframe_${id}"
+      id="iframe_${id}"
+      sandbox="allow-scripts allow-popups allow-same-origin allow-forms allow-pointer-lock allow-downloads"
+      seamless="seamless"
+      frameborder="0"
+      src="${html_file}"
+      height="${height}"
+      width="${width}"
+      title="${title}"
+      />
+  <input type="hidden" name="input_${id}" id="input_${id}"
+    waitfor=""
+    value="${value}"/>
+
+  <br/>
+    <p id="answer_${id}" class="answer"></p>
+
+    <div class="indicator-container">
+        <%include file="status_span.html" args="status=status, status_id=id"/>
+    </div>
+
+    <div class="error_message" style="padding: 5px 5px 5px 5px; background-color:#FA6666; height:60px;width:400px; display: none"></div>
+
+  % if status in ['unsubmitted', 'submitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+  </div>
+  % endif
+
+  % if msg:
+      <span class="message" tabindex="-1">${HTML(msg)}</span>
+  % endif
+</div>

xmodule/capa/templates/mathstring.html

@@ -0,0 +1,8 @@
+<section class="math-string">
+ % if isinline:
+   <span>[mathjaxinline]${mathstr}[/mathjaxinline]</span>
+ % else:
+    <span>[mathjax]${mathstr}[/mathjax]</span>
+ % endif
+    <span>${tail}</span>
+</section>

xmodule/capa/templates/matlabinput.html

@@ -0,0 +1,141 @@
+<%page expression_filter="h"/>
+<section id="textbox_${id}" class="capa_inputtype cminput">
+<div class="script_placeholder" data-src="${matlab_editor_js}"></div>
+<textarea
+  rows="${rows}"
+  cols="${cols}"
+  name="input_${id}"
+  aria-describedby="answer_${id}"
+  id="input_${id}"
+  data-tabsize="${tabsize}"
+  data-mode="octave"
+  % if linenumbers:
+  data-linenums="true"
+  % endif
+ % if hidden:
+  style="display:none;"
+ % endif
+  >${value}</textarea>
+
+  <div class="grader-status" tabindex="-1">
+
+    <%include file="status_span.html" args="status=status, status_id=id"/>
+
+    % if status == 'queued':
+      <span style="display:none;" class="xqueue" id="${id}">${queue_len}</span>
+    % endif
+
+    % if hidden:
+      <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+    % endif
+
+    <p class="debug">${status.display_name}</p>
+  </div>
+
+  <span id="answer_${id}"></span>
+
+  <div class="external-grader-message" aria-live="polite">
+    ${msg|n, decode.utf8}
+  </div>
+  <div class="ungraded-matlab-result" aria-live="polite">
+    ${queue_msg|n, decode.utf8}
+  </div>
+
+  % if button_enabled:
+  <div class="plot-button">
+      <input type="button" class="save" name="plot-button" id="plot_${id}" value="Run Code" />
+  </div>
+  %endif
+
+  <script type="text/javascript">
+    $(function(){
+      var gentle_alert = function (parent_elt, msg) {
+          if($(parent_elt).find('.capa_alert').length) {
+              $(parent_elt).find('.capa_alert').remove();
+          }
+          var alert_elem = $(edx.HtmlUtils.joinHtml(
+            edx.HtmlUtils.HTML("<div>"),
+            msg,
+            edx.HtmlUtils.HTML("</div>")
+          ).toString());
+          alert_elem.addClass('capa_alert').addClass('is-fading-in');
+          // xss-lint: disable=javascript-jquery-insertion
+          $(parent_elt).find('.action').after(alert_elem);
+      };
+
+      // hook up the plot button
+      var plot = function(event) {
+          var problem_elt = $(event.target).closest('.problems-wrapper');
+          url = $(event.target).closest('.problems-wrapper').data('url');
+          input_id = "${id|n, decode.utf8}";
+
+          // save the codemirror text to the textarea
+          // since there could be multiple codemirror instances on the page,
+          // save all of them.
+          $('.CodeMirror').each(function(i, el){
+              el.CodeMirror.save();
+          });
+          var input = $("#input_${id|n, decode.utf8}");
+
+          // pull out the coded text
+          submission = input.val();
+
+          answer = input.serialize();
+
+          // a chain of callbacks, each querying the server on success of the previous one
+          var poll = function(prev_timeout) {
+              $.postWithPrefix(url + "/problem_get", function(response) {
+                   var new_result_elem = $(response.html).find(".ungraded-matlab-result").html();
+                   var external_grader_msg = $(response.html).find(".external-grader-message").html();
+                   var result_elem = $(problem_elt).find(".ungraded-matlab-result");
+                   result_elem.addClass("is-fading-in");
+                   edx.HtmlUtils.setHtml(result_elem, new_result_elem);
+                   var external_grader_msg_elem = $(problem_elt).find(".external-grader-message");
+                   external_grader_msg_elem.addClass("is-fading-in");
+                   edx.HtmlUtils.setHtml(external_grader_msg_elem, external_grader_msg);
+                   // If we have a message about waiting for the external grader.
+                   if (external_grader_msg.trim()) {
+                        result_elem.html('');
+                        // Setup the polling for the next round
+                        var next_timeout = prev_timeout * 2;
+                        // The XML parsing that capa uses doesn't handle the greater-than symbol properly here, so we are forced to work around it.
+                        // The backend MatlabInput code will also terminate after 35 seconds, so this is mostly a protective measure.
+                        if (next_timeout === 64000) {
+                            gentle_alert(problem_elt, gettext("Your code is still being run. Refresh the page to see updates."));
+                        }
+                        window.setTimeout(function(){ poll(next_timeout); }, next_timeout);
+                   }
+              });
+          };
+
+          var plot_callback = function(response) {
+              if(response.success) {
+                  // If successful, start polling.
+                  // If we change the initial polling value, we will also need to change the check within poll (next_time === 64000) to match it.
+                  poll(1000);
+              } else {
+                  // Used response.message because input_ajax is returning "message"
+                  gentle_alert(problem_elt, response.message);
+              }
+          };
+
+          var save_callback = function(response) {
+              if(response.success) {
+                  // send information to the problem's plot functionality
+                  Problem.inputAjax(url, input_id, 'plot',
+                      {'submission': submission}, plot_callback);
+              }
+              else {
+                  // Used response.msg because problem_save is returning "msg" instead of "message"
+                  gentle_alert(problem_elt, response.msg);
+              }
+          };
+
+          // save the answer
+          $.postWithPrefix(url + '/problem_save', answer, save_callback);
+      };
+      $('#plot_${id|n, decode.utf8}').click(plot);
+
+    });
+  </script>
+</section>

xmodule/capa/templates/optioninput.html

@@ -0,0 +1,32 @@
+<%page expression_filter="h"/>
+<%! from openedx.core.djangolib.markup import HTML %>
+<% doinline = "inline" if inline else "" %>
+
+<div class="inputtype option-input ${doinline}">
+  % if response_data['label']:
+      <label class="problem-group-label" for="input_${id}" id="label_${id}">${response_data['label']}</label>
+  % endif
+
+  % for description_id, description_text in response_data['descriptions'].items():
+    <p class="question-description" id="${description_id}">${description_text}</p>
+  % endfor
+
+   <select name="input_${id}" id="input_${id}" ${describedby_html}>
+      <option value="option_${id}_dummy_default">${default_option_text}</option>
+      % for option_id, option_description in options:
+          <option value="${option_id}"
+          % if (option_id == value or option_id == answervariable):
+        	selected="true"
+          % endif
+          > ${option_description}</option>
+      % endfor
+    </select>
+
+  <div class="indicator-container">
+    <%include file="status_span.html" args="status=status, status_id=id"/>
+  </div>
+  <p class="answer" id="answer_${id}"></p>
+  % if msg:
+      <span class="message" aria-describedby="label_${id}" tabindex="-1">${HTML(msg)}</span>
+  % endif
+</div>

xmodule/capa/templates/schematicinput.html

@@ -0,0 +1,24 @@
+<%! from xmodule.capa.util import remove_markup %>
+<div>
+  <div class="script_placeholder" data-src="${setup_script}"/>
+  <input type="hidden"
+    class="schematic"
+    height="${height}"
+    width="${width}"
+    parts="${parts}"
+    analyses="${analyses}"
+    name="input_${id}"
+    id="input_${id}"
+    aria-label="${remove_markup(response_data['label'])}"
+    aria-describedby="answer_${id}"
+    value="${value|h}"
+    initial_value="${initial_value|h}"
+    submit_analyses="${submit_analyses|h}"
+    />
+
+  <span id="answer_${id}"></span>
+  <div class="indicator-container">
+    <%include file="status_span.html" args="status=status, status_id=id"/>
+
+  </div>
+</div>

xmodule/capa/templates/solutionspan.html

@@ -0,0 +1,4 @@
+<%page expression_filter="h"/>
+<div class="solution-span">
+ <span id="solution_${id}"></span>
+</div>

xmodule/capa/templates/status_span.html

@@ -0,0 +1,13 @@
+<%page expression_filter="h" args="status, status_id='', hide_correctness=False"/>
+
+% if status_id == '':
+<span class="status ${'' if hide_correctness == True else status.classname}"
+      data-tooltip="${'' if hide_correctness == True else status.display_tooltip}">
+% else:
+<span class="status ${'' if hide_correctness == True else status.classname}" id="status_${status_id}"
+      data-tooltip="${'' if hide_correctness == True else status.display_tooltip}">
+% endif
+  % if hide_correctness == False:
+  <span class="sr">${status.display_name}</span><span class="status-icon" aria-hidden="true"></span>
+  % endif
+</span>

xmodule/capa/templates/textline.html

@@ -0,0 +1,53 @@
+<%page expression_filter="h"/>
+<%! from openedx.core.djangolib.markup import HTML %>
+<% doinline = "inline" if inline else "" %>
+
+<div id="inputtype_${id}" class="${'text-input-dynamath' if do_math else ''} capa_inputtype ${doinline} textline">
+% if preprocessor is not None:
+    <div class="text-input-dynamath_data ${doinline}" data-preprocessor="${preprocessor['class_name']}"/>
+    <div class="script_placeholder" data-src="${preprocessor['script_src']}"/>
+% endif
+
+% if status in ('unsubmitted', 'submitted', 'correct', 'incorrect', 'partially-correct', 'incomplete'):
+    <div class="${status.classname} ${doinline}">
+% endif
+
+% if hidden:
+    <div style="display:none;" name="${hidden}" inputid="input_${id}" />
+% endif
+
+% if response_data['label']:
+    <label class="problem-group-label" for="input_${id}" id="label_${id}">${response_data['label']}</label>
+% endif
+
+% for description_id, description_text in response_data['descriptions'].items():
+    <p class="question-description" id="${description_id}">${description_text}</p>
+% endfor
+<input type="text" class="mw-100 ${'math' if do_math else ''}" name="input_${id}" id="input_${id}" ${describedby_html} value="${value}"
+    % if size:
+        size="${size}"
+    % endif
+    % if hidden:
+        style="display:none;"
+    % endif
+/>
+<span class="trailing_text" id="trailing_text_${id}">${trailing_text}</span>
+
+<%include file="status_span.html" args="status=status, status_id=id"/>
+
+<p id="answer_${id}" class="answer"></p>
+
+% if do_math:
+    <div id="display_${id}" class="equation">`{::}`</div>
+    <textarea style="display:none" id="input_${id}_dynamath" name="input_${id}_dynamath"></textarea>
+% endif
+
+% if status in ('unsubmitted', 'submitted', 'correct', 'incorrect', 'partially-correct', 'incomplete'):
+    </div>
+% endif
+
+  % if msg:
+      <span class="message" aria-describedby="label_${id}" tabindex="-1">${HTML(msg)}</span>
+  % endif
+
+</div>

xmodule/capa/templates/vsepr_input.html

@@ -0,0 +1,35 @@
+<%! from openedx.core.djangolib.markup import HTML %>
+<section id="inputtype_${id}" class="capa_inputtype" >
+     <table><tr><td height='600'>
+    <div id="vsepr_div_${id}" style="position:relative;" data-molecules="${molecules}" data-geometries="${geometries}">
+    <canvas id="vsepr_canvas_${id}" width="${width}" height="${height}">
+    </canvas>
+    </div>
+    </td><td valign ='top'>
+    <select class="molecule_select" id="molecule_select_${id}" size="18">
+    </select>
+    </td></tr></table>
+
+    <div class="script_placeholder" data-src="/static/js/vsepr/vsepr.js"></div>
+
+    % if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+    <div class="${status.classname}" id="status_${id}">
+    % endif
+
+  <input type="text" name="input_${id}" id="input_${id}" aria-describedby="answer_${id}" value="${value|h}"
+      style="display:none;"
+   />
+
+      <p class="status">
+         <span class="sr">${status.display_name}</span>
+      </p>
+
+  <p id="answer_${id}" class="answer"></p>
+
+  % if msg:
+      <span class="message" tabindex="-1">${HTML(msg)}</span>
+  % endif
+% if status in ['unsubmitted', 'correct', 'incorrect', 'partially-correct', 'incomplete']:
+</div>
+% endif
+</section>

xmodule/capa/tests/helpers.py

@@ -0,0 +1,122 @@
+"""Tools for helping with testing capa."""
+
+
+import gettext
+import io
+import os
+import os.path
+import xml.sax.saxutils as saxutils
+
+import fs.osfs
+import six
+from mako.lookup import TemplateLookup
+from mock import MagicMock, Mock
+from path import Path
+
+from xmodule.capa.capa_problem import LoncapaProblem, LoncapaSystem
+from xmodule.capa.inputtypes import Status
+
+TEST_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def get_template(template_name):
+    """
+    Return template for a capa inputtype.
+    """
+    return TemplateLookup(
+        directories=[Path(__file__).dirname().dirname() / 'templates'],
+        default_filters=['decode.utf8']
+    ).get_template(template_name)
+
+
+def capa_render_template(template, context):
+    """
+    Render template for a capa inputtype.
+    """
+    return get_template(template).render_unicode(**context)
+
+
+def tst_render_template(template, context):  # pylint: disable=unused-argument
+    """
+    A test version of render to template.  Renders to the repr of the context, completely ignoring
+    the template name.  To make the output valid xml, quotes the content, and wraps it in a <div>
+    """
+    return '<div>{0}</div>'.format(saxutils.escape(repr(context)))
+
+
+class StubXQueueService:
+    """
+    Stubs out the XQueueService for Capa problem tests.
+    """
+    def __init__(self):
+        self.interface = MagicMock()
+        self.interface.send_to_queue.return_value = (0, 'Success!')
+        self.default_queuename = 'testqueue'
+        self.waittime = 10
+
+    def construct_callback(self, dispatch='score_update'):
+        """A callback url method to use in tests."""
+        return dispatch
+
+
+def test_capa_system(render_template=None):
+    """
+    Construct a mock LoncapaSystem instance.
+
+    """
+    the_system = Mock(
+        spec=LoncapaSystem,
+        ajax_url='/dummy-ajax-url',
+        anonymous_student_id='student',
+        cache=None,
+        can_execute_unsafe_code=lambda: False,
+        get_python_lib_zip=lambda: None,
+        DEBUG=True,
+        i18n=gettext.NullTranslations(),
+        render_template=render_template or tst_render_template,
+        resources_fs=fs.osfs.OSFS(os.path.join(TEST_DIR, "test_files")),
+        seed=0,
+        STATIC_URL='/dummy-static/',
+        STATUS_CLASS=Status,
+        xqueue=StubXQueueService(),
+    )
+    return the_system
+
+
+def mock_capa_module():
+    """
+    capa response types needs just two things from the capa_module: location and track_function.
+    """
+    def mock_location_text(self):  # lint-amnesty, pylint: disable=unused-argument
+        """
+        Mock implementation of __unicode__ or __str__ for the module's location.
+        """
+        return 'i4x://Foo/bar/mock/abc'
+
+    capa_module = Mock()
+    if six.PY2:
+        capa_module.location.__unicode__ = mock_location_text
+    else:
+        capa_module.location.__str__ = mock_location_text
+    # The following comes into existence by virtue of being called
+    # capa_module.runtime.track_function
+    return capa_module
+
+
+def new_loncapa_problem(xml, problem_id='1', capa_system=None, seed=723, use_capa_render_template=False):
+    """Construct a `LoncapaProblem` suitable for unit tests."""
+    render_template = capa_render_template if use_capa_render_template else None
+    return LoncapaProblem(xml, id=problem_id, seed=seed, capa_system=capa_system or test_capa_system(render_template),
+                          capa_module=mock_capa_module())
+
+
+def load_fixture(relpath):
+    """
+    Return a `unicode` object representing the contents
+    of the fixture file at the given path within a test_files directory
+    in the same directory as the test file.
+    """
+    abspath = os.path.join(os.path.dirname(__file__), 'test_files', relpath)
+    with io.open(abspath, encoding="utf-8") as fixture_file:
+        contents = fixture_file.read()
+        return contents

xmodule/capa/tests/response_xml_factory.py

@@ -0,0 +1,948 @@
+# lint-amnesty, pylint: disable=missing-module-docstring
+
+from abc import ABCMeta, abstractmethod
+
+import six
+from lxml import etree
+from six.moves import range, zip
+
+
+class ResponseXMLFactory(six.with_metaclass(ABCMeta, object)):
+    """ Abstract base class for capa response XML factories.
+    Subclasses override create_response_element and
+    create_input_element to produce XML of particular response types"""
+
+    @abstractmethod
+    def create_response_element(self, **kwargs):
+        """ Subclasses override to return an etree element
+        representing the capa response XML
+        (e.g. <numericalresponse>).
+
+        The tree should NOT contain any input elements
+        (such as <textline />) as these will be added later."""
+        return None
+
+    @abstractmethod
+    def create_input_element(self, **kwargs):
+        """ Subclasses override this to return an etree element
+        representing the capa input XML (such as <textline />)"""
+        return None
+
+    def build_xml(self, **kwargs):
+        """ Construct an XML string for a capa response
+        based on **kwargs.
+
+        **kwargs is a dictionary that will be passed
+        to create_response_element() and create_input_element().
+        See the subclasses below for other keyword arguments
+        you can specify.
+
+        For all response types, **kwargs can contain:
+
+        *question_text*: The text of the question to display,
+            wrapped in <label> tags.
+
+        *explanation_text*: The detailed explanation that will
+            be shown if the user answers incorrectly.
+
+        *script*: The embedded Python script (a string)
+
+        *num_responses*: The number of responses to create [DEFAULT: 1]
+
+        *num_inputs*: The number of input elements
+            to create [DEFAULT: 1]
+
+        *credit_type*: String of comma-separated words specifying the
+            partial credit grading scheme.
+
+        Returns a string representation of the XML tree.
+        """
+
+        # Retrieve keyward arguments
+        question_text = kwargs.get('question_text', '')
+        explanation_text = kwargs.get('explanation_text', '')
+        script = kwargs.get('script', None)
+        num_responses = kwargs.get('num_responses', 1)
+        num_inputs = kwargs.get('num_inputs', 1)
+        credit_type = kwargs.get('credit_type', None)
+
+        # The root is <problem>
+        root = etree.Element("problem")
+
+        # Add a script if there is one
+        if script:
+            script_element = etree.SubElement(root, "script")
+            script_element.set("type", "loncapa/python")
+            script_element.text = str(script)
+
+        # Add the response(s)
+        for __ in range(int(num_responses)):
+            response_element = self.create_response_element(**kwargs)
+
+            # Set partial credit
+            if credit_type is not None:
+                response_element.set('partial_credit', str(credit_type))
+
+            root.append(response_element)
+
+            # Add the question label
+            question = etree.SubElement(response_element, "label")
+            question.text = question_text
+
+            # Add input elements
+            for __ in range(int(num_inputs)):
+                input_element = self.create_input_element(**kwargs)
+                if input_element is not None:
+                    response_element.append(input_element)
+
+            # The problem has an explanation of the solution
+            if explanation_text:
+                explanation = etree.SubElement(root, "solution")
+                explanation_div = etree.SubElement(explanation, "div")
+                explanation_div.set("class", "detailed-solution")
+                explanation_div.text = explanation_text
+
+        return etree.tostring(root).decode('utf-8')
+
+    @staticmethod
+    def textline_input_xml(**kwargs):
+        """ Create a <textline/> XML element
+
+        Uses **kwargs:
+
+        *math_display*: If True, then includes a MathJax display of user input
+
+        *size*: An integer representing the width of the text line
+        """
+        math_display = kwargs.get('math_display', False)
+        size = kwargs.get('size', None)
+        input_element_label = kwargs.get('input_element_label', '')
+
+        input_element = etree.Element('textline')
+
+        if input_element_label:
+            input_element.set('label', input_element_label)
+
+        if math_display:
+            input_element.set('math', '1')
+
+        if size:
+            input_element.set('size', str(size))
+
+        return input_element
+
+    @staticmethod
+    def choicegroup_input_xml(**kwargs):
+        """ Create a <choicegroup> XML element
+
+        Uses **kwargs:
+
+        *choice_type*: Can be "checkbox", "radio", or "multiple"
+
+        *choices*: List of True/False values indicating whether
+                            a particular choice is correct or not.
+                            Users must choose *all* correct options in order
+                            to be marked correct.
+                            DEFAULT: [True]
+
+        *choice_names": List of strings identifying the choices.
+                        If specified, you must ensure that
+                        len(choice_names) == len(choices)
+
+        *points*: List of strings giving partial credit values (0-1)
+                  for each choice. Interpreted as floats in problem.
+                  If specified, ensure len(points) == len(choices)
+        """
+        # Names of group elements
+        group_element_names = {
+            'checkbox': 'checkboxgroup',
+            'radio': 'radiogroup',
+            'multiple': 'choicegroup'
+        }
+
+        # Retrieve **kwargs
+        choices = kwargs.get('choices', [True])
+        choice_type = kwargs.get('choice_type', 'multiple')
+        choice_names = kwargs.get('choice_names', [None] * len(choices))
+        points = kwargs.get('points', [None] * len(choices))
+
+        # Create the <choicegroup>, <checkboxgroup>, or <radiogroup> element
+        assert choice_type in group_element_names
+        group_element = etree.Element(group_element_names[choice_type])
+
+        # Create the <choice> elements
+        for (correct_val, name, pointval) in zip(choices, choice_names, points):
+            choice_element = etree.SubElement(group_element, "choice")
+            if correct_val is True:
+                correctness = 'true'
+            elif correct_val is False:
+                correctness = 'false'
+            elif 'partial' in correct_val:
+                correctness = 'partial'
+            else:
+                correctness = correct_val
+
+            choice_element.set('correct', correctness)
+
+            # Add a name identifying the choice, if one exists
+            # For simplicity, we use the same string as both the
+            # name attribute and the text of the element
+            if name:
+                choice_element.text = str(name)
+                choice_element.set("name", str(name))
+
+            # Add point values for partially-correct choices.
+            if pointval:
+                choice_element.set("point_value", str(pointval))
+
+        return group_element
+
+
+class NumericalResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <numericalresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <numericalresponse> XML element.
+        Uses **kwarg keys:
+
+        *answer*: The correct answer (e.g. "5")
+
+        *correcthint*: The feedback describing correct answer.
+
+        *additional_answers*: A dict of additional answers along with their correcthint.
+
+        *tolerance*: The tolerance within which a response
+        is considered correct.  Can be a decimal (e.g. "0.01")
+        or percentage (e.g. "2%")
+
+        *credit_type*: String of comma-separated words specifying the
+        partial credit grading scheme.
+
+        *partial_range*: The multiplier for the tolerance that will
+        still provide partial credit in the "close" grading style
+
+        *partial_answers*: A string of comma-separated alternate
+        answers that will receive partial credit in the "list" style
+        """
+
+        answer = kwargs.get('answer', None)
+        correcthint = kwargs.get('correcthint', '')
+        additional_answers = kwargs.get('additional_answers', {})
+        tolerance = kwargs.get('tolerance', None)
+        credit_type = kwargs.get('credit_type', None)
+        partial_range = kwargs.get('partial_range', None)
+        partial_answers = kwargs.get('partial_answers', None)
+
+        response_element = etree.Element('numericalresponse')
+
+        if answer:
+            if isinstance(answer, float):
+                response_element.set('answer', repr(answer))
+            else:
+                response_element.set('answer', str(answer))
+
+        for additional_answer, additional_correcthint in additional_answers.items():
+            additional_element = etree.SubElement(response_element, 'additional_answer')
+            additional_element.set('answer', str(additional_answer))
+            if additional_correcthint:
+                correcthint_element = etree.SubElement(additional_element, 'correcthint')
+                correcthint_element.text = str(additional_correcthint)
+
+        if tolerance:
+            responseparam_element = etree.SubElement(response_element, 'responseparam')
+            responseparam_element.set('type', 'tolerance')
+            responseparam_element.set('default', str(tolerance))
+            if partial_range is not None and 'close' in credit_type:
+                responseparam_element.set('partial_range', str(partial_range))
+
+        if partial_answers is not None and 'list' in credit_type:
+            # The line below throws a false positive pylint violation, so it's excepted.
+            responseparam_element = etree.SubElement(response_element, 'responseparam')
+            responseparam_element.set('partial_answers', partial_answers)
+
+        if correcthint:
+            correcthint_element = etree.SubElement(response_element, 'correcthint')
+            correcthint_element.text = str(correcthint)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class CustomResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <customresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <customresponse> XML element.
+
+        Uses **kwargs:
+
+        *cfn*: the Python code to run.  Can be inline code,
+        or the name of a function defined in earlier <script> tags.
+
+        Should have the form: cfn(expect, answer_given, student_answers)
+        where expect is a value (see below),
+        answer_given is a single value (for 1 input)
+        or a list of values (for multiple inputs),
+        and student_answers is a dict of answers by input ID.
+
+        *expect*: The value passed to the function cfn
+
+        *answer*: Inline script that calculates the answer
+
+        *answer_attr*: The "answer" attribute on the tag itself (treated as an
+        alias to "expect", though "expect" takes priority if both are given)
+        """
+
+        # Retrieve **kwargs
+        cfn = kwargs.get('cfn', None)
+        expect = kwargs.get('expect', None)
+        answer_attr = kwargs.get('answer_attr', None)
+        answer = kwargs.get('answer', None)
+        options = kwargs.get('options', None)
+        cfn_extra_args = kwargs.get('cfn_extra_args', None)
+
+        # Create the response element
+        response_element = etree.Element("customresponse")
+
+        if cfn:
+            response_element.set('cfn', str(cfn))
+
+        if expect:
+            response_element.set('expect', str(expect))
+
+        if answer_attr:
+            response_element.set('answer', str(answer_attr))
+
+        if answer:
+            answer_element = etree.SubElement(response_element, "answer")
+            answer_element.text = str(answer)
+
+        if options:
+            response_element.set('options', str(options))
+
+        if cfn_extra_args:
+            response_element.set('cfn_extra_args', str(cfn_extra_args))
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class SchematicResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <schematicresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <schematicresponse> XML element.
+
+        Uses *kwargs*:
+
+        *answer*: The Python script used to evaluate the answer.
+        """
+        answer_script = kwargs.get('answer', None)
+
+        # Create the <schematicresponse> element
+        response_element = etree.Element("schematicresponse")
+
+        # Insert the <answer> script if one is provided
+        if answer_script:
+            answer_element = etree.SubElement(response_element, "answer")
+            answer_element.set("type", "loncapa/python")
+            answer_element.text = str(answer_script)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        """ Create the <schematic> XML element.
+
+        Although <schematic> can have several attributes,
+        (*height*, *width*, *parts*, *analyses*, *submit_analysis*, and *initial_value*),
+        none of them are used in the capa module.
+        For testing, we create a bare-bones version of <schematic>."""
+        return etree.Element("schematic")
+
+
+class CodeResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <coderesponse> XML trees """
+
+    def build_xml(self, **kwargs):
+        # Since we are providing an <answer> tag,
+        # we should override the default behavior
+        # of including a <solution> tag as well
+        kwargs['explanation_text'] = None
+        return super(CodeResponseXMLFactory, self).build_xml(**kwargs)  # lint-amnesty, pylint: disable=super-with-arguments
+
+    def create_response_element(self, **kwargs):
+        """
+        Create a <coderesponse> XML element.
+
+        Uses **kwargs:
+
+        *initial_display*: The code that initially appears in the textbox
+                            [DEFAULT: "Enter code here"]
+        *answer_display*: The answer to display to the student
+                            [DEFAULT: "This is the correct answer!"]
+        *grader_payload*: A JSON-encoded string sent to the grader
+                            [DEFAULT: empty dict string]
+        *allowed_files*: A space-separated string of file names.
+                            [DEFAULT: None]
+        *required_files*: A space-separated string of file names.
+                            [DEFAULT: None]
+
+        """
+        # Get **kwargs
+        initial_display = kwargs.get("initial_display", "Enter code here")
+        answer_display = kwargs.get("answer_display", "This is the correct answer!")
+        grader_payload = kwargs.get("grader_payload", '{}')
+        allowed_files = kwargs.get("allowed_files", None)
+        required_files = kwargs.get("required_files", None)
+
+        # Create the <coderesponse> element
+        response_element = etree.Element("coderesponse")
+
+        # If files are involved, create the <filesubmission> element.
+        has_files = allowed_files or required_files
+        if has_files:
+            filesubmission_element = etree.SubElement(response_element, "filesubmission")
+            if allowed_files:
+                filesubmission_element.set("allowed_files", allowed_files)
+            if required_files:
+                filesubmission_element.set("required_files", required_files)
+
+        # Create the <codeparam> element.
+        codeparam_element = etree.SubElement(response_element, "codeparam")
+
+        # Set the initial display text
+        initial_element = etree.SubElement(codeparam_element, "initial_display")
+        initial_element.text = str(initial_display)
+
+        # Set the answer display text
+        answer_element = etree.SubElement(codeparam_element, "answer_display")
+        answer_element.text = str(answer_display)
+
+        # Set the grader payload string
+        grader_element = etree.SubElement(codeparam_element, "grader_payload")
+        grader_element.text = str(grader_payload)
+
+        # Create the input within the response
+        if not has_files:
+            input_element = etree.SubElement(response_element, "textbox")
+            input_element.set("mode", "python")
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        # Since we create this in create_response_element(),
+        # return None here
+        return None
+
+
+class ChoiceResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <choiceresponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <choiceresponse> element """
+        return etree.Element("choiceresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create a <checkboxgroup> element."""
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+
+class FormulaResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <formularesponse> XML trees """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <formularesponse> element.
+
+        *sample_dict*: A dictionary of the form:
+                        { VARIABLE_NAME: (MIN, MAX), ....}
+
+                        This specifies the range within which
+                        to numerically sample each variable to check
+                        student answers.
+                        [REQUIRED]
+
+        *num_samples*: The number of times to sample the student's answer
+                        to numerically compare it to the correct answer.
+
+        *tolerance*: The tolerance within which answers will be accepted
+                        [DEFAULT: 0.01]
+
+        *answer*: The answer to the problem.  Can be a formula string
+                    or a Python variable defined in a script
+                    (e.g. "$calculated_answer" for a Python variable
+                    called calculated_answer)
+                    [REQUIRED]
+
+        *hints*: List of (hint_prompt, hint_name, hint_text) tuples
+                Where *hint_prompt* is the formula for which we show the hint,
+                *hint_name* is an internal identifier for the hint,
+                and *hint_text* is the text we show for the hint.
+        """
+        # Retrieve kwargs
+        sample_dict = kwargs.get("sample_dict", None)
+        num_samples = kwargs.get("num_samples", None)
+        tolerance = kwargs.get("tolerance", 0.01)
+        answer = kwargs.get("answer", None)
+        hint_list = kwargs.get("hints", None)
+
+        assert answer
+        assert sample_dict and num_samples
+
+        # Create the <formularesponse> element
+        response_element = etree.Element("formularesponse")
+
+        # Set the sample information
+        sample_str = self._sample_str(sample_dict, num_samples, tolerance)
+        response_element.set("samples", sample_str)
+
+        # Set the tolerance
+        responseparam_element = etree.SubElement(response_element, "responseparam")
+        responseparam_element.set("type", "tolerance")
+        responseparam_element.set("default", str(tolerance))
+
+        # Set the answer
+        response_element.set("answer", str(answer))
+
+        # Include hints, if specified
+        if hint_list:
+            hintgroup_element = etree.SubElement(response_element, "hintgroup")
+
+            for (hint_prompt, hint_name, hint_text) in hint_list:
+
+                # For each hint, create a <formulahint> element
+                formulahint_element = etree.SubElement(hintgroup_element, "formulahint")
+
+                # We could sample a different range, but for simplicity,
+                # we use the same sample string for the hints
+                # that we used previously.
+                formulahint_element.set("samples", sample_str)
+
+                formulahint_element.set("answer", str(hint_prompt))
+                formulahint_element.set("name", str(hint_name))
+
+                # For each hint, create a <hintpart> element
+                # corresponding to the <formulahint>
+                hintpart_element = etree.SubElement(hintgroup_element, "hintpart")
+                hintpart_element.set("on", str(hint_name))
+                text_element = etree.SubElement(hintpart_element, "text")
+                text_element.text = str(hint_text)
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+    def _sample_str(self, sample_dict, num_samples, tolerance):  # lint-amnesty, pylint: disable=missing-function-docstring, unused-argument
+        # Loncapa uses a special format for sample strings:
+        # "x,y,z@4,5,3:10,12,8#4" means plug in values for (x,y,z)
+        # from within the box defined by points (4,5,3) and (10,12,8)
+        # The "#4" means to repeat 4 times.
+        low_range_vals = [str(f[0]) for f in sample_dict.values()]
+        high_range_vals = [str(f[1]) for f in sample_dict.values()]
+        sample_str = (
+            ",".join(list(sample_dict.keys())) + "@" +
+            ",".join(low_range_vals) + ":" +
+            ",".join(high_range_vals) +
+            "#" + str(num_samples)
+        )
+        return sample_str
+
+
+class ImageResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <imageresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <imageresponse> element."""
+        return etree.Element("imageresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create the <imageinput> element.
+
+        Uses **kwargs:
+
+        *src*: URL for the image file [DEFAULT: "/static/image.jpg"]
+
+        *width*: Width of the image [DEFAULT: 100]
+
+        *height*: Height of the image [DEFAULT: 100]
+
+        *rectangle*: String representing the rectangles the user should select.
+
+                    Take the form "(x1,y1)-(x2,y2)", where the two (x,y)
+                    tuples define the corners of the rectangle.
+
+                    Can include multiple rectangles separated by a semicolon, e.g.
+                    "(490,11)-(556,98);(242,202)-(296,276)"
+
+        *regions*: String representing the regions a user can select
+
+                    Take the form "[ [[x1,y1], [x2,y2], [x3,y3]],
+                                    [[x1,y1], [x2,y2], [x3,y3]] ]"
+                    (Defines two regions, each with 3 points)
+
+        REQUIRED: Either *rectangle* or *region* (or both)
+        """
+
+        # Get the **kwargs
+        src = kwargs.get("src", "/static/image.jpg")
+        width = kwargs.get("width", 100)
+        height = kwargs.get("height", 100)
+        rectangle = kwargs.get('rectangle', None)
+        regions = kwargs.get('regions', None)
+
+        assert rectangle or regions
+
+        # Create the <imageinput> element
+        input_element = etree.Element("imageinput")
+        input_element.set("src", str(src))
+        input_element.set("width", str(width))
+        input_element.set("height", str(height))
+
+        if rectangle:
+            input_element.set("rectangle", rectangle)
+
+        if regions:
+            input_element.set("regions", regions)
+
+        return input_element
+
+
+class JSInputXMLFactory(CustomResponseXMLFactory):
+    """
+    Factory for producing <jsinput> XML.
+    Note that this factory currently does not create a functioning problem.
+    It will only create an empty iframe.
+    """
+
+    def create_input_element(self, **kwargs):
+        """ Create the <jsinput> element """
+        return etree.Element("jsinput")
+
+
+class MultipleChoiceResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <multiplechoiceresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <multiplechoiceresponse> element"""
+        return etree.Element('multiplechoiceresponse')
+
+    def create_input_element(self, **kwargs):
+        """ Create the <choicegroup> element"""
+        kwargs['choice_type'] = 'multiple'
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+
+class TrueFalseResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <truefalseresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create the <truefalseresponse> element"""
+        return etree.Element('truefalseresponse')
+
+    def create_input_element(self, **kwargs):
+        """ Create the <choicegroup> element"""
+        kwargs['choice_type'] = 'multiple'
+        return ResponseXMLFactory.choicegroup_input_xml(**kwargs)
+
+
+class OptionResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <optionresponse> XML"""
+
+    def create_response_element(self, **kwargs):
+        """ Create the <optionresponse> element"""
+        return etree.Element("optionresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create the <optioninput> element.
+
+        Uses **kwargs:
+
+        *options*: a list of possible options the user can choose from [REQUIRED]
+                    You must specify at least 2 options.
+        *correct_option*: the correct choice from the list of options [REQUIRED]
+        """
+
+        options_list = kwargs.get('options', None)
+        correct_option = kwargs.get('correct_option', None)
+
+        assert options_list and correct_option
+        assert len(options_list) > 1
+        assert correct_option in options_list
+
+        # Create the <optioninput> element
+        optioninput_element = etree.Element("optioninput")
+
+        # Set the "options" attribute
+        # Format: "('first', 'second', 'third')"
+        options_attr_string = ",".join(["'{}'".format(o) for o in options_list])
+        options_attr_string = "({})".format(options_attr_string)
+        optioninput_element.set('options', options_attr_string)
+
+        # Set the "correct" attribute
+        optioninput_element.set('correct', str(correct_option))
+
+        return optioninput_element
+
+
+class StringResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <stringresponse> XML """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <stringresponse> XML element.
+
+            Uses **kwargs:
+
+            *answer*: The correct answer (a string) [REQUIRED]
+
+            *case_sensitive*: Whether the response is case-sensitive (True/False)
+                            [DEFAULT: True]
+
+            *hints*: List of (hint_prompt, hint_name, hint_text) tuples
+                Where *hint_prompt* is the string for which we show the hint,
+                *hint_name* is an internal identifier for the hint,
+                and *hint_text* is the text we show for the hint.
+
+            *hintfn*: The name of a function in the script to use for hints.
+
+            *regexp*: Whether the response is regexp
+
+            *additional_answers*: list of additional answers.
+
+            *non_attribute_answers*: list of additional answers to be coded in the
+                non-attribute format
+
+        """
+        # Retrieve the **kwargs
+        answer = kwargs.get("answer", None)
+        case_sensitive = kwargs.get("case_sensitive", None)
+        hint_list = kwargs.get('hints', None)
+        hint_fn = kwargs.get('hintfn', None)
+        regexp = kwargs.get('regexp', None)
+        additional_answers = kwargs.get('additional_answers', [])
+        non_attribute_answers = kwargs.get('non_attribute_answers', [])
+        assert answer
+
+        # Create the <stringresponse> element
+        response_element = etree.Element("stringresponse")
+
+        # Set the answer attribute
+        response_element.set("answer", six.text_type(answer))
+
+        # Set the case sensitivity and regexp:
+        type_value = ''
+        if case_sensitive is not None:
+            type_value += "cs" if case_sensitive else "ci"
+        type_value += ' regexp' if regexp else ''
+        if type_value:
+            response_element.set("type", type_value.strip())
+
+        # Add the hints if specified
+        if hint_list or hint_fn:
+            hintgroup_element = etree.SubElement(response_element, "hintgroup")
+            if hint_list:
+                assert not hint_fn
+                for (hint_prompt, hint_name, hint_text) in hint_list:
+                    stringhint_element = etree.SubElement(hintgroup_element, "stringhint")
+                    stringhint_element.set("answer", str(hint_prompt))
+                    stringhint_element.set("name", str(hint_name))
+
+                    hintpart_element = etree.SubElement(hintgroup_element, "hintpart")
+                    hintpart_element.set("on", str(hint_name))
+
+                    hint_text_element = etree.SubElement(hintpart_element, "text")
+                    hint_text_element.text = str(hint_text)
+
+            if hint_fn:
+                assert not hint_list
+                hintgroup_element.set("hintfn", hint_fn)
+
+        for additional_answer in additional_answers:
+            additional_node = etree.SubElement(response_element, "additional_answer")
+            additional_node.set("answer", additional_answer)
+
+        for answer in non_attribute_answers:
+            additional_node = etree.SubElement(response_element, "additional_answer")
+            additional_node.text = answer
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class AnnotationResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for creating <annotationresponse> XML trees """
+    def create_response_element(self, **kwargs):
+        """ Create a <annotationresponse> element """
+        return etree.Element("annotationresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create a <annotationinput> element."""
+
+        input_element = etree.Element("annotationinput")
+
+        text_children = [
+            {'tag': 'title', 'text': kwargs.get('title', 'super cool annotation')},
+            {'tag': 'text', 'text': kwargs.get('text', 'texty text')},
+            {'tag': 'comment', 'text': kwargs.get('comment', 'blah blah erudite comment blah blah')},
+            {'tag': 'comment_prompt', 'text': kwargs.get('comment_prompt', 'type a commentary below')},
+            {'tag': 'tag_prompt', 'text': kwargs.get('tag_prompt', 'select one tag')}
+        ]
+
+        for child in text_children:
+            etree.SubElement(input_element, child['tag']).text = child['text']
+
+        default_options = [('green', 'correct'), ('eggs', 'incorrect'), ('ham', 'partially-correct')]
+        options = kwargs.get('options', default_options)
+        options_element = etree.SubElement(input_element, 'options')
+
+        for (description, correctness) in options:
+            option_element = etree.SubElement(options_element, 'option', {'choice': correctness})
+            option_element.text = description
+
+        return input_element
+
+
+class SymbolicResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <symbolicresponse> xml """
+
+    def create_response_element(self, **kwargs):
+        """ Build the <symbolicresponse> XML element.
+
+        Uses **kwargs:
+
+        *expect*: The correct answer (a sympy string)
+
+        *options*: list of option strings to pass to symmath_check
+            (e.g. 'matrix', 'qbit', 'imaginary', 'numerical')"""
+
+        # Retrieve **kwargs
+        expect = kwargs.get('expect', '')
+        options = kwargs.get('options', [])
+
+        # Symmath check expects a string of options
+        options_str = ",".join(options)
+
+        # Construct the <symbolicresponse> element
+        response_element = etree.Element('symbolicresponse')
+
+        if expect:
+            response_element.set('expect', str(expect))
+
+        if options_str:
+            response_element.set('options', str(options_str))
+
+        return response_element
+
+    def create_input_element(self, **kwargs):
+        return ResponseXMLFactory.textline_input_xml(**kwargs)
+
+
+class ChoiceTextResponseXMLFactory(ResponseXMLFactory):
+    """ Factory for producing <choicetextresponse> xml """
+
+    def create_response_element(self, **kwargs):
+        """ Create a <choicetextresponse> element """
+        return etree.Element("choicetextresponse")
+
+    def create_input_element(self, **kwargs):
+        """ Create a <checkboxgroup> element.
+        choices can be specified in the following format:
+        [("true", [{"answer": "5", "tolerance": 0}]),
+        ("false", [{"answer": "5", "tolerance": 0}])
+        ]
+
+        This indicates that the first checkbox/radio is correct and it
+        contains a numtolerance_input with an answer of 5 and a tolerance of 0
+
+        It also indicates that the second has a second incorrect radiobutton
+        or checkbox with a numtolerance_input.
+        """
+        choices = kwargs.get('choices', [("true", {})])
+        choice_inputs = []
+        # Ensure that the first element of choices is an ordered
+        # collection. It will start as a list, a tuple, or not a Container.
+        if not isinstance(choices[0], (list, tuple)):
+            choices = [choices]
+
+        for choice in choices:
+            correctness, answers = choice
+            numtolerance_inputs = []
+            # If the current `choice` contains any("answer": number)
+            # elements, turn those into numtolerance_inputs
+            if answers:
+                # `answers` will be a list or tuple of answers or a single
+                # answer, representing the answers for numtolerance_inputs
+                # inside of this specific choice.
+
+                # Make sure that `answers` is an ordered collection for
+                # convenience.
+                if not isinstance(answers, (list, tuple)):
+                    answers = [answers]
+
+                numtolerance_inputs = [
+                    self._create_numtolerance_input_element(answer)
+                    for answer in answers
+                ]
+
+            choice_inputs.append(
+                self._create_choice_element(
+                    correctness=correctness,
+                    inputs=numtolerance_inputs
+                )
+            )
+        # Default type is 'radiotextgroup'
+        input_type = kwargs.get('type', 'radiotextgroup')
+        input_element = etree.Element(input_type)
+
+        for ind, choice in enumerate(choice_inputs):
+            # Give each choice text equal to it's position(0,1,2...)
+            choice.text = "choice_{0}".format(ind)
+            input_element.append(choice)
+
+        return input_element
+
+    def _create_choice_element(self, **kwargs):
+        """
+        Creates a choice element for a choictextproblem.
+        Defaults to a correct choice with no numtolerance_input
+        """
+        text = kwargs.get('text', '')
+        correct = kwargs.get('correctness', "true")
+        inputs = kwargs.get('inputs', [])
+        choice_element = etree.Element("choice")
+        choice_element.set("correct", correct)
+        choice_element.text = text
+        for inp in inputs:
+            # Add all of the inputs as children of this choice
+            choice_element.append(inp)
+
+        return choice_element
+
+    def _create_numtolerance_input_element(self, params):
+        """
+        Creates a <numtolerance_input/>  or <decoy_input/> element with
+        optionally specified tolerance and answer.
+        """
+        answer = params['answer'] if 'answer' in params else None
+        # If there is not an answer specified, Then create a <decoy_input/>
+        # otherwise create a <numtolerance_input/> and set its tolerance
+        # and answer attributes.
+        if answer:
+            text_input = etree.Element("numtolerance_input")
+            text_input.set('answer', answer)
+            # If tolerance was specified, was specified use it, otherwise
+            # Set the tolerance to "0"
+            text_input.set(
+                'tolerance',
+                params['tolerance'] if 'tolerance' in params else "0"
+            )
+
+        else:
+            text_input = etree.Element("decoy_input")
+
+        return text_input

xmodule/capa/tests/test_answer_pool.py

@@ -0,0 +1,653 @@
+"""
+Tests the logic of the "answer-pool" attribute, e.g.
+  <choicegroup answer-pool="4">
+"""
+
+
+import textwrap
+import unittest
+
+from xmodule.capa.responsetypes import LoncapaProblemError
+from xmodule.capa.tests.helpers import new_loncapa_problem, test_capa_system
+
+
+class CapaAnswerPoolTest(unittest.TestCase):
+    """Capa Answer Pool Test"""
+    def setUp(self):
+        super(CapaAnswerPoolTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.system = test_capa_system()
+
+    # XML problem setup used by a few tests.
+    common_question_xml = textwrap.dedent("""
+        <problem>
+
+        <p>What is the correct answer?</p>
+        <multiplechoiceresponse>
+          <choicegroup type="MultipleChoice" answer-pool="4">
+            <choice correct="false">wrong-1</choice>
+            <choice correct="false">wrong-2</choice>
+            <choice correct="true" explanation-id="solution1">correct-1</choice>
+            <choice correct="false">wrong-3</choice>
+            <choice correct="false">wrong-4</choice>
+            <choice correct="true" explanation-id="solution2">correct-2</choice>
+          </choicegroup>
+        </multiplechoiceresponse>
+
+        <solutionset>
+            <solution explanation-id="solution1">
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the 1st solution</p>
+                <p>Not much to explain here, sorry!</p>
+            </div>
+            </solution>
+
+            <solution explanation-id="solution2">
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the 2nd solution</p>
+            </div>
+            </solution>
+        </solutionset>
+    </problem>
+    """)
+
+    def test_answer_pool_4_choices_1_multiplechoiceresponse_seed1(self):
+        problem = new_loncapa_problem(self.common_question_xml, seed=723)
+        the_html = problem.get_html()
+        # [('choice_3', 'wrong-3'), ('choice_5', 'correct-2'), ('choice_1', 'wrong-2'), ('choice_4', 'wrong-4')]
+        self.assertRegex(the_html, r"<div>.*\[.*'wrong-3'.*'correct-2'.*'wrong-2'.*'wrong-4'.*\].*</div>")
+        self.assertRegex(the_html, r"<div>\{.*'1_solution_2'.*\}</div>")
+        assert the_html == problem.get_html(), 'should be able to call get_html() twice'
+        # Check about masking
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert response.has_answerpool()
+        assert response.unmask_order() == ['choice_3', 'choice_5', 'choice_1', 'choice_4']
+
+    def test_answer_pool_4_choices_1_multiplechoiceresponse_seed2(self):
+        problem = new_loncapa_problem(self.common_question_xml, seed=9)
+        the_html = problem.get_html()
+        # [('choice_0', 'wrong-1'), ('choice_4', 'wrong-4'), ('choice_3', 'wrong-3'), ('choice_2', 'correct-1')]
+        self.assertRegex(the_html, r"<div>.*\[.*'wrong-1'.*'wrong-4'.*'wrong-3'.*'correct-1'.*\].*</div>")
+        self.assertRegex(the_html, r"<div>\{.*'1_solution_1'.*\}</div>")
+        # Check about masking
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert hasattr(response, 'has_answerpool')
+        assert response.unmask_order() == ['choice_0', 'choice_4', 'choice_3', 'choice_2']
+
+    def test_no_answer_pool_4_choices_1_multiplechoiceresponse(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+        </problem>
+        """)
+
+        problem = new_loncapa_problem(xml_str)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'wrong-1'.*'wrong-2'.*'correct-1'.*'wrong-3'.*'wrong-4'.*'correct-2'.*\].*</div>")  # lint-amnesty, pylint: disable=line-too-long
+        self.assertRegex(the_html, r"<div>\{.*'1_solution_1'.*'1_solution_2'.*\}</div>")
+        assert the_html == problem.get_html(), 'should be able to call get_html() twice'
+        # Check about masking
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert not response.has_answerpool()
+
+    def test_0_answer_pool_4_choices_1_multiplechoiceresponse(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="0">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+        </problem>
+        """)
+
+        problem = new_loncapa_problem(xml_str)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'wrong-1'.*'wrong-2'.*'correct-1'.*'wrong-3'.*'wrong-4'.*'correct-2'.*\].*</div>")  # lint-amnesty, pylint: disable=line-too-long
+        self.assertRegex(the_html, r"<div>\{.*'1_solution_1'.*'1_solution_2'.*\}</div>")
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert not response.has_answerpool()
+
+    def test_invalid_answer_pool_value(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="2.3">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+        </problem>
+        """)
+
+        with self.assertRaisesRegex(LoncapaProblemError, "answer-pool"):
+            new_loncapa_problem(xml_str)
+
+    def test_invalid_answer_pool_none_correct(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="4">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="false">wrong!!</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+        </problem>
+        """)
+        with self.assertRaisesRegex(LoncapaProblemError, "1 correct.*1 incorrect"):
+            new_loncapa_problem(xml_str)
+
+    def test_invalid_answer_pool_all_correct(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="4">
+                <choice correct="true">!wrong-1</choice>
+                <choice correct="true">!wrong-2</choice>
+                <choice correct="true">!wrong-3</choice>
+                <choice correct="true">!wrong-4</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+        </problem>
+        """)
+        with self.assertRaisesRegex(LoncapaProblemError, "1 correct.*1 incorrect"):
+            new_loncapa_problem(xml_str)
+
+    def test_answer_pool_5_choices_1_multiplechoiceresponse_seed1(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="5">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+        </problem>
+        """)
+
+        problem = new_loncapa_problem(xml_str, seed=723)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'correct-2'.*'wrong-1'.*'wrong-2'.*.*'wrong-3'.*'wrong-4'.*\].*</div>")
+        self.assertRegex(the_html, r"<div>\{.*'1_solution_2'.*\}</div>")
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert response.unmask_order() == ['choice_5', 'choice_0', 'choice_1', 'choice_3', 'choice_4']
+
+    def test_answer_pool_2_multiplechoiceresponses_seed1(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="4">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="3">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+        </problem>
+        """)
+        problem = new_loncapa_problem(xml_str)
+        the_html = problem.get_html()
+
+        str1 = r"<div>.*\[.*'wrong-3'.*'correct-2'.*'wrong-2'.*'wrong-4'.*\].*</div>"
+        str2 = r"<div>.*\[.*'wrong-2'.*'wrong-1'.*'correct-2'.*\].*</div>"    # rng shared
+        # str2 = r"<div>.*\[.*'correct-2'.*'wrong-2'.*'wrong-3'.*\].*</div>"  # rng independent
+
+        str3 = r"<div>\{.*'1_solution_2'.*\}</div>"
+        str4 = r"<div>\{.*'1_solution_4'.*\}</div>"
+
+        self.assertRegex(the_html, str1)
+        self.assertRegex(the_html, str2)
+        self.assertRegex(the_html, str3)
+        self.assertRegex(the_html, str4)
+
+        without_new_lines = the_html.replace("\n", "")
+
+        self.assertRegex(without_new_lines, str1 + r".*" + str2)
+        self.assertRegex(without_new_lines, str3 + r".*" + str4)
+
+    def test_answer_pool_2_multiplechoiceresponses_seed2(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="3">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="4">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+        </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=9)
+        the_html = problem.get_html()
+
+        str1 = r"<div>.*\[.*'wrong-4'.*'wrong-3'.*'correct-1'.*\].*</div>"
+        str2 = r"<div>.*\[.*'wrong-2'.*'wrong-3'.*'wrong-4'.*'correct-2'.*\].*</div>"
+        str3 = r"<div>\{.*'1_solution_1'.*\}</div>"
+        str4 = r"<div>\{.*'1_solution_4'.*\}</div>"
+
+        self.assertRegex(the_html, str1)
+        self.assertRegex(the_html, str2)
+        self.assertRegex(the_html, str3)
+        self.assertRegex(the_html, str4)
+
+        without_new_lines = the_html.replace("\n", "")
+
+        self.assertRegex(without_new_lines, str1 + r".*" + str2)
+        self.assertRegex(without_new_lines, str3 + r".*" + str4)
+
+    def test_answer_pool_random_consistent(self):
+        """
+        The point of this test is to make sure that the exact randomization
+        per seed does not change.
+        """
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="2">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true">correct-2</choice>
+                <choice correct="true">correct-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="3">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true">correct-2</choice>
+                <choice correct="true">correct-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="2">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true">correct-2</choice>
+                <choice correct="true">correct-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="3">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true">correct-2</choice>
+                <choice correct="true">correct-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+        </problem>
+        """)
+
+        problem = new_loncapa_problem(xml_str)
+        the_html = problem.get_html()
+        str1 = (r"<div>.*\[.*'correct-2'.*'wrong-2'.*\].*</div>.*" +
+                r"<div>.*\[.*'wrong-1'.*'correct-2'.*'wrong-4'.*\].*</div>.*" +
+                r"<div>.*\[.*'correct-1'.*'wrong-4'.*\].*</div>.*" +
+                r"<div>.*\[.*'wrong-1'.*'wrong-2'.*'correct-1'.*\].*</div>")
+        without_new_lines = the_html.replace("\n", "")
+        self.assertRegex(without_new_lines, str1)
+
+    def test_no_answer_pool(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+        </problem>
+        """)
+
+        problem = new_loncapa_problem(xml_str, seed=723)
+        the_html = problem.get_html()
+
+        str1 = r"<div>.*\[.*'wrong-1'.*'wrong-2'.*'correct-1'.*'wrong-3'.*'wrong-4'.*\].*</div>"
+
+        self.assertRegex(the_html, str1)
+        # attributes *not* present
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert not response.has_answerpool()
+
+    def test_answer_pool_and_no_answer_pool(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solution>
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the solution</p>
+                <p>Not much to explain here, sorry!</p>
+            </div>
+            </solution>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="4">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true" explanation-id="solution1">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true" explanation-id="solution2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solutionset>
+                <solution explanation-id="solution1">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 1st solution</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+
+                <solution explanation-id="solution2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the 2nd solution</p>
+                </div>
+                </solution>
+            </solutionset>
+
+        </problem>
+        """)
+
+        problem = new_loncapa_problem(xml_str, seed=723)
+        the_html = problem.get_html()
+
+        str1 = r"<div>.*\[.*'wrong-1'.*'wrong-2'.*'correct-1'.*'wrong-3'.*'wrong-4'.*\].*</div>"
+        str2 = r"<div>.*\[.*'wrong-3'.*'correct-2'.*'wrong-2'.*'wrong-4'.*\].*</div>"
+        str3 = r"<div>\{.*'1_solution_1'.*\}</div>"
+        str4 = r"<div>\{.*'1_solution_3'.*\}</div>"
+
+        self.assertRegex(the_html, str1)
+        self.assertRegex(the_html, str2)
+        self.assertRegex(the_html, str3)
+        self.assertRegex(the_html, str4)
+
+        without_new_lines = the_html.replace("\n", "")
+
+        self.assertRegex(without_new_lines, str1 + r".*" + str2)
+        self.assertRegex(without_new_lines, str3 + r".*" + str4)
+
+    def test_answer_pool_without_solutionset(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" answer-pool="4">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+                <choice correct="false">wrong-4</choice>
+                <choice correct="true">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <solution>
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the solution</p>
+                <p>Not much to explain here, sorry!</p>
+            </div>
+            </solution>
+
+        </problem>
+        """)
+
+        problem = new_loncapa_problem(xml_str, seed=723)
+        the_html = problem.get_html()
+
+        self.assertRegex(the_html, r"<div>.*\[.*'wrong-3'.*'correct-2'.*'wrong-2'.*'wrong-4'.*\].*</div>")
+        self.assertRegex(the_html, r"<div>\{.*'1_solution_1'.*\}</div>")

xmodule/capa/tests/test_capa_problem.py

@@ -0,0 +1,736 @@
+"""
+Test capa problem.
+"""
+
+
+import textwrap
+import unittest
+import pytest
+import ddt
+import six
+from lxml import etree
+from markupsafe import Markup
+from mock import patch
+
+from xmodule.capa.responsetypes import LoncapaProblemError
+from xmodule.capa.tests.helpers import new_loncapa_problem
+from openedx.core.djangolib.markup import HTML
+
+
+@ddt.ddt
+class CAPAProblemTest(unittest.TestCase):
+    """ CAPA problem related tests"""
+
+    @ddt.unpack
+    @ddt.data(
+        {'question': 'Select the correct synonym of paranoid?'},
+        {'question': 'Select the correct <em>synonym</em> of <strong>paranoid</strong>?'},
+    )
+    def test_label_and_description_inside_responsetype(self, question):
+        """
+        Verify that
+        * label is extracted
+        * <label> tag is removed to avoid duplication
+
+        This is the case when we have a problem with single question or
+        problem with multiple-questions separated as per the new format.
+        """
+        xml = """
+        <problem>
+            <choiceresponse>
+                <label>{question}</label>
+                <description>Only the paranoid survive.</description>
+                <checkboxgroup>
+                    <choice correct="true">over-suspicious</choice>
+                    <choice correct="false">funny</choice>
+                </checkboxgroup>
+            </choiceresponse>
+        </problem>
+        """.format(question=question)
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data ==\
+               {'1_2_1': {'label': question, 'descriptions': {'description_1_1_1': 'Only the paranoid survive.'}}}
+        assert len(problem.tree.xpath('//label')) == 0
+
+    @ddt.unpack
+    @ddt.data(
+        {
+            'question': 'Once we become predictable, we become ______?',
+            'label_attr': 'Once we become predictable, we become ______?'
+        },
+        {
+            'question': 'Once we become predictable, we become ______?<img src="img/src"/>',
+            'label_attr': 'Once we become predictable, we become ______?'
+        },
+    )
+    def test_legacy_problem(self, question, label_attr):
+        """
+        Verify that legacy problem is handled correctly.
+        """
+        xml = """
+        <problem>
+            <p>Be sure to check your spelling.</p>
+            <p>{}</p>
+            <stringresponse answer="vulnerable" type="ci">
+                <textline label="{}" size="40"/>
+            </stringresponse>
+        </problem>
+        """.format(question, label_attr)
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data == {'1_2_1': {'label': question, 'descriptions': {}}}
+        assert len(problem.tree.xpath("//*[normalize-space(text())='{}']".format(question))) == 0
+
+    @ddt.unpack
+    @ddt.data(
+        {
+            'question1': 'People who say they have nothing to ____ almost always do?',
+            'question2': 'Select the correct synonym of paranoid?'
+        },
+        {
+            'question1': '<b>People</b> who say they have <mark>nothing</mark> to ____ almost always do?',
+            'question2': 'Select the <sup>correct</sup> synonym of <mark>paranoid</mark>?'
+        },
+    )
+    def test_neither_label_tag_nor_attribute(self, question1, question2):
+        """
+        Verify that label is extracted correctly.
+
+        This is the case when we have a markdown problem with multiple-questions.
+        In this case when markdown is converted to xml, there will be no label
+        tag and label attribute inside responsetype. But we have a label tag
+        before the responsetype.
+        """
+        xml = """
+        <problem>
+            <p>Be sure to check your spelling.</p>
+            <label>{}</label>
+            <stringresponse answer="hide" type="ci">
+                <textline size="40"/>
+            </stringresponse>
+            <choiceresponse>
+                <label>{}</label>
+                <checkboxgroup>
+                    <choice correct="true">over-suspicious</choice>
+                    <choice correct="false">funny</choice>
+                </checkboxgroup>
+            </choiceresponse>
+        </problem>
+        """.format(question1, question2)
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data ==\
+               {'1_2_1': {'label': question1, 'descriptions': {}}, '1_3_1': {'label': question2, 'descriptions': {}}}
+        for question in (question1, question2):
+            assert len(problem.tree.xpath('//label[text()="{}"]'.format(question))) == 0
+
+    def test_multiple_descriptions(self):
+        """
+        Verify that multiple descriptions are handled correctly.
+        """
+        desc1 = "The problem with trying to be the <em>bad guy</em>, there's always someone <strong>worse</strong>."
+        desc2 = "Anyone who looks the world as if it was a game of chess deserves to lose."
+        xml = """
+        <problem>
+            <p>Be sure to check your spelling.</p>
+            <stringresponse answer="War" type="ci">
+                <label>___ requires sacrifices.</label>
+                <description>{}</description>
+                <description>{}</description>
+                <textline size="40"/>
+            </stringresponse>
+        </problem>
+        """.format(desc1, desc2)
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data ==\
+               {'1_2_1': {'label': '___ requires sacrifices.',
+                          'descriptions': {'description_1_1_1': desc1, 'description_1_1_2': desc2}}}
+
+    def test_additional_answer_is_skipped_from_resulting_html(self):
+        """Tests that additional_answer element is not present in transformed HTML"""
+        xml = """
+        <problem>
+            <p>Be sure to check your spelling.</p>
+            <stringresponse answer="War" type="ci">
+                <label>___ requires sacrifices.</label>
+                <description>Anyone who looks the world as if it was a game of chess deserves to lose.</description>
+                <additional_answer answer="optional acceptable variant of the correct answer"/>
+                <textline size="40"/>
+            </stringresponse>
+        </problem>
+        """
+        problem = new_loncapa_problem(xml)
+        assert len(problem.extracted_tree.xpath('//additional_answer')) == 0
+        assert 'additional_answer' not in problem.get_html()
+
+    def test_non_accessible_inputtype(self):
+        """
+        Verify that tag with question text is not removed when inputtype is not fully accessible.
+        """
+        question = "Click the country which is home to the Pyramids."
+        # lint-amnesty, pylint: disable=duplicate-string-formatting-argument
+        xml = """
+        <problem>
+            <p>{}</p>
+            <imageresponse>
+                <imageinput label="{}"
+                src="/static/Africa.png" width="600" height="638" rectangle="(338,98)-(412,168)"/>
+            </imageresponse>
+        </problem>
+        """.format(question, question)
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data == {'1_2_1': {'label': question, 'descriptions': {}}}
+        # <p> tag with question text should not be deleted
+        assert problem.tree.xpath("string(p[text()='{}'])".format(question)) == question
+
+    def test_label_is_empty_if_no_label_attribute(self):
+        """
+        Verify that label in response_data is empty string when label
+        attribute is missing and responsetype is not fully accessible.
+        """
+        question = "Click the country which is home to the Pyramids."
+        xml = """
+        <problem>
+            <p>{}</p>
+            <imageresponse>
+                <imageinput
+                src="/static/Africa.png" width="600" height="638" rectangle="(338,98)-(412,168)"/>
+            </imageresponse>
+        </problem>
+        """.format(question)
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data == {'1_2_1': {'label': '', 'descriptions': {}}}
+
+    def test_multiple_questions_problem(self):
+        """
+        For a problem with multiple questions verify that for each question
+        * label is extracted
+        * descriptions info is constructed
+        * <label> tag is removed to avoid duplication
+        """
+        xml = """
+        <problem>
+            <choiceresponse>
+                <label>Select the correct synonym of paranoid?</label>
+                <description>Only the paranoid survive.</description>
+                <checkboxgroup>
+                    <choice correct="true">over-suspicious</choice>
+                    <choice correct="false">funny</choice>
+                </checkboxgroup>
+            </choiceresponse>
+            <multiplechoiceresponse>
+                <p>one more question</p>
+                <label>What Apple device competed with the portable CD player?</label>
+                <description>Device looks like an egg plant.</description>
+                <choicegroup type="MultipleChoice">
+                    <choice correct="false">The iPad</choice>
+                    <choice correct="false">Napster</choice>
+                    <choice correct="true">The iPod</choice>
+                    <choice correct="false">The vegetable peeler</choice>
+                </choicegroup>
+            </multiplechoiceresponse>
+        </problem>
+        """
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data ==\
+               {'1_2_1': {'label': 'Select the correct synonym of paranoid?',
+                          'descriptions': {'description_1_1_1': 'Only the paranoid survive.'}},
+                '1_3_1': {'label': 'What Apple device competed with the portable CD player?',
+                          'descriptions': {'description_1_2_1': 'Device looks like an egg plant.'}}}
+        assert len(problem.tree.xpath('//label')) == 0
+
+    def test_question_title_not_removed_got_children(self):
+        """
+        Verify that <p> question text before responsetype not deleted when
+        it contains other children and label is picked from label attribute of inputtype
+
+        This is the case when author updated the <p> immediately before
+        responsetype to contain other elements. We do not want to delete information in that case.
+        """
+        question = 'Is egg plant a fruit?'
+        xml = """
+        <problem>
+            <p>Choose wisely.</p>
+            <p>Select the correct synonym of paranoid?</p>
+            <p><img src="" /></p>
+            <choiceresponse>
+                <checkboxgroup label="{}">
+                    <choice correct="true">over-suspicious</choice>
+                    <choice correct="false">funny</choice>
+                </checkboxgroup>
+            </choiceresponse>
+        </problem>
+        """.format(question)
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data == {'1_2_1': {'label': '', 'descriptions': {}}}
+        assert len(problem.tree.xpath('//p/img')) == 1
+
+    @ddt.unpack
+    @ddt.data(
+        {'group_label': 'Choose the correct color'},
+        {'group_label': 'Choose the <b>correct</b> <mark>color</mark>'},
+    )
+    def test_multiple_inputtypes(self, group_label):
+        """
+        Verify that group label and labels for individual inputtypes are extracted correctly.
+        """
+        input1_label = 'What color is the sky?'
+        input2_label = 'What color are pine needles?'
+        xml = """
+        <problem>
+            <optionresponse>
+                <label>{}</label>
+                <optioninput options="('yellow','blue','green')" correct="blue" label="{}"/>
+                <optioninput options="('yellow','blue','green')" correct="green" label="{}"/>
+            </optionresponse>
+        </problem>
+        """.format(group_label, input1_label, input2_label)
+
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data ==\
+               {'1_2_1': {'group_label': group_label, 'label': input1_label, 'descriptions': {}},
+                '1_2_2': {'group_label': group_label, 'label': input2_label, 'descriptions': {}}}
+
+    def test_single_inputtypes(self):
+        """
+        Verify that HTML is correctly rendered when there is single inputtype.
+        """
+        question = 'Enter sum of 1+2'
+        xml = textwrap.dedent("""
+        <problem>
+            <customresponse cfn="test_sum" expect="3">
+        <script type="loncapa/python">
+        def test_sum(expect, ans):
+            return int(expect) == int(ans)
+        </script>
+                <label>{}</label>
+                <textline size="20" correct_answer="3" />
+            </customresponse>
+        </problem>
+        """.format(question))
+        problem = new_loncapa_problem(xml, use_capa_render_template=True)
+        problem_html = etree.XML(problem.get_html())
+
+        # verify that only no multi input group div is present
+        multi_inputs_group = problem_html.xpath('//div[@class="multi-inputs-group"]')
+        assert len(multi_inputs_group) == 0
+
+        # verify that question is rendered only once
+        question = problem_html.xpath("//*[normalize-space(text())='{}']".format(question))
+        assert len(question) == 1
+
+    def assert_question_tag(self, question1, question2, tag, label_attr=False):
+        """
+        Verify question tag correctness.
+        """
+        question1_tag = '<{tag}>{}</{tag}>'.format(question1, tag=tag) if question1 else ''
+        question2_tag = '<{tag}>{}</{tag}>'.format(question2, tag=tag) if question2 else ''
+        question1_label_attr = 'label="{}"'.format(question1) if label_attr else ''
+        question2_label_attr = 'label="{}"'.format(question2) if label_attr else ''
+        xml = """
+        <problem>
+            {question1_tag}
+            <choiceresponse>
+                <checkboxgroup {question1_label_attr}>
+                    <choice correct="true">choice1</choice>
+                    <choice correct="false">choice2</choice>
+                </checkboxgroup>
+            </choiceresponse>
+            {question2_tag}
+            <multiplechoiceresponse>
+                <choicegroup type="MultipleChoice" {question2_label_attr}>
+                    <choice correct="false">choice1</choice>
+                    <choice correct="true">choice2</choice>
+                </choicegroup>
+            </multiplechoiceresponse>
+        </problem>
+        """.format(
+            question1_tag=question1_tag,
+            question2_tag=question2_tag,
+            question1_label_attr=question1_label_attr,
+            question2_label_attr=question2_label_attr,
+        )
+        problem = new_loncapa_problem(xml)
+        assert problem.problem_data ==\
+               {'1_2_1': {'label': question1, 'descriptions': {}}, '1_3_1': {'label': question2, 'descriptions': {}}}
+        assert len(problem.tree.xpath('//{}'.format(tag))) == 0
+
+    @ddt.unpack
+    @ddt.data(
+        {'question1': 'question 1 label', 'question2': 'question 2 label'},
+        {'question1': '', 'question2': 'question 2 label'},
+        {'question1': 'question 1 label', 'question2': ''}
+    )
+    def test_correct_question_tag_is_picked(self, question1, question2):
+        """
+        For a problem with multiple questions verify that correct question tag is picked.
+        """
+        self.assert_question_tag(question1, question2, tag='label', label_attr=False)
+        self.assert_question_tag(question1, question2, tag='p', label_attr=True)
+
+    def test_optionresponse_xml_compatibility(self):
+        """
+        Verify that an optionresponse problem with multiple correct answers is not instantiated.
+
+        Scenario:
+        Given an optionresponse/Dropdown problem
+        If there are multiple correct answers
+        Then the problem is not instantiated
+        And Loncapa problem error exception is raised
+        If the problem is corrected by including only one correct answer
+        Then the problem is created successfully
+        """
+        xml = """
+        <problem>
+            <optionresponse>
+              <p>You can use this template as a guide to the simple editor markdown and OLX markup to use for dropdown problems. Edit this component to replace this template with your own assessment.</p>
+            <label>Add the question text, or prompt, here. This text is required.</label>
+            <description>You can add an optional tip or note related to the prompt like this. </description>
+            <optioninput>
+                <option correct="False">an incorrect answer</option>
+                <option correct="True">the correct answer</option>
+                <option correct="{correctness}">an incorrect answer</option>
+              </optioninput>
+            </optionresponse>
+        </problem>
+        """
+        with pytest.raises(LoncapaProblemError):
+            new_loncapa_problem(xml.format(correctness=True))
+        problem = new_loncapa_problem(xml.format(correctness=False))
+        assert problem is not None
+
+    def test_optionresponse_option_with_empty_text(self):
+        """
+        Verify successful instantiation of an optionresponse problem
+        with an option with empty text
+        """
+        xml = """
+        <problem>
+            <optionresponse>
+                <label>Select True or False</label>
+                <optioninput>
+                    <option correct="False">True <optionhint>Not this one</optionhint></option>
+                    <option correct="True">False</option>
+                    <option correct="False"><optionhint>Not this empty one either</optionhint></option>
+                </optioninput>
+            </optionresponse>
+        </problem>
+        """
+        problem = new_loncapa_problem(xml)
+        assert problem is not None
+
+
+@ddt.ddt
+class CAPAMultiInputProblemTest(unittest.TestCase):
+    """ TestCase for CAPA problems with multiple inputtypes """
+
+    def capa_problem(self, xml):
+        """
+        Create capa problem.
+        """
+        return new_loncapa_problem(xml, use_capa_render_template=True)
+
+    def assert_problem_data(self, problem_data):
+        """Verify problem data is in expected state"""
+        for problem_value in six.viewvalues(problem_data):
+            assert isinstance(problem_value['label'], Markup)
+
+    def assert_problem_html(self, problem_html, group_label, *input_labels):
+        """
+        Verify that correct html is rendered for multiple inputtypes.
+
+        Arguments:
+            problem_html (str): problem HTML
+            group_label (str or None): multi input group label or None if label is not present
+            input_labels (tuple): individual input labels
+        """
+        html = etree.XML(problem_html)
+
+        # verify that only one multi input group div is present at correct path
+        multi_inputs_group = html.xpath(
+            '//div[@class="wrapper-problem-response"]/div[@class="multi-inputs-group"]'
+        )
+        assert len(multi_inputs_group) == 1
+
+        if group_label is None:
+            # if multi inputs group label is not present then there shouldn't be `aria-labelledby` attribute
+            assert multi_inputs_group[0].attrib.get('aria-labelledby') is None
+        else:
+            # verify that multi input group label <p> tag exists and its
+            # id matches with correct multi input group aria-labelledby
+            multi_inputs_group_label_id = multi_inputs_group[0].attrib.get('aria-labelledby')
+            multi_inputs_group_label = html.xpath('//p[@id="{}"]'.format(multi_inputs_group_label_id))
+            assert len(multi_inputs_group_label) == 1
+            assert multi_inputs_group_label[0].text == group_label
+
+        # verify that label for each input comes only once
+        for input_label in input_labels:
+            # normalize-space is used to remove whitespace around the text
+            input_label_element = multi_inputs_group[0].xpath('//*[normalize-space(text())="{}"]'.format(input_label))
+            assert len(input_label_element) == 1
+
+    @ddt.unpack
+    @ddt.data(
+        {'label_html': '<label>Choose the correct color</label>', 'group_label': 'Choose the correct color'},
+        {'label_html': '', 'group_label': None}
+    )
+    def test_optionresponse(self, label_html, group_label):
+        """
+        Verify that optionresponse problem with multiple inputtypes is rendered correctly.
+        """
+        input1_label = 'What color is the sky?'
+        input2_label = 'What color are pine needles?'
+        xml = """
+        <problem>
+            <optionresponse>
+                {label_html}
+                <optioninput options="('yellow','blue','green')" correct="blue" label="{input1_label}"/>
+                <optioninput options="('yellow','blue','green')" correct="green" label="{input2_label}"/>
+            </optionresponse>
+        </problem>
+        """.format(label_html=label_html, input1_label=input1_label, input2_label=input2_label)
+        problem = self.capa_problem(xml)
+        self.assert_problem_html(problem.get_html(), group_label, input1_label, input2_label)
+        self.assert_problem_data(problem.problem_data)
+
+    @ddt.unpack
+    @ddt.data(
+        {'inputtype': 'textline'},
+        {'inputtype': 'formulaequationinput'}
+    )
+    def test_customresponse(self, inputtype):
+        """
+        Verify that customresponse problem with multiple textline
+        and formulaequationinput inputtypes is rendered correctly.
+        """
+        group_label = 'Enter two integers that sum to 10.'
+        input1_label = 'Integer 1'
+        input2_label = 'Integer 2'
+        xml = textwrap.dedent("""
+        <problem>
+            <customresponse cfn="test_add_to_ten">
+        <script type="loncapa/python">
+        def test_add_to_ten(expect, ans):
+            return test_add(10, ans)
+        </script>
+                <label>{}</label>
+                <{inputtype} size="40" correct_answer="3" label="{}" /><br/>
+                <{inputtype} size="40" correct_answer="7" label="{}" />
+            </customresponse>
+        </problem>
+        """.format(group_label, input1_label, input2_label, inputtype=inputtype))
+        problem = self.capa_problem(xml)
+        self.assert_problem_html(problem.get_html(), group_label, input1_label, input2_label)
+        self.assert_problem_data(problem.problem_data)
+
+    @ddt.unpack
+    @ddt.data(
+        {
+            'descriptions': ('desc1', 'desc2'),
+            'descriptions_html': '<description>desc1</description><description>desc2</description>'
+        },
+        {
+            'descriptions': (),
+            'descriptions_html': ''
+        }
+    )
+    def test_descriptions(self, descriptions, descriptions_html):
+        """
+        Verify that groups descriptions are rendered correctly.
+        """
+        xml = """
+        <problem>
+            <optionresponse>
+                <label>group label</label>
+                {descriptions_html}
+                <optioninput options="('yellow','blue','green')" correct="blue" label="first label"/>
+                <optioninput options="('yellow','blue','green')" correct="green" label="second label"/>
+            </optionresponse>
+        </problem>
+        """.format(descriptions_html=descriptions_html)
+        problem = self.capa_problem(xml)
+        problem_html = etree.XML(problem.get_html())
+
+        multi_inputs_group = problem_html.xpath('//div[@class="multi-inputs-group"]')[0]
+        description_ids = multi_inputs_group.attrib.get('aria-describedby', '').split()
+
+        # Verify that number of descriptions matches description_ids
+        assert len(description_ids) == len(descriptions)
+
+        # For each description, check its order and text is correct
+        for index, description_id in enumerate(description_ids):
+            description_element = multi_inputs_group.xpath('//p[@id="{}"]'.format(description_id))
+            assert len(description_element) == 1
+            assert description_element[0].text == descriptions[index]
+
+
+@ddt.ddt
+class CAPAProblemReportHelpersTest(unittest.TestCase):
+    """ TestCase for CAPA methods for finding question labels and answer text """
+
+    @ddt.data(
+        ('answerid_2_1', 'label', 'label'),
+        ('answerid_2_2', 'label <some>html</some>', 'label html'),
+        ('answerid_2_2', '<more html="yes"/>label <some>html</some>', 'label html'),
+        ('answerid_2_3', None, 'Question 1'),
+        ('answerid_2_3', '', 'Question 1'),
+        ('answerid_3_3', '', 'Question 2'),
+    )
+    @ddt.unpack
+    def test_find_question_label(self, answer_id, label, stripped_label):
+        problem = new_loncapa_problem(
+            '<problem><some-problem id="{}"/></problem>'.format(answer_id)
+        )
+        mock_problem_data = {
+            answer_id: {
+                'label': HTML(label) if label else ''
+            }
+        }
+        with patch.object(problem, 'problem_data', mock_problem_data):
+            assert problem.find_question_label(answer_id) == stripped_label
+
+    @ddt.data(None, {}, [None])
+    def test_find_answer_test_not_implemented(self, current_answer):
+        problem = new_loncapa_problem('<problem/>')
+        self.assertRaises(NotImplementedError, problem.find_answer_text, '', current_answer)
+
+    @ddt.data(
+        ('1_2_1', 'choice_0', 'over-suspicious'),
+        ('1_2_1', 'choice_1', 'funny'),
+        ('1_3_1', 'choice_0', 'The iPad'),
+        ('1_3_1', 'choice_2', 'The iPod'),
+        ('1_3_1', ['choice_0', 'choice_1'], 'The iPad, Napster'),
+        ('1_4_1', 'yellow', 'yellow'),
+        ('1_4_1', 'blue', 'blue'),
+    )
+    @ddt.unpack
+    def test_find_answer_text_choices(self, answer_id, choice_id, answer_text):
+        problem = new_loncapa_problem(
+            """
+            <problem>
+                <choiceresponse>
+                    <checkboxgroup label="Select the correct synonym of paranoid?">
+                        <choice correct="true">over-suspicious</choice>
+                        <choice correct="false">funny</choice>
+                    </checkboxgroup>
+                </choiceresponse>
+                <multiplechoiceresponse>
+                    <choicegroup type="MultipleChoice">
+                        <choice correct="false">The iPad</choice>
+                        <choice correct="false">Napster</choice>
+                        <choice correct="true">The iPod</choice>
+                    </choicegroup>
+                </multiplechoiceresponse>
+                <optionresponse>
+                    <optioninput options="('yellow','blue','green')" correct="blue" label="Color_1"/>
+                </optionresponse>
+            </problem>
+            """
+        )
+        assert problem.find_answer_text(answer_id, choice_id) == answer_text
+
+    @ddt.data(
+        # Test for ChoiceResponse
+        ('1_2_1', 'choice_0', 'Answer Text Missing'),
+        ('1_2_1', 'choice_1', 'funny'),
+        # Test for MultipleChoiceResponse
+        ('1_3_1', 'choice_0', 'The iPad'),
+        ('1_3_1', 'choice_2', 'Answer Text Missing'),
+        ('1_3_1', ['choice_0', 'choice_1'], 'The iPad, Answer Text Missing'),
+        # Test for OptionResponse
+        ('1_4_1', '', 'Answer Text Missing'),
+    )
+    @ddt.unpack
+    def test_find_answer_text_choices_with_missing_text(self, answer_id, choice_id, answer_text):
+        problem = new_loncapa_problem(
+            """
+            <problem>
+                <choiceresponse>
+                    <checkboxgroup label="Select the correct synonym of paranoid?">
+                        <choice correct="true"></choice>
+                        <choice correct="false">funny</choice>
+                    </checkboxgroup>
+                </choiceresponse>
+                <multiplechoiceresponse>
+                    <choicegroup type="MultipleChoice">
+                        <choice correct="false">The iPad</choice>
+                        <choice correct="false"></choice>
+                        <choice correct="true"></choice>
+                    </choicegroup>
+                </multiplechoiceresponse>
+                <optionresponse>
+                    <optioninput options="('yellow','blue','green')" correct="blue" label="Color_1"/>
+                </optionresponse>
+            </problem>
+            """
+        )
+        assert problem.find_answer_text(answer_id, choice_id) == answer_text
+
+    @ddt.data(
+        # Test for ChoiceResponse
+        ('1_2_1', 'over-suspicious'),
+        # Test for MultipleChoiceResponse
+        ('1_3_1', 'The iPad, Napster'),
+        # Test for OptionResponse
+        ('1_4_1', 'blue'),
+    )
+    @ddt.unpack
+    def test_find_correct_answer_text_choices(self, answer_id, answer_text):
+        """
+        Verify that ``find_correct_answer_text`` can find the correct answer for
+        ChoiceResponse, MultipleChoiceResponse and OptionResponse problems.
+        """
+        problem = new_loncapa_problem(
+            """
+            <problem>
+                <choiceresponse>
+                    <checkboxgroup label="Select the correct synonym of paranoid?">
+                        <choice correct="true">over-suspicious</choice>
+                        <choice correct="false">funny</choice>
+                    </checkboxgroup>
+                </choiceresponse>
+                <multiplechoiceresponse>
+                    <choicegroup type="MultipleChoice">
+                        <choice correct="true">The iPad</choice>
+                        <choice correct="true">Napster</choice>
+                        <choice correct="false">The iPod</choice>
+                    </choicegroup>
+                </multiplechoiceresponse>
+                <optionresponse>
+                    <optioninput options="('yellow','blue','green')" correct="blue" label="Color_1"/>
+                </optionresponse>
+            </problem>
+            """
+        )
+        assert problem.find_correct_answer_text(answer_id) == answer_text
+
+    def test_find_answer_text_textinput(self):
+        problem = new_loncapa_problem(
+            """
+            <problem>
+                <stringresponse answer="hide" type="ci">
+                    <textline size="40"/>
+                </stringresponse>
+            </problem>
+            """
+        )
+        assert problem.find_answer_text('1_2_1', 'hide') == 'hide'
+
+    def test_get_question_answer(self):
+        problem = new_loncapa_problem(
+            """
+            <problem>
+                <optionresponse>
+                    <optioninput options="('yellow','blue','green')" correct="blue" label="Color_1"/>
+                </optionresponse>
+                <solution>
+                    <div class="detailed-solution">
+                        <p>Explanation</p>
+                        <p>Blue is the answer.</p>
+                    </div>
+                </solution>
+            </problem>
+            """
+        )
+
+        # Ensure that the answer is a string so that the dict returned from this
+        # function can eventualy be serialized to json without issues.
+        assert isinstance(problem.get_question_answers()['1_solution_1'], six.text_type)

xmodule/capa/tests/test_correctmap.py

@@ -0,0 +1,223 @@
+"""
+Tests to verify that CorrectMap behaves correctly
+"""
+
+
+import datetime
+import unittest
+import pytest
+from xmodule.capa.correctmap import CorrectMap
+
+
+class CorrectMapTest(unittest.TestCase):
+    """
+    Tests to verify that CorrectMap behaves correctly
+    """
+
+    def setUp(self):
+        super(CorrectMapTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.cmap = CorrectMap()
+
+    def test_set_input_properties(self):
+        # Set the correctmap properties for three inputs
+        self.cmap.set(
+            answer_id='1_2_1',
+            correctness='correct',
+            npoints=5,
+            msg='Test message',
+            hint='Test hint',
+            hintmode='always',
+            queuestate={
+                'key': 'secretstring',
+                'time': '20130228100026'
+            }
+        )
+
+        self.cmap.set(
+            answer_id='2_2_1',
+            correctness='incorrect',
+            npoints=None,
+            msg=None,
+            hint=None,
+            hintmode=None,
+            queuestate=None
+        )
+
+        self.cmap.set(
+            answer_id='3_2_1',
+            correctness='partially-correct',
+            npoints=3,
+            msg=None,
+            hint=None,
+            hintmode=None,
+            queuestate=None
+        )
+
+        # Assert that each input has the expected properties
+        assert self.cmap.is_correct('1_2_1')
+        assert not self.cmap.is_correct('2_2_1')
+        assert self.cmap.is_correct('3_2_1')
+
+        assert self.cmap.is_partially_correct('3_2_1')
+        assert not self.cmap.is_partially_correct('2_2_1')
+
+        # Intentionally testing an item that's not in cmap.
+        assert not self.cmap.is_partially_correct('9_2_1')
+
+        assert self.cmap.get_correctness('1_2_1') == 'correct'
+        assert self.cmap.get_correctness('2_2_1') == 'incorrect'
+        assert self.cmap.get_correctness('3_2_1') == 'partially-correct'
+
+        assert self.cmap.get_npoints('1_2_1') == 5
+        assert self.cmap.get_npoints('2_2_1') == 0
+        assert self.cmap.get_npoints('3_2_1') == 3
+
+        assert self.cmap.get_msg('1_2_1') == 'Test message'
+        assert self.cmap.get_msg('2_2_1') is None
+
+        assert self.cmap.get_hint('1_2_1') == 'Test hint'
+        assert self.cmap.get_hint('2_2_1') is None
+
+        assert self.cmap.get_hintmode('1_2_1') == 'always'
+        assert self.cmap.get_hintmode('2_2_1') is None
+
+        assert self.cmap.is_queued('1_2_1')
+        assert not self.cmap.is_queued('2_2_1')
+
+        assert self.cmap.get_queuetime_str('1_2_1') == '20130228100026'
+        assert self.cmap.get_queuetime_str('2_2_1') is None
+
+        assert self.cmap.is_right_queuekey('1_2_1', 'secretstring')
+        assert not self.cmap.is_right_queuekey('1_2_1', 'invalidstr')
+        assert not self.cmap.is_right_queuekey('1_2_1', '')
+        assert not self.cmap.is_right_queuekey('1_2_1', None)
+
+        assert not self.cmap.is_right_queuekey('2_2_1', 'secretstring')
+        assert not self.cmap.is_right_queuekey('2_2_1', 'invalidstr')
+        assert not self.cmap.is_right_queuekey('2_2_1', '')
+        assert not self.cmap.is_right_queuekey('2_2_1', None)
+
+    def test_get_npoints(self):
+        # Set the correctmap properties for 4 inputs
+        # 1) correct, 5 points
+        # 2) correct, None points
+        # 3) incorrect, 5 points
+        # 4) incorrect, None points
+        # 5) correct, 0 points
+        # 4) partially correct, 2.5 points
+        # 5) partially correct, None points
+        self.cmap.set(
+            answer_id='1_2_1',
+            correctness='correct',
+            npoints=5.3
+        )
+
+        self.cmap.set(
+            answer_id='2_2_1',
+            correctness='correct',
+            npoints=None
+        )
+
+        self.cmap.set(
+            answer_id='3_2_1',
+            correctness='incorrect',
+            npoints=5
+        )
+
+        self.cmap.set(
+            answer_id='4_2_1',
+            correctness='incorrect',
+            npoints=None
+        )
+
+        self.cmap.set(
+            answer_id='5_2_1',
+            correctness='correct',
+            npoints=0
+        )
+
+        self.cmap.set(
+            answer_id='6_2_1',
+            correctness='partially-correct',
+            npoints=2.5
+        )
+
+        self.cmap.set(
+            answer_id='7_2_1',
+            correctness='partially-correct',
+            npoints=None
+        )
+
+        # Assert that we get the expected points
+        # If points assigned --> npoints
+        # If no points assigned and correct --> 1 point
+        # If no points assigned and partially correct --> 1 point
+        # If no points assigned and incorrect --> 0 points
+        assert self.cmap.get_npoints('1_2_1') == 5.3
+        assert self.cmap.get_npoints('2_2_1') == 1
+        assert self.cmap.get_npoints('3_2_1') == 5
+        assert self.cmap.get_npoints('4_2_1') == 0
+        assert self.cmap.get_npoints('5_2_1') == 0
+        assert self.cmap.get_npoints('6_2_1') == 2.5
+        assert self.cmap.get_npoints('7_2_1') == 1
+
+    def test_set_overall_message(self):
+
+        # Default is an empty string string
+        assert self.cmap.get_overall_message() == ''
+
+        # Set a message that applies to the whole question
+        self.cmap.set_overall_message("Test message")
+
+        # Retrieve the message
+        assert self.cmap.get_overall_message() == 'Test message'
+
+        # Setting the message to None --> empty string
+        self.cmap.set_overall_message(None)
+        assert self.cmap.get_overall_message() == ''
+
+    def test_update_from_correctmap(self):
+        # Initialize a CorrectMap with some properties
+        self.cmap.set(
+            answer_id='1_2_1',
+            correctness='correct',
+            npoints=5,
+            msg='Test message',
+            hint='Test hint',
+            hintmode='always',
+            queuestate={
+                'key': 'secretstring',
+                'time': '20130228100026'
+            }
+        )
+
+        self.cmap.set_overall_message("Test message")
+
+        # Create a second cmap, then update it to have the same properties
+        # as the first cmap
+        other_cmap = CorrectMap()
+        other_cmap.update(self.cmap)
+
+        # Assert that it has all the same properties
+        assert other_cmap.get_overall_message() == self.cmap.get_overall_message()
+
+        assert other_cmap.get_dict() == self.cmap.get_dict()
+
+    def test_update_from_invalid(self):
+        # Should get an exception if we try to update() a CorrectMap
+        # with a non-CorrectMap value
+        invalid_list = [None, "string", 5, datetime.datetime.today()]
+
+        for invalid in invalid_list:
+            with pytest.raises(Exception):
+                self.cmap.update(invalid)
+
+    def test_set_none_state(self):
+        """
+        Test that if an invalid state is set to correct map, the state does not
+        update at all.
+        """
+        invalid_list = [None, "", False, 0]
+        for invalid in invalid_list:
+            self.cmap.set_dict(invalid)
+            assert not self.cmap.get_dict()

xmodule/capa/tests/test_customrender.py

@@ -0,0 +1,80 @@
+# lint-amnesty, pylint: disable=missing-module-docstring
+
+import unittest
+import xml.sax.saxutils as saxutils
+
+from lxml import etree
+
+from xmodule.capa import customrender
+from xmodule.capa.tests.helpers import test_capa_system
+
+# just a handy shortcut
+lookup_tag = customrender.registry.get_class_for_tag
+
+
+def extract_context(xml):
+    """
+    Given an xml element corresponding to the output of test_capa_system.render_template, get back the
+    original context
+    """
+    return eval(xml.text)  # lint-amnesty, pylint: disable=eval-used
+
+
+def quote_attr(s):
+    return saxutils.quoteattr(s)[1:-1]  # don't want the outer quotes
+
+
+class HelperTest(unittest.TestCase):
+    '''
+    Make sure that our helper function works!
+    '''
+    def check(self, d):
+        xml = etree.XML(test_capa_system().render_template('blah', d))
+        assert d == extract_context(xml)
+
+    def test_extract_context(self):
+        self.check({})
+        self.check({1, 2})
+        self.check({'id', 'an id'})
+        self.check({'with"quote', 'also"quote'})
+
+
+class SolutionRenderTest(unittest.TestCase):
+    '''
+    Make sure solutions render properly.
+    '''
+
+    def test_rendering(self):
+        solution = 'To compute unicorns, count them.'
+        xml_str = """<solution id="solution_12">{s}</solution>""".format(s=solution)
+        element = etree.fromstring(xml_str)
+
+        renderer = lookup_tag('solution')(test_capa_system(), element)
+
+        assert renderer.id == 'solution_12'
+
+        # Our test_capa_system "renders" templates to a div with the repr of the context.
+        xml = renderer.get_html()
+        context = extract_context(xml)
+        assert context == {'id': 'solution_12'}
+
+
+class MathRenderTest(unittest.TestCase):
+    '''
+    Make sure math renders properly.
+    '''
+
+    def check_parse(self, latex_in, mathjax_out):  # lint-amnesty, pylint: disable=missing-function-docstring
+        xml_str = """<math>{tex}</math>""".format(tex=latex_in)
+        element = etree.fromstring(xml_str)
+
+        renderer = lookup_tag('math')(test_capa_system(), element)
+
+        assert renderer.mathstr == mathjax_out
+
+    def test_parsing(self):
+        self.check_parse('$abc$', '[mathjaxinline]abc[/mathjaxinline]')
+        self.check_parse('$abc', '$abc')
+        self.check_parse(r'$\displaystyle 2+2$', '[mathjax] 2+2[/mathjax]')
+
+    # NOTE: not testing get_html yet because I don't understand why it's doing what it's doing.

xmodule/capa/tests/test_files/dynamath_input.txt

@@ -0,0 +1,43 @@
+<math xmlns="http://www.w3.org/1998/Math/MathML">                                                           
+  <mstyle displaystyle="true">                                                                              
+    <mrow>                                                                                                  
+      <mi>cos</mi>                                                                                          
+      <mrow><mo>(</mo><mi>&#x3B8;</mi><mo>)</mo></mrow>                                                     
+    </mrow>                                                                                                 
+    <mo>&#x22C5;</mo>                                                                                       
+    <mrow>                                                                                                  
+      <mo>[</mo>                                                                                            
+      <mtable>                                                                                              
+        <mtr>                                                                                               
+          <mtd><mn>1</mn></mtd><mtd><mn>0</mn></mtd>                                                        
+        </mtr>                                                                                              
+        <mtr>                                                                                               
+          <mtd><mn>0</mn></mtd><mtd><mn>1</mn></mtd>                                                        
+        </mtr>                                                                                              
+      </mtable>                                                                                             
+      <mo>]</mo>                                                                                            
+    </mrow>                                                                                                 
+    <mo>+</mo>                                                                                              
+    <mi>i</mi>                                                                                              
+    <mo>&#x22C5;</mo>                                                                                       
+    <mrow>                                                                                                  
+      <mi>sin</mi>                                                                                          
+      <mrow>                                                                                                
+        <mo>(</mo><mi>&#x3B8;</mi><mo>)</mo>                                                                
+      </mrow>                                                                                               
+    </mrow>                                                                                                 
+    <mo>&#x22C5;</mo>                                                                                       
+    <mrow>                                                                                                  
+      <mo>[</mo>                                                                                            
+      <mtable>                                                                                              
+        <mtr>                                                                                               
+          <mtd><mn>0</mn></mtd><mtd><mn>1</mn></mtd>                                                        
+        </mtr>                                                                                              
+        <mtr>                                                                                               
+          <mtd><mn>1</mn></mtd><mtd><mn>0</mn></mtd>                                                        
+        </mtr>                                                                                              
+      </mtable>                                                                                             
+      <mo>]</mo>                                                                                            
+    </mrow>                                                                                                 
+  </mstyle>                                                                                                 
+</math>  

xmodule/capa/tests/test_files/extended_hints.xml

@@ -0,0 +1,50 @@
+<problem>
+    <p>What is the correct answer?</p>
+    <multiplechoiceresponse targeted-feedback="">
+      <choicegroup type="MultipleChoice">
+        <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+        <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+        <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+        <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+      </choicegroup>
+    </multiplechoiceresponse>
+
+    <targetedfeedbackset>
+        <targetedfeedback explanation-id="feedback1">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 1st WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedback2">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 2nd WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedback3">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 3rd WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedbackC">
+        <div class="detailed-targeted-feedback-correct">
+            <p>Targeted Feedback</p>
+            <p>Feedback on your correct solution...</p>
+        </div>
+        </targetedfeedback>
+
+    </targetedfeedbackset>
+
+    <solution explanation-id="feedbackC">
+    <div class="detailed-solution">
+        <p>Explanation</p>
+        <p>This is the solution explanation</p>
+        <p>Not much to explain here, sorry!</p>
+    </div>
+    </solution>
+</problem>

xmodule/capa/tests/test_files/extended_hints_checkbox.xml

@@ -0,0 +1,119 @@
+<problem>
+    <p>In retrospect, the wordiness of these tests increases the dizziness!</p>
+    <choiceresponse>
+      <label>Select all the fruits from the list</label>
+      <checkboxgroup>
+        <choice correct="true" id="alpha">Apple
+                   <choicehint selected="TrUe">You are right that apple is a fruit.
+                   </choicehint>
+                   <choicehint selected="false">Remember that apple is also a fruit.
+                   </choicehint>
+        </choice>
+        <choice correct="false">Mushroom
+                   <choicehint selected="true">Mushroom is a fungus, not a fruit.
+                   </choicehint>
+                   <choicehint selected="false">You are right that mushrooms are not fruit
+                   </choicehint>
+        </choice>
+        <choice correct="true">Grape
+                   <choicehint selected="true">You are right that grape is a fruit
+                   </choicehint>
+                   <choicehint selected="false">Remember that grape is also a fruit.
+                   </choicehint>
+        </choice>
+        <choice correct="false">Mustang</choice>
+        <choice correct="false">Camero
+                   <choicehint selected="true">I do not know what a Camero is but it is not a fruit.
+                   </choicehint>
+                   <choicehint selected="false">What is a camero anyway?
+                   </choicehint>
+        </choice>
+        <compoundhint value="alpha B" label="Almost right"> You are right that apple is a fruit, but there is one you are missing. Also, mushroom is not a fruit.
+        </compoundhint>
+        <compoundhint value="  c     b   "> You are right that grape is a fruit, but there is one you are missing. Also, mushroom is not a fruit.
+        </compoundhint>
+      </checkboxgroup>
+    </choiceresponse>
+
+    <choiceresponse>
+      <label>Select all the vegetables from the list</label>
+      <checkboxgroup>
+        <choice correct="false">Banana
+                   <choicehint selected="true">No, sorry, a banana is a fruit.
+                   </choicehint>
+                   <choicehint selected="false">poor banana.
+                   </choicehint>
+        </choice>
+        <choice correct="false">Ice Cream</choice>
+        <choice correct="false">Mushroom
+                   <choicehint selected="true">Mushroom is a fungus, not a vegetable.
+                   </choicehint>
+                   <choicehint selected="false">You are right that mushrooms are not vegatbles
+                   </choicehint>
+        </choice>
+        <choice correct="true">
+        Brussel Sprout
+                   <choicehint selected="true">
+
+                   Brussel sprouts are vegetables.
+                   </choicehint>
+                   <choicehint selected="false">
+
+                   Brussel sprout is the only vegetable in this list.
+                   </choicehint>
+        </choice>
+        <compoundhint value="A B" label="Very funny"> Making a banana split?
+        </compoundhint>
+        <compoundhint value="B D"> That will make a horrible dessert: a brussel sprout split?
+        </compoundhint>
+      </checkboxgroup>
+    </choiceresponse>
+
+    <p>Compoundhint vs. correctness</p>
+    <choiceresponse>
+      <checkboxgroup>
+        <choice correct="true">A</choice>
+        <choice correct="false">B</choice>
+        <choice correct="true">C</choice>
+        <compoundhint value="A B">AB</compoundhint>
+        <compoundhint value="A C">AC</compoundhint>
+      </checkboxgroup>
+    </choiceresponse>
+    <p>If one label matches we use it, otherwise go with the default, and whitespace scattered around.</p>
+    <choiceresponse>
+      <checkboxgroup>
+        <choice correct="true">
+
+              A
+          <choicehint selected="true" label="AA">
+
+        aa
+
+        </choicehint></choice>
+        <choice correct="true">
+        B <choicehint selected="false" label="BB">
+           bb
+
+        </choicehint></choice>
+      </checkboxgroup>
+    </choiceresponse>
+
+    <p>Blank labels</p>
+    <choiceresponse>
+      <checkboxgroup>
+        <choice correct="true">A <choicehint selected="true" label="">aa</choicehint></choice>
+        <choice correct="true">B <choicehint selected="true">bb</choicehint></choice>
+        <compoundhint value="A B" label="">compoundo</compoundhint>
+      </checkboxgroup>
+    </choiceresponse>
+
+    <p>Case where the student selects nothing, but there's feedback</p>
+    <choiceresponse>
+      <checkboxgroup>
+        <choice correct="true">A <choicehint selected="true">aa</choicehint></choice>
+        <choice correct="false">B <choicehint selected="false">bb</choicehint></choice>
+      </checkboxgroup>
+    </choiceresponse>
+
+
+</problem>

xmodule/capa/tests/test_files/extended_hints_dropdown.xml

@@ -0,0 +1,42 @@
+<problem>
+<p>Translation between Dropdown and ________ is straightforward.
+And not confused by whitespace around the answer.</p>
+<optionresponse>
+    <optioninput>
+        <option correct="True"> Multiple Choice
+           <optionhint label="Good Job">Yes, multiple choice is the right answer.
+           </optionhint> </option>
+        <option correct="False">  Text Input
+           <optionhint>No, text input problems do not present options.
+           </optionhint> </option>
+        <option correct="False">   Numerical Input
+           <optionhint>No, numerical input problems do not present options.
+           </optionhint> </option>
+    </optioninput>
+</optionresponse>
+<p>Clowns have funny _________ to make people laugh.</p>
+<optionresponse>
+    <optioninput>
+        <option correct="False">dogs
+           <optionhint label="NOPE">Not dogs, not cats, not toads
+           </optionhint> </option>
+        <option correct="True">FACES
+           <optionhint>With lots of makeup, doncha know?
+           </optionhint> </option>
+        <option correct="False">money
+           <optionhint>Clowns do not have any money, of course
+           </optionhint> </option>
+    </optioninput>
+</optionresponse>
+<p>Regression case where feedback includes answer substring, confusing the match logic</p>
+<optionresponse>
+    <optioninput>
+        <option correct="False">AAA
+           <optionhint>AAABBB1
+           </optionhint> </option>
+        <option correct="True">BBB
+           <optionhint>AAABBB2
+           </optionhint> </option>
+    </optioninput>
+</optionresponse>
+</problem>

xmodule/capa/tests/test_files/extended_hints_multiple_choice.xml

@@ -0,0 +1,35 @@
+<problem>
+<p>(note the blank line before mushroom -- be sure to include this test case)</p>
+<multiplechoiceresponse>
+  <label>Select the fruit from the list</label>
+  <choicegroup type="MultipleChoice">
+    <choice correct="false">Mushroom
+        <choicehint label="">Mushroom is a fungus, not a fruit.
+        </choicehint>
+    </choice>
+    <choice correct="false">Potato</choice>
+    <choice correct="true">Apple
+        <choicehint label="OUTSTANDING">Apple is indeed a fruit.
+        </choicehint>
+    </choice>
+  </choicegroup>
+</multiplechoiceresponse>
+
+<multiplechoiceresponse>
+  <label>Select the vegetables from the list</label>
+  <choicegroup type="MultipleChoice">
+    <choice correct="false">Mushroom
+        <choicehint>Mushroom is a fungus, not a vegetable.
+        </choicehint>
+    </choice>
+    <choice correct="true">Potato
+        <choicehint>Potato is a root vegetable.
+        </choicehint>
+    </choice>
+    <choice correct="false">Apple
+        <choicehint label="OOPS">Apple is a fruit.
+        </choicehint>
+    </choice>
+  </choicegroup>
+</multiplechoiceresponse>
+</problem>

xmodule/capa/tests/test_files/extended_hints_multiple_choice_with_html.xml

@@ -0,0 +1,15 @@
+<problem>
+<multiplechoiceresponse>
+  <label>Select the fruit from the list</label>
+  <choicegroup type="MultipleChoice">
+    <choice correct="false">Mushroom
+        <choicehint>Mushroom <img src="#" ale="#"/>is a fungus, not a fruit.</choicehint>
+    </choice>
+    <choice correct="false">Potato
+        <choicehint>Potato is <img src="#" ale="#"/> not a fruit.</choicehint></choice>
+    <choice correct="true">Apple
+        <choicehint><a href="#">Apple</a> is a fruit.</choicehint>
+    </choice>
+  </choicegroup>
+</multiplechoiceresponse>
+</problem>

xmodule/capa/tests/test_files/extended_hints_numeric_input.xml

@@ -0,0 +1,41 @@
+<problem>
+    <numericalresponse answer="1.141">
+        <label>What value when squared is approximately equal to 2 (give your answer to 2 decimal places)?</label>
+        <responseparam default=".01" type="tolerance"/>
+        <formulaequationinput/>
+        <additional_answer answer="10">
+            <correcthint>This is an additional hint.</correcthint>
+        </additional_answer>
+
+        <correcthint label="Nice">
+            The square root of two turns up in the strangest places.
+        </correcthint>
+
+
+    </numericalresponse>
+
+    <numericalresponse answer="4">
+        <label>What is 2 + 2?</label>
+        <responseparam default=".01" type="tolerance"/>
+        <formulaequationinput/>
+        <correcthint>
+            Pretty easy, uh?.
+        </correcthint>
+    </numericalresponse>
+<!-- I don't think we're supporting these yet
+
+also not multiple correcthint
+
+        <numerichint answer="-1.141" tolerance="0.01">
+            Yes, squaring a negative number yields a positive
+        </numerichint>
+
+        <numerichint answer="7">
+            7 x 7 = 49 which is much too high.
+        </numerichint>
+
+        <lehint answer="1">
+            Much too low. You may be rounding down.
+        </lehint>
+-->
+</problem>

xmodule/capa/tests/test_files/extended_hints_text_input.xml

@@ -0,0 +1,81 @@
+<problem>
+    <p>In which country would you find the city of Paris?</p>
+
+    <stringresponse answer="FranceΩ" type="ci" >
+        <label>In which country would you find the city of Paris?</label>
+        <textline size="20"/>
+        <correcthint>
+            Viva la France!Ω
+        </correcthint>
+
+        <additional_answer answer="USAΩ">
+             <correcthint>Less well known, but yes, there is a Paris, Texas.Ω</correcthint>
+        </additional_answer>
+
+        <stringequalhint answer="GermanyΩ">
+            I do not think so.Ω
+        </stringequalhint>
+
+        <regexphint answer=".*landΩ">
+             The country name does not end in LANDΩ
+        </regexphint>
+    </stringresponse>
+
+    <p>What color is the sky? A minimal example, case sensitive, not regex.</p>
+    <stringresponse answer="Blue">
+        <label>What color is the sky?</label>
+        <correcthint >The red light is scattered by water molecules leaving only blue light.
+        </correcthint>
+        <textline size="20"/>
+    </stringresponse>
+
+    <p>(This question will cause an illegal regular expression exception)</p>
+    <stringresponse answer="Bonk">
+        <label>Why not?</label>
+        <correcthint >This hint should never appear.
+        </correcthint>
+        <textline size="20"/>
+        <regexphint answer="[">
+             This hint should never appear either because the regex is illegal.
+        </regexphint>
+    </stringresponse>
+
+    <!-- string response with extended hints + case_insensitive + blank labels -->
+    <p>Meh</p>
+    <stringresponse answer="A" type="ci">
+        <correcthint label="Woo Hoo">hint1</correcthint>
+        <additional_answer answer="B"> <correcthint label=""> hint2</correcthint> </additional_answer>
+        <stringequalhint answer="C" label=""> hint4</stringequalhint>
+        <regexphint answer="FG+" label=""> hint6 </regexphint>
+        <regexphint answer="(abc"> erroneous regex don't match anything </regexphint>
+        <textline size="20"/>
+    </stringresponse>
+
+    <!-- string response with extended hints + case_insensitive = False -->
+    <stringresponse answer="A">
+        <correcthint>hint1</correcthint>
+        <additional_answer answer="B">  <correcthint> hint2 </correcthint>   </additional_answer>
+        <stringequalhint answer="C"> hint4 </stringequalhint>
+        <regexphint answer="FG+"> hint6 </regexphint>
+        <textline size="20"/>
+    </stringresponse>
+
+    <!-- backward compatibility for additional_answer: old and new format together in
+         a problem, scored correclty and new style has a hint -->
+    <stringresponse answer="A">
+        <correcthint>hint1</correcthint>
+        <additional_answer>B</additional_answer>
+        <additional_answer answer="C"><correcthint> hint2 </correcthint> </additional_answer>
+        <additional_answer>&lt;&amp;"'&gt;</additional_answer>
+        <textline size="20"/>
+    </stringresponse>
+
+    <!-- type regexp with extended hints -->
+    <stringresponse answer="AB+C" type="ci regexp">
+        <correcthint>hint1</correcthint>
+        <additional_answer answer="B+"><correcthint> hint2 </correcthint> </additional_answer>
+        <stringequalhint answer="C"> hint4 </stringequalhint>
+        <regexphint answer="D"> hint6 </regexphint>
+        <textline size="20"/>
+    </stringresponse>
+</problem>

xmodule/capa/tests/test_files/extended_hints_with_errors.xml

@@ -0,0 +1,14 @@
+<problem>
+    <choiceresponse>
+      <label>Select all the vegetables from the list</label>
+      <checkboxgroup>
+        <choice correct="false">Banana
+                   <choicehint selected="true">No, sorry, a banana is a fruit.
+                   </choicehint>
+                   <choicehint selected="false">poor banana.
+                   </choicehint>
+        </choice>
+        <badElement>  this element is not a legal sibling of 'choice' and 'booleanhint' </badElement>
+      </checkboxgroup>
+    </choiceresponse>
+</problem>

xmodule/capa/tests/test_files/filename_convert_test.txt

@@ -0,0 +1 @@
+This file is used to test converting file handles to filenames in the capa utility module

xmodule/capa/tests/test_files/js/mersenne-twister-min.js

@@ -0,0 +1,203 @@
+/*
+  I've wrapped Makoto Matsumoto and Takuji Nishimura's code in a namespace
+  so it's better encapsulated. Now you can have multiple random number generators
+  and they won't stomp all over eachother's state.
+  
+  If you want to use this as a substitute for Math.random(), use the random()
+  method like so:
+  
+  var m = new MersenneTwister();
+  var randomNumber = m.random();
+  
+  You can also call the other genrand_{foo}() methods on the instance.
+
+  If you want to use a specific seed in order to get a repeatable random
+  sequence, pass an integer into the constructor:
+
+  var m = new MersenneTwister(123);
+
+  and that will always produce the same random sequence.
+
+  Sean McCullough (banksean@gmail.com)
+*/
+
+/* 
+   A C-program for MT19937, with initialization improved 2002/1/26.
+   Coded by Takuji Nishimura and Makoto Matsumoto.
+ 
+   Before using, initialize the state by using init_genrand(seed)  
+   or init_by_array(init_key, key_length).
+ 
+   Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
+   All rights reserved.                          
+ 
+   Redistribution and use in source and binary forms, with or without
+   modification, are permitted provided that the following conditions
+   are met:
+ 
+     1. Redistributions of source code must retain the above copyright
+        notice, this list of conditions and the following disclaimer.
+ 
+     2. Redistributions in binary form must reproduce the above copyright
+        notice, this list of conditions and the following disclaimer in the
+        documentation and/or other materials provided with the distribution.
+ 
+     3. The names of its contributors may not be used to endorse or promote 
+        products derived from this software without specific prior written 
+        permission.
+ 
+   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+   A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+   PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+   LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+   NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+   SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ 
+ 
+   Any feedback is very welcome.
+   http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html
+   email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space)
+*/
+
+/* eslint-disable */
+var MersenneTwister = function(seed) {
+  if (seed == undefined) {
+    seed = new Date().getTime();
+  } 
+  /* Period parameters */  
+  this.N = 624;
+  this.M = 397;
+  this.MATRIX_A = 0x9908b0df;   /* constant vector a */
+  this.UPPER_MASK = 0x80000000; /* most significant w-r bits */
+  this.LOWER_MASK = 0x7fffffff; /* least significant r bits */
+ 
+  this.mt = new Array(this.N); /* the array for the state vector */
+  this.mti=this.N+1; /* mti==N+1 means mt[N] is not initialized */
+
+  this.init_genrand(seed);
+}  
+ 
+/* initializes mt[N] with a seed */
+MersenneTwister.prototype.init_genrand = function(s) {
+  this.mt[0] = s >>> 0;
+  for (this.mti=1; this.mti<this.N; this.mti++) {
+      var s = this.mt[this.mti-1] ^ (this.mt[this.mti-1] >>> 30);
+   this.mt[this.mti] = (((((s & 0xffff0000) >>> 16) * 1812433253) << 16) + (s & 0x0000ffff) * 1812433253)
+  + this.mti;
+      /* See Knuth TAOCP Vol2. 3rd Ed. P.106 for multiplier. */
+      /* In the previous versions, MSBs of the seed affect   */
+      /* only MSBs of the array mt[].                        */
+      /* 2002/01/09 modified by Makoto Matsumoto             */
+      this.mt[this.mti] >>>= 0;
+      /* for >32 bit machines */
+  }
+}
+ 
+/* initialize by an array with array-length */
+/* init_key is the array for initializing keys */
+/* key_length is its length */
+/* slight change for C++, 2004/2/26 */
+MersenneTwister.prototype.init_by_array = function(init_key, key_length) {
+  var i, j, k;
+  this.init_genrand(19650218);
+  i=1; j=0;
+  k = (this.N>key_length ? this.N : key_length);
+  for (; k; k--) {
+    var s = this.mt[i-1] ^ (this.mt[i-1] >>> 30)
+    this.mt[i] = (this.mt[i] ^ (((((s & 0xffff0000) >>> 16) * 1664525) << 16) + ((s & 0x0000ffff) * 1664525)))
+      + init_key[j] + j; /* non linear */
+    this.mt[i] >>>= 0; /* for WORDSIZE > 32 machines */
+    i++; j++;
+    if (i>=this.N) { this.mt[0] = this.mt[this.N-1]; i=1; }
+    if (j>=key_length) j=0;
+  }
+  for (k=this.N-1; k; k--) {
+    var s = this.mt[i-1] ^ (this.mt[i-1] >>> 30);
+    this.mt[i] = (this.mt[i] ^ (((((s & 0xffff0000) >>> 16) * 1566083941) << 16) + (s & 0x0000ffff) * 1566083941))
+      - i; /* non linear */
+    this.mt[i] >>>= 0; /* for WORDSIZE > 32 machines */
+    i++;
+    if (i>=this.N) { this.mt[0] = this.mt[this.N-1]; i=1; }
+  }
+
+  this.mt[0] = 0x80000000; /* MSB is 1; assuring non-zero initial array */ 
+}
+ 
+/* generates a random number on [0,0xffffffff]-interval */
+MersenneTwister.prototype.genrand_int32 = function() {
+  var y;
+  var mag01 = new Array(0x0, this.MATRIX_A);
+  /* mag01[x] = x * MATRIX_A  for x=0,1 */
+
+  if (this.mti >= this.N) { /* generate N words at one time */
+    var kk;
+
+    if (this.mti == this.N+1)   /* if init_genrand() has not been called, */
+      this.init_genrand(5489); /* a default initial seed is used */
+
+    for (kk=0;kk<this.N-this.M;kk++) {
+      y = (this.mt[kk]&this.UPPER_MASK)|(this.mt[kk+1]&this.LOWER_MASK);
+      this.mt[kk] = this.mt[kk+this.M] ^ (y >>> 1) ^ mag01[y & 0x1];
+    }
+    for (;kk<this.N-1;kk++) {
+      y = (this.mt[kk]&this.UPPER_MASK)|(this.mt[kk+1]&this.LOWER_MASK);
+      this.mt[kk] = this.mt[kk+(this.M-this.N)] ^ (y >>> 1) ^ mag01[y & 0x1];
+    }
+    y = (this.mt[this.N-1]&this.UPPER_MASK)|(this.mt[0]&this.LOWER_MASK);
+    this.mt[this.N-1] = this.mt[this.M-1] ^ (y >>> 1) ^ mag01[y & 0x1];
+
+    this.mti = 0;
+  }
+
+  y = this.mt[this.mti++];
+
+  /* Tempering */
+  y ^= (y >>> 11);
+  y ^= (y << 7) & 0x9d2c5680;
+  y ^= (y << 15) & 0xefc60000;
+  y ^= (y >>> 18);
+
+  return y >>> 0;
+}
+ 
+/* generates a random number on [0,0x7fffffff]-interval */
+MersenneTwister.prototype.genrand_int31 = function() {
+  return (this.genrand_int32()>>>1);
+}
+ 
+/* generates a random number on [0,1]-real-interval */
+MersenneTwister.prototype.genrand_real1 = function() {
+  return this.genrand_int32()*(1.0/4294967295.0); 
+  /* divided by 2^32-1 */ 
+}
+
+/* generates a random number on [0,1)-real-interval */
+MersenneTwister.prototype.random = function() {
+  return this.genrand_int32()*(1.0/4294967296.0); 
+  /* divided by 2^32 */
+}
+ 
+/* generates a random number on (0,1)-real-interval */
+MersenneTwister.prototype.genrand_real3 = function() {
+  return (this.genrand_int32() + 0.5)*(1.0/4294967296.0); 
+  /* divided by 2^32 */
+}
+ 
+/* generates a random number on [0,1) with 53-bit resolution*/
+MersenneTwister.prototype.genrand_res53 = function() { 
+  var a=this.genrand_int32()>>>5, b=this.genrand_int32()>>>6; 
+  return(a*67108864.0+b)*(1.0/9007199254740992.0); 
+} 
+
+/* These real versions are due to Isaku Wada, 2002/01/09 added */
+if(typeof exports == 'undefined'){
+  var root = this;
+} else {
+  var root = exports;
+}
+root.MersenneTwister = MersenneTwister;

xmodule/capa/tests/test_files/js/test_problem_display.js

@@ -0,0 +1,55 @@
+/*
+ * decaffeinate suggestions:
+ * DS001: Remove Babel/TypeScript constructor workaround
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS205: Consider reworking code to avoid use of IIFEs
+ * DS207: Consider shorter variations of null checks
+ * DS208: Avoid top-level this
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+class MinimaxProblemDisplay extends XProblemDisplay {
+
+  constructor(state, submission, evaluation, container, submissionField, parameters) {
+
+    {
+      // Hack: trick Babel/TypeScript into allowing this before super.
+      if (false) { super(); }
+      let thisFn = (() => { this; }).toString();
+      let thisName = thisFn.slice(thisFn.indexOf('{') + 1, thisFn.indexOf(';')).trim();
+      eval(`${thisName} = this;`);
+    }
+    this.state = state;
+    this.submission = submission;
+    this.evaluation = evaluation;
+    this.container = container;
+    this.submissionField = submissionField;
+    if (parameters == null) { parameters = {}; }
+    this.parameters = parameters;
+    super(this.state, this.submission, this.evaluation, this.container, this.submissionField, this.parameters);
+  }
+
+  render() {}
+
+  createSubmission() {
+
+    this.newSubmission = {};
+
+    if (this.submission != null) {
+      return (() => {
+        const result = [];
+        for (let id in this.submission) {
+          const value = this.submission[id];
+          result.push(this.newSubmission[id] = value);
+        }
+        return result;
+      })();
+    }
+  }
+
+  getCurrentSubmission() {
+    return this.newSubmission;
+  }
+}
+
+const root = typeof exports !== 'undefined' && exports !== null ? exports : this;
+root.TestProblemDisplay = TestProblemDisplay;

xmodule/capa/tests/test_files/js/test_problem_generator.js

@@ -0,0 +1,33 @@
+/*
+ * decaffeinate suggestions:
+ * DS001: Remove Babel/TypeScript constructor workaround
+ * DS207: Consider shorter variations of null checks
+ * DS208: Avoid top-level this
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+class TestProblemGenerator extends XProblemGenerator {
+
+  constructor(seed, parameters) {
+
+    {
+      // Hack: trick Babel/TypeScript into allowing this before super.
+      if (false) { super(); }
+      let thisFn = (() => { this; }).toString();
+      let thisName = thisFn.slice(thisFn.indexOf('{') + 1, thisFn.indexOf(';')).trim();
+      eval(`${thisName} = this;`);
+    }
+    if (parameters == null) { parameters = {}; }
+    this.parameters = parameters;
+    super(seed, this.parameters);
+  }
+
+  generate() {
+
+    this.problemState.value = this.parameters.value;
+
+    return this.problemState;
+  }
+}
+
+const root = typeof exports !== 'undefined' && exports !== null ? exports : this;
+root.generatorClass = TestProblemGenerator;

xmodule/capa/tests/test_files/js/test_problem_grader.js

@@ -0,0 +1,54 @@
+/*
+ * decaffeinate suggestions:
+ * DS001: Remove Babel/TypeScript constructor workaround
+ * DS102: Remove unnecessary code created because of implicit returns
+ * DS207: Consider shorter variations of null checks
+ * DS208: Avoid top-level this
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+class TestProblemGrader extends XProblemGrader {
+
+  constructor(submission, problemState, parameters) {
+
+    {
+      // Hack: trick Babel/TypeScript into allowing this before super.
+      if (false) { super(); }
+      let thisFn = (() => { this; }).toString();
+      let thisName = thisFn.slice(thisFn.indexOf('{') + 1, thisFn.indexOf(';')).trim();
+      eval(`${thisName} = this;`);
+    }
+    this.submission = submission;
+    this.problemState = problemState;
+    if (parameters == null) { parameters = {}; }
+    this.parameters = parameters;
+    super(this.submission, this.problemState, this.parameters);
+  }
+
+  solve() {
+
+    return this.solution = {0: this.problemState.value};
+  }
+
+  grade() {
+
+    if ((this.solution == null)) {
+      this.solve();
+    }
+
+    let allCorrect = true;
+
+    for (let id in this.solution) {
+      const value = this.solution[id];
+      const valueCorrect = (this.submission != null) ? (value === this.submission[id]) : false;
+      this.evaluation[id] = valueCorrect;
+      if (!valueCorrect) {
+        allCorrect = false;
+      }
+    }
+
+    return allCorrect;
+  }
+}
+
+const root = typeof exports !== 'undefined' && exports !== null ? exports : this;
+root.graderClass = TestProblemGrader;

xmodule/capa/tests/test_files/js/xproblem.js

@@ -0,0 +1,78 @@
+/*
+ * decaffeinate suggestions:
+ * DS207: Consider shorter variations of null checks
+ * DS208: Avoid top-level this
+ * Full docs: https://github.com/decaffeinate/decaffeinate/blob/master/docs/suggestions.md
+ */
+class XProblemGenerator {
+
+  constructor(seed, parameters) {
+
+    if (parameters == null) { parameters = {}; }
+    this.parameters = parameters;
+    this.random = new MersenneTwister(seed);
+
+    this.problemState = {};
+  }
+
+  generate() {
+
+    console.error("Abstract method called: XProblemGenerator.generate");
+  }
+}
+
+class XProblemDisplay {
+
+  constructor(state, submission, evaluation, container, submissionField, parameters) {
+    this.state = state;
+    this.submission = submission;
+    this.evaluation = evaluation;
+    this.container = container;
+    this.submissionField = submissionField;
+    if (parameters == null) { parameters = {}; }
+    this.parameters = parameters;
+  }
+
+  render() {
+
+    console.error("Abstract method called: XProblemDisplay.render");
+  }
+
+  updateSubmission() {
+
+    this.submissionField.val(JSON.stringify(this.getCurrentSubmission()));
+  }
+
+  getCurrentSubmission() {
+    console.error("Abstract method called: XProblemDisplay.getCurrentSubmission");
+  }
+}
+
+class XProblemGrader {
+
+  constructor(submission, problemState, parameters) {
+
+    this.submission = submission;
+    this.problemState = problemState;
+    if (parameters == null) { parameters = {}; }
+    this.parameters = parameters;
+    this.solution   = null;
+    this.evaluation = {};
+  }
+
+  solve() {
+
+    console.error("Abstract method called: XProblemGrader.solve");
+  }
+
+  grade() {
+
+    console.error("Abstract method called: XProblemGrader.grade");
+  }
+}
+
+const root = typeof exports !== 'undefined' && exports !== null ? exports : this;
+
+root.XProblemGenerator = XProblemGenerator;
+root.XProblemDisplay   = XProblemDisplay;
+root.XProblemGrader    = XProblemGrader;

xmodule/capa/tests/test_files/snuggletex_2x+3y.xml

@@ -0,0 +1,235 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html
+  PUBLIC "-//W3C//DTD XHTML 1.1 plus MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/xhtml-math11-f.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+   <head>
+      <meta content="application/xhtml+xml; charset=UTF-8" http-equiv="Content-Type" />
+      <meta content="SnuggleTeX" name="Generator" />
+      <meta content="SnuggleTeX Documentation" name="description" />
+      <meta content="David McKain" name="author" />
+      <meta content="The University of Edinburgh" name="publisher" />
+      <link href="/snuggletex-webapp-1.2.2/includes/core.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/webapp.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/snuggletex.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.css"
+            rel="stylesheet" /><script src="/snuggletex-webapp-1.2.2/includes/jquery.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.js"
+              type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/webapp.js" type="text/javascript"></script><title>SnuggleTeX - ASCIIMathML Enrichment Demo</title><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathML.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathMLwidget.js"
+              type="text/javascript"></script></head>
+   <body id="asciiMathMLUpConversionDemo">
+      <table border="0" cellpadding="0" cellspacing="0" id="header" width="100%">
+         <tr>
+            <td align="left" id="logo" valign="top"><a class="headertext" href="http://www.ed.ac.uk"><img alt="The University of Edinburgh" height="84"
+                       src="/snuggletex-webapp-1.2.2/includes/uoe_logo.jpg"
+                       width="84" /></a></td>
+            <td align="left">
+               <h3>THE UNIVERSITY of EDINBURGH</h3>
+               <h1>SCHOOL OF PHYSICS AND ASTRONOMY</h1>
+            </td>
+         </tr>
+      </table>
+      <h1 id="location"><a href="/snuggletex-webapp-1.2.2">SnuggleTeX (1.2.2)</a></h1>
+      <div id="content">
+         <div id="skipnavigation"><a href="#maincontent">Skip Navigation</a></div>
+         <div id="navigation">
+            <div id="navinner">
+               <h2>About SnuggleTeX</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/overview-and-features.html">Overview &amp; Features</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/use-cases.html">Why Use SnuggleTeX?</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/license.html">License</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a></li>
+               </ul>
+               <h2>Demos &amp; Samples</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/MathInputDemo">Simple Math Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/FullLaTeXInputDemo">Full LaTeX Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichment Demo</a></li>
+                  <li><a class="selected" href="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo">ASCIIMathML Enrichment Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output-samples.html">Web Output Samples</a></li>
+               </ul>
+               <h2>User Guide</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/getting-snuggletex.html">Getting SnuggleTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/software-requirements.html">Software Requirements</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/your-classpath.html">Setting up Your ClassPath</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/examples.html">Examples</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/basic-usage.html">Basic Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/inputs.html">Parsing LaTeX Inputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/xml-or-dom-output.html">Creating XML String or DOM Outputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output.html">Creating Web Pages</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/error-reporting.html">Error Reporting</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/supported-latex.html">Supported LaTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/advanced-usage.html">Advanced Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">Semantic Enrichment</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/migrating-from-older-versions.html">Migrating from older versions</a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/apidocs/index.html">API Documentation<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/xref/index.html">Source Code Cross-Reference<span class="extlink"> </span></a></li>
+               </ul>
+               <h2>SnuggleTeX Project Links</h2>
+               <ul>
+                  <li><a href="http://sourceforge.net/project/showfiles.php?group_id=221375">Download from SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://sourceforge.net/projects/snuggletex/">SnuggleTeX on SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/">SnuggleTeX Maven Developer Reports<span class="extlink"> </span></a></li>
+                  <li><a href="https://www.wiki.ed.ac.uk/display/Physics/SnuggleTeX">SnuggleTeX Wiki<span class="extlink"> </span></a></li>
+               </ul>
+            </div>
+         </div>
+         <div id="maincontent">
+            <div id="popup"></div>
+            <div id="maininner">
+               <h2>ASCIIMathML Enrichment Demo</h2>
+               <h3>Input</h3>
+               <p>
+                  This demo is similar to the
+                  <a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichnment Demo</a>
+                  but uses
+                  <a href="http://www1.chapman.edu/~jipsen/asciimath.html">ASCIIMathML</a> as
+                  an alternative input format, which provides real-time feedback as you
+                  type but can often generate MathML with odd semantics in it.
+                  SnuggleTeX includes some functionality that can to convert this raw MathML into
+                  something equivalent to its own MathML output, thereby allowing you to
+                  <a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">semantically enrich</a> it in
+                  certain simple cases, making ASCIIMathML a possibly viable input format
+                  for simple semantic maths.
+                  
+               </p>
+               <p>
+                  To try the demo, simply enter some some ASCIIMathML into the box below.
+                  You should see a real time preview of this while you type.
+                  Then hit <tt>Go!</tt> to use SnuggleTeX to semantically enrich your
+                  input.
+                  
+               </p>
+               <form action="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo" class="input"
+                     method="post">
+                  <div class="inputBox">
+                     ASCIIMath Input:
+                     <input id="asciiMathInput" name="asciiMathInput" type="text" value="" /><input id="asciiMathML" name="asciiMathML" type="hidden" /><input type="submit" value="Go!" /></div>
+               </form>
+               <h3>Live Preview</h3>
+               <p>
+                  This is a MathML rendering of your input, generated by ASCIIMathML as you type.
+                  
+               </p>
+               <div class="result">
+                  <div id="preview"> </div>
+               </div>
+               <p>
+                  This is the underlying MathML source generated by ASCIIMathML, again updated in real time.
+                  
+               </p>
+               <div class="result"><pre id="previewSource"> </pre></div><script type="text/javascript">
+          registerASCIIMathMLInputWidget('asciiMathInput', 'preview', 'asciiMathML', 'previewSource');
+          var inputChanged = false;
+          // Hide any existing output stuff in page on first change, as it will no longer be in sync
+          jQuery(document).ready(function() {
+            jQuery('#asciiMathInput').bind('keydown', function() {
+              if (!inputChanged) jQuery('.outputContainer').css('visibility', 'hidden');
+              inputChanged = true;
+            });
+          });
+        </script><div class="outputContainer">
+                  <h3>Enhanced Presentation MathML</h3>
+                  <p>
+                     This shows the result of attempting to enrich the raw Presentation MathML
+                     generated by ASCIIMathML:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;mrow&gt;
+      &lt;mrow&gt;
+         &lt;mn&gt;2&lt;/mn&gt;
+         &lt;mo&gt;*&lt;/mo&gt;
+         &lt;mi&gt;x&lt;/mi&gt;
+      &lt;/mrow&gt;
+      &lt;mo&gt;+&lt;/mo&gt;
+      &lt;mrow&gt;
+         &lt;mn&gt;3&lt;/mn&gt;
+         &lt;mo&gt;*&lt;/mo&gt;
+         &lt;mi&gt;y&lt;/mi&gt;
+      &lt;/mrow&gt;
+   &lt;/mrow&gt;
+&lt;/math&gt;</pre><h3>Content MathML</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/content-mathml.html">conversion to Content MathML</a>:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;apply&gt;
+      &lt;plus/&gt;
+      &lt;apply&gt;
+         &lt;times/&gt;
+         &lt;cn&gt;2&lt;/cn&gt;
+         &lt;ci&gt;x&lt;/ci&gt;
+      &lt;/apply&gt;
+      &lt;apply&gt;
+         &lt;times/&gt;
+         &lt;cn&gt;3&lt;/cn&gt;
+         &lt;ci&gt;y&lt;/ci&gt;
+      &lt;/apply&gt;
+   &lt;/apply&gt;
+&lt;/math&gt;</pre><h3>Maxima Input Form</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/maxima-input.html">conversion to Maxima Input syntax</a>:
+                     
+                  </p><pre class="result">(2 * x) + (3 * y)</pre><h3>MathML Parallel Markup</h3>
+                  <p>
+                     This shows the enhanced Presentation MathML with other forms encapsulated
+                     as annotations:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;semantics&gt;
+      &lt;mrow&gt;
+         &lt;mrow&gt;
+            &lt;mn&gt;2&lt;/mn&gt;
+            &lt;mo&gt;*&lt;/mo&gt;
+            &lt;mi&gt;x&lt;/mi&gt;
+         &lt;/mrow&gt;
+         &lt;mo&gt;+&lt;/mo&gt;
+         &lt;mrow&gt;
+            &lt;mn&gt;3&lt;/mn&gt;
+            &lt;mo&gt;*&lt;/mo&gt;
+            &lt;mi&gt;y&lt;/mi&gt;
+         &lt;/mrow&gt;
+      &lt;/mrow&gt;
+      &lt;annotation-xml encoding="MathML-Content"&gt;
+         &lt;apply&gt;
+            &lt;plus/&gt;
+            &lt;apply&gt;
+               &lt;times/&gt;
+               &lt;cn&gt;2&lt;/cn&gt;
+               &lt;ci&gt;x&lt;/ci&gt;
+            &lt;/apply&gt;
+            &lt;apply&gt;
+               &lt;times/&gt;
+               &lt;cn&gt;3&lt;/cn&gt;
+               &lt;ci&gt;y&lt;/ci&gt;
+            &lt;/apply&gt;
+         &lt;/apply&gt;
+      &lt;/annotation-xml&gt;
+      &lt;annotation encoding="ASCIIMathInput"/&gt;
+      &lt;annotation encoding="Maxima"&gt;(2 * x) + (3 * y)&lt;/annotation&gt;
+   &lt;/semantics&gt;
+&lt;/math&gt;</pre></div>
+            </div>
+         </div>
+      </div>
+      <div id="copyright">
+         <p>
+            SnuggleTeX Release 1.2.2 —
+            <a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a><br />
+            Copyright © 2009
+            <a href="http://www.ph.ed.ac.uk">The School of Physics and Astronomy</a>,
+            <a href="http://www.ed.ac.uk">The University of Edinburgh</a>.
+            <br />
+            For more information, contact
+            <a href="http://www.ph.ed.ac.uk/elearning/contacts/#dmckain">David McKain</a>.
+            
+         </p>
+         <p>
+            The University of Edinburgh is a charitable body, registered in Scotland,
+            with registration number SC005336.
+            
+         </p>
+      </div>
+   </body>
+</html>

xmodule/capa/tests/test_files/snuggletex_correct.html

@@ -0,0 +1,480 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html
+  PUBLIC "-//W3C//DTD XHTML 1.1 plus MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/xhtml-math11-f.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+   <head>
+      <meta content="application/xhtml+xml; charset=UTF-8" http-equiv="Content-Type" />
+      <meta content="SnuggleTeX" name="Generator" />
+      <meta content="SnuggleTeX Documentation" name="description" />
+      <meta content="David McKain" name="author" />
+      <meta content="The University of Edinburgh" name="publisher" />
+      <link href="/snuggletex-webapp-1.2.2/includes/core.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/webapp.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/snuggletex.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.css"
+            rel="stylesheet" /><script src="/snuggletex-webapp-1.2.2/includes/jquery.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.js"
+              type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/webapp.js" type="text/javascript"></script><title>SnuggleTeX - ASCIIMathML Enrichment Demo</title><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathML.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathMLwidget.js"
+              type="text/javascript"></script></head>
+   <body id="asciiMathMLUpConversionDemo">
+      <table border="0" cellpadding="0" cellspacing="0" id="header" width="100%">
+         <tr>
+            <td align="left" id="logo" valign="top"><a class="headertext" href="http://www.ed.ac.uk"><img alt="The University of Edinburgh" height="84"
+                       src="/snuggletex-webapp-1.2.2/includes/uoe_logo.jpg"
+                       width="84" /></a></td>
+            <td align="left">
+               <h3>THE UNIVERSITY of EDINBURGH</h3>
+               <h1>SCHOOL OF PHYSICS AND ASTRONOMY</h1>
+            </td>
+         </tr>
+      </table>
+      <h1 id="location"><a href="/snuggletex-webapp-1.2.2">SnuggleTeX (1.2.2)</a></h1>
+      <div id="content">
+         <div id="skipnavigation"><a href="#maincontent">Skip Navigation</a></div>
+         <div id="navigation">
+            <div id="navinner">
+               <h2>About SnuggleTeX</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/overview-and-features.html">Overview &amp; Features</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/use-cases.html">Why Use SnuggleTeX?</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/license.html">License</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a></li>
+               </ul>
+               <h2>Demos &amp; Samples</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/MathInputDemo">Simple Math Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/FullLaTeXInputDemo">Full LaTeX Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichment Demo</a></li>
+                  <li><a class="selected" href="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo">ASCIIMathML Enrichment Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output-samples.html">Web Output Samples</a></li>
+               </ul>
+               <h2>User Guide</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/getting-snuggletex.html">Getting SnuggleTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/software-requirements.html">Software Requirements</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/your-classpath.html">Setting up Your ClassPath</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/examples.html">Examples</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/basic-usage.html">Basic Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/inputs.html">Parsing LaTeX Inputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/xml-or-dom-output.html">Creating XML String or DOM Outputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output.html">Creating Web Pages</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/error-reporting.html">Error Reporting</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/supported-latex.html">Supported LaTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/advanced-usage.html">Advanced Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">Semantic Enrichment</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/migrating-from-older-versions.html">Migrating from older versions</a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/apidocs/index.html">API Documentation<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/xref/index.html">Source Code Cross-Reference<span class="extlink"> </span></a></li>
+               </ul>
+               <h2>SnuggleTeX Project Links</h2>
+               <ul>
+                  <li><a href="http://sourceforge.net/project/showfiles.php?group_id=221375">Download from SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://sourceforge.net/projects/snuggletex/">SnuggleTeX on SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/">SnuggleTeX Maven Developer Reports<span class="extlink"> </span></a></li>
+                  <li><a href="https://www.wiki.ed.ac.uk/display/Physics/SnuggleTeX">SnuggleTeX Wiki<span class="extlink"> </span></a></li>
+               </ul>
+            </div>
+         </div>
+         <div id="maincontent">
+            <div id="popup"></div>
+            <div id="maininner">
+               <h2>ASCIIMathML Enrichment Demo</h2>
+               <h3>Input</h3>
+               <p>
+                  This demo is similar to the
+                  <a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichnment Demo</a>
+                  but uses
+                  <a href="http://www1.chapman.edu/~jipsen/asciimath.html">ASCIIMathML</a> as
+                  an alternative input format, which provides real-time feedback as you
+                  type but can often generate MathML with odd semantics in it.
+                  SnuggleTeX includes some functionality that can to convert this raw MathML into
+                  something equivalent to its own MathML output, thereby allowing you to
+                  <a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">semantically enrich</a> it in
+                  certain simple cases, making ASCIIMathML a possibly viable input format
+                  for simple semantic maths.
+                  
+               </p>
+               <p>
+                  To try the demo, simply enter some some ASCIIMathML into the box below.
+                  You should see a real time preview of this while you type.
+                  Then hit <tt>Go!</tt> to use SnuggleTeX to semantically enrich your
+                  input.
+                  
+               </p>
+               <form action="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo" class="input"
+                     method="post">
+                  <div class="inputBox">
+                     ASCIIMath Input:
+                     <input id="asciiMathInput" name="asciiMathInput" type="text" value="" /><input id="asciiMathML" name="asciiMathML" type="hidden" /><input type="submit" value="Go!" /></div>
+               </form>
+               <h3>Live Preview</h3>
+               <p>
+                  This is a MathML rendering of your input, generated by ASCIIMathML as you type.
+                  
+               </p>
+               <div class="result">
+                  <div id="preview"> </div>
+               </div>
+               <p>
+                  This is the underlying MathML source generated by ASCIIMathML, again updated in real time.
+                  
+               </p>
+               <div class="result"><pre id="previewSource"> </pre></div><script type="text/javascript">
+          registerASCIIMathMLInputWidget('asciiMathInput', 'preview', 'asciiMathML', 'previewSource');
+          var inputChanged = false;
+          // Hide any existing output stuff in page on first change, as it will no longer be in sync
+          jQuery(document).ready(function() {
+            jQuery('#asciiMathInput').bind('keydown', function() {
+              if (!inputChanged) jQuery('.outputContainer').css('visibility', 'hidden');
+              inputChanged = true;
+            });
+          });
+        </script><div class="outputContainer">
+                  <h3>Enhanced Presentation MathML</h3>
+                  <p>
+                     This shows the result of attempting to enrich the raw Presentation MathML
+                     generated by ASCIIMathML:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;mrow&gt;
+      &lt;mrow&gt;
+         &lt;mrow&gt;
+            &lt;mi&gt;cos&lt;/mi&gt;
+            &lt;mo&gt;&amp;ApplyFunction;&lt;/mo&gt;
+            &lt;mfenced close=")" open="("&gt;
+               &lt;mi&gt;theta&lt;/mi&gt;
+            &lt;/mfenced&gt;
+         &lt;/mrow&gt;
+         &lt;mo&gt;&amp;sdot;&lt;/mo&gt;
+         &lt;mfenced close="]" open="["&gt;
+            &lt;mtable&gt;
+               &lt;mtr&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;1&lt;/mn&gt;
+                  &lt;/mtd&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;0&lt;/mn&gt;
+                  &lt;/mtd&gt;
+               &lt;/mtr&gt;
+               &lt;mtr&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;0&lt;/mn&gt;
+                  &lt;/mtd&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;1&lt;/mn&gt;
+                  &lt;/mtd&gt;
+               &lt;/mtr&gt;
+            &lt;/mtable&gt;
+         &lt;/mfenced&gt;
+      &lt;/mrow&gt;
+      &lt;mo&gt;+&lt;/mo&gt;
+      &lt;mrow&gt;
+         &lt;mi&gt;i&lt;/mi&gt;
+         &lt;mo&gt;&amp;sdot;&lt;/mo&gt;
+         &lt;mrow&gt;
+            &lt;mi&gt;sin&lt;/mi&gt;
+            &lt;mo&gt;&amp;ApplyFunction;&lt;/mo&gt;
+            &lt;mfenced close=")" open="("&gt;
+               &lt;mi&gt;theta&lt;/mi&gt;
+            &lt;/mfenced&gt;
+         &lt;/mrow&gt;
+         &lt;mo&gt;&amp;sdot;&lt;/mo&gt;
+         &lt;mfenced close="]" open="["&gt;
+            &lt;mtable&gt;
+               &lt;mtr&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;0&lt;/mn&gt;
+                  &lt;/mtd&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;1&lt;/mn&gt;
+                  &lt;/mtd&gt;
+               &lt;/mtr&gt;
+               &lt;mtr&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;1&lt;/mn&gt;
+                  &lt;/mtd&gt;
+                  &lt;mtd&gt;
+                     &lt;mn&gt;0&lt;/mn&gt;
+                  &lt;/mtd&gt;
+               &lt;/mtr&gt;
+            &lt;/mtable&gt;
+         &lt;/mfenced&gt;
+      &lt;/mrow&gt;
+   &lt;/mrow&gt;
+&lt;/math&gt;</pre><h3>Content MathML</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/content-mathml.html">conversion to Content MathML</a>:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;apply&gt;
+      &lt;plus/&gt;
+      &lt;apply&gt;
+         &lt;times/&gt;
+         &lt;apply&gt;
+            &lt;cos/&gt;
+            &lt;ci&gt;theta&lt;/ci&gt;
+         &lt;/apply&gt;
+         &lt;list&gt;
+            &lt;matrix&gt;
+               &lt;vector&gt;
+                  &lt;cn&gt;1&lt;/cn&gt;
+                  &lt;cn&gt;0&lt;/cn&gt;
+               &lt;/vector&gt;
+               &lt;vector&gt;
+                  &lt;cn&gt;0&lt;/cn&gt;
+                  &lt;cn&gt;1&lt;/cn&gt;
+               &lt;/vector&gt;
+            &lt;/matrix&gt;
+         &lt;/list&gt;
+      &lt;/apply&gt;
+      &lt;apply&gt;
+         &lt;times/&gt;
+         &lt;ci&gt;i&lt;/ci&gt;
+         &lt;apply&gt;
+            &lt;sin/&gt;
+            &lt;ci&gt;theta&lt;/ci&gt;
+         &lt;/apply&gt;
+         &lt;list&gt;
+            &lt;matrix&gt;
+               &lt;vector&gt;
+                  &lt;cn&gt;0&lt;/cn&gt;
+                  &lt;cn&gt;1&lt;/cn&gt;
+               &lt;/vector&gt;
+               &lt;vector&gt;
+                  &lt;cn&gt;1&lt;/cn&gt;
+                  &lt;cn&gt;0&lt;/cn&gt;
+               &lt;/vector&gt;
+            &lt;/matrix&gt;
+         &lt;/list&gt;
+      &lt;/apply&gt;
+   &lt;/apply&gt;
+&lt;/math&gt;</pre><h3>Maxima Input Form</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/maxima-input.html">conversion to Maxima Input syntax</a>:
+                     
+                  </p>
+                  <p>
+                     The conversion from Content MathML to Maxima Input was not successful for
+                     this input.
+                     
+                  </p>
+                  <table class="failures">
+                     <thead>
+                        <tr>
+                           <th>Failure Code</th>
+                           <th>Message</th>
+                           <th>XPath</th>
+                           <th>Context</th>
+                        </tr>
+                     </thead>
+                     <tbody>
+                        <tr>
+                           <td><a href="/snuggletex-webapp-1.2.2/documentation/error-codes.html#UMFG00">UMFG00</a></td>
+                           <td>Content MathML element matrix not supported</td>
+                           <td>apply[1]/apply[1]/list[1]/matrix[1]</td>
+                           <td><pre>&lt;matrix&gt;
+   &lt;vector&gt;
+      &lt;cn&gt;1&lt;/cn&gt;
+      &lt;cn&gt;0&lt;/cn&gt;
+   &lt;/vector&gt;
+   &lt;vector&gt;
+      &lt;cn&gt;0&lt;/cn&gt;
+      &lt;cn&gt;1&lt;/cn&gt;
+   &lt;/vector&gt;
+&lt;/matrix&gt;</pre></td>
+                        </tr>
+                        <tr>
+                           <td><a href="/snuggletex-webapp-1.2.2/documentation/error-codes.html#UMFG00">UMFG00</a></td>
+                           <td>Content MathML element matrix not supported</td>
+                           <td>apply[1]/apply[2]/list[1]/matrix[1]</td>
+                           <td><pre>&lt;matrix&gt;
+   &lt;vector&gt;
+      &lt;cn&gt;0&lt;/cn&gt;
+      &lt;cn&gt;1&lt;/cn&gt;
+   &lt;/vector&gt;
+   &lt;vector&gt;
+      &lt;cn&gt;1&lt;/cn&gt;
+      &lt;cn&gt;0&lt;/cn&gt;
+   &lt;/vector&gt;
+&lt;/matrix&gt;</pre></td>
+                        </tr>
+                     </tbody>
+                  </table>
+                  <h3>MathML Parallel Markup</h3>
+                  <p>
+                     This shows the enhanced Presentation MathML with other forms encapsulated
+                     as annotations:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;semantics&gt;
+      &lt;mrow&gt;
+         &lt;mrow&gt;
+            &lt;mrow&gt;
+               &lt;mi&gt;cos&lt;/mi&gt;
+               &lt;mo&gt;&amp;ApplyFunction;&lt;/mo&gt;
+               &lt;mfenced close=")" open="("&gt;
+                  &lt;mi&gt;theta&lt;/mi&gt;
+               &lt;/mfenced&gt;
+            &lt;/mrow&gt;
+            &lt;mo&gt;&amp;sdot;&lt;/mo&gt;
+            &lt;mfenced close="]" open="["&gt;
+               &lt;mtable&gt;
+                  &lt;mtr&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;1&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;0&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                  &lt;/mtr&gt;
+                  &lt;mtr&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;0&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;1&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                  &lt;/mtr&gt;
+               &lt;/mtable&gt;
+            &lt;/mfenced&gt;
+         &lt;/mrow&gt;
+         &lt;mo&gt;+&lt;/mo&gt;
+         &lt;mrow&gt;
+            &lt;mi&gt;i&lt;/mi&gt;
+            &lt;mo&gt;&amp;sdot;&lt;/mo&gt;
+            &lt;mrow&gt;
+               &lt;mi&gt;sin&lt;/mi&gt;
+               &lt;mo&gt;&amp;ApplyFunction;&lt;/mo&gt;
+               &lt;mfenced close=")" open="("&gt;
+                  &lt;mi&gt;theta&lt;/mi&gt;
+               &lt;/mfenced&gt;
+            &lt;/mrow&gt;
+            &lt;mo&gt;&amp;sdot;&lt;/mo&gt;
+            &lt;mfenced close="]" open="["&gt;
+               &lt;mtable&gt;
+                  &lt;mtr&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;0&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;1&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                  &lt;/mtr&gt;
+                  &lt;mtr&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;1&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                     &lt;mtd&gt;
+                        &lt;mn&gt;0&lt;/mn&gt;
+                     &lt;/mtd&gt;
+                  &lt;/mtr&gt;
+               &lt;/mtable&gt;
+            &lt;/mfenced&gt;
+         &lt;/mrow&gt;
+      &lt;/mrow&gt;
+      &lt;annotation-xml encoding="MathML-Content"&gt;
+         &lt;apply&gt;
+            &lt;plus/&gt;
+            &lt;apply&gt;
+               &lt;times/&gt;
+               &lt;apply&gt;
+                  &lt;cos/&gt;
+                  &lt;ci&gt;theta&lt;/ci&gt;
+               &lt;/apply&gt;
+               &lt;list&gt;
+                  &lt;matrix&gt;
+                     &lt;vector&gt;
+                        &lt;cn&gt;1&lt;/cn&gt;
+                        &lt;cn&gt;0&lt;/cn&gt;
+                     &lt;/vector&gt;
+                     &lt;vector&gt;
+                        &lt;cn&gt;0&lt;/cn&gt;
+                        &lt;cn&gt;1&lt;/cn&gt;
+                     &lt;/vector&gt;
+                  &lt;/matrix&gt;
+               &lt;/list&gt;
+            &lt;/apply&gt;
+            &lt;apply&gt;
+               &lt;times/&gt;
+               &lt;ci&gt;i&lt;/ci&gt;
+               &lt;apply&gt;
+                  &lt;sin/&gt;
+                  &lt;ci&gt;theta&lt;/ci&gt;
+               &lt;/apply&gt;
+               &lt;list&gt;
+                  &lt;matrix&gt;
+                     &lt;vector&gt;
+                        &lt;cn&gt;0&lt;/cn&gt;
+                        &lt;cn&gt;1&lt;/cn&gt;
+                     &lt;/vector&gt;
+                     &lt;vector&gt;
+                        &lt;cn&gt;1&lt;/cn&gt;
+                        &lt;cn&gt;0&lt;/cn&gt;
+                     &lt;/vector&gt;
+                  &lt;/matrix&gt;
+               &lt;/list&gt;
+            &lt;/apply&gt;
+         &lt;/apply&gt;
+      &lt;/annotation-xml&gt;
+      &lt;annotation encoding="ASCIIMathInput"/&gt;
+      &lt;annotation-xml encoding="Maxima-upconversion-failures"&gt;
+         &lt;s:fail xmlns:s="http://www.ph.ed.ac.uk/snuggletex" code="UMFG00"
+                 message="Content MathML element matrix not supported"&gt;
+            &lt;s:arg&gt;matrix&lt;/s:arg&gt;
+            &lt;s:xpath&gt;apply[1]/apply[1]/list[1]/matrix[1]&lt;/s:xpath&gt;
+            &lt;s:context&gt;
+               &lt;matrix&gt;
+                  &lt;vector&gt;
+                     &lt;cn&gt;1&lt;/cn&gt;
+                     &lt;cn&gt;0&lt;/cn&gt;
+                  &lt;/vector&gt;
+                  &lt;vector&gt;
+                     &lt;cn&gt;0&lt;/cn&gt;
+                     &lt;cn&gt;1&lt;/cn&gt;
+                  &lt;/vector&gt;
+               &lt;/matrix&gt;
+            &lt;/s:context&gt;
+         &lt;/s:fail&gt;
+         &lt;s:fail xmlns:s="http://www.ph.ed.ac.uk/snuggletex" code="UMFG00"
+                 message="Content MathML element matrix not supported"&gt;
+            &lt;s:arg&gt;matrix&lt;/s:arg&gt;
+            &lt;s:xpath&gt;apply[1]/apply[2]/list[1]/matrix[1]&lt;/s:xpath&gt;
+            &lt;s:context&gt;
+               &lt;matrix&gt;
+                  &lt;vector&gt;
+                     &lt;cn&gt;0&lt;/cn&gt;
+                     &lt;cn&gt;1&lt;/cn&gt;
+                  &lt;/vector&gt;
+                  &lt;vector&gt;
+                     &lt;cn&gt;1&lt;/cn&gt;
+                     &lt;cn&gt;0&lt;/cn&gt;
+                  &lt;/vector&gt;
+               &lt;/matrix&gt;
+            &lt;/s:context&gt;
+         &lt;/s:fail&gt;
+      &lt;/annotation-xml&gt;
+   &lt;/semantics&gt;
+&lt;/math&gt;</pre></div>
+            </div>
+         </div>
+      </div>
+      <div id="copyright">
+         <p>
+            SnuggleTeX Release 1.2.2 —
+            <a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a><br />
+            Copyright © 2009
+            <a href="http://www.ph.ed.ac.uk">The School of Physics and Astronomy</a>,
+            <a href="http://www.ed.ac.uk">The University of Edinburgh</a>.
+            <br />
+            For more information, contact
+            <a href="http://www.ph.ed.ac.uk/elearning/contacts/#dmckain">David McKain</a>.
+            
+         </p>
+         <p>
+            The University of Edinburgh is a charitable body, registered in Scotland,
+            with registration number SC005336.
+            
+         </p>
+      </div>
+   </body>
+</html>

xmodule/capa/tests/test_files/snuggletex_wrong.html

@@ -0,0 +1,187 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html
+  PUBLIC "-//W3C//DTD XHTML 1.1 plus MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/xhtml-math11-f.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+   <head>
+      <meta content="application/xhtml+xml; charset=UTF-8" http-equiv="Content-Type" />
+      <meta content="SnuggleTeX" name="Generator" />
+      <meta content="SnuggleTeX Documentation" name="description" />
+      <meta content="David McKain" name="author" />
+      <meta content="The University of Edinburgh" name="publisher" />
+      <link href="/snuggletex-webapp-1.2.2/includes/core.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/webapp.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/snuggletex.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.css"
+            rel="stylesheet" /><script src="/snuggletex-webapp-1.2.2/includes/jquery.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.js"
+              type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/webapp.js" type="text/javascript"></script><title>SnuggleTeX - ASCIIMathML Enrichment Demo</title><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathML.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathMLwidget.js"
+              type="text/javascript"></script></head>
+   <body id="asciiMathMLUpConversionDemo">
+      <table border="0" cellpadding="0" cellspacing="0" id="header" width="100%">
+         <tr>
+            <td align="left" id="logo" valign="top"><a class="headertext" href="http://www.ed.ac.uk"><img alt="The University of Edinburgh" height="84"
+                       src="/snuggletex-webapp-1.2.2/includes/uoe_logo.jpg"
+                       width="84" /></a></td>
+            <td align="left">
+               <h3>THE UNIVERSITY of EDINBURGH</h3>
+               <h1>SCHOOL OF PHYSICS AND ASTRONOMY</h1>
+            </td>
+         </tr>
+      </table>
+      <h1 id="location"><a href="/snuggletex-webapp-1.2.2">SnuggleTeX (1.2.2)</a></h1>
+      <div id="content">
+         <div id="skipnavigation"><a href="#maincontent">Skip Navigation</a></div>
+         <div id="navigation">
+            <div id="navinner">
+               <h2>About SnuggleTeX</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/overview-and-features.html">Overview &amp; Features</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/use-cases.html">Why Use SnuggleTeX?</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/license.html">License</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a></li>
+               </ul>
+               <h2>Demos &amp; Samples</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/MathInputDemo">Simple Math Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/FullLaTeXInputDemo">Full LaTeX Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichment Demo</a></li>
+                  <li><a class="selected" href="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo">ASCIIMathML Enrichment Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output-samples.html">Web Output Samples</a></li>
+               </ul>
+               <h2>User Guide</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/getting-snuggletex.html">Getting SnuggleTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/software-requirements.html">Software Requirements</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/your-classpath.html">Setting up Your ClassPath</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/examples.html">Examples</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/basic-usage.html">Basic Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/inputs.html">Parsing LaTeX Inputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/xml-or-dom-output.html">Creating XML String or DOM Outputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output.html">Creating Web Pages</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/error-reporting.html">Error Reporting</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/supported-latex.html">Supported LaTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/advanced-usage.html">Advanced Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">Semantic Enrichment</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/migrating-from-older-versions.html">Migrating from older versions</a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/apidocs/index.html">API Documentation<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/xref/index.html">Source Code Cross-Reference<span class="extlink"> </span></a></li>
+               </ul>
+               <h2>SnuggleTeX Project Links</h2>
+               <ul>
+                  <li><a href="http://sourceforge.net/project/showfiles.php?group_id=221375">Download from SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://sourceforge.net/projects/snuggletex/">SnuggleTeX on SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/">SnuggleTeX Maven Developer Reports<span class="extlink"> </span></a></li>
+                  <li><a href="https://www.wiki.ed.ac.uk/display/Physics/SnuggleTeX">SnuggleTeX Wiki<span class="extlink"> </span></a></li>
+               </ul>
+            </div>
+         </div>
+         <div id="maincontent">
+            <div id="popup"></div>
+            <div id="maininner">
+               <h2>ASCIIMathML Enrichment Demo</h2>
+               <h3>Input</h3>
+               <p>
+                  This demo is similar to the
+                  <a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichnment Demo</a>
+                  but uses
+                  <a href="http://www1.chapman.edu/~jipsen/asciimath.html">ASCIIMathML</a> as
+                  an alternative input format, which provides real-time feedback as you
+                  type but can often generate MathML with odd semantics in it.
+                  SnuggleTeX includes some functionality that can to convert this raw MathML into
+                  something equivalent to its own MathML output, thereby allowing you to
+                  <a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">semantically enrich</a> it in
+                  certain simple cases, making ASCIIMathML a possibly viable input format
+                  for simple semantic maths.
+                  
+               </p>
+               <p>
+                  To try the demo, simply enter some some ASCIIMathML into the box below.
+                  You should see a real time preview of this while you type.
+                  Then hit <tt>Go!</tt> to use SnuggleTeX to semantically enrich your
+                  input.
+                  
+               </p>
+               <form action="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo" class="input"
+                     method="post">
+                  <div class="inputBox">
+                     ASCIIMath Input:
+                     <input id="asciiMathInput" name="asciiMathInput" type="text" value="" /><input id="asciiMathML" name="asciiMathML" type="hidden" /><input type="submit" value="Go!" /></div>
+               </form>
+               <h3>Live Preview</h3>
+               <p>
+                  This is a MathML rendering of your input, generated by ASCIIMathML as you type.
+                  
+               </p>
+               <div class="result">
+                  <div id="preview"> </div>
+               </div>
+               <p>
+                  This is the underlying MathML source generated by ASCIIMathML, again updated in real time.
+                  
+               </p>
+               <div class="result"><pre id="previewSource"> </pre></div><script type="text/javascript">
+          registerASCIIMathMLInputWidget('asciiMathInput', 'preview', 'asciiMathML', 'previewSource');
+          var inputChanged = false;
+          // Hide any existing output stuff in page on first change, as it will no longer be in sync
+          jQuery(document).ready(function() {
+            jQuery('#asciiMathInput').bind('keydown', function() {
+              if (!inputChanged) jQuery('.outputContainer').css('visibility', 'hidden');
+              inputChanged = true;
+            });
+          });
+        </script><div class="outputContainer">
+                  <h3>Enhanced Presentation MathML</h3>
+                  <p>
+                     This shows the result of attempting to enrich the raw Presentation MathML
+                     generated by ASCIIMathML:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;mn&gt;2&lt;/mn&gt;
+&lt;/math&gt;</pre><h3>Content MathML</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/content-mathml.html">conversion to Content MathML</a>:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;cn&gt;2&lt;/cn&gt;
+&lt;/math&gt;</pre><h3>Maxima Input Form</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/maxima-input.html">conversion to Maxima Input syntax</a>:
+                     
+                  </p><pre class="result">2</pre><h3>MathML Parallel Markup</h3>
+                  <p>
+                     This shows the enhanced Presentation MathML with other forms encapsulated
+                     as annotations:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;semantics&gt;
+      &lt;mn&gt;2&lt;/mn&gt;
+      &lt;annotation-xml encoding="MathML-Content"&gt;
+         &lt;cn&gt;2&lt;/cn&gt;
+      &lt;/annotation-xml&gt;
+      &lt;annotation encoding="ASCIIMathInput"/&gt;
+      &lt;annotation encoding="Maxima"&gt;2&lt;/annotation&gt;
+   &lt;/semantics&gt;
+&lt;/math&gt;</pre></div>
+            </div>
+         </div>
+      </div>
+      <div id="copyright">
+         <p>
+            SnuggleTeX Release 1.2.2 —
+            <a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a><br />
+            Copyright © 2009
+            <a href="http://www.ph.ed.ac.uk">The School of Physics and Astronomy</a>,
+            <a href="http://www.ed.ac.uk">The University of Edinburgh</a>.
+            <br />
+            For more information, contact
+            <a href="http://www.ph.ed.ac.uk/elearning/contacts/#dmckain">David McKain</a>.
+            
+         </p>
+         <p>
+            The University of Edinburgh is a charitable body, registered in Scotland,
+            with registration number SC005336.
+            
+         </p>
+      </div>
+   </body>
+</html>

xmodule/capa/tests/test_files/snuggletex_x+x+3y.xml

@@ -0,0 +1,225 @@
+<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE html
+  PUBLIC "-//W3C//DTD XHTML 1.1 plus MathML 2.0//EN" "http://www.w3.org/Math/DTD/mathml2/xhtml-math11-f.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en">
+   <head>
+      <meta content="application/xhtml+xml; charset=UTF-8" http-equiv="Content-Type" />
+      <meta content="SnuggleTeX" name="Generator" />
+      <meta content="SnuggleTeX Documentation" name="description" />
+      <meta content="David McKain" name="author" />
+      <meta content="The University of Edinburgh" name="publisher" />
+      <link href="/snuggletex-webapp-1.2.2/includes/core.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/webapp.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/snuggletex.css" rel="stylesheet" />
+      <link href="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.css"
+            rel="stylesheet" /><script src="/snuggletex-webapp-1.2.2/includes/jquery.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/jquery-ui-1.7.2.custom.js"
+              type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/webapp.js" type="text/javascript"></script><title>SnuggleTeX - ASCIIMathML Enrichment Demo</title><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathML.js" type="text/javascript"></script><script src="/snuggletex-webapp-1.2.2/includes/ASCIIMathMLwidget.js"
+              type="text/javascript"></script></head>
+   <body id="asciiMathMLUpConversionDemo">
+      <table border="0" cellpadding="0" cellspacing="0" id="header" width="100%">
+         <tr>
+            <td align="left" id="logo" valign="top"><a class="headertext" href="http://www.ed.ac.uk"><img alt="The University of Edinburgh" height="84"
+                       src="/snuggletex-webapp-1.2.2/includes/uoe_logo.jpg"
+                       width="84" /></a></td>
+            <td align="left">
+               <h3>THE UNIVERSITY of EDINBURGH</h3>
+               <h1>SCHOOL OF PHYSICS AND ASTRONOMY</h1>
+            </td>
+         </tr>
+      </table>
+      <h1 id="location"><a href="/snuggletex-webapp-1.2.2">SnuggleTeX (1.2.2)</a></h1>
+      <div id="content">
+         <div id="skipnavigation"><a href="#maincontent">Skip Navigation</a></div>
+         <div id="navigation">
+            <div id="navinner">
+               <h2>About SnuggleTeX</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/overview-and-features.html">Overview &amp; Features</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/use-cases.html">Why Use SnuggleTeX?</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/license.html">License</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a></li>
+               </ul>
+               <h2>Demos &amp; Samples</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/MathInputDemo">Simple Math Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/FullLaTeXInputDemo">Full LaTeX Input Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichment Demo</a></li>
+                  <li><a class="selected" href="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo">ASCIIMathML Enrichment Demo</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output-samples.html">Web Output Samples</a></li>
+               </ul>
+               <h2>User Guide</h2>
+               <ul>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/getting-snuggletex.html">Getting SnuggleTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/software-requirements.html">Software Requirements</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/your-classpath.html">Setting up Your ClassPath</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/examples.html">Examples</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/basic-usage.html">Basic Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/inputs.html">Parsing LaTeX Inputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/xml-or-dom-output.html">Creating XML String or DOM Outputs</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/web-output.html">Creating Web Pages</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/error-reporting.html">Error Reporting</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/supported-latex.html">Supported LaTeX</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/advanced-usage.html">Advanced Usage</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">Semantic Enrichment</a></li>
+                  <li><a href="/snuggletex-webapp-1.2.2/documentation/migrating-from-older-versions.html">Migrating from older versions</a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/apidocs/index.html">API Documentation<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/xref/index.html">Source Code Cross-Reference<span class="extlink"> </span></a></li>
+               </ul>
+               <h2>SnuggleTeX Project Links</h2>
+               <ul>
+                  <li><a href="http://sourceforge.net/project/showfiles.php?group_id=221375">Download from SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://sourceforge.net/projects/snuggletex/">SnuggleTeX on SourceForge.net<span class="extlink"> </span></a></li>
+                  <li><a href="http://snuggletex.sourceforge.net/maven/">SnuggleTeX Maven Developer Reports<span class="extlink"> </span></a></li>
+                  <li><a href="https://www.wiki.ed.ac.uk/display/Physics/SnuggleTeX">SnuggleTeX Wiki<span class="extlink"> </span></a></li>
+               </ul>
+            </div>
+         </div>
+         <div id="maincontent">
+            <div id="popup"></div>
+            <div id="maininner">
+               <h2>ASCIIMathML Enrichment Demo</h2>
+               <h3>Input</h3>
+               <p>
+                  This demo is similar to the
+                  <a href="/snuggletex-webapp-1.2.2/UpConversionDemo">MathML Semantic Enrichnment Demo</a>
+                  but uses
+                  <a href="http://www1.chapman.edu/~jipsen/asciimath.html">ASCIIMathML</a> as
+                  an alternative input format, which provides real-time feedback as you
+                  type but can often generate MathML with odd semantics in it.
+                  SnuggleTeX includes some functionality that can to convert this raw MathML into
+                  something equivalent to its own MathML output, thereby allowing you to
+                  <a href="/snuggletex-webapp-1.2.2/documentation/semantic-enrichment.html">semantically enrich</a> it in
+                  certain simple cases, making ASCIIMathML a possibly viable input format
+                  for simple semantic maths.
+                  
+               </p>
+               <p>
+                  To try the demo, simply enter some some ASCIIMathML into the box below.
+                  You should see a real time preview of this while you type.
+                  Then hit <tt>Go!</tt> to use SnuggleTeX to semantically enrich your
+                  input.
+                  
+               </p>
+               <form action="/snuggletex-webapp-1.2.2/ASCIIMathMLUpConversionDemo" class="input"
+                     method="post">
+                  <div class="inputBox">
+                     ASCIIMath Input:
+                     <input id="asciiMathInput" name="asciiMathInput" type="text" value="" /><input id="asciiMathML" name="asciiMathML" type="hidden" /><input type="submit" value="Go!" /></div>
+               </form>
+               <h3>Live Preview</h3>
+               <p>
+                  This is a MathML rendering of your input, generated by ASCIIMathML as you type.
+                  
+               </p>
+               <div class="result">
+                  <div id="preview"> </div>
+               </div>
+               <p>
+                  This is the underlying MathML source generated by ASCIIMathML, again updated in real time.
+                  
+               </p>
+               <div class="result"><pre id="previewSource"> </pre></div><script type="text/javascript">
+          registerASCIIMathMLInputWidget('asciiMathInput', 'preview', 'asciiMathML', 'previewSource');
+          var inputChanged = false;
+          // Hide any existing output stuff in page on first change, as it will no longer be in sync
+          jQuery(document).ready(function() {
+            jQuery('#asciiMathInput').bind('keydown', function() {
+              if (!inputChanged) jQuery('.outputContainer').css('visibility', 'hidden');
+              inputChanged = true;
+            });
+          });
+        </script><div class="outputContainer">
+                  <h3>Enhanced Presentation MathML</h3>
+                  <p>
+                     This shows the result of attempting to enrich the raw Presentation MathML
+                     generated by ASCIIMathML:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;mrow&gt;
+      &lt;mi&gt;x&lt;/mi&gt;
+      &lt;mo&gt;+&lt;/mo&gt;
+      &lt;mi&gt;x&lt;/mi&gt;
+      &lt;mo&gt;+&lt;/mo&gt;
+      &lt;mrow&gt;
+         &lt;mn&gt;3&lt;/mn&gt;
+         &lt;mo&gt;*&lt;/mo&gt;
+         &lt;mi&gt;y&lt;/mi&gt;
+      &lt;/mrow&gt;
+   &lt;/mrow&gt;
+&lt;/math&gt;</pre><h3>Content MathML</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/content-mathml.html">conversion to Content MathML</a>:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;apply&gt;
+      &lt;plus/&gt;
+      &lt;ci&gt;x&lt;/ci&gt;
+      &lt;ci&gt;x&lt;/ci&gt;
+      &lt;apply&gt;
+         &lt;times/&gt;
+         &lt;cn&gt;3&lt;/cn&gt;
+         &lt;ci&gt;y&lt;/ci&gt;
+      &lt;/apply&gt;
+   &lt;/apply&gt;
+&lt;/math&gt;</pre><h3>Maxima Input Form</h3>
+                  <p>
+                     This shows the result of an attempted
+                     <a href="documentation/maxima-input.html">conversion to Maxima Input syntax</a>:
+                     
+                  </p><pre class="result">x + x + (3 * y)</pre><h3>MathML Parallel Markup</h3>
+                  <p>
+                     This shows the enhanced Presentation MathML with other forms encapsulated
+                     as annotations:
+                     
+                  </p><pre class="result">&lt;math xmlns="http://www.w3.org/1998/Math/MathML"&gt;
+   &lt;semantics&gt;
+      &lt;mrow&gt;
+         &lt;mi&gt;x&lt;/mi&gt;
+         &lt;mo&gt;+&lt;/mo&gt;
+         &lt;mi&gt;x&lt;/mi&gt;
+         &lt;mo&gt;+&lt;/mo&gt;
+         &lt;mrow&gt;
+            &lt;mn&gt;3&lt;/mn&gt;
+            &lt;mo&gt;*&lt;/mo&gt;
+            &lt;mi&gt;y&lt;/mi&gt;
+         &lt;/mrow&gt;
+      &lt;/mrow&gt;
+      &lt;annotation-xml encoding="MathML-Content"&gt;
+         &lt;apply&gt;
+            &lt;plus/&gt;
+            &lt;ci&gt;x&lt;/ci&gt;
+            &lt;ci&gt;x&lt;/ci&gt;
+            &lt;apply&gt;
+               &lt;times/&gt;
+               &lt;cn&gt;3&lt;/cn&gt;
+               &lt;ci&gt;y&lt;/ci&gt;
+            &lt;/apply&gt;
+         &lt;/apply&gt;
+      &lt;/annotation-xml&gt;
+      &lt;annotation encoding="ASCIIMathInput"/&gt;
+      &lt;annotation encoding="Maxima"&gt;x + x + (3 * y)&lt;/annotation&gt;
+   &lt;/semantics&gt;
+&lt;/math&gt;</pre></div>
+            </div>
+         </div>
+      </div>
+      <div id="copyright">
+         <p>
+            SnuggleTeX Release 1.2.2 —
+            <a href="/snuggletex-webapp-1.2.2/documentation/release-notes.html">Release Notes</a><br />
+            Copyright © 2009
+            <a href="http://www.ph.ed.ac.uk">The School of Physics and Astronomy</a>,
+            <a href="http://www.ed.ac.uk">The University of Edinburgh</a>.
+            <br />
+            For more information, contact
+            <a href="http://www.ph.ed.ac.uk/elearning/contacts/#dmckain">David McKain</a>.
+            
+         </p>
+         <p>
+            The University of Edinburgh is a charitable body, registered in Scotland,
+            with registration number SC005336.
+            
+         </p>
+      </div>
+   </body>
+</html>

xmodule/capa/tests/test_files/targeted_feedback.xml

@@ -0,0 +1,50 @@
+<problem>
+    <p>What is the correct answer?</p>
+    <multiplechoiceresponse targeted-feedback="">
+      <choicegroup type="MultipleChoice">
+        <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+        <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+        <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+        <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+      </choicegroup>
+    </multiplechoiceresponse>
+
+    <targetedfeedbackset>
+        <targetedfeedback explanation-id="feedback1">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 1st WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedback2">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 2nd WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedback3">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 3rd WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedbackC">
+        <div class="detailed-targeted-feedback-correct">
+            <p>Targeted Feedback</p>
+            <p>Feedback on your correct solution...</p>
+        </div>
+        </targetedfeedback>
+
+    </targetedfeedbackset>
+
+    <solution explanation-id="feedbackC">
+    <div class="detailed-solution">
+        <p>Explanation</p>
+        <p>This is the solution explanation</p>
+        <p>Not much to explain here, sorry!</p>
+    </div>
+    </solution>
+</problem>

xmodule/capa/tests/test_files/targeted_feedback_multiple.xml

@@ -0,0 +1,91 @@
+<problem>
+    <p>Q1</p>
+    <multiplechoiceresponse targeted-feedback="">
+      <choicegroup type="MultipleChoice">
+        <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+        <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+        <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+        <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+      </choicegroup>
+    </multiplechoiceresponse>
+
+    <targetedfeedbackset>
+        <targetedfeedback explanation-id="feedback1">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 1st WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedback3">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 3rd WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedbackC">
+        <div class="detailed-targeted-feedback-correct">
+            <p>Targeted Feedback</p>
+            <p>Feedback on your correct solution...</p>
+        </div>
+        </targetedfeedback>
+
+    </targetedfeedbackset>
+
+    <solutionset>
+        <solution explanation-id="feedbackC">
+        <div class="detailed-solution">
+            <p>Explanation</p>
+            <p>This is the solution explanation</p>
+            <p>Not much to explain here, sorry!</p>
+        </div>
+        </solution>
+    </solutionset>
+
+    <hr/>
+
+    <p>Q2</p>
+    <multiplechoiceresponse targeted-feedback="">
+      <choicegroup type="MultipleChoice" answer-pool="3">
+        <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+        <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+        <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+        <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+      </choicegroup>
+    </multiplechoiceresponse>
+
+    <targetedfeedbackset>
+        <targetedfeedback explanation-id="feedback1">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 1st WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedback3">
+        <div class="detailed-targeted-feedback">
+            <p>Targeted Feedback</p>
+            <p>This is the 3rd WRONG solution</p>
+        </div>
+        </targetedfeedback>
+
+        <targetedfeedback explanation-id="feedbackC">
+        <div class="detailed-targeted-feedback-correct">
+            <p>Targeted Feedback</p>
+            <p>Feedback on your correct solution...</p>
+        </div>
+        </targetedfeedback>
+
+    </targetedfeedbackset>
+
+    <solutionset>
+        <solution explanation-id="feedbackC">
+        <div class="detailed-solution">
+            <p>Explanation</p>
+            <p>This is the solution explanation</p>
+            <p>Not much to explain here, sorry!</p>
+        </div>
+        </solution>
+    </solutionset>
+</problem>

xmodule/capa/tests/test_hint_functionality.py

@@ -0,0 +1,543 @@
+# -*- coding: utf-8 -*-
+"""
+Tests of extended hints
+"""
+
+
+import unittest
+import pytest
+from ddt import data, ddt, unpack
+
+from xmodule.capa.tests.helpers import load_fixture, new_loncapa_problem
+
+# With the use of ddt, some of the data expected_string cases below are naturally long stretches
+# of text text without whitespace. I think it's best to leave such lines intact
+# in the test code. Therefore:
+# pylint: disable=line-too-long
+# For out many ddt data cases, prefer a compact form of { .. }
+
+
+class HintTest(unittest.TestCase):
+    """Base class for tests of extended hinting functionality."""
+
+    def correctness(self, problem_id, choice):
+        """Grades the problem and returns the 'correctness' string from cmap."""
+        student_answers = {problem_id: choice}
+        cmap = self.problem.grade_answers(answers=student_answers)    # pylint: disable=no-member
+        return cmap[problem_id]['correctness']
+
+    def get_hint(self, problem_id, choice):
+        """Grades the problem and returns its hint from cmap or the empty string."""
+        student_answers = {problem_id: choice}
+        cmap = self.problem.grade_answers(answers=student_answers)    # pylint: disable=no-member
+        adict = cmap.cmap.get(problem_id)
+        if adict:
+            return adict['msg']
+        else:
+            return ''
+
+
+# It is a little surprising how much more complicated TextInput is than all the other cases.
+@ddt
+class TextInputHintsTest(HintTest):
+    """
+    Test Text Input Hints Test
+    """
+    xml = load_fixture('extended_hints_text_input.xml')
+    problem = new_loncapa_problem(xml)
+
+    def test_tracking_log(self):
+        """Test that the tracking log comes out right."""
+        self.problem.capa_module.reset_mock()
+        self.get_hint('1_3_1', 'Blue')
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'module_id': 'i4x://Foo/bar/mock/abc',
+             'problem_part_id': '1_2',
+             'trigger_type': 'single',
+             'hint_label': 'Correct:',
+             'correctness': True,
+             'student_answer': ['Blue'],
+             'question_type': 'stringresponse',
+             'hints': [{'text': 'The red light is scattered by water molecules leaving only blue light.'}]}
+        )
+
+    @data(
+        {'problem_id': '1_2_1', 'choice': 'GermanyΩ',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">I do not think so.&#937;</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'franceΩ',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">Viva la France!&#937;</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'FranceΩ',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">Viva la France!&#937;</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'Mexico',
+         'expected_string': ''},
+        {'problem_id': '1_2_1', 'choice': 'USAΩ',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">Less well known, but yes, there is a Paris, Texas.&#937;</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'usaΩ',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">Less well known, but yes, there is a Paris, Texas.&#937;</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'uSAxΩ',
+         'expected_string': ''},
+        {'problem_id': '1_2_1', 'choice': 'NICKLANDΩ',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">The country name does not end in LAND&#937;</div></div>'},
+        {'problem_id': '1_3_1', 'choice': 'Blue',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">The red light is scattered by water molecules leaving only blue light.</div></div>'},
+        {'problem_id': '1_3_1', 'choice': 'blue',
+         'expected_string': ''},
+        {'problem_id': '1_3_1', 'choice': 'b',
+         'expected_string': ''},
+    )
+    @unpack
+    def test_text_input_hints(self, problem_id, choice, expected_string):
+        hint = self.get_hint(problem_id, choice)
+        assert hint == expected_string
+
+
+@ddt
+class TextInputExtendedHintsCaseInsensitive(HintTest):
+    """Test Text Input Extended hints Case Insensitive"""
+    xml = load_fixture('extended_hints_text_input.xml')
+    problem = new_loncapa_problem(xml)
+
+    @data(
+        {'problem_id': '1_5_1', 'choice': 'abc', 'expected_string': ''},  # wrong answer yielding no hint
+        {'problem_id': '1_5_1', 'choice': 'A', 'expected_string':
+         '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Woo Hoo </span><div class="hint-text">hint1</div></div>'},
+        {'problem_id': '1_5_1', 'choice': 'a', 'expected_string':
+         '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Woo Hoo </span><div class="hint-text">hint1</div></div>'},
+        {'problem_id': '1_5_1', 'choice': 'B', 'expected_string':
+         '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><div class="hint-text">hint2</div></div>'},
+        {'problem_id': '1_5_1', 'choice': 'b', 'expected_string':
+         '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><div class="hint-text">hint2</div></div>'},
+        {'problem_id': '1_5_1', 'choice': 'C', 'expected_string':
+         '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><div class="hint-text">hint4</div></div>'},
+        {'problem_id': '1_5_1', 'choice': 'c', 'expected_string':
+         '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><div class="hint-text">hint4</div></div>'},
+        # regexp cases
+        {'problem_id': '1_5_1', 'choice': 'FGGG', 'expected_string':
+         '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><div class="hint-text">hint6</div></div>'},
+        {'problem_id': '1_5_1', 'choice': 'fgG', 'expected_string':
+         '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><div class="hint-text">hint6</div></div>'},
+    )
+    @unpack
+    def test_text_input_hints(self, problem_id, choice, expected_string):
+        hint = self.get_hint(problem_id, choice)
+        assert hint == expected_string
+
+
+@ddt
+class TextInputExtendedHintsCaseSensitive(HintTest):
+    """Sometimes the semantics can be encoded in the class name."""
+    xml = load_fixture('extended_hints_text_input.xml')
+    problem = new_loncapa_problem(xml)
+
+    @data(
+        {'problem_id': '1_6_1', 'choice': 'abc', 'expected_string': ''},
+        {'problem_id': '1_6_1', 'choice': 'A', 'expected_string':
+         '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint1</div></div>'},
+        {'problem_id': '1_6_1', 'choice': 'a', 'expected_string': ''},
+        {'problem_id': '1_6_1', 'choice': 'B', 'expected_string':
+         '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint2</div></div>'},
+        {'problem_id': '1_6_1', 'choice': 'b', 'expected_string': ''},
+        {'problem_id': '1_6_1', 'choice': 'C', 'expected_string':
+         '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">hint4</div></div>'},
+        {'problem_id': '1_6_1', 'choice': 'c', 'expected_string': ''},
+        # regexp cases
+        {'problem_id': '1_6_1', 'choice': 'FGG', 'expected_string':
+         '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">hint6</div></div>'},
+        {'problem_id': '1_6_1', 'choice': 'fgG', 'expected_string': ''},
+    )
+    @unpack
+    def test_text_input_hints(self, problem_id, choice, expected_string):
+        message_text = self.get_hint(problem_id, choice)
+        assert message_text == expected_string
+
+
+@ddt
+class TextInputExtendedHintsCompatible(HintTest):
+    """
+    Compatibility test with mixed old and new style additional_answer tags.
+    """
+    xml = load_fixture('extended_hints_text_input.xml')
+    problem = new_loncapa_problem(xml)
+
+    @data(
+        {'problem_id': '1_7_1', 'choice': 'A', 'correct': 'correct',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint1</div></div>'},
+        {'problem_id': '1_7_1', 'choice': 'B', 'correct': 'correct', 'expected_string': ''},
+        {'problem_id': '1_7_1', 'choice': 'C', 'correct': 'correct',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint2</div></div>'},
+        {'problem_id': '1_7_1', 'choice': 'D', 'correct': 'incorrect', 'expected_string': ''},
+        # check going through conversion with difficult chars
+        {'problem_id': '1_7_1', 'choice': """<&"'>""", 'correct': 'correct', 'expected_string': ''},
+    )
+    @unpack
+    def test_text_input_hints(self, problem_id, choice, correct, expected_string):
+        message_text = self.get_hint(problem_id, choice)
+        assert message_text == expected_string
+        assert self.correctness(problem_id, choice) == correct
+
+
+@ddt
+class TextInputExtendedHintsRegex(HintTest):
+    """
+    Extended hints where the answer is regex mode.
+    """
+    xml = load_fixture('extended_hints_text_input.xml')
+    problem = new_loncapa_problem(xml)
+
+    @data(
+        {'problem_id': '1_8_1', 'choice': 'ABwrong', 'correct': 'incorrect', 'expected_string': ''},
+        {'problem_id': '1_8_1', 'choice': 'ABC', 'correct': 'correct',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint1</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'ABBBBC', 'correct': 'correct',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint1</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'aBc', 'correct': 'correct',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint1</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'BBBB', 'correct': 'correct',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint2</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'bbb', 'correct': 'correct',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">hint2</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'C', 'correct': 'incorrect',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">hint4</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'c', 'correct': 'incorrect',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">hint4</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'D', 'correct': 'incorrect',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">hint6</div></div>'},
+        {'problem_id': '1_8_1', 'choice': 'd', 'correct': 'incorrect',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">hint6</div></div>'},
+    )
+    @unpack
+    def test_text_input_hints(self, problem_id, choice, correct, expected_string):
+        message_text = self.get_hint(problem_id, choice)
+        assert message_text == expected_string
+        assert self.correctness(problem_id, choice) == correct
+
+
+@ddt
+class NumericInputHintsTest(HintTest):
+    """
+    This class consists of a suite of test cases to be run on the numeric input problem represented by the XML below.
+    """
+    xml = load_fixture('extended_hints_numeric_input.xml')
+    problem = new_loncapa_problem(xml)          # this problem is properly constructed
+
+    def test_tracking_log(self):
+        self.get_hint('1_2_1', '1.141')
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'module_id': 'i4x://Foo/bar/mock/abc', 'problem_part_id': '1_1', 'trigger_type': 'single',
+             'hint_label': 'Nice',
+             'correctness': True,
+             'student_answer': ['1.141'],
+             'question_type': 'numericalresponse',
+             'hints': [{'text': 'The square root of two turns up in the strangest places.'}]}
+        )
+
+    @data(
+        {'problem_id': '1_2_1', 'choice': '1.141',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Nice </span><div class="hint-text">The square root of two turns up in the strangest places.</div></div>'},
+        # additional answer
+        {'problem_id': '1_2_1', 'choice': '10',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">This is an additional hint.</div></div>'},
+        {'problem_id': '1_3_1', 'choice': '4',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">Pretty easy, uh?.</div></div>'},
+        # should get hint, when correct via numeric-tolerance
+        {'problem_id': '1_2_1', 'choice': '1.15',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Nice </span><div class="hint-text">The square root of two turns up in the strangest places.</div></div>'},
+        # when they answer wrong, nothing
+        {'problem_id': '1_2_1', 'choice': '2', 'expected_string': ''},
+    )
+    @unpack
+    def test_numeric_input_hints(self, problem_id, choice, expected_string):
+        hint = self.get_hint(problem_id, choice)
+        assert hint == expected_string
+
+
+@ddt
+class CheckboxHintsTest(HintTest):
+    """
+    This class consists of a suite of test cases to be run on the checkbox problem represented by the XML below.
+    """
+    xml = load_fixture('extended_hints_checkbox.xml')
+    problem = new_loncapa_problem(xml)          # this problem is properly constructed
+
+    @data(
+        {'problem_id': '1_2_1', 'choice': ['choice_0'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">You are right that apple is a fruit.</div><div class="hint-text">You are right that mushrooms are not fruit</div><div class="hint-text">Remember that grape is also a fruit.</div><div class="hint-text">What is a camero anyway?</div></div></div>'},
+        {'problem_id': '1_2_1', 'choice': ['choice_1'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">Remember that apple is also a fruit.</div><div class="hint-text">Mushroom is a fungus, not a fruit.</div><div class="hint-text">Remember that grape is also a fruit.</div><div class="hint-text">What is a camero anyway?</div></div></div>'},
+        {'problem_id': '1_2_1', 'choice': ['choice_2'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">Remember that apple is also a fruit.</div><div class="hint-text">You are right that mushrooms are not fruit</div><div class="hint-text">You are right that grape is a fruit</div><div class="hint-text">What is a camero anyway?</div></div></div>'},
+        {'problem_id': '1_2_1', 'choice': ['choice_3'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">Remember that apple is also a fruit.</div><div class="hint-text">You are right that mushrooms are not fruit</div><div class="hint-text">Remember that grape is also a fruit.</div><div class="hint-text">What is a camero anyway?</div></div></div>'},
+        {'problem_id': '1_2_1', 'choice': ['choice_4'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">Remember that apple is also a fruit.</div><div class="hint-text">You are right that mushrooms are not fruit</div><div class="hint-text">Remember that grape is also a fruit.</div><div class="hint-text">I do not know what a Camero is but it is not a fruit.</div></div></div>'},
+        {'problem_id': '1_2_1', 'choice': ['choice_0', 'choice_1'],  # compound
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Almost right </span><div class="hint-text">You are right that apple is a fruit, but there is one you are missing. Also, mushroom is not a fruit.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': ['choice_1', 'choice_2'],  # compound
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">You are right that grape is a fruit, but there is one you are missing. Also, mushroom is not a fruit.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': ['choice_0', 'choice_2'],
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="feedback-hint-multi"><div class="hint-text">You are right that apple is a fruit.</div><div class="hint-text">You are right that mushrooms are not fruit</div><div class="hint-text">You are right that grape is a fruit</div><div class="hint-text">What is a camero anyway?</div></div></div>'},
+        {'problem_id': '1_3_1', 'choice': ['choice_0'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">No, sorry, a banana is a fruit.</div><div class="hint-text">You are right that mushrooms are not vegatbles</div><div class="hint-text">Brussel sprout is the only vegetable in this list.</div></div></div>'},
+        {'problem_id': '1_3_1', 'choice': ['choice_1'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">poor banana.</div><div class="hint-text">You are right that mushrooms are not vegatbles</div><div class="hint-text">Brussel sprout is the only vegetable in this list.</div></div></div>'},
+        {'problem_id': '1_3_1', 'choice': ['choice_2'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">poor banana.</div><div class="hint-text">Mushroom is a fungus, not a vegetable.</div><div class="hint-text">Brussel sprout is the only vegetable in this list.</div></div></div>'},
+        {'problem_id': '1_3_1', 'choice': ['choice_3'],
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="feedback-hint-multi"><div class="hint-text">poor banana.</div><div class="hint-text">You are right that mushrooms are not vegatbles</div><div class="hint-text">Brussel sprouts are vegetables.</div></div></div>'},
+        {'problem_id': '1_3_1', 'choice': ['choice_0', 'choice_1'],  # compound
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Very funny </span><div class="hint-text">Making a banana split?</div></div>'},
+        {'problem_id': '1_3_1', 'choice': ['choice_1', 'choice_2'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">poor banana.</div><div class="hint-text">Mushroom is a fungus, not a vegetable.</div><div class="hint-text">Brussel sprout is the only vegetable in this list.</div></div></div>'},
+        {'problem_id': '1_3_1', 'choice': ['choice_0', 'choice_2'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">No, sorry, a banana is a fruit.</div><div class="hint-text">Mushroom is a fungus, not a vegetable.</div><div class="hint-text">Brussel sprout is the only vegetable in this list.</div></div></div>'},
+
+        # check for interaction between compoundhint and correct/incorrect
+        {'problem_id': '1_4_1', 'choice': ['choice_0', 'choice_1'],  # compound
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">AB</div></div>'},
+        {'problem_id': '1_4_1', 'choice': ['choice_0', 'choice_2'],  # compound
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">AC</div></div>'},
+
+        # check for labeling where multiple child hints have labels
+        # These are some tricky cases
+        {'problem_id': '1_5_1', 'choice': ['choice_0', 'choice_1'],
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">AA </span><div class="feedback-hint-multi"><div class="hint-text">aa</div></div></div>'},
+        {'problem_id': '1_5_1', 'choice': ['choice_0'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">aa</div><div class="hint-text">bb</div></div></div>'},
+        {'problem_id': '1_5_1', 'choice': ['choice_1'],
+         'expected_string': ''},
+        {'problem_id': '1_5_1', 'choice': [],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">BB </span><div class="feedback-hint-multi"><div class="hint-text">bb</div></div></div>'},
+
+        {'problem_id': '1_6_1', 'choice': ['choice_0'],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><div class="feedback-hint-multi"><div class="hint-text">aa</div></div></div>'},
+        {'problem_id': '1_6_1', 'choice': ['choice_0', 'choice_1'],
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><div class="hint-text">compoundo</div></div>'},
+
+        # The user selects *nothing*, but can still get "unselected" feedback
+        {'problem_id': '1_7_1', 'choice': [],
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="feedback-hint-multi"><div class="hint-text">bb</div></div></div>'},
+        # 100% not match of sel/unsel feedback
+        {'problem_id': '1_7_1', 'choice': ['choice_1'],
+         'expected_string': ''},
+        # Here we have the correct combination, and that makes feedback too
+        {'problem_id': '1_7_1', 'choice': ['choice_0'],
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="feedback-hint-multi"><div class="hint-text">aa</div><div class="hint-text">bb</div></div></div>'},
+    )
+    @unpack
+    def test_checkbox_hints(self, problem_id, choice, expected_string):
+        self.maxDiff = None  # pylint: disable=invalid-name
+        hint = self.get_hint(problem_id, choice)
+        assert hint == expected_string
+
+
+class CheckboxHintsTestTracking(HintTest):
+    """
+    Test the rather complicated tracking log output for checkbox cases.
+    """
+    xml = """
+    <problem>
+        <p>question</p>
+        <choiceresponse>
+        <checkboxgroup>
+            <choice correct="true">Apple
+              <choicehint selected="true">A true</choicehint>
+              <choicehint selected="false">A false</choicehint>
+            </choice>
+            <choice correct="false">Banana
+            </choice>
+            <choice correct="true">Cronut
+              <choicehint selected="true">C true</choicehint>
+            </choice>
+            <compoundhint value="A C">A C Compound</compoundhint>
+        </checkboxgroup>
+        </choiceresponse>
+    </problem>
+    """
+    problem = new_loncapa_problem(xml)
+
+    def test_tracking_log(self):
+        """Test checkbox tracking log - by far the most complicated case"""
+        # A -> 1 hint
+        self.get_hint('1_2_1', ['choice_0'])
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'hint_label': 'Incorrect:',
+             'module_id': 'i4x://Foo/bar/mock/abc',
+             'problem_part_id': '1_1',
+             'choice_all': ['choice_0', 'choice_1', 'choice_2'],
+             'correctness': False,
+             'trigger_type': 'single',
+             'student_answer': ['choice_0'],
+             'hints': [{'text': 'A true', 'trigger': [{'choice': 'choice_0', 'selected': True}]}],
+             'question_type': 'choiceresponse'}
+        )
+
+        # B C -> 2 hints
+        self.problem.capa_module.runtime.track_function.reset_mock()
+        self.get_hint('1_2_1', ['choice_1', 'choice_2'])
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'hint_label': 'Incorrect:',
+             'module_id': 'i4x://Foo/bar/mock/abc',
+             'problem_part_id': '1_1',
+             'choice_all': ['choice_0', 'choice_1', 'choice_2'],
+             'correctness': False,
+             'trigger_type': 'single',
+             'student_answer': ['choice_1', 'choice_2'],
+             'hints': [
+                 {'text': 'A false', 'trigger': [{'choice': 'choice_0', 'selected': False}]},
+                 {'text': 'C true', 'trigger': [{'choice': 'choice_2', 'selected': True}]}
+             ],
+             'question_type': 'choiceresponse'}
+        )
+
+        # A C -> 1 Compound hint
+        self.problem.capa_module.runtime.track_function.reset_mock()
+        self.get_hint('1_2_1', ['choice_0', 'choice_2'])
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'hint_label': 'Correct:',
+             'module_id': 'i4x://Foo/bar/mock/abc',
+             'problem_part_id': '1_1',
+             'choice_all': ['choice_0', 'choice_1', 'choice_2'],
+             'correctness': True,
+             'trigger_type': 'compound',
+             'student_answer': ['choice_0', 'choice_2'],
+             'hints': [
+                 {'text': 'A C Compound',
+                  'trigger': [{'choice': 'choice_0', 'selected': True}, {'choice': 'choice_2', 'selected': True}]}
+             ],
+             'question_type': 'choiceresponse'}
+        )
+
+
+@ddt
+class MultpleChoiceHintsTest(HintTest):
+    """
+    This class consists of a suite of test cases to be run on the multiple choice problem represented by the XML below.
+    """
+    xml = load_fixture('extended_hints_multiple_choice.xml')
+    problem = new_loncapa_problem(xml)
+
+    def test_tracking_log(self):
+        """Test that the tracking log comes out right."""
+        self.problem.capa_module.reset_mock()
+        self.get_hint('1_3_1', 'choice_2')
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'module_id': 'i4x://Foo/bar/mock/abc', 'problem_part_id': '1_2', 'trigger_type': 'single',
+             'student_answer': ['choice_2'], 'correctness': False, 'question_type': 'multiplechoiceresponse',
+             'hint_label': 'OOPS', 'hints': [{'text': 'Apple is a fruit.'}]}
+        )
+
+    @data(
+        {'problem_id': '1_2_1', 'choice': 'choice_0',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><div class="hint-text">Mushroom is a fungus, not a fruit.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'choice_1',
+         'expected_string': ''},
+        {'problem_id': '1_3_1', 'choice': 'choice_1',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">Potato is a root vegetable.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'choice_2',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">OUTSTANDING </span><div class="hint-text">Apple is indeed a fruit.</div></div>'},
+        {'problem_id': '1_3_1', 'choice': 'choice_2',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">OOPS </span><div class="hint-text">Apple is a fruit.</div></div>'},
+        {'problem_id': '1_3_1', 'choice': 'choice_9',
+         'expected_string': ''},
+    )
+    @unpack
+    def test_multiplechoice_hints(self, problem_id, choice, expected_string):
+        hint = self.get_hint(problem_id, choice)
+        assert hint == expected_string
+
+
+@ddt
+class MultpleChoiceHintsWithHtmlTest(HintTest):
+    """
+    This class consists of a suite of test cases to be run on the multiple choice problem represented by the XML below.
+
+    """
+    xml = load_fixture('extended_hints_multiple_choice_with_html.xml')
+    problem = new_loncapa_problem(xml)
+
+    def test_tracking_log(self):
+        """Test that the tracking log comes out right."""
+        self.problem.capa_module.reset_mock()
+        self.get_hint('1_2_1', 'choice_0')
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'module_id': 'i4x://Foo/bar/mock/abc', 'problem_part_id': '1_1', 'trigger_type': 'single',
+             'student_answer': ['choice_0'], 'correctness': False, 'question_type': 'multiplechoiceresponse',
+             'hint_label': 'Incorrect:', 'hints': [{'text': 'Mushroom <img src="#" ale="#"/>is a fungus, not a fruit.'}]}
+        )
+
+    @data(
+        {'problem_id': '1_2_1', 'choice': 'choice_0',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">Mushroom <img src="#" ale="#"/>is a fungus, not a fruit.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'choice_1',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">Potato is <img src="#" ale="#"/> not a fruit.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'choice_2',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text"><a href="#">Apple</a> is a fruit.</div></div>'}
+    )
+    @unpack
+    def test_multiplechoice_hints(self, problem_id, choice, expected_string):
+        hint = self.get_hint(problem_id, choice)
+        assert hint == expected_string
+
+
+@ddt
+class DropdownHintsTest(HintTest):
+    """
+    This class consists of a suite of test cases to be run on the drop down problem represented by the XML below.
+    """
+    xml = load_fixture('extended_hints_dropdown.xml')
+    problem = new_loncapa_problem(xml)
+
+    def test_tracking_log(self):
+        """Test that the tracking log comes out right."""
+        self.problem.capa_module.reset_mock()
+        self.get_hint('1_3_1', 'FACES')
+        self.problem.capa_module.runtime.track_function.assert_called_with(
+            'edx.problem.hint.feedback_displayed',
+            {'module_id': 'i4x://Foo/bar/mock/abc', 'problem_part_id': '1_2', 'trigger_type': 'single',
+             'student_answer': ['FACES'], 'correctness': True, 'question_type': 'optionresponse',
+             'hint_label': 'Correct:', 'hints': [{'text': 'With lots of makeup, doncha know?'}]}
+        )
+
+    @data(
+        {'problem_id': '1_2_1', 'choice': 'Multiple Choice',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Good Job </span><div class="hint-text">Yes, multiple choice is the right answer.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'Text Input',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">No, text input problems do not present options.</div></div>'},
+        {'problem_id': '1_2_1', 'choice': 'Numerical Input',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">No, numerical input problems do not present options.</div></div>'},
+        {'problem_id': '1_3_1', 'choice': 'FACES',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">With lots of makeup, doncha know?</div></div>'},
+        {'problem_id': '1_3_1', 'choice': 'dogs',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">NOPE </span><div class="hint-text">Not dogs, not cats, not toads</div></div>'},
+        {'problem_id': '1_3_1', 'choice': 'wrongo',
+         'expected_string': ''},
+
+        # Regression case where feedback includes answer substring
+        {'problem_id': '1_4_1', 'choice': 'AAA',
+         'expected_string': '<div class="feedback-hint-incorrect"><div class="explanation-title">Answer</div><span class="hint-label">Incorrect: </span><div class="hint-text">AAABBB1</div></div>'},
+        {'problem_id': '1_4_1', 'choice': 'BBB',
+         'expected_string': '<div class="feedback-hint-correct"><div class="explanation-title">Answer</div><span class="hint-label">Correct: </span><div class="hint-text">AAABBB2</div></div>'},
+        {'problem_id': '1_4_1', 'choice': 'not going to match',
+         'expected_string': ''},
+    )
+    @unpack
+    def test_dropdown_hints(self, problem_id, choice, expected_string):
+        hint = self.get_hint(problem_id, choice)
+        assert hint == expected_string
+
+
+class ErrorConditionsTest(HintTest):
+    """
+    Erroneous xml should raise exception.
+    """
+    def test_error_conditions_illegal_element(self):
+        xml_with_errors = load_fixture('extended_hints_with_errors.xml')
+        with pytest.raises(Exception):
+            new_loncapa_problem(xml_with_errors)    # this problem is improperly constructed

xmodule/capa/tests/test_html_render.py

@@ -0,0 +1,307 @@
+"""
+CAPA HTML rendering tests.
+"""
+
+
+import os
+import textwrap
+import unittest
+
+import ddt
+import mock
+from lxml import etree
+from xmodule.capa.tests.helpers import new_loncapa_problem, test_capa_system
+from openedx.core.djangolib.markup import HTML
+
+from .response_xml_factory import CustomResponseXMLFactory, StringResponseXMLFactory
+
+
+@ddt.ddt
+class CapaHtmlRenderTest(unittest.TestCase):
+    """
+    CAPA HTML rendering tests class.
+    """
+
+    def setUp(self):
+        super(CapaHtmlRenderTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.capa_system = test_capa_system()
+
+    def test_blank_problem(self):
+        """
+        It's important that blank problems don't break, since that's
+        what you start with in studio.
+        """
+        xml_str = "<problem> </problem>"
+
+        # Create the problem
+        problem = new_loncapa_problem(xml_str)
+
+        # Render the HTML
+        etree.XML(problem.get_html())
+        # TODO: This test should inspect the rendered html and assert one or more things about it
+
+    def test_include_html(self):
+        # Create a test file to include
+        self._create_test_file(
+            'test_include.xml',
+            '<test>Test include</test>'
+        )
+
+        # Generate some XML with an <include>
+        xml_str = textwrap.dedent("""
+            <problem>
+                <include file="test_include.xml"/>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = new_loncapa_problem(xml_str, capa_system=self.capa_system)
+
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the include file was embedded in the problem
+        test_element = rendered_html.find("test")
+        assert test_element.tag == 'test'
+        assert test_element.text == 'Test include'
+
+    def test_process_outtext(self):
+        # Generate some XML with <startouttext /> and <endouttext />
+        xml_str = textwrap.dedent("""
+            <problem>
+            <startouttext/>Test text<endouttext/>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = new_loncapa_problem(xml_str)
+
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the <startouttext /> and <endouttext />
+        # were converted to <span></span> tags
+        span_element = rendered_html.find('span')
+        assert span_element.text == 'Test text'
+
+    def test_anonymous_student_id(self):
+        # make sure anonymous_student_id is rendered properly as a context variable
+        xml_str = textwrap.dedent("""
+            <problem>
+            <span>Welcome $anonymous_student_id</span>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = new_loncapa_problem(xml_str)
+
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the anonymous_student_id was converted to "student"
+        span_element = rendered_html.find('span')
+        assert span_element.text == 'Welcome student'
+
+    def test_render_script(self):
+        # Generate some XML with a <script> tag
+        xml_str = textwrap.dedent("""
+            <problem>
+                <script>test=True</script>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = new_loncapa_problem(xml_str)
+
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the script element has been removed from the rendered HTML
+        script_element = rendered_html.find('script')
+        assert script_element is None
+
+    def test_render_javascript(self):
+        # Generate some XML with a <script> tag
+        xml_str = textwrap.dedent("""
+            <problem>
+                <script type="text/javascript">function(){}</script>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = new_loncapa_problem(xml_str)
+
+        # Render the HTML
+        rendered_html = etree.XML(problem.get_html())
+
+        # expect the javascript is still present in the rendered html
+        assert '<script type="text/javascript">function(){}</script>' in etree.tostring(rendered_html).decode('utf-8')
+
+    def test_render_response_xml(self):
+        # Generate some XML for a string response
+        kwargs = {
+            'question_text': "Test question",
+            'explanation_text': "Test explanation",
+            'answer': 'Test answer',
+            'hints': [('test prompt', 'test_hint', 'test hint text')]
+        }
+        xml_str = StringResponseXMLFactory().build_xml(**kwargs)
+
+        # Mock out the template renderer
+        the_system = test_capa_system()
+        the_system.render_template = mock.Mock()
+        the_system.render_template.return_value = "<div class='input-template-render'>Input Template Render</div>"
+
+        # Create the problem and render the HTML
+        problem = new_loncapa_problem(xml_str, capa_system=the_system)
+        rendered_html = etree.XML(problem.get_html())
+        # Expect problem has been turned into a <div>
+        assert rendered_html.tag == 'div'
+
+        # Expect that the response has been turned into a <div> with correct attributes
+        response_element = rendered_html.find('div')
+
+        assert response_element.tag == 'div'
+        assert response_element.attrib['aria-label'] == 'Question 1'
+
+        # Expect that the response div.wrapper-problem-response
+        # that contains a <div> for the textline
+        textline_element = response_element.find('div')
+        assert textline_element.text == 'Input Template Render'
+
+        # Expect a child <div> for the solution
+        # with the rendered template
+        solution_element = rendered_html.xpath('//div[@class="input-template-render"]')[0]
+        assert solution_element.text == 'Input Template Render'
+
+        # Expect that the template renderer was called with the correct
+        # arguments, once for the textline input and once for
+        # the solution
+        expected_textline_context = {
+            'STATIC_URL': '/dummy-static/',
+            'status': the_system.STATUS_CLASS('unsubmitted'),
+            'value': '',
+            'preprocessor': None,
+            'msg': '',
+            'inline': False,
+            'hidden': False,
+            'do_math': False,
+            'id': '1_2_1',
+            'trailing_text': '',
+            'size': None,
+            'response_data': {'label': 'Test question', 'descriptions': {}},
+            'describedby_html': HTML('aria-describedby="status_1_2_1"')
+        }
+
+        expected_solution_context = {'id': '1_solution_1'}
+
+        expected_calls = [
+            mock.call('textline.html', expected_textline_context),
+            mock.call('solutionspan.html', expected_solution_context),
+            mock.call('textline.html', expected_textline_context),
+            mock.call('solutionspan.html', expected_solution_context)
+        ]
+
+        assert the_system.render_template.call_args_list == expected_calls
+
+    def test_correct_aria_label(self):
+        xml = """
+                 <problem>
+                     <choiceresponse>
+                         <checkboxgroup>
+                             <choice correct="true">over-suspicious</choice>
+                             <choice correct="false">funny</choice>
+                         </checkboxgroup>
+                     </choiceresponse>
+                     <choiceresponse>
+                         <checkboxgroup>
+                             <choice correct="true">Urdu</choice>
+                             <choice correct="false">Finnish</choice>
+                         </checkboxgroup>
+                     </choiceresponse>
+                 </problem>
+                 """
+        problem = new_loncapa_problem(xml)
+        rendered_html = etree.XML(problem.get_html())
+        response_elements = rendered_html.findall('div')
+        assert response_elements[0].attrib['aria-label'] == 'Question 1'
+        assert response_elements[1].attrib['aria-label'] == 'Question 2'
+
+    def test_render_response_with_overall_msg(self):
+        # CustomResponse script that sets an overall_message
+        script = textwrap.dedent("""
+            def check_func(*args):
+                msg = '<p>Test message 1<br /></p><p>Test message 2</p>'
+                return {'overall_message': msg,
+                        'input_list': [ {'ok': True, 'msg': '' } ] }
+        """)
+
+        # Generate some XML for a CustomResponse
+        kwargs = {'script': script, 'cfn': 'check_func'}
+        xml_str = CustomResponseXMLFactory().build_xml(**kwargs)
+
+        # Create the problem and render the html
+        problem = new_loncapa_problem(xml_str)
+
+        # Grade the problem
+        problem.grade_answers({'1_2_1': 'test'})
+
+        # Render the html
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that there is a <div> within the response <div>
+        # with css class response_message
+        msg_div_element = rendered_html.find(".//div[@class='response_message']")
+        assert msg_div_element.tag == 'div'
+        assert msg_div_element.get('class') == 'response_message'
+
+        # Expect that the <div> contains our message (as part of the XML tree)
+        msg_p_elements = msg_div_element.findall('p')
+        assert msg_p_elements[0].tag == 'p'
+        assert msg_p_elements[0].text == 'Test message 1'
+
+        assert msg_p_elements[1].tag == 'p'
+        assert msg_p_elements[1].text == 'Test message 2'
+
+    def test_substitute_python_vars(self):
+        # Generate some XML with Python variables defined in a script
+        # and used later as attributes
+        xml_str = textwrap.dedent("""
+            <problem>
+                <script>test="TEST"</script>
+                <span attr="$test"></span>
+            </problem>
+        """)
+
+        # Create the problem and render the HTML
+        problem = new_loncapa_problem(xml_str)
+        rendered_html = etree.XML(problem.get_html())
+
+        # Expect that the variable $test has been replaced with its value
+        span_element = rendered_html.find('span')
+        assert span_element.get('attr') == 'TEST'
+
+    def test_xml_comments_and_other_odd_things(self):
+        # Comments and processing instructions should be skipped.
+        xml_str = textwrap.dedent("""\
+            <?xml version="1.0" encoding="utf-8"?>
+            <!DOCTYPE html []>
+            <problem>
+            <!-- A commment. -->
+            <?ignore this processing instruction. ?>
+            </problem>
+        """)
+
+        # Create the problem
+        problem = new_loncapa_problem(xml_str)
+
+        # Render the HTML
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>\s*</div>")
+
+    def _create_test_file(self, path, content_str):  # lint-amnesty, pylint: disable=missing-function-docstring
+        test_fp = self.capa_system.resources_fs.open(path, "w")
+        test_fp.write(content_str)
+        test_fp.close()
+
+        self.addCleanup(lambda: os.remove(test_fp.name))

xmodule/capa/tests/test_input_templates.py

@@ -0,0 +1,1217 @@
+"""
+Tests for the logic in input type mako templates.
+"""
+
+
+import json
+import unittest
+from collections import OrderedDict
+
+from lxml import etree
+from mako import exceptions
+from six.moves import range
+
+from xmodule.capa.inputtypes import Status
+from xmodule.capa.tests.helpers import capa_render_template
+from openedx.core.djangolib.markup import HTML
+from xmodule.stringify import stringify_children
+
+
+class TemplateError(Exception):
+    """
+    Error occurred while rendering a Mako template.
+    """
+    pass  # lint-amnesty, pylint: disable=unnecessary-pass
+
+
+class TemplateTestCase(unittest.TestCase):
+    """
+    Utilities for testing templates.
+    """
+
+    # Subclasses override this to specify the file name of the template
+    # to be loaded from capa/templates.
+    # The template name should include the .html extension:
+    # for example: choicegroup.html
+    TEMPLATE_NAME = None
+    DESCRIBEDBY = 'aria-describedby="desc-1 desc-2"'
+    DESCRIPTIONS = OrderedDict(
+        [
+            ('desc-1', 'description text 1'),
+            ('desc-2', '<em>description</em> <mark>text</mark> 2')
+        ]
+    )
+    DESCRIPTION_IDS = ' '.join(list(DESCRIPTIONS.keys()))
+    RESPONSE_DATA = {
+        'label': 'question text 101',
+        'descriptions': DESCRIPTIONS
+    }
+
+    def setUp(self):
+        """
+        Initialize the context.
+        """
+        super(TemplateTestCase, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {}
+
+    def render_to_xml(self, context_dict):
+        """
+        Render the template using the `context_dict` dict.
+        Returns an `etree` XML element.
+        """
+        # add dummy STATIC_URL to template context
+        context_dict.setdefault("STATIC_URL", "/dummy-static/")
+        try:
+            xml_str = capa_render_template(self.TEMPLATE_NAME, context_dict)
+        except:
+            raise TemplateError(exceptions.text_error_template().render())  # lint-amnesty, pylint: disable=raise-missing-from
+
+        # Attempt to construct an XML tree from the template
+        # This makes it easy to use XPath to make assertions, rather
+        # than dealing with a string.
+        # We modify the string slightly by wrapping it in <test>
+        # tags, to ensure it has one root element.
+        try:
+            xml = etree.fromstring("<test>" + xml_str + "</test>")
+        except Exception as exc:
+            raise TemplateError("Could not parse XML from '{0}': {1}".format(  # lint-amnesty, pylint: disable=raise-missing-from
+                                xml_str, str(exc)))
+        else:
+            return xml
+
+    def assert_has_xpath(self, xml_root, xpath, context_dict, exact_num=1):
+        """
+        Asserts that the xml tree has an element satisfying `xpath`.
+
+        `xml_root` is an etree XML element
+        `xpath` is an XPath string, such as `'/foo/bar'`
+        `context` is used to print a debugging message
+        `exact_num` is the exact number of matches to expect.
+        """
+        message = ("XML does not have %d match(es) for xpath '%s'\nXML: %s\nContext: %s"
+                   % (exact_num, str(xpath), etree.tostring(xml_root), str(context_dict)))
+
+        assert len(xml_root.xpath(xpath)) == exact_num, message
+
+    def assert_no_xpath(self, xml_root, xpath, context_dict):
+        """
+        Asserts that the xml tree does NOT have an element
+        satisfying `xpath`.
+
+        `xml_root` is an etree XML element
+        `xpath` is an XPath string, such as `'/foo/bar'`
+        `context` is used to print a debugging message
+        """
+        self.assert_has_xpath(xml_root, xpath, context_dict, exact_num=0)
+
+    def assert_has_text(self, xml_root, xpath, text, exact=True):
+        """
+        Find the element at `xpath` in `xml_root` and assert
+        that its text is `text`.
+
+        `xml_root` is an etree XML element
+        `xpath` is an XPath string, such as `'/foo/bar'`
+        `text` is the expected text that the element should contain
+
+        If multiple elements are found, checks the first one.
+        If no elements are found, the assertion fails.
+        """
+        element_list = xml_root.xpath(xpath)
+        assert len(element_list) > 0, ("Could not find element at '%s'\n%s" % (str(xpath), etree.tostring(xml_root)))
+        if exact:
+            assert text == element_list[0].text.strip()
+        else:
+            assert text in element_list[0].text.strip()
+
+    def assert_description(self, describedby_xpaths):
+        """
+        Verify that descriptions information is correct.
+
+        Arguments:
+            describedby_xpaths (list): list of xpaths to check aria-describedby attribute
+        """
+        xml = self.render_to_xml(self.context)
+
+        # Verify that each description <p> tag has correct id, text and order
+        descriptions = OrderedDict(
+            (tag.get('id'), stringify_children(tag)) for tag in xml.xpath('//p[@class="question-description"]')
+        )
+        assert self.DESCRIPTIONS == descriptions
+
+        # for each xpath verify that description_ids are set correctly
+        for describedby_xpath in describedby_xpaths:
+            describedbys = xml.xpath(describedby_xpath)
+
+            # aria-describedby attributes must have ids
+            assert describedbys
+
+            for describedby in describedbys:
+                assert describedby == self.DESCRIPTION_IDS
+
+    def assert_describedby_attribute(self, describedby_xpaths):
+        """
+        Verify that an element has no aria-describedby attribute if there are no descriptions.
+
+        Arguments:
+            describedby_xpaths (list): list of xpaths to check aria-describedby attribute
+        """
+        self.context['describedby_html'] = ''
+        xml = self.render_to_xml(self.context)
+
+        # for each xpath verify that description_ids are set correctly
+        for describedby_xpath in describedby_xpaths:
+            describedbys = xml.xpath(describedby_xpath)
+            assert not describedbys
+
+    def assert_status(self, status_div=False, status_class=False):
+        """
+        Verify status information.
+
+        Arguments:
+            status_div (bool): check presence of status div
+            status_class (bool): check presence of status class
+        """
+        cases = [
+            ('correct', 'correct'),
+            ('unsubmitted', 'unanswered'),
+            ('submitted', 'submitted'),
+            ('incorrect', 'incorrect'),
+            ('incomplete', 'incorrect')
+        ]
+
+        for context_status, div_class in cases:
+            self.context['status'] = Status(context_status)
+            xml = self.render_to_xml(self.context)
+
+            # Expect that we get a <div> with correct class
+            if status_div:
+                xpath = "//div[normalize-space(@class)='%s']" % div_class
+                self.assert_has_xpath(xml, xpath, self.context)
+
+            # Expect that we get a <span> with class="status"
+            # (used to by CSS to draw the green check / red x)
+            self.assert_has_text(
+                xml,
+                "//span[@class='status {}']/span[@class='sr']".format(
+                    div_class if status_class else ''
+                ),
+                self.context['status'].display_name
+            )
+
+    def assert_label(self, xpath=None, aria_label=False):
+        """
+        Verify label is rendered correctly.
+
+        Arguments:
+            xpath (str): xpath expression for label element
+            aria_label (bool): check aria-label attribute value
+        """
+        labels = [
+            {
+                'actual': "You see, but you do not observe. The distinction is clear.",
+                'expected': "You see, but you do not observe. The distinction is clear.",
+            },
+            {
+                'actual': "I choose to have <mark>faith</mark> because without that, I have <em>nothing</em>.",
+                'expected': "I choose to have faith because without that, I have nothing.",
+            }
+        ]
+
+        response_data = {
+            'response_data': {
+                'descriptions': {},
+                'label': ''
+            }
+        }
+        self.context.update(response_data)
+
+        for label in labels:
+            self.context['response_data']['label'] = label['actual']
+            xml = self.render_to_xml(self.context)
+
+            if aria_label:
+                self.assert_has_xpath(xml, "//*[@aria-label='%s']" % label['expected'], self.context)
+            else:
+                element_list = xml.xpath(xpath)
+                assert len(element_list) == 1
+                assert stringify_children(element_list[0]) == label['actual']
+
+
+class ChoiceGroupTemplateTest(TemplateTestCase):
+    """
+    Test mako template for `<choicegroup>` input.
+    """
+
+    TEMPLATE_NAME = 'choicegroup.html'
+
+    def setUp(self):
+        super(ChoiceGroupTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        choices = [('1', 'choice 1'), ('2', 'choice 2'), ('3', 'choice 3')]
+        self.context = {
+            'id': '1',
+            'choices': choices,
+            'status': Status('correct'),
+            'input_type': 'checkbox',
+            'name_array_suffix': '1',
+            'value': '3',
+            'response_data': self.RESPONSE_DATA,
+            'describedby_html': HTML(self.DESCRIBEDBY),
+        }
+
+    def test_problem_marked_correct(self):
+        """
+        Test conditions under which the entire problem
+        (not a particular option) is marked correct.
+        """
+
+        self.context['status'] = Status('correct')
+        self.context['input_type'] = 'checkbox'
+        self.context['value'] = ['1', '2']
+
+        # Should mark the entire problem correct
+        xml = self.render_to_xml(self.context)
+        xpath = "//div[@class='indicator-container']/span[@class='status correct']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        # Should NOT mark individual options
+        self.assert_no_xpath(xml, "//label[@class='choicegroup_incorrect']",
+                             self.context)
+
+        self.assert_no_xpath(xml, "//label[@class='choicegroup_correct']",
+                             self.context)
+
+    def test_problem_marked_incorrect(self):
+        """
+        Test all conditions under which the entire problem
+        (not a particular option) is marked incorrect.
+        """
+        conditions = [
+            {'status': Status('incorrect'), 'input_type': 'checkbox', 'value': []},
+            {'status': Status('incorrect'), 'input_type': 'checkbox', 'value': ['2']},
+            {'status': Status('incorrect'), 'input_type': 'checkbox', 'value': ['2', '3']},
+            {'status': Status('incomplete'), 'input_type': 'checkbox', 'value': []},
+            {'status': Status('incomplete'), 'input_type': 'checkbox', 'value': ['2']},
+            {'status': Status('incomplete'), 'input_type': 'checkbox', 'value': ['2', '3']}]
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//div[@class='indicator-container']/span[@class='status incorrect']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should NOT mark individual options
+            self.assert_no_xpath(xml,
+                                 "//label[@class='choicegroup_incorrect']",
+                                 self.context)
+
+            self.assert_no_xpath(xml,
+                                 "//label[@class='choicegroup_correct']",
+                                 self.context)
+
+    def test_problem_marked_unsubmitted(self):
+        """
+        Test all conditions under which the entire problem
+        (not a particular option) is marked unanswered.
+        """
+        conditions = [
+            {'status': Status('unsubmitted'), 'input_type': 'radio', 'value': ''},
+            {'status': Status('unsubmitted'), 'input_type': 'radio', 'value': []},
+            {'status': Status('unsubmitted'), 'input_type': 'checkbox', 'value': []},
+            {'input_type': 'radio', 'value': ''},
+            {'input_type': 'radio', 'value': []},
+            {'input_type': 'checkbox', 'value': []},
+            {'input_type': 'checkbox', 'value': ['1']},
+            {'input_type': 'checkbox', 'value': ['1', '2']}]
+
+        self.context['status'] = Status('unanswered')
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//div[@class='indicator-container']/span[@class='status unanswered']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should NOT mark individual options
+            self.assert_no_xpath(xml,
+                                 "//label[@class='choicegroup_incorrect']",
+                                 self.context)
+
+            self.assert_no_xpath(xml,
+                                 "//label[@class='choicegroup_correct']",
+                                 self.context)
+
+    def test_option_marked_correct(self):
+        """
+        Test conditions under which a particular option
+        and the entire problem is marked correct.
+        """
+        conditions = [
+            {'input_type': 'radio', 'value': '2'},
+            {'input_type': 'radio', 'value': ['2']}]
+
+        self.context['status'] = Status('correct')
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//label[contains(@class, 'choicegroup_correct')]"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should also mark the whole problem
+            xpath = "//div[@class='indicator-container']/span[@class='status correct']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_option_marked_incorrect(self):
+        """
+        Test conditions under which a particular option
+        and the entire problem is marked incorrect.
+        """
+        conditions = [
+            {'input_type': 'radio', 'value': '2'},
+            {'input_type': 'radio', 'value': ['2']}]
+
+        self.context['status'] = Status('incorrect')
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//label[contains(@class, 'choicegroup_incorrect')]"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should also mark the whole problem
+            xpath = "//div[@class='indicator-container']/span[@class='status incorrect']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_never_show_correctness(self):
+        """
+        Test conditions under which we tell the template to
+        NOT show correct/incorrect, but instead show a message.
+
+        This is used, for example, by the Justice course to ask
+        questions without specifying a correct answer.  When
+        the student responds, the problem displays "Thank you
+        for your response"
+        """
+
+        conditions = [
+            {'input_type': 'radio', 'status': Status('correct'), 'value': ''},
+            {'input_type': 'radio', 'status': Status('correct'), 'value': '2'},
+            {'input_type': 'radio', 'status': Status('correct'), 'value': ['2']},
+            {'input_type': 'radio', 'status': Status('incorrect'), 'value': '2'},
+            {'input_type': 'radio', 'status': Status('incorrect'), 'value': []},
+            {'input_type': 'radio', 'status': Status('incorrect'), 'value': ['2']},
+            {'input_type': 'checkbox', 'status': Status('correct'), 'value': []},
+            {'input_type': 'checkbox', 'status': Status('correct'), 'value': ['2']},
+            {'input_type': 'checkbox', 'status': Status('incorrect'), 'value': []},
+            {'input_type': 'checkbox', 'status': Status('incorrect'), 'value': ['2']}]
+
+        self.context['show_correctness'] = 'never'
+        self.context['submitted_message'] = 'Test message'
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+
+            # Should NOT mark the entire problem correct/incorrect
+            xpath = "//div[@class='indicator-container']/span[@class='status correct']"
+            self.assert_no_xpath(xml, xpath, self.context)
+
+            xpath = "//div[@class='indicator-container']/span[@class='status incorrect']"
+            self.assert_no_xpath(xml, xpath, self.context)
+
+            # Should NOT mark individual options
+            self.assert_no_xpath(xml,
+                                 "//label[@class='choicegroup_incorrect']",
+                                 self.context)
+
+            self.assert_no_xpath(xml,
+                                 "//label[@class='choicegroup_correct']",
+                                 self.context)
+
+            # Expect to see the message
+            self.assert_has_text(xml, "//div[@class='capa_alert']",
+                                 self.context['submitted_message'])
+
+    def test_no_message_before_submission(self):
+        """
+        Ensure that we don't show the `submitted_message`
+        before submitting.
+        """
+
+        conditions = [
+            {'input_type': 'radio', 'status': Status('unsubmitted'), 'value': ''},
+            {'input_type': 'radio', 'status': Status('unsubmitted'), 'value': []},
+            {'input_type': 'checkbox', 'status': Status('unsubmitted'), 'value': []},
+
+            # These tests expose bug #365
+            # When the bug is fixed, uncomment these cases.
+            #{'input_type': 'radio', 'status': 'unsubmitted', 'value': '2'},
+            #{'input_type': 'radio', 'status': 'unsubmitted', 'value': ['2']},
+            #{'input_type': 'radio', 'status': 'unsubmitted', 'value': '2'},
+            #{'input_type': 'radio', 'status': 'unsubmitted', 'value': ['2']},
+            #{'input_type': 'checkbox', 'status': 'unsubmitted', 'value': ['2']},
+            #{'input_type': 'checkbox', 'status': 'unsubmitted', 'value': ['2']}]
+        ]
+
+        self.context['show_correctness'] = 'never'
+        self.context['submitted_message'] = 'Test message'
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+
+            # Expect that we do NOT see the message yet
+            self.assert_no_xpath(xml, "//div[@class='capa_alert']", self.context)
+
+    def test_label(self):
+        """
+        Verify label element value rendering.
+        """
+        self.assert_label(xpath="//legend")
+
+    def test_description(self):
+        """
+        Test that correct description information is set on desired elements.
+        """
+        xpaths = ['//fieldset/@aria-describedby', '//label/@aria-describedby']
+        self.assert_description(xpaths)
+        self.assert_describedby_attribute(xpaths)
+
+    def test_status(self):
+        """
+        Verify status information.
+        """
+        self.assert_status(status_class=True)
+
+
+class TextlineTemplateTest(TemplateTestCase):
+    """
+    Test mako template for `<textline>` input.
+    """
+
+    TEMPLATE_NAME = 'textline.html'
+
+    def setUp(self):
+        super(TextlineTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {
+            'id': '1',
+            'status': Status('correct'),
+            'value': '3',
+            'preprocessor': None,
+            'trailing_text': None,
+            'response_data': self.RESPONSE_DATA,
+            'describedby_html': HTML(self.DESCRIBEDBY),
+        }
+
+    def test_section_class(self):
+        cases = [({}, ' capa_inputtype  textline'),
+                 ({'do_math': True}, 'text-input-dynamath capa_inputtype  textline'),
+                 ({'inline': True}, ' capa_inputtype inline textline'),
+                 ({'do_math': True, 'inline': True}, 'text-input-dynamath capa_inputtype inline textline'), ]
+
+        for (context, css_class) in cases:
+            base_context = self.context.copy()
+            base_context.update(context)
+            xml = self.render_to_xml(base_context)
+            xpath = "//div[@class='%s']" % css_class
+            self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_status(self):
+        """
+        Verify status information.
+        """
+        self.assert_status(status_class=True)
+
+    def test_label(self):
+        """
+        Verify label element value rendering.
+        """
+        self.assert_label(xpath="//label[@class='problem-group-label']")
+
+    def test_hidden(self):
+        self.context['hidden'] = True
+        xml = self.render_to_xml(self.context)
+
+        xpath = "//div[@style='display:none;']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        xpath = "//input[@style='display:none;']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_do_math(self):
+        self.context['do_math'] = True
+        xml = self.render_to_xml(self.context)
+
+        xpath = "//input[@class='mw-100 math']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        xpath = "//div[@class='equation']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        xpath = "//textarea[@id='input_1_dynamath']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_size(self):
+        self.context['size'] = '20'
+        xml = self.render_to_xml(self.context)
+
+        xpath = "//input[@size='20']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_preprocessor(self):
+        self.context['preprocessor'] = {'class_name': 'test_class',
+                                        'script_src': 'test_script'}
+        xml = self.render_to_xml(self.context)
+
+        xpath = "//div[contains(@class, 'text-input-dynamath_data') and @data-preprocessor='test_class']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        xpath = "//div[@class='script_placeholder' and @data-src='test_script']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_do_inline_and_preprocessor(self):
+        self.context['preprocessor'] = {'class_name': 'test_class',
+                                        'script_src': 'test_script'}
+        self.context['inline'] = True
+        xml = self.render_to_xml(self.context)
+
+        xpath = "//div[contains(@class, 'text-input-dynamath_data inline') and @data-preprocessor='test_class']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_do_inline(self):
+        cases = [('correct', 'correct'),
+                 ('unsubmitted', 'unanswered'),
+                 ('incorrect', 'incorrect'),
+                 ('incomplete', 'incorrect')]
+
+        self.context['inline'] = True
+
+        for (context_status, div_class) in cases:
+            self.context['status'] = Status(context_status)
+            xml = self.render_to_xml(self.context)
+
+            # Expect that we get a <div> with correct class
+            xpath = "//div[@class='%s inline']" % div_class
+            self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_message(self):
+        self.context['msg'] = "Test message"
+        xml = self.render_to_xml(self.context)
+
+        xpath = "//span[@class='message']"
+        self.assert_has_text(xml, xpath, self.context['msg'])
+
+    def test_description(self):
+        """
+        Test that correct description information is set on desired elements.
+        """
+        xpaths = ['//input/@aria-describedby']
+        self.assert_description(xpaths)
+        self.assert_describedby_attribute(xpaths)
+
+
+class FormulaEquationInputTemplateTest(TemplateTestCase):
+    """
+    Test make template for `<formulaequationinput>`s.
+    """
+    TEMPLATE_NAME = 'formulaequationinput.html'
+
+    def setUp(self):
+        super(FormulaEquationInputTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {
+            'id': 2,
+            'value': 'PREFILLED_VALUE',
+            'status': Status('unsubmitted'),
+            'previewer': 'file.js',
+            'reported_status': 'REPORTED_STATUS',
+            'trailing_text': None,
+            'response_data': self.RESPONSE_DATA,
+            'describedby_html': HTML(self.DESCRIBEDBY),
+        }
+
+    def test_no_size(self):
+        xml = self.render_to_xml(self.context)
+        self.assert_no_xpath(xml, "//input[@size]", self.context)
+
+    def test_size(self):
+        self.context['size'] = '40'
+        xml = self.render_to_xml(self.context)
+
+        self.assert_has_xpath(xml, "//input[@size='40']", self.context)
+
+    def test_description(self):
+        """
+        Test that correct description information is set on desired elements.
+        """
+        xpaths = ['//input/@aria-describedby']
+        self.assert_description(xpaths)
+        self.assert_describedby_attribute(xpaths)
+
+    def test_status(self):
+        """
+        Verify status information.
+        """
+        self.assert_status(status_class=True)
+
+    def test_label(self):
+        """
+        Verify label element value rendering.
+        """
+        self.assert_label(xpath="//label[@class='problem-group-label']")
+
+
+class AnnotationInputTemplateTest(TemplateTestCase):
+    """
+    Test mako template for `<annotationinput>` input.
+    """
+
+    TEMPLATE_NAME = 'annotationinput.html'
+
+    def setUp(self):
+        super(AnnotationInputTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {
+            'id': 2,
+            'value': '<p>Test value</p>',
+            'title': '<h1>This is a title</h1>',
+            'text': '<p><b>This</b> is a test.</p>',
+            'comment': '<p>This is a test comment</p>',
+            'comment_prompt': '<p>This is a test comment prompt</p>',
+            'comment_value': '<p>This is the value of a test comment</p>',
+            'tag_prompt': '<p>This is a tag prompt</p>',
+            'options': [],
+            'has_options_value': False,
+            'debug': False,
+            'status': Status('unsubmitted'),
+            'return_to_annotation': False,
+            'msg': '<p>This is a test message</p>',
+        }
+
+    def test_return_to_annotation(self):
+        """
+        Test link for `Return to Annotation` appears if and only if
+        the flag is set.
+        """
+
+        xpath = "//a[@class='annotation-return']"
+
+        # If return_to_annotation set, then show the link
+        self.context['return_to_annotation'] = True
+        xml = self.render_to_xml(self.context)
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        # Otherwise, do not show the links
+        self.context['return_to_annotation'] = False
+        xml = self.render_to_xml(self.context)
+        self.assert_no_xpath(xml, xpath, self.context)
+
+    def test_option_selection(self):
+        """
+        Test that selected options are selected.
+        """
+
+        # Create options 0-4 and select option 2
+        self.context['options_value'] = [2]
+        self.context['options'] = [
+            {'id': id_num,
+             'choice': 'correct',
+             'description': '<p>Unescaped <b>HTML {0}</b></p>'.format(id_num)}
+            for id_num in range(5)]
+
+        xml = self.render_to_xml(self.context)
+
+        # Expect that each option description is visible
+        # with unescaped HTML.
+        # Since the HTML is unescaped, we can traverse the XML tree
+        for id_num in range(5):
+            xpath = "//span[@data-id='{0}']/p/b".format(id_num)
+            self.assert_has_text(xml, xpath, 'HTML {0}'.format(id_num), exact=False)
+
+        # Expect that the correct option is selected
+        xpath = "//span[contains(@class,'selected')]/p/b"
+        self.assert_has_text(xml, xpath, 'HTML 2', exact=False)
+
+    def test_submission_status(self):
+        """
+        Test that the submission status displays correctly.
+        """
+
+        # Test cases of `(input_status, expected_css_class)` tuples
+        test_cases = [('unsubmitted', 'unanswered'),
+                      ('incomplete', 'incorrect'),
+                      ('incorrect', 'incorrect')]
+
+        for (input_status, expected_css_class) in test_cases:
+            self.context['status'] = Status(input_status)
+            xml = self.render_to_xml(self.context)
+
+            xpath = "//span[@class='status {0}']".format(expected_css_class)
+            self.assert_has_xpath(xml, xpath, self.context)
+
+        # If individual options are being marked, then expect
+        # just the option to be marked incorrect, not the whole problem
+        self.context['has_options_value'] = True
+        self.context['status'] = Status('incorrect')
+        xpath = "//span[@class='incorrect']"
+        xml = self.render_to_xml(self.context)
+        self.assert_no_xpath(xml, xpath, self.context)
+
+    def test_display_html_comment(self):
+        """
+        Test that HTML comment and comment prompt render.
+        """
+        self.context['comment'] = "<p>Unescaped <b>comment HTML</b></p>"
+        self.context['comment_prompt'] = "<p>Prompt <b>prompt HTML</b></p>"
+        self.context['text'] = "<p>Unescaped <b>text</b></p>"
+        xml = self.render_to_xml(self.context)
+
+        # Because the HTML is unescaped, we should be able to
+        # descend to the <b> tag
+        xpath = "//div[@class='block']/p/b"
+        self.assert_has_text(xml, xpath, 'prompt HTML')
+
+        xpath = "//div[@class='block block-comment']/p/b"
+        self.assert_has_text(xml, xpath, 'comment HTML')
+
+        xpath = "//div[@class='block block-highlight']/p/b"
+        self.assert_has_text(xml, xpath, 'text')
+
+    def test_display_html_tag_prompt(self):
+        """
+        Test that HTML tag prompts render.
+        """
+        self.context['tag_prompt'] = "<p>Unescaped <b>HTML</b></p>"
+        xml = self.render_to_xml(self.context)
+
+        # Because the HTML is unescaped, we should be able to
+        # descend to the <b> tag
+        xpath = "//div[@class='block']/p/b"
+        self.assert_has_text(xml, xpath, 'HTML')
+
+
+class MathStringTemplateTest(TemplateTestCase):
+    """
+    Test mako template for `<mathstring>` input.
+    """
+
+    TEMPLATE_NAME = 'mathstring.html'
+
+    def setUp(self):
+        super(MathStringTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {'isinline': False, 'mathstr': '', 'tail': ''}
+
+    def test_math_string_inline(self):
+        self.context['isinline'] = True
+        self.context['mathstr'] = 'y = ax^2 + bx + c'
+
+        xml = self.render_to_xml(self.context)
+        xpath = "//section[@class='math-string']/span[1]"
+        self.assert_has_text(xml, xpath,
+                             '[mathjaxinline]y = ax^2 + bx + c[/mathjaxinline]')
+
+    def test_math_string_not_inline(self):
+        self.context['isinline'] = False
+        self.context['mathstr'] = 'y = ax^2 + bx + c'
+
+        xml = self.render_to_xml(self.context)
+        xpath = "//section[@class='math-string']/span[1]"
+        self.assert_has_text(xml, xpath,
+                             '[mathjax]y = ax^2 + bx + c[/mathjax]')
+
+    def test_tail_html(self):
+        self.context['tail'] = "<p>This is some <b>tail</b> <em>HTML</em></p>"
+        xml = self.render_to_xml(self.context)
+
+        # HTML from `tail` should NOT be escaped.
+        # We should be able to traverse it as part of the XML tree
+        xpath = "//section[@class='math-string']/span[2]/p/b"
+        self.assert_has_text(xml, xpath, 'tail')
+
+        xpath = "//section[@class='math-string']/span[2]/p/em"
+        self.assert_has_text(xml, xpath, 'HTML')
+
+
+class OptionInputTemplateTest(TemplateTestCase):
+    """
+    Test mako template for `<optioninput>` input.
+    """
+
+    TEMPLATE_NAME = 'optioninput.html'
+
+    def setUp(self):
+        super(OptionInputTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {
+            'id': 2,
+            'options': [],
+            'status': Status('unsubmitted'),
+            'value': 0,
+            'default_option_text': 'Select an option',
+            'response_data': self.RESPONSE_DATA,
+            'describedby_html': HTML(self.DESCRIBEDBY),
+        }
+
+    def test_select_options(self):
+
+        # Create options 0-4, and select option 2
+        self.context['options'] = [(id_num, 'Option {0}'.format(id_num))
+                                   for id_num in range(5)]
+        self.context['value'] = 2
+
+        xml = self.render_to_xml(self.context)
+
+        # Should have a dummy default
+        xpath = "//option[@value='option_2_dummy_default']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        for id_num in range(5):
+            xpath = "//option[@value='{0}']".format(id_num)
+            self.assert_has_text(xml, xpath, 'Option {0}'.format(id_num))
+
+        # Should have the correct option selected
+        xpath = "//option[@selected='true']"
+        self.assert_has_text(xml, xpath, 'Option 2')
+
+    def test_status(self):
+        """
+        Verify status information.
+        """
+        self.assert_status(status_class=True)
+
+    def test_label(self):
+        """
+        Verify label element value rendering.
+        """
+        self.assert_label(xpath="//label[@class='problem-group-label']")
+
+    def test_description(self):
+        """
+        Test that correct description information is set on desired elements.
+        """
+        xpaths = ['//select/@aria-describedby']
+        self.assert_description(xpaths)
+        self.assert_describedby_attribute(xpaths)
+
+
+class DragAndDropTemplateTest(TemplateTestCase):
+    """
+    Test mako template for `<draganddropinput>` input.
+    """
+
+    TEMPLATE_NAME = 'drag_and_drop_input.html'
+
+    def setUp(self):
+        super(DragAndDropTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {'id': 2,
+                        'drag_and_drop_json': '',
+                        'value': 0,
+                        'status': Status('unsubmitted'),
+                        'msg': ''}
+
+    def test_status(self):
+
+        # Test cases, where each tuple represents
+        # `(input_status, expected_css_class, expected_text)`
+        test_cases = [('unsubmitted', 'unanswered', 'unanswered'),
+                      ('correct', 'correct', 'correct'),
+                      ('incorrect', 'incorrect', 'incorrect'),
+                      ('incomplete', 'incorrect', 'incomplete')]
+
+        for (input_status, expected_css_class, expected_text) in test_cases:
+            self.context['status'] = Status(input_status)
+            xml = self.render_to_xml(self.context)
+
+            # Expect a <div> with the status
+            xpath = "//div[@class='{0}']".format(expected_css_class)
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Expect a <span> with the status
+            xpath = "//span[@class='status {0}']/span[@class='sr']".format(expected_css_class)
+            self.assert_has_text(xml, xpath, expected_text, exact=False)
+
+    def test_drag_and_drop_json_html(self):
+
+        json_with_html = json.dumps({'test': '<p>Unescaped <b>HTML</b></p>'})
+        self.context['drag_and_drop_json'] = json_with_html
+        xml = self.render_to_xml(self.context)
+
+        # Assert that the JSON-encoded string was inserted without
+        # escaping the HTML.  We should be able to traverse the XML tree.
+        xpath = "//div[@class='drag_and_drop_problem_json']/p/b"
+        self.assert_has_text(xml, xpath, 'HTML')
+
+
+class ChoiceTextGroupTemplateTest(TemplateTestCase):
+    """Test mako template for `<choicetextgroup>` input"""
+
+    TEMPLATE_NAME = 'choicetext.html'
+    VALUE_DICT = {'1_choiceinput_0bc': '1_choiceinput_0bc', '1_choiceinput_0_textinput_0': '0',
+                  '1_choiceinput_1_textinput_0': '0'}
+    EMPTY_DICT = {'1_choiceinput_0_textinput_0': '',
+                  '1_choiceinput_1_textinput_0': ''}
+    BOTH_CHOICE_CHECKBOX = {'1_choiceinput_0bc': 'choiceinput_0',
+                            '1_choiceinput_1bc': 'choiceinput_1',
+                            '1_choiceinput_0_textinput_0': '0',
+                            '1_choiceinput_1_textinput_0': '0'}
+    WRONG_CHOICE_CHECKBOX = {'1_choiceinput_1bc': 'choiceinput_1',
+                             '1_choiceinput_0_textinput_0': '0',
+                             '1_choiceinput_1_textinput_0': '0'}
+
+    def setUp(self):
+        super(ChoiceTextGroupTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        choices = [
+            (
+                '1_choiceinput_0bc',
+                [
+                    {'tail_text': '', 'type': 'text', 'value': '', 'contents': ''},
+                    {'tail_text': '', 'type': 'textinput', 'value': '', 'contents': 'choiceinput_0_textinput_0'},
+                ]
+            ),
+            (
+                '1_choiceinput_1bc',
+                [
+                    {'tail_text': '', 'type': 'text', 'value': '', 'contents': ''},
+                    {'tail_text': '', 'type': 'textinput', 'value': '', 'contents': 'choiceinput_1_textinput_0'},
+                ]
+            )
+        ]
+        self.context = {
+            'id': '1',
+            'choices': choices,
+            'status': Status('correct'),
+            'input_type': 'radio',
+            'value': self.VALUE_DICT,
+            'response_data': self.RESPONSE_DATA
+        }
+
+    def test_grouping_tag(self):
+        """
+        Tests whether we are using a section or a label to wrap choice elements.
+        Section is used for checkbox, so inputting text does not deselect
+        """
+        input_tags = ('radio', 'checkbox')
+        self.context['status'] = Status('correct')
+        xpath = "//section[@id='forinput1_choiceinput_0bc']"
+
+        self.context['value'] = {}
+        for input_type in input_tags:
+            self.context['input_type'] = input_type
+            xml = self.render_to_xml(self.context)
+            self.assert_has_xpath(xml, xpath, self.context)
+
+    def test_problem_marked_correct(self):
+        """Test conditions under which the entire problem
+        (not a particular option) is marked correct"""
+
+        self.context['status'] = Status('correct')
+        self.context['input_type'] = 'checkbox'
+        self.context['value'] = self.VALUE_DICT
+
+        # Should mark the entire problem correct
+        xml = self.render_to_xml(self.context)
+        xpath = "//div[@class='indicator-container']/span[@class='status correct']"
+        self.assert_has_xpath(xml, xpath, self.context)
+
+        # Should NOT mark individual options
+        self.assert_no_xpath(xml, "//label[@class='choicetextgroup_incorrect']",
+                             self.context)
+
+        self.assert_no_xpath(xml, "//label[@class='choicetextgroup_correct']",
+                             self.context)
+
+    def test_problem_marked_incorrect(self):
+        """Test all conditions under which the entire problem
+        (not a particular option) is marked incorrect"""
+        grouping_tags = {'radio': 'label', 'checkbox': 'section'}
+        conditions = [
+            {'status': Status('incorrect'), 'input_type': 'radio', 'value': {}},
+            {'status': Status('incorrect'), 'input_type': 'checkbox', 'value': self.WRONG_CHOICE_CHECKBOX},
+            {'status': Status('incorrect'), 'input_type': 'checkbox', 'value': self.BOTH_CHOICE_CHECKBOX},
+            {'status': Status('incorrect'), 'input_type': 'checkbox', 'value': self.VALUE_DICT},
+            {'status': Status('incomplete'), 'input_type': 'radio', 'value': {}},
+            {'status': Status('incomplete'), 'input_type': 'checkbox', 'value': self.WRONG_CHOICE_CHECKBOX},
+            {'status': Status('incomplete'), 'input_type': 'checkbox', 'value': self.BOTH_CHOICE_CHECKBOX},
+            {'status': Status('incomplete'), 'input_type': 'checkbox', 'value': self.VALUE_DICT}]
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//div[@class='indicator-container']/span[@class='status incorrect']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should NOT mark individual options
+            grouping_tag = grouping_tags[test_conditions['input_type']]
+            self.assert_no_xpath(xml,
+                                 "//{0}[@class='choicetextgroup_incorrect']".format(grouping_tag),
+                                 self.context)
+
+            self.assert_no_xpath(xml,
+                                 "//{0}[@class='choicetextgroup_correct']".format(grouping_tag),
+                                 self.context)
+
+    def test_problem_marked_unsubmitted(self):
+        """Test all conditions under which the entire problem
+        (not a particular option) is marked unanswered"""
+        grouping_tags = {'radio': 'label', 'checkbox': 'section'}
+
+        conditions = [
+            {'status': Status('unsubmitted'), 'input_type': 'radio', 'value': {}},
+            {'status': Status('unsubmitted'), 'input_type': 'radio', 'value': self.EMPTY_DICT},
+            {'status': Status('unsubmitted'), 'input_type': 'checkbox', 'value': {}},
+            {'status': Status('unsubmitted'), 'input_type': 'checkbox', 'value': self.EMPTY_DICT},
+            {'status': Status('unsubmitted'), 'input_type': 'checkbox', 'value': self.VALUE_DICT},
+            {'status': Status('unsubmitted'), 'input_type': 'checkbox', 'value': self.BOTH_CHOICE_CHECKBOX},
+        ]
+
+        self.context['status'] = Status('unanswered')
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//div[@class='indicator-container']/span[@class='status unanswered']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should NOT mark individual options
+            grouping_tag = grouping_tags[test_conditions['input_type']]
+            self.assert_no_xpath(xml,
+                                 "//{0}[@class='choicetextgroup_incorrect']".format(grouping_tag),
+                                 self.context)
+
+            self.assert_no_xpath(xml,
+                                 "//{0}[@class='choicetextgroup_correct']".format(grouping_tag),
+                                 self.context)
+
+    def test_option_marked_correct(self):
+        """Test conditions under which a particular option
+        (not the entire problem) is marked correct."""
+
+        conditions = [
+            {'input_type': 'radio', 'value': self.VALUE_DICT}]
+
+        self.context['status'] = Status('correct')
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//section[@id='forinput1_choiceinput_0bc' and\
+                    @class='choicetextgroup_correct']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should NOT mark the whole problem
+            xpath = "//div[@class='indicator-container']/span"
+            self.assert_no_xpath(xml, xpath, self.context)
+
+    def test_option_marked_incorrect(self):
+        """Test conditions under which a particular option
+        (not the entire problem) is marked incorrect."""
+
+        conditions = [
+            {'input_type': 'radio', 'value': self.VALUE_DICT}]
+
+        self.context['status'] = Status('incorrect')
+
+        for test_conditions in conditions:
+            self.context.update(test_conditions)
+            xml = self.render_to_xml(self.context)
+            xpath = "//section[@id='forinput1_choiceinput_0bc' and\
+                    @class='choicetextgroup_incorrect']"
+            self.assert_has_xpath(xml, xpath, self.context)
+
+            # Should NOT mark the whole problem
+            xpath = "//div[@class='indicator-container']/span"
+            self.assert_no_xpath(xml, xpath, self.context)
+
+    def test_aria_label(self):
+        """
+        Verify aria-label attribute rendering.
+        """
+        self.assert_label(aria_label=True)
+
+
+class ChemicalEquationTemplateTest(TemplateTestCase):
+    """Test mako template for `<chemicalequationinput>` input"""
+
+    TEMPLATE_NAME = 'chemicalequationinput.html'
+
+    def setUp(self):
+        super(ChemicalEquationTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {
+            'id': '1',
+            'status': Status('correct'),
+            'previewer': 'dummy.js',
+            'value': '101',
+        }
+
+    def test_aria_label(self):
+        """
+        Verify aria-label attribute rendering.
+        """
+        self.assert_label(aria_label=True)
+
+
+class SchematicInputTemplateTest(TemplateTestCase):
+    """Test mako template for `<schematic>` input"""
+
+    TEMPLATE_NAME = 'schematicinput.html'
+
+    def setUp(self):
+        super(SchematicInputTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {
+            'id': '1',
+            'status': Status('correct'),
+            'previewer': 'dummy.js',
+            'value': '101',
+            'STATIC_URL': '/dummy-static/',
+            'msg': '',
+            'initial_value': 'two large batteries',
+            'width': '100',
+            'height': '100',
+            'parts': 'resistors, capacitors, and flowers',
+            'setup_script': '/dummy-static/js/capa/schematicinput.js',
+            'analyses': 'fast, slow, and pink',
+            'submit_analyses': 'maybe',
+        }
+
+    def test_aria_label(self):
+        """
+        Verify aria-label attribute rendering.
+        """
+        self.assert_label(aria_label=True)
+
+
+class CodeinputTemplateTest(TemplateTestCase):
+    """
+    Test mako template for `<textbox>` input
+    """
+
+    TEMPLATE_NAME = 'codeinput.html'
+
+    def setUp(self):
+        super(CodeinputTemplateTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.context = {
+            'id': '1',
+            'status': Status('correct'),
+            'mode': 'parrot',
+            'linenumbers': 'false',
+            'rows': '37',
+            'cols': '11',
+            'tabsize': '7',
+            'hidden': '',
+            'msg': '',
+            'value': 'print "good evening"',
+            'aria_label': 'python editor',
+            'code_mirror_exit_message': 'Press ESC then TAB or click outside of the code editor to exit',
+            'response_data': self.RESPONSE_DATA,
+            'describedby': HTML(self.DESCRIBEDBY),
+        }
+
+    def test_label(self):
+        """
+        Verify question label is rendered correctly.
+        """
+        self.assert_label(xpath="//label[@class='problem-group-label']")
+
+    def test_editor_exit_message(self):
+        """
+        Verify that editor exit message is rendered.
+        """
+        xml = self.render_to_xml(self.context)
+        self.assert_has_text(xml, '//span[@id="cm-editor-exit-message-1"]', self.context['code_mirror_exit_message'])

xmodule/capa/tests/test_inputtypes.py

@@ -0,0 +1,1682 @@
+# -*- coding: utf-8 -*-
+"""
+Tests of input types.
+
+TODO:
+- refactor: so much repetive code (have factory methods that build xml elements directly, etc)
+
+- test error cases
+
+- check rendering -- e.g. msg should appear in the rendered output.  If possible, test that
+  templates are escaping things properly.
+
+
+- test unicode in values, parameters, etc.
+- test various html escapes
+- test funny xml chars -- should never get xml parse error if things are escaped properly.
+
+"""
+
+
+import json
+import textwrap
+import unittest
+import xml.sax.saxutils as saxutils
+from collections import OrderedDict
+
+import pytest
+import six
+from lxml import etree
+from lxml.html import fromstring
+from mock import ANY, patch
+from pyparsing import ParseException
+from six.moves import zip
+
+from xmodule.capa import inputtypes
+from xmodule.capa.checker import DemoSystem
+from xmodule.capa.tests.helpers import test_capa_system
+from xmodule.capa.xqueue_interface import XQUEUE_TIMEOUT
+from openedx.core.djangolib.markup import HTML
+
+# just a handy shortcut
+lookup_tag = inputtypes.registry.get_class_for_tag
+
+
+DESCRIBEDBY = HTML('aria-describedby="status_{status_id} desc-1 desc-2"')
+# Use TRAILING_TEXT_DESCRIBEDBY when trailing_text is not null
+TRAILING_TEXT_DESCRIBEDBY = HTML('aria-describedby="trailing_text_{trailing_text_id} status_{status_id} desc-1 desc-2"')
+DESCRIPTIONS = OrderedDict([('desc-1', 'description text 1'), ('desc-2', 'description text 2')])
+RESPONSE_DATA = {
+    'label': 'question text 101',
+    'descriptions': DESCRIPTIONS
+}
+
+
+def quote_attr(s):
+    return saxutils.quoteattr(s)[1:-1]  # don't want the outer quotes
+
+
+class OptionInputTest(unittest.TestCase):
+    """
+    Make sure option inputs work
+    """
+
+    def test_rendering(self):
+        xml_str = """<optioninput options="('Up','Down','Don't know')" id="sky_input" correct="Up"/>"""
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'Down',
+            'id': 'sky_input',
+            'status': 'answered',
+            'default_option_text': 'Select an option',
+            'response_data': RESPONSE_DATA
+        }
+        option_input = lookup_tag('optioninput')(test_capa_system(), element, state)
+
+        context = option_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'sky_input'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'value': 'Down',
+            'options': [('Up', 'Up'), ('Down', 'Down'), ('Don\'t know', 'Don\'t know')],
+            'status': inputtypes.Status('answered'),
+            'msg': '',
+            'inline': False,
+            'id': prob_id,
+            'default_option_text': 'Select an option',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id='sky_input')
+        }
+
+        assert context == expected
+
+    def test_option_parsing(self):
+        f = inputtypes.OptionInput.parse_options
+
+        def check(input, options):  # lint-amnesty, pylint: disable=redefined-builtin
+            """
+            Take list of options, confirm that output is in the silly doubled format
+            """
+            expected = [(o, o) for o in options]
+            assert f(input) == expected
+
+        check("('a','b')", ['a', 'b'])
+        check("('a', 'b')", ['a', 'b'])
+        check("('a b','b')", ['a b', 'b'])
+        check("('My \"quoted\"place','b')", ['My \"quoted\"place', 'b'])
+        check("('б','в')", ['б', 'в'])
+        check("('б', 'в')", ['б', 'в'])
+        check("('б в','в')", ['б в', 'в'])
+        check("('Мой \"кавыки\"место','в')", ['Мой \"кавыки\"место', 'в'])
+
+        # check that escaping single quotes with leading backslash (\') properly works
+        # note: actual input by user will be hasn\'t but json parses it as hasn\\'t
+        check("('hasnt','hasn't')", ['hasnt', 'hasn\'t'])
+
+
+class ChoiceGroupTest(unittest.TestCase):
+    """
+    Test choice groups, radio groups, and checkbox groups
+    """
+
+    def check_group(self, tag, expected_input_type, expected_suffix):  # lint-amnesty, pylint: disable=missing-function-docstring
+        xml_str = """
+  <{tag}>
+    <choice correct="false" name="foil1"><text>This is foil One.</text></choice>
+    <choice correct="false" name="foil2"><text>This is foil Two.</text></choice>
+    <choice correct="true" name="foil3">This is foil Three.</choice>
+    <choice correct="false" name="foil4">This is <b>foil</b> Four.</choice>
+  </{tag}>
+        """.format(tag=tag)
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'foil3',
+            'id': 'sky_input',
+            'status': 'answered',
+            'response_data': RESPONSE_DATA
+        }
+
+        the_input = lookup_tag(tag)(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': 'sky_input',
+            'value': 'foil3',
+            'status': inputtypes.Status('answered'),
+            'msg': '',
+            'input_type': expected_input_type,
+            'choices': [('foil1', '<text>This is foil One.</text>'),
+                        ('foil2', '<text>This is foil Two.</text>'),
+                        ('foil3', 'This is foil Three.'),
+                        ('foil4', 'This is <b>foil</b> Four.'), ],
+            'show_correctness': 'always',
+            'submitted_message': 'Answer received.',
+            'name_array_suffix': expected_suffix,   # what is this for??
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id='sky_input')
+        }
+
+        assert context == expected
+
+    def test_choicegroup(self):
+        self.check_group('choicegroup', 'radio', '')
+
+    def test_radiogroup(self):
+        self.check_group('radiogroup', 'radio', '[]')
+
+    def test_checkboxgroup(self):
+        self.check_group('checkboxgroup', 'checkbox', '[]')
+
+
+class JSInputTest(unittest.TestCase):
+    """
+    Test context variables passed into the jsinput template.
+    """
+
+    def test_rendering_default_values(self):
+        """
+        Tests the default values passed through to render.
+        """
+        xml_str = '<jsinput id="prob_1_2"/>'
+        expected = {
+            'html_file': None,
+            'gradefn': "gradefn",
+            'get_statefn': None,
+            'set_statefn': None,
+            'initial_state': None,
+            'width': "400",
+            'height': "300",
+            'title': "Problem Remote Content",
+            'sop': None
+        }
+
+        self._render_context_test(xml_str, expected)
+
+    def test_rendering_provided_values(self):
+        """
+        Tests that values provided by course authors are passed through to render.
+        """
+        xml_str = """
+        <jsinput id="prob_1_2"
+            gradefn="WebGLDemo.getGrade" get_statefn="WebGLDemo.getState" set_statefn="WebGLDemo.setState"
+            initial_state='{"selectedObjects":{"cube":true,"cylinder":false}}'
+            width="1000" height="1200"
+            html_file="https://studio.edx.org/c4x/edX/DemoX/asset/webGLDemo.html"
+            sop="false" title="Awesome and fun!"
+        />
+        """
+
+        expected = {
+            'html_file': "https://studio.edx.org/c4x/edX/DemoX/asset/webGLDemo.html",
+            'gradefn': "WebGLDemo.getGrade",
+            'get_statefn': "WebGLDemo.getState",
+            'set_statefn': "WebGLDemo.setState",
+            'initial_state': '{"selectedObjects":{"cube":true,"cylinder":false}}',
+            'width': "1000",
+            'height': "1200",
+            'title': "Awesome and fun!",
+            'sop': 'false'
+        }
+
+        self._render_context_test(xml_str, expected)
+
+    def _render_context_test(self, xml_str, expected_context):
+        """
+        Helper method for testing context based on the provided XML string.
+        """
+        element = etree.fromstring(xml_str)
+        state = {
+            'value': 103,
+            'response_data': RESPONSE_DATA
+        }
+        the_input = lookup_tag('jsinput')(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+
+        full_expected_context = {
+            'STATIC_URL': '/dummy-static/',
+            'id': 'prob_1_2',
+            'status': inputtypes.Status('unanswered'),
+            'describedby_html': DESCRIBEDBY.format(status_id='prob_1_2'),
+            'msg': "",
+            'params': None,
+            'jschannel_loader': '/dummy-static/js/capa/src/jschannel.js',
+            'jsinput_loader': '/dummy-static/js/capa/src/jsinput.js',
+            'saved_state': 103,
+            'response_data': RESPONSE_DATA,
+            'value': 103
+        }
+        full_expected_context.update(expected_context)
+
+        assert full_expected_context == context
+
+
+class TextLineTest(unittest.TestCase):
+    """
+    Check that textline inputs work, with and without math.
+    """
+
+    def test_rendering(self):
+        size = "42"
+        xml_str = """<textline id="prob_1_2" size="{size}"/>""".format(size=size)
+
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'BumbleBee',
+            'response_data': RESPONSE_DATA
+        }
+        the_input = lookup_tag('textline')(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'BumbleBee',
+            'status': inputtypes.Status('unanswered'),
+            'size': size,
+            'msg': '',
+            'hidden': False,
+            'inline': False,
+            'do_math': False,
+            'trailing_text': '',
+            'preprocessor': None,
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+        assert context == expected
+
+    def test_math_rendering(self):
+        size = "42"
+        preprocessorClass = "preParty"
+        script = "foo/party.js"
+
+        xml_str = """<textline math="True" id="prob_1_2" size="{size}"
+        preprocessorClassName="{pp}"
+        preprocessorSrc="{sc}"/>""".format(size=size, pp=preprocessorClass, sc=script)
+
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'BumbleBee',
+            'response_data': RESPONSE_DATA
+        }
+        the_input = lookup_tag('textline')(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'BumbleBee',
+            'status': inputtypes.Status('unanswered'),
+            'size': size,
+            'msg': '',
+            'hidden': False,
+            'inline': False,
+            'trailing_text': '',
+            'do_math': True,
+            'preprocessor': {
+                'class_name': preprocessorClass,
+                'script_src': script,
+            },
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+        assert context == expected
+
+    def test_trailing_text_rendering(self):
+        size = "42"
+        # store (xml_text, expected)
+        trailing_text = []
+        # standard trailing text
+        trailing_text.append(('m/s', 'm/s'))
+        # unicode trailing text
+        trailing_text.append(('\xc3', '\xc3'))
+        # html escaped trailing text
+        # this is the only one we expect to change
+        trailing_text.append(('a &lt; b', 'a < b'))
+
+        for xml_text, expected_text in trailing_text:
+            xml_str = """<textline id="prob_1_2"
+                            size="{size}"
+                            trailing_text="{tt}"
+                            />""".format(size=size, tt=xml_text)
+
+            element = etree.fromstring(xml_str)
+
+            state = {
+                'value': 'BumbleBee',
+                'response_data': RESPONSE_DATA
+            }
+            the_input = lookup_tag('textline')(test_capa_system(), element, state)
+
+            context = the_input._get_render_context()  # pylint: disable=protected-access
+            prob_id = 'prob_1_2'
+            expected = {
+                'STATIC_URL': '/dummy-static/',
+                'id': prob_id,
+                'value': 'BumbleBee',
+                'status': inputtypes.Status('unanswered'),
+                'size': size,
+                'msg': '',
+                'hidden': False,
+                'inline': False,
+                'do_math': False,
+                'trailing_text': expected_text,
+                'preprocessor': None,
+                'response_data': RESPONSE_DATA,
+                'describedby_html': TRAILING_TEXT_DESCRIBEDBY.format(trailing_text_id=prob_id, status_id=prob_id)
+            }
+            assert context == expected
+
+
+class FileSubmissionTest(unittest.TestCase):
+    """
+    Check that file submission inputs work
+    """
+
+    def test_rendering(self):
+        allowed_files = "runme.py nooooo.rb ohai.java"
+        required_files = "cookies.py"
+
+        xml_str = """<filesubmission id="prob_1_2"
+        allowed_files="{af}"
+        required_files="{rf}"
+        />""".format(af=allowed_files,
+                     rf=required_files,)
+
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'BumbleBee.py',
+            'status': 'incomplete',
+            'feedback': {'message': '3'},
+            'response_data': RESPONSE_DATA
+        }
+        input_class = lookup_tag('filesubmission')
+        the_input = input_class(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'status': inputtypes.Status('queued'),
+            'msg': the_input.submitted_msg,
+            'value': 'BumbleBee.py',
+            'queue_len': '3',
+            'allowed_files': '["runme.py", "nooooo.rb", "ohai.java"]',
+            'required_files': '["cookies.py"]',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+
+class CodeInputTest(unittest.TestCase):
+    """
+    Check that codeinput inputs work
+    """
+
+    def test_rendering(self):
+        mode = "parrot"
+        linenumbers = 'false'
+        rows = '37'
+        cols = '11'
+        tabsize = '7'
+
+        xml_str = """<codeinput id="prob_1_2"
+        mode="{m}"
+        cols="{c}"
+        rows="{r}"
+        linenumbers="{ln}"
+        tabsize="{ts}"
+        />""".format(m=mode, c=cols, r=rows, ln=linenumbers, ts=tabsize)
+
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'print "good evening"',
+            'status': 'incomplete',
+            'feedback': {'message': '3'},
+            'response_data': RESPONSE_DATA
+        }
+
+        input_class = lookup_tag('codeinput')
+        the_input = input_class(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'print "good evening"',
+            'status': inputtypes.Status('queued'),
+            'msg': the_input.submitted_msg,
+            'mode': mode,
+            'linenumbers': linenumbers,
+            'rows': rows,
+            'cols': cols,
+            'hidden': '',
+            'tabsize': int(tabsize),
+            'queue_len': '3',
+            'aria_label': '{mode} editor'.format(mode=mode),
+            'code_mirror_exit_message': 'Press ESC then TAB or click outside of the code editor to exit',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+
+class MatlabTest(unittest.TestCase):
+    """
+    Test Matlab input types
+    """
+    def setUp(self):
+        super(MatlabTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.rows = '10'
+        self.cols = '80'
+        self.tabsize = '4'
+        self.mode = ""
+        self.payload = "payload"
+        self.linenumbers = 'true'
+        self.xml = """<matlabinput id="prob_1_2"
+            rows="{r}" cols="{c}"
+            tabsize="{tabsize}" mode="{m}"
+            linenumbers="{ln}">
+                <plot_payload>
+                    {payload}
+                </plot_payload>
+            </matlabinput>""".format(r=self.rows,
+                                     c=self.cols,
+                                     tabsize=self.tabsize,
+                                     m=self.mode,
+                                     payload=self.payload,
+                                     ln=self.linenumbers)
+        elt = etree.fromstring(self.xml)
+        state = {
+            'value': 'print "good evening"',
+            'status': 'incomplete',
+            'feedback': {'message': '3'},
+            'response_data': {}
+        }
+
+        self.input_class = lookup_tag('matlabinput')
+        self.the_input = self.input_class(test_capa_system(), elt, state)
+
+    def test_rendering(self):
+        context = self.the_input._get_render_context()  # pylint: disable=protected-access
+
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': 'prob_1_2',
+            'value': 'print "good evening"',
+            'status': inputtypes.Status('queued'),
+            'msg': self.the_input.submitted_msg,
+            'mode': self.mode,
+            'rows': self.rows,
+            'cols': self.cols,
+            'queue_msg': '',
+            'linenumbers': 'true',
+            'hidden': '',
+            'tabsize': int(self.tabsize),
+            'button_enabled': True,
+            'queue_len': '3',
+            'matlab_editor_js': '/dummy-static/js/vendor/CodeMirror/octave.js',
+            'response_data': {},
+            'describedby_html': HTML('aria-describedby="status_prob_1_2"')
+        }
+
+        assert context == expected
+
+    def test_rendering_with_state(self):
+        state = {
+            'value': 'print "good evening"',
+            'status': 'incomplete',
+            'input_state': {'queue_msg': 'message'},
+            'feedback': {'message': '3'},
+            'response_data': RESPONSE_DATA
+        }
+        elt = etree.fromstring(self.xml)
+
+        the_input = self.input_class(test_capa_system(), elt, state)
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'print "good evening"',
+            'status': inputtypes.Status('queued'),
+            'msg': the_input.submitted_msg,
+            'mode': self.mode,
+            'rows': self.rows,
+            'cols': self.cols,
+            'queue_msg': 'message',
+            'linenumbers': 'true',
+            'hidden': '',
+            'tabsize': int(self.tabsize),
+            'button_enabled': True,
+            'queue_len': '3',
+            'matlab_editor_js': '/dummy-static/js/vendor/CodeMirror/octave.js',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+    def test_rendering_when_completed(self):
+        for status in ['correct', 'incorrect']:
+            state = {
+                'value': 'print "good evening"',
+                'status': status,
+                'input_state': {},
+                'response_data': RESPONSE_DATA
+            }
+            elt = etree.fromstring(self.xml)
+            prob_id = 'prob_1_2'
+            the_input = self.input_class(test_capa_system(), elt, state)
+            context = the_input._get_render_context()  # pylint: disable=protected-access
+            expected = {
+                'STATIC_URL': '/dummy-static/',
+                'id': prob_id,
+                'value': 'print "good evening"',
+                'status': inputtypes.Status(status),
+                'msg': '',
+                'mode': self.mode,
+                'rows': self.rows,
+                'cols': self.cols,
+                'queue_msg': '',
+                'linenumbers': 'true',
+                'hidden': '',
+                'tabsize': int(self.tabsize),
+                'button_enabled': False,
+                'queue_len': '0',
+                'matlab_editor_js': '/dummy-static/js/vendor/CodeMirror/octave.js',
+                'response_data': RESPONSE_DATA,
+                'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+            }
+
+            assert context == expected
+
+    @patch('xmodule.capa.inputtypes.time.time', return_value=10)
+    def test_rendering_while_queued(self, time):  # lint-amnesty, pylint: disable=unused-argument
+        state = {
+            'value': 'print "good evening"',
+            'status': 'incomplete',
+            'input_state': {'queuestate': 'queued', 'queuetime': 5},
+            'response_data': RESPONSE_DATA
+        }
+        elt = etree.fromstring(self.xml)
+        prob_id = 'prob_1_2'
+        the_input = self.input_class(test_capa_system(), elt, state)
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'print "good evening"',
+            'status': inputtypes.Status('queued'),
+            'msg': the_input.submitted_msg,
+            'mode': self.mode,
+            'rows': self.rows,
+            'cols': self.cols,
+            'queue_msg': '',
+            'linenumbers': 'true',
+            'hidden': '',
+            'tabsize': int(self.tabsize),
+            'button_enabled': True,
+            'queue_len': '1',
+            'matlab_editor_js': '/dummy-static/js/vendor/CodeMirror/octave.js',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+    def test_plot_data(self):
+        data = {'submission': 'x = 1234;'}
+        response = self.the_input.handle_ajax("plot", data)
+        self.the_input.capa_system.xqueue.interface.send_to_queue.assert_called_with(header=ANY, body=ANY)
+        assert response['success']
+        assert self.the_input.input_state['queuekey'] is not None
+        assert self.the_input.input_state['queuestate'] == 'queued'
+
+    def test_plot_data_failure(self):
+        data = {'submission': 'x = 1234;'}
+        error_message = 'Error message!'
+        self.the_input.capa_system.xqueue.interface.send_to_queue.return_value = (1, error_message)
+        response = self.the_input.handle_ajax("plot", data)
+        assert not response['success']
+        assert response['message'] == error_message
+        assert 'queuekey' not in self.the_input.input_state
+        assert 'queuestate' not in self.the_input.input_state
+
+    @patch('xmodule.capa.inputtypes.time.time', return_value=10)
+    def test_ungraded_response_success(self, time):  # lint-amnesty, pylint: disable=unused-argument
+        queuekey = 'abcd'
+        input_state = {'queuekey': queuekey, 'queuestate': 'queued', 'queuetime': 5}
+        state = {'value': 'print "good evening"',
+                 'status': 'incomplete',
+                 'input_state': input_state,
+                 'feedback': {'message': '3'}, }
+        elt = etree.fromstring(self.xml)
+
+        the_input = self.input_class(test_capa_system(), elt, state)
+        inner_msg = 'hello!'
+        queue_msg = json.dumps({'msg': inner_msg})
+
+        the_input.ungraded_response(queue_msg, queuekey)
+        assert input_state['queuekey'] is None
+        assert input_state['queuestate'] is None
+        assert input_state['queue_msg'] == inner_msg
+
+    @patch('xmodule.capa.inputtypes.time.time', return_value=10)
+    def test_ungraded_response_key_mismatch(self, time):  # lint-amnesty, pylint: disable=unused-argument
+        queuekey = 'abcd'
+        input_state = {'queuekey': queuekey, 'queuestate': 'queued', 'queuetime': 5}
+        state = {'value': 'print "good evening"',
+                 'status': 'incomplete',
+                 'input_state': input_state,
+                 'feedback': {'message': '3'}, }
+        elt = etree.fromstring(self.xml)
+
+        the_input = self.input_class(test_capa_system(), elt, state)
+        inner_msg = 'hello!'
+        queue_msg = json.dumps({'msg': inner_msg})
+
+        the_input.ungraded_response(queue_msg, 'abc')
+        assert input_state['queuekey'] == queuekey
+        assert input_state['queuestate'] == 'queued'
+        assert 'queue_msg' not in input_state
+
+    @patch('xmodule.capa.inputtypes.time.time', return_value=20)
+    def test_matlab_response_timeout_not_exceeded(self, time):  # lint-amnesty, pylint: disable=unused-argument
+
+        state = {'input_state': {'queuestate': 'queued', 'queuetime': 5}}
+        elt = etree.fromstring(self.xml)
+
+        the_input = self.input_class(test_capa_system(), elt, state)
+        assert the_input.status == 'queued'
+
+    @patch('xmodule.capa.inputtypes.time.time', return_value=45)
+    def test_matlab_response_timeout_exceeded(self, time):  # lint-amnesty, pylint: disable=unused-argument
+
+        state = {'input_state': {'queuestate': 'queued', 'queuetime': 5}}
+        elt = etree.fromstring(self.xml)
+
+        the_input = self.input_class(test_capa_system(), elt, state)
+        assert the_input.status == 'unsubmitted'
+        assert the_input.msg == 'No response from Xqueue within {} seconds. Aborted.'.format(XQUEUE_TIMEOUT)
+
+    @patch('xmodule.capa.inputtypes.time.time', return_value=20)
+    def test_matlab_response_migration_of_queuetime(self, time):  # lint-amnesty, pylint: disable=unused-argument
+        """
+        Test if problem was saved before queuetime was introduced.
+        """
+        state = {'input_state': {'queuestate': 'queued'}}
+        elt = etree.fromstring(self.xml)
+
+        the_input = self.input_class(test_capa_system(), elt, state)
+        assert the_input.status == 'unsubmitted'
+
+    def test_matlab_api_key(self):
+        """
+        Test that api_key ends up in the xqueue payload
+        """
+        elt = etree.fromstring(self.xml)
+        system = test_capa_system()
+        system.matlab_api_key = 'test_api_key'
+        the_input = lookup_tag('matlabinput')(system, elt, {})
+
+        data = {'submission': 'x = 1234;'}
+        response = the_input.handle_ajax("plot", data)  # lint-amnesty, pylint: disable=unused-variable
+
+        body = system.xqueue.interface.send_to_queue.call_args[1]['body']
+        payload = json.loads(body)
+        assert 'test_api_key' == payload['token']
+        assert '2' == payload['endpoint_version']
+
+    def test_get_html(self):
+        # usual output
+        output = self.the_input.get_html()
+        output_string = etree.tostring(output).decode('utf-8')
+        assert output_string.startswith('<div>{')
+        assert output_string.endswith('}</div>')
+        output_string = output_string.replace('}</div>', '')
+        output_string = output_string.replace('<div>{', '')
+        output_list = output_string.split(',')
+        for index, value in enumerate(output_list):
+            output_list[index] = value.replace('u\'', '\'').strip()
+
+        expected_string = """
+        \'matlab_editor_js\': \'/dummy-static/js/vendor/CodeMirror/octave.js\',
+        \'value\': \'print "good evening"\', \'hidden\': \'\',
+        \'msg\': \'Submitted. As soon as a response is returned, this message will be replaced by that feedback.\',
+        \'status\': Status(\'queued\'), \'response_data\': {}, \'queue_msg\': \'\', \'mode\': \'\',
+        \'id\': \'prob_1_2\', \'queue_len\': \'3\', \'tabsize\': 4, \'STATIC_URL\': \'/dummy-static/\',
+        \'linenumbers\': \'true\', \'cols\': \'80\', \'button_enabled\': True, \'rows\': \'10\',
+        \'describedby_html\': Markup(\'aria-describedby="status_prob_1_2"\')"""
+        expected_list = (textwrap.dedent(expected_string).replace('\n', ' ').strip()).split(',')
+        for index, value in enumerate(expected_list):
+            expected_list[index] = value.replace('u\'', '\'').strip()
+        six.assertCountEqual(self, output_list, expected_list)
+
+        # test html, that is correct HTML5 html, but is not parsable by XML parser.
+        old_render_template = self.the_input.capa_system.render_template
+        self.the_input.capa_system.render_template = lambda *args: textwrap.dedent("""
+                <div class='matlabResponse'><div id='mwAudioPlaceHolder'>
+                <audio controls autobuffer autoplay src='data:audio/wav;base64='>Audio is not supported on this browser.</audio>
+                <div>Right click <a href=https://endpoint.mss-mathworks.com/media/filename.wav>here</a> and click \"Save As\" to download the file</div></div>
+                <div style='white-space:pre' class='commandWindowOutput'></div><ul></ul></div>
+            """).replace('\n', '')
+
+        output = self.the_input.get_html()
+        elements = []
+        element_tags = []
+        element_keys = []
+        for element in output.iter():
+            elements.append(element)
+            element_tags.append(element.tag)
+            element_keys.append(element.keys())
+        assert element_tags.count('div') == 4
+        assert element_tags.count('audio') == 1
+        audio_index = element_tags.index('audio')
+
+        six.assertCountEqual(self, element_keys[audio_index], ['autobuffer', 'controls', 'autoplay', 'src'])
+        assert elements[audio_index].get('src') == 'data:audio/wav;base64='
+        assert elements[audio_index].text == 'Audio is not supported on this browser.'
+        href_index = element_keys.index(['href'])
+        assert elements[href_index].get('href') == 'https://endpoint.mss-mathworks.com/media/filename.wav'
+        id_index = element_keys.index(['id'])
+        assert elements[id_index].get('id') == 'mwAudioPlaceHolder'
+        output_string = etree.tostring(output).decode('utf-8')
+
+        # check that exception is raised during parsing for html.
+        self.the_input.capa_system.render_template = lambda *args: "<aaa"
+        with pytest.raises(etree.XMLSyntaxError):
+            self.the_input.get_html()
+
+        self.the_input.capa_system.render_template = old_render_template
+
+    def test_malformed_queue_msg(self):
+        # an actual malformed response
+        queue_msg = textwrap.dedent("""
+    <div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'> <strong>if</strong> Conditionally execute statements.
+    The general form of the <strong>if</strong> statement is
+
+       <strong>if</strong> expression
+         statements
+       ELSEIF expression
+         statements
+       ELSE
+         statements
+       END
+
+    The statements are executed if the real part of the expression
+    has all non-zero elements. The ELSE and ELSEIF parts are optional.
+    Zero or more ELSEIF parts can be used as well as nested <strong>if</strong>'s.
+    The expression is usually of the form expr rop expr where
+    rop is ==, <, >, <=, >=, or ~=.
+    <img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAjAAAAGkCAIAAACgj==" />
+
+    Example
+       if I == J
+         A(I,J) = 2;
+       elseif abs(I-J) == 1
+         A(I,J) = -1;
+       else
+         A(I,J) = 0;
+       end
+
+    See also <a href="matlab:help relop">relop</a>, <a href="matlab:help else">else</a>, <a href="matlab:help elseif">elseif</a>, <a href="matlab:help end">end</a>, <a href="matlab:help for">for</a>, <a href="matlab:help while">while</a>, <a href="matlab:help switch">switch</a>.
+
+    Reference page in Help browser
+       <a href="matlab:doc if">doc if</a>
+
+    </div><ul></ul></div>
+        """)
+
+        state = {
+            'value': 'print "good evening"',
+            'status': 'incomplete',
+            'input_state': {'queue_msg': queue_msg},
+            'feedback': {'message': '3'},
+            'response_data': RESPONSE_DATA
+        }
+        elt = etree.fromstring(self.xml)
+
+        the_input = self.input_class(test_capa_system(), elt, state)
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        self.maxDiff = None
+        expected = fromstring('\n<div class="matlabResponse"><div class="commandWindowOutput" style="white-space: pre;"> <strong>if</strong> Conditionally execute statements.\nThe general form of the <strong>if</strong> statement is\n\n   <strong>if</strong> expression\n     statements\n   ELSEIF expression\n     statements\n   ELSE\n     statements\n   END\n\nThe statements are executed if the real part of the expression \nhas all non-zero elements. The ELSE and ELSEIF parts are optional.\nZero or more ELSEIF parts can be used as well as nested <strong>if</strong>\'s.\nThe expression is usually of the form expr rop expr where \nrop is ==, &lt;, &gt;, &lt;=, &gt;=, or ~=.\n<img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAjAAAAGkCAIAAACgj==">\n\nExample\n   if I == J\n     A(I,J) = 2;\n   elseif abs(I-J) == 1\n     A(I,J) = -1;\n   else\n     A(I,J) = 0;\n   end\n\nSee also <a>relop</a>, <a>else</a>, <a>elseif</a>, <a>end</a>, <a>for</a>, <a>while</a>, <a>switch</a>.\n\nReference page in Help browser\n   <a>doc if</a>\n\n</div><ul></ul></div>\n')  # lint-amnesty, pylint: disable=line-too-long
+        received = fromstring(context['queue_msg'])
+        html_tree_equal(received, expected)
+
+    def test_rendering_with_invalid_queue_msg(self):
+        self.the_input.queue_msg = ("<div class='matlabResponse'><div style='white-space:pre' class='commandWindowOutput'>"  # lint-amnesty, pylint: disable=line-too-long
+                                    "\nans =\n\n\u0002\n\n</div><ul></ul></div>")
+        context = self.the_input._get_render_context()  # pylint: disable=protected-access
+
+        self.maxDiff = None
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'print "good evening"',
+            'status': inputtypes.Status('queued'),
+            'msg': self.the_input.submitted_msg,
+            'mode': self.mode,
+            'rows': self.rows,
+            'cols': self.cols,
+            'queue_msg': "<span>Error running code.</span>",
+            'linenumbers': 'true',
+            'hidden': '',
+            'tabsize': int(self.tabsize),
+            'button_enabled': True,
+            'queue_len': '3',
+            'matlab_editor_js': '/dummy-static/js/vendor/CodeMirror/octave.js',
+            'response_data': {},
+            'describedby_html': 'aria-describedby="status_{id}"'.format(id=prob_id)
+        }
+        assert context == expected
+        self.the_input.capa_system.render_template = DemoSystem().render_template
+        self.the_input.get_html()  # Should not raise an exception
+
+    def test_matlab_queue_message_allowed_tags(self):
+        """
+        Test allowed tags.
+        """
+        allowed_tags = ['div', 'p', 'audio', 'pre', 'span']
+        for tag in allowed_tags:
+            queue_msg = "<{0}>Test message</{0}>".format(tag)
+            state = {
+                'input_state': {'queue_msg': queue_msg},
+                'status': 'queued',
+            }
+            elt = etree.fromstring(self.xml)
+            the_input = self.input_class(test_capa_system(), elt, state)
+            assert the_input.queue_msg == queue_msg
+
+    def test_matlab_queue_message_not_allowed_tag(self):
+        """
+        Test not allowed tag.
+        """
+        not_allowed_tag = 'script'
+        queue_msg = "<{0}>Test message</{0}>".format(not_allowed_tag)
+        state = {
+            'input_state': {'queue_msg': queue_msg},
+            'status': 'queued',
+        }
+        elt = etree.fromstring(self.xml)
+        the_input = self.input_class(test_capa_system(), elt, state)
+        expected = "&lt;script&gt;Test message&lt;/script&gt;"
+        assert the_input.queue_msg == expected
+
+    def test_matlab_sanitize_msg(self):
+        """
+        Check that the_input.msg is sanitized.
+        """
+        not_allowed_tag = 'script'
+        self.the_input.msg = "<{0}>Test message</{0}>".format(not_allowed_tag)
+        expected = "&lt;script&gt;Test message&lt;/script&gt;"
+        assert self.the_input._get_render_context()['msg'] == expected  # pylint: disable=protected-access
+
+
+def html_tree_equal(received, expected):
+    """
+    Returns whether two etree Elements are the same, with insensitivity to attribute order.
+    """
+    for attr in ('tag', 'attrib', 'text', 'tail'):
+        if getattr(received, attr) != getattr(expected, attr):
+            return False
+    if len(received) != len(expected):
+        return False
+    if any(not html_tree_equal(rec, exp) for rec, exp in zip(received, expected)):
+        return False
+    return True
+
+
+class SchematicTest(unittest.TestCase):
+    """
+    Check that schematic inputs work
+    """
+    def test_rendering(self):
+        height = '12'
+        width = '33'
+        parts = 'resistors, capacitors, and flowers'
+        analyses = 'fast, slow, and pink'
+        initial_value = 'two large batteries'
+        submit_analyses = 'maybe'
+
+        xml_str = """<schematic id="prob_1_2"
+        height="{h}"
+        width="{w}"
+        parts="{p}"
+        analyses="{a}"
+        initial_value="{iv}"
+        submit_analyses="{sa}"
+        />""".format(h=height, w=width, p=parts, a=analyses,
+                     iv=initial_value, sa=submit_analyses)
+
+        element = etree.fromstring(xml_str)
+
+        value = 'three resistors and an oscilating pendulum'
+        state = {
+            'value': value,
+            'status': 'unsubmitted',
+            'response_data': RESPONSE_DATA
+        }
+
+        the_input = lookup_tag('schematic')(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': value,
+            'status': inputtypes.Status('unsubmitted'),
+            'msg': '',
+            'initial_value': initial_value,
+            'width': width,
+            'height': height,
+            'parts': parts,
+            'setup_script': '/dummy-static/js/capa/schematicinput.js',
+            'analyses': analyses,
+            'submit_analyses': submit_analyses,
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+
+class ImageInputTest(unittest.TestCase):
+    """
+    Check that image inputs work
+    """
+    def check(self, value, egx, egy):  # lint-amnesty, pylint: disable=missing-function-docstring
+        height = '78'
+        width = '427'
+        src = 'http://www.edx.org/cowclicker.jpg'
+
+        xml_str = """<imageinput id="prob_1_2"
+        src="{s}"
+        height="{h}"
+        width="{w}"
+        />""".format(s=src, h=height, w=width)
+
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': value,
+            'status': 'unsubmitted',
+            'response_data': RESPONSE_DATA
+        }
+
+        the_input = lookup_tag('imageinput')(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': value,
+            'status': inputtypes.Status('unsubmitted'),
+            'width': width,
+            'height': height,
+            'src': src,
+            'gx': egx,
+            'gy': egy,
+            'msg': '',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+    def test_with_value(self):
+        # Check that compensating for the dot size works properly.
+        self.check('[50,40]', 35, 25)
+
+    def test_without_value(self):
+        self.check('', 0, 0)
+
+    def test_corrupt_values(self):
+        self.check('[12', 0, 0)
+        self.check('[12, a]', 0, 0)
+        self.check('[12 10]', 0, 0)
+        self.check('[12]', 0, 0)
+        self.check('[12 13 14]', 0, 0)
+
+
+class CrystallographyTest(unittest.TestCase):
+    """
+    Check that crystallography inputs work
+    """
+    def test_rendering(self):
+        height = '12'
+        width = '33'
+
+        xml_str = """<crystallography id="prob_1_2"
+        height="{h}"
+        width="{w}"
+        />""".format(h=height, w=width)
+
+        element = etree.fromstring(xml_str)
+
+        value = 'abc'
+        state = {
+            'value': value,
+            'status': 'unsubmitted',
+            'response_data': RESPONSE_DATA
+        }
+
+        the_input = lookup_tag('crystallography')(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': value,
+            'status': inputtypes.Status('unsubmitted'),
+            'msg': '',
+            'width': width,
+            'height': height,
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+
+class VseprTest(unittest.TestCase):
+    """
+    Check that vsepr inputs work
+    """
+    def test_rendering(self):
+        height = '12'
+        width = '33'
+        molecules = "H2O, C2O"
+        geometries = "AX12,TK421"
+
+        xml_str = """<vsepr id="prob_1_2"
+        height="{h}"
+        width="{w}"
+        molecules="{m}"
+        geometries="{g}"
+        />""".format(h=height, w=width, m=molecules, g=geometries)
+
+        element = etree.fromstring(xml_str)
+
+        value = 'abc'
+        state = {
+            'value': value,
+            'status': 'unsubmitted',
+            'response_data': RESPONSE_DATA
+        }
+
+        the_input = lookup_tag('vsepr_input')(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': value,
+            'status': inputtypes.Status('unsubmitted'),
+            'msg': '',
+            'width': width,
+            'height': height,
+            'molecules': molecules,
+            'geometries': geometries,
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        assert context == expected
+
+
+class ChemicalEquationTest(unittest.TestCase):
+    """
+    Check that chemical equation inputs work.
+    """
+    def setUp(self):
+        super(ChemicalEquationTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.size = "42"
+        xml_str = """<chemicalequationinput id="prob_1_2" size="{size}"/>""".format(size=self.size)
+
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'H2OYeah',
+            'response_data': RESPONSE_DATA
+        }
+        self.the_input = lookup_tag('chemicalequationinput')(test_capa_system(), element, state)
+
+    def test_rendering(self):
+        """
+        Verify that the render context matches the expected render context
+        """
+        context = self.the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'H2OYeah',
+            'status': inputtypes.Status('unanswered'),
+            'msg': '',
+            'size': self.size,
+            'previewer': '/dummy-static/js/capa/chemical_equation_preview.js',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+        assert context == expected
+
+    def test_chemcalc_ajax_sucess(self):
+        """
+        Verify that using the correct dispatch and valid data produces a valid response
+        """
+        data = {'formula': "H"}
+        response = self.the_input.handle_ajax("preview_chemcalc", data)
+
+        assert 'preview' in response
+        assert response['preview'] != ''
+        assert response['error'] == ''
+
+    def test_ajax_bad_method(self):
+        """
+        With a bad dispatch, we shouldn't receive anything
+        """
+        response = self.the_input.handle_ajax("obviously_not_real", {})
+        assert response == {}
+
+    def test_ajax_no_formula(self):
+        """
+        When we ask for a formula rendering, there should be an error if no formula
+        """
+        response = self.the_input.handle_ajax("preview_chemcalc", {})
+        assert 'error' in response
+        assert response['error'] == 'No formula specified.'
+
+    def test_ajax_parse_err(self):
+        """
+        With parse errors, ChemicalEquationInput should give an error message
+        """
+        # Simulate answering a problem that raises the exception
+        with patch('xmodule.capa.inputtypes.chemcalc.render_to_html') as mock_render:
+            mock_render.side_effect = ParseException("ȧƈƈḗƞŧḗḓ ŧḗẋŧ ƒǿř ŧḗşŧīƞɠ")
+            response = self.the_input.handle_ajax(
+                "preview_chemcalc",
+                {'formula': 'H2O + invalid chemistry'}
+            )
+
+        assert 'error' in response
+        assert "Couldn't parse formula" in response['error']
+
+    @patch('xmodule.capa.inputtypes.log')
+    def test_ajax_other_err(self, mock_log):
+        """
+        With other errors, test that ChemicalEquationInput also logs it
+        """
+        with patch('xmodule.capa.inputtypes.chemcalc.render_to_html') as mock_render:
+            mock_render.side_effect = Exception()
+            response = self.the_input.handle_ajax(
+                "preview_chemcalc",
+                {'formula': 'H2O + superterrible chemistry'}
+            )
+        mock_log.warning.assert_called_once_with(
+            "Error while previewing chemical formula", exc_info=True
+        )
+        assert 'error' in response
+        assert response['error'] == 'Error while rendering preview'
+
+
+class FormulaEquationTest(unittest.TestCase):
+    """
+    Check that formula equation inputs work.
+    """
+    def setUp(self):
+        super(FormulaEquationTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.size = "42"
+        xml_str = """<formulaequationinput id="prob_1_2" size="{size}"/>""".format(size=self.size)
+
+        element = etree.fromstring(xml_str)
+
+        state = {
+            'value': 'x^2+1/2',
+            'response_data': RESPONSE_DATA
+        }
+        self.the_input = lookup_tag('formulaequationinput')(test_capa_system(), element, state)
+
+    def test_rendering(self):
+        """
+        Verify that the render context matches the expected render context
+        """
+        context = self.the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'prob_1_2'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': 'x^2+1/2',
+            'status': inputtypes.Status('unanswered'),
+            'msg': '',
+            'size': self.size,
+            'previewer': '/dummy-static/js/capa/src/formula_equation_preview.js',
+            'inline': False,
+            'trailing_text': '',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+        assert context == expected
+
+    def test_trailing_text_rendering(self):
+        """
+        Verify that the render context matches the expected render context with trailing_text
+        """
+        size = "42"
+        # store (xml_text, expected)
+        trailing_text = []
+        # standard trailing text
+        trailing_text.append(('m/s', 'm/s'))
+        # unicode trailing text
+        trailing_text.append(('\xc3', '\xc3'))
+        # html escaped trailing text
+        # this is the only one we expect to change
+        trailing_text.append(('a &lt; b', 'a < b'))
+
+        for xml_text, expected_text in trailing_text:
+            xml_str = """<formulaequationinput id="prob_1_2"
+                            size="{size}"
+                            trailing_text="{tt}"
+                            />""".format(size=size, tt=xml_text)
+
+            element = etree.fromstring(xml_str)
+
+            state = {
+                'value': 'x^2+1/2',
+                'response_data': RESPONSE_DATA
+            }
+            the_input = lookup_tag('formulaequationinput')(test_capa_system(), element, state)
+
+            context = the_input._get_render_context()  # pylint: disable=protected-access
+            prob_id = 'prob_1_2'
+            expected = {
+                'STATIC_URL': '/dummy-static/',
+                'id': prob_id,
+                'value': 'x^2+1/2',
+                'status': inputtypes.Status('unanswered'),
+                'msg': '',
+                'size': size,
+                'previewer': '/dummy-static/js/capa/src/formula_equation_preview.js',
+                'inline': False,
+                'trailing_text': expected_text,
+                'response_data': RESPONSE_DATA,
+                'describedby_html': TRAILING_TEXT_DESCRIBEDBY.format(trailing_text_id=prob_id, status_id=prob_id)
+            }
+
+            assert context == expected
+
+    def test_formcalc_ajax_sucess(self):
+        """
+        Verify that using the correct dispatch and valid data produces a valid response
+        """
+        data = {'formula': "x^2+1/2", 'request_start': 0}
+        response = self.the_input.handle_ajax("preview_formcalc", data)
+
+        assert 'preview' in response
+        assert response['preview'] != ''
+        assert response['error'] == ''
+        assert response['request_start'] == data['request_start']
+
+    def test_ajax_bad_method(self):
+        """
+        With a bad dispatch, we shouldn't receive anything
+        """
+        response = self.the_input.handle_ajax("obviously_not_real", {})
+        assert response == {}
+
+    def test_ajax_no_formula(self):
+        """
+        When we ask for a formula rendering, there should be an error if no formula
+        """
+        response = self.the_input.handle_ajax(
+            "preview_formcalc",
+            {'request_start': 1, }
+        )
+        assert 'error' in response
+        assert response['error'] == 'No formula specified.'
+
+    def test_ajax_parse_err(self):
+        """
+        With parse errors, FormulaEquationInput should give an error message
+        """
+        # Simulate answering a problem that raises the exception
+        with patch('xmodule.capa.inputtypes.latex_preview') as mock_preview:
+            mock_preview.side_effect = ParseException("Oopsie")
+            response = self.the_input.handle_ajax(
+                "preview_formcalc",
+                {'formula': 'x^2+1/2', 'request_start': 1, }
+            )
+
+        assert 'error' in response
+        assert response['error'] == "Sorry, couldn't parse formula"
+
+    @patch('xmodule.capa.inputtypes.log')
+    def test_ajax_other_err(self, mock_log):
+        """
+        With other errors, test that FormulaEquationInput also logs it
+        """
+        with patch('xmodule.capa.inputtypes.latex_preview') as mock_preview:
+            mock_preview.side_effect = Exception()
+            response = self.the_input.handle_ajax(
+                "preview_formcalc",
+                {'formula': 'x^2+1/2', 'request_start': 1, }
+            )
+        mock_log.warning.assert_called_once_with(
+            "Error while previewing formula", exc_info=True
+        )
+        assert 'error' in response
+        assert response['error'] == 'Error while rendering preview'
+
+
+class DragAndDropTest(unittest.TestCase):
+    """
+    Check that drag and drop inputs work
+    """
+    def test_rendering(self):
+        path_to_images = '/dummy-static/images/'
+
+        xml_str = """
+        <drag_and_drop_input id="prob_1_2" img="{path}about_1.png" target_outline="false">
+            <draggable id="1" label="Label 1"/>
+            <draggable id="name_with_icon" label="cc" icon="{path}cc.jpg"/>
+            <draggable id="with_icon" label="arrow-left" icon="{path}arrow-left.png" />
+            <draggable id="5" label="Label2" />
+            <draggable id="2" label="Mute" icon="{path}mute.png" />
+            <draggable id="name_label_icon3" label="spinner" icon="{path}spinner.gif" />
+            <draggable id="name4" label="Star" icon="{path}volume.png" />
+            <draggable id="7" label="Label3" />
+
+            <target id="t1" x="210" y="90" w="90" h="90"/>
+            <target id="t2" x="370" y="160" w="90" h="90"/>
+
+        </drag_and_drop_input>
+        """.format(path=path_to_images)
+
+        element = etree.fromstring(xml_str)
+
+        value = 'abc'
+        state = {
+            'value': value,
+            'status': 'unsubmitted',
+            'response_data': RESPONSE_DATA
+        }
+
+        user_input = {  # order matters, for string comparison
+                        "target_outline": "false",
+                        "base_image": "/dummy-static/images/about_1.png",
+                        "draggables": [
+                            {"can_reuse": "", "label": "Label 1", "id": "1", "icon": "", "target_fields": []},
+                            {"can_reuse": "", "label": "cc", "id": "name_with_icon", "icon": "/dummy-static/images/cc.jpg", "target_fields": []},  # lint-amnesty, pylint: disable=line-too-long
+                            {"can_reuse": "", "label": "arrow-left", "id": "with_icon", "icon": "/dummy-static/images/arrow-left.png", "target_fields": []},  # lint-amnesty, pylint: disable=line-too-long
+                            {"can_reuse": "", "label": "Label2", "id": "5", "icon": "", "target_fields": []},
+                            {"can_reuse": "", "label": "Mute", "id": "2", "icon": "/dummy-static/images/mute.png", "target_fields": []},  # lint-amnesty, pylint: disable=line-too-long
+                            {"can_reuse": "", "label": "spinner", "id": "name_label_icon3", "icon": "/dummy-static/images/spinner.gif", "target_fields": []},  # lint-amnesty, pylint: disable=line-too-long
+                            {"can_reuse": "", "label": "Star", "id": "name4", "icon": "/dummy-static/images/volume.png", "target_fields": []},  # lint-amnesty, pylint: disable=line-too-long
+                            {"can_reuse": "", "label": "Label3", "id": "7", "icon": "", "target_fields": []}],
+                        "one_per_target": "True",
+                        "targets": [
+                            {"y": "90", "x": "210", "id": "t1", "w": "90", "h": "90"},
+                            {"y": "160", "x": "370", "id": "t2", "w": "90", "h": "90"}
+                        ]
+        }
+
+        the_input = lookup_tag('drag_and_drop_input')(test_capa_system(), element, state)
+        prob_id = 'prob_1_2'
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'value': value,
+            'status': inputtypes.Status('unsubmitted'),
+            'msg': '',
+            'drag_and_drop_json': json.dumps(user_input),
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        # as we are dumping 'draggables' dicts while dumping user_input, string
+        # comparison will fail, as order of keys is random.
+        assert json.loads(context['drag_and_drop_json']) == user_input
+        context.pop('drag_and_drop_json')
+        expected.pop('drag_and_drop_json')
+        assert context == expected
+
+
+class AnnotationInputTest(unittest.TestCase):
+    """
+    Make sure option inputs work
+    """
+    def test_rendering(self):
+        xml_str = '''
+<annotationinput>
+    <title>foo</title>
+    <text>bar</text>
+    <comment>my comment</comment>
+    <comment_prompt>type a commentary</comment_prompt>
+    <tag_prompt>select a tag</tag_prompt>
+    <options>
+        <option choice="correct">x</option>
+        <option choice="incorrect">y</option>
+        <option choice="partially-correct">z</option>
+    </options>
+</annotationinput>
+'''
+        element = etree.fromstring(xml_str)
+
+        value = {"comment": "blah blah", "options": [1]}
+        json_value = json.dumps(value)
+        state = {
+            'value': json_value,
+            'id': 'annotation_input',
+            'status': 'answered',
+            'response_data': RESPONSE_DATA
+        }
+
+        tag = 'annotationinput'
+
+        the_input = lookup_tag(tag)(test_capa_system(), element, state)
+
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        prob_id = 'annotation_input'
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'id': prob_id,
+            'status': inputtypes.Status('answered'),
+            'msg': '',
+            'title': 'foo',
+            'text': 'bar',
+            'comment': 'my comment',
+            'comment_prompt': 'type a commentary',
+            'tag_prompt': 'select a tag',
+            'options': [
+                {'id': 0, 'description': 'x', 'choice': 'correct'},
+                {'id': 1, 'description': 'y', 'choice': 'incorrect'},
+                {'id': 2, 'description': 'z', 'choice': 'partially-correct'}
+            ],
+            'value': json_value,
+            'options_value': value['options'],
+            'has_options_value': len(value['options']) > 0,
+            'comment_value': value['comment'],
+            'debug': False,
+            'return_to_annotation': True,
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+
+        self.maxDiff = None
+        self.assertDictEqual(context, expected)
+
+
+class TestChoiceText(unittest.TestCase):
+    """
+    Tests for checkboxtextgroup inputs
+    """
+    @staticmethod
+    def build_choice_element(node_type, contents, tail_text, value):
+        """
+        Builds a content node for a choice.
+        """
+        # When xml is being parsed numtolerance_input and decoy_input tags map to textinput type
+        # in order to provide the template with correct rendering information.
+        if node_type in ('numtolerance_input', 'decoy_input'):
+            node_type = 'textinput'
+        choice = {'type': node_type, 'contents': contents, 'tail_text': tail_text, 'value': value}
+        return choice
+
+    def check_group(self, tag, choice_tag, expected_input_type):
+        """
+        Build a radio or checkbox group, parse it and check the resuls against the
+        expected output.
+
+        `tag` should be 'checkboxtextgroup' or 'radiotextgroup'
+        `choice_tag` is either 'choice' for proper xml, or any other value to trigger an error.
+        `expected_input_type` is either 'radio' or 'checkbox'.
+        """
+        xml_str = """
+  <{tag}>
+      <{choice_tag} correct="false" name="choiceinput_0">this is<numtolerance_input name="choiceinput_0_textinput_0"/>false</{choice_tag}>
+      <choice correct="true" name="choiceinput_1">Is a number<decoy_input name="choiceinput_1_textinput_0"/><text>!</text></choice>
+  </{tag}>
+        """.format(tag=tag, choice_tag=choice_tag)
+        element = etree.fromstring(xml_str)
+        prob_id = 'choicetext_input'
+        state = {
+            'value': '{}',
+            'id': prob_id,
+            'status': inputtypes.Status('answered'),
+            'response_data': RESPONSE_DATA
+        }
+
+        first_input = self.build_choice_element('numtolerance_input', 'choiceinput_0_textinput_0', 'false', '')
+        second_input = self.build_choice_element('decoy_input', 'choiceinput_1_textinput_0', '', '')
+        first_choice_content = self.build_choice_element('text', 'this is', '', '')
+        second_choice_content = self.build_choice_element('text', 'Is a number', '', '')
+        second_choice_text = self.build_choice_element('text', "!", '', '')
+
+        choices = [
+            ('choiceinput_0', [first_choice_content, first_input]),
+            ('choiceinput_1', [second_choice_content, second_input, second_choice_text])
+        ]
+        expected = {
+            'STATIC_URL': '/dummy-static/',
+            'msg': '',
+            'input_type': expected_input_type,
+            'choices': choices,
+            'show_correctness': 'always',
+            'submitted_message': 'Answer received.',
+            'response_data': RESPONSE_DATA,
+            'describedby_html': DESCRIBEDBY.format(status_id=prob_id)
+        }
+        expected.update(state)
+        the_input = lookup_tag(tag)(test_capa_system(), element, state)
+        context = the_input._get_render_context()  # pylint: disable=protected-access
+        assert context == expected
+
+    def test_radiotextgroup(self):
+        """
+        Test that a properly formatted radiotextgroup problem generates
+        expected ouputs
+        """
+        self.check_group('radiotextgroup', 'choice', 'radio')
+
+    def test_checkboxtextgroup(self):
+        """
+        Test that a properly formatted checkboxtextgroup problem generates
+        expected ouput
+        """
+        self.check_group('checkboxtextgroup', 'choice', 'checkbox')
+
+    def test_invalid_tag(self):
+        """
+        Test to ensure that an unrecognized inputtype tag causes an error
+        """
+        with pytest.raises(Exception):
+            self.check_group('invalid', 'choice', 'checkbox')
+
+    def test_invalid_input_tag(self):
+        """
+        Test to ensure having a tag other than <choice> inside of
+        a checkbox or radiotextgroup problem raises an error.
+        """
+        with self.assertRaisesRegex(Exception, "Error in xml"):
+            self.check_group('checkboxtextgroup', 'invalid', 'checkbox')
+
+
+class TestStatus(unittest.TestCase):
+    """
+    Tests for Status class
+    """
+    def test_str(self):
+        """
+        Test stringifing Status objects
+        """
+        statobj = inputtypes.Status('test')
+        assert str(statobj) == 'test'
+        assert six.text_type(statobj) == 'test'
+
+    def test_classes(self):
+        """
+        Test that css classnames are correct
+        """
+        css_classes = [
+            ('unsubmitted', 'unanswered'),
+            ('incomplete', 'incorrect'),
+            ('queued', 'processing'),
+            ('correct', 'correct'),
+            ('test', 'test'),
+        ]
+        for status, classname in css_classes:
+            statobj = inputtypes.Status(status)
+            assert statobj.classname == classname
+
+    def test_display_names(self):
+        """
+        Test that display names are correct
+        """
+        names = [
+            ('correct', 'correct'),
+            ('incorrect', 'incorrect'),
+            ('incomplete', 'incomplete'),
+            ('unanswered', 'unanswered'),
+            ('unsubmitted', 'unanswered'),
+            ('queued', 'processing'),
+            ('dave', 'dave'),
+        ]
+        for status, display_name in names:
+            statobj = inputtypes.Status(status)
+            assert statobj.display_name == display_name
+
+    def test_translated_names(self):
+        """
+        Test that display names are "translated"
+        """
+        func = lambda t: t.upper()
+        # status is in the mapping
+        statobj = inputtypes.Status('queued', func)
+        assert statobj.display_name == 'PROCESSING'
+
+        # status is not in the mapping
+        statobj = inputtypes.Status('test', func)
+        assert statobj.display_name == 'test'
+        assert str(statobj) == 'test'
+        assert statobj.classname == 'test'

xmodule/capa/tests/test_responsetypes.py

@@ -0,0 +1,2824 @@
+# -*- coding: utf-8 -*-
+"""
+Tests of responsetypes
+"""
+
+
+import io
+import json
+import os
+import textwrap
+import unittest
+import zipfile
+from datetime import datetime
+
+import pytest
+import calc
+import mock
+import pyparsing
+import random2 as random
+import requests
+import six
+from pytz import UTC
+from six import text_type
+
+from xmodule.capa.correctmap import CorrectMap
+from xmodule.capa.responsetypes import LoncapaProblemError, ResponseError, StudentInputError
+from xmodule.capa.tests.helpers import load_fixture, new_loncapa_problem, test_capa_system
+from xmodule.capa.tests.response_xml_factory import (
+    AnnotationResponseXMLFactory,
+    ChoiceResponseXMLFactory,
+    ChoiceTextResponseXMLFactory,
+    CodeResponseXMLFactory,
+    CustomResponseXMLFactory,
+    FormulaResponseXMLFactory,
+    ImageResponseXMLFactory,
+    MultipleChoiceResponseXMLFactory,
+    NumericalResponseXMLFactory,
+    OptionResponseXMLFactory,
+    SchematicResponseXMLFactory,
+    StringResponseXMLFactory,
+    SymbolicResponseXMLFactory,
+    TrueFalseResponseXMLFactory
+)
+from xmodule.capa.util import convert_files_to_filenames
+from xmodule.capa.xqueue_interface import dateformat
+
+
+class ResponseTest(unittest.TestCase):
+    """Base class for tests of capa responses."""
+
+    xml_factory_class = None
+
+    # If something is wrong, show it to us.
+    maxDiff = None
+
+    def setUp(self):
+        super(ResponseTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        if self.xml_factory_class:
+            self.xml_factory = self.xml_factory_class()  # lint-amnesty, pylint: disable=not-callable
+
+    def build_problem(self, capa_system=None, **kwargs):
+        xml = self.xml_factory.build_xml(**kwargs)
+        return new_loncapa_problem(xml, capa_system=capa_system)
+
+    # pylint: disable=missing-function-docstring
+    def assert_grade(self, problem, submission, expected_correctness, msg=None):
+        input_dict = {'1_2_1': submission}
+        correct_map = problem.grade_answers(input_dict)
+        if msg is None:
+            assert correct_map.get_correctness('1_2_1') == expected_correctness
+        else:
+            assert correct_map.get_correctness('1_2_1') == expected_correctness, msg
+
+    def assert_answer_format(self, problem):
+        answers = problem.get_question_answers()
+        assert answers['1_2_1'] is not None
+
+    # pylint: disable=missing-function-docstring
+    def assert_multiple_grade(self, problem, correct_answers, incorrect_answers):
+        for input_str in correct_answers:
+            result = problem.grade_answers({'1_2_1': input_str}).get_correctness('1_2_1')
+            assert result == 'correct'
+
+        for input_str in incorrect_answers:
+            result = problem.grade_answers({'1_2_1': input_str}).get_correctness('1_2_1')
+            assert result == 'incorrect'
+
+    def assert_multiple_partial(self, problem, correct_answers, incorrect_answers, partial_answers):
+        """
+        Runs multiple asserts for varying correct, incorrect,
+        and partially correct answers, all passed as lists.
+        """
+        for input_str in correct_answers:
+            result = problem.grade_answers({'1_2_1': input_str}).get_correctness('1_2_1')
+            assert result == 'correct'
+
+        for input_str in incorrect_answers:
+            result = problem.grade_answers({'1_2_1': input_str}).get_correctness('1_2_1')
+            assert result == 'incorrect'
+
+        for input_str in partial_answers:
+            result = problem.grade_answers({'1_2_1': input_str}).get_correctness('1_2_1')
+            assert result == 'partially-correct'
+
+    def _get_random_number_code(self):
+        """Returns code to be used to generate a random result."""
+        return "str(random.randint(0, 1e9))"
+
+    def _get_random_number_result(self, seed_value):
+        """Returns a result that should be generated using the random_number_code."""
+        rand = random.Random(seed_value)
+        return str(rand.randint(0, 1e9))
+
+
+class MultiChoiceResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = MultipleChoiceResponseXMLFactory
+
+    def test_multiple_choice_grade(self):
+        problem = self.build_problem(choices=[False, True, False])
+
+        # Ensure that we get the expected grades
+        self.assert_grade(problem, 'choice_0', 'incorrect')
+        self.assert_grade(problem, 'choice_1', 'correct')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+
+    def test_partial_multiple_choice_grade(self):
+        problem = self.build_problem(choices=[False, True, 'partial'], credit_type='points')
+
+        # Ensure that we get the expected grades
+        self.assert_grade(problem, 'choice_0', 'incorrect')
+        self.assert_grade(problem, 'choice_1', 'correct')
+        self.assert_grade(problem, 'choice_2', 'partially-correct')
+
+    def test_named_multiple_choice_grade(self):
+        problem = self.build_problem(choices=[False, True, False],
+                                     choice_names=["foil_1", "foil_2", "foil_3"])
+
+        # Ensure that we get the expected grades
+        self.assert_grade(problem, 'choice_foil_1', 'incorrect')
+        self.assert_grade(problem, 'choice_foil_2', 'correct')
+        self.assert_grade(problem, 'choice_foil_3', 'incorrect')
+
+    def test_multiple_choice_valid_grading_schemes(self):
+        # Multiple Choice problems only allow one partial credit scheme.
+        # Change this test if that changes.
+        problem = self.build_problem(choices=[False, True, 'partial'], credit_type='points,points')
+        with pytest.raises(LoncapaProblemError):
+            input_dict = {'1_2_1': 'choice_1'}
+            problem.grade_answers(input_dict)
+
+        # 'bongo' is not a valid grading scheme.
+        problem = self.build_problem(choices=[False, True, 'partial'], credit_type='bongo')
+        with pytest.raises(LoncapaProblemError):
+            input_dict = {'1_2_1': 'choice_1'}
+            problem.grade_answers(input_dict)
+
+    def test_partial_points_multiple_choice_grade(self):
+        problem = self.build_problem(
+            choices=['partial', 'partial', 'partial'],
+            credit_type='points',
+            points=['1', '0.6', '0']
+        )
+
+        # Ensure that we get the expected number of points
+        # Using assertAlmostEqual to avoid floating point issues
+        correct_map = problem.grade_answers({'1_2_1': 'choice_0'})
+        assert round(correct_map.get_npoints('1_2_1') - 1, 7) >= 0
+
+        correct_map = problem.grade_answers({'1_2_1': 'choice_1'})
+        assert round(correct_map.get_npoints('1_2_1') - 0.6, 7) >= 0
+
+        correct_map = problem.grade_answers({'1_2_1': 'choice_2'})
+        assert round(correct_map.get_npoints('1_2_1') - 0, 7) >= 0
+
+    def test_contextualized_choices(self):
+        script = textwrap.dedent("""
+            a = 2
+            b = 9
+            c = a + b
+
+            ok0 = c % 2 == 0 # check remainder modulo 2
+            text0 = "$a + $b is even"
+
+            ok1 = c % 2 == 1 # check remainder modulo 2
+            text1 = "$a + $b is odd"
+
+            ok2 = "partial"
+            text2 = "infinity may be both"
+        """)
+        choices = ["$ok0", "$ok1", "$ok2"]
+        choice_names = ["$text0 ... (should be $ok0)",
+                        "$text1 ... (should be $ok1)",
+                        "$text2 ... (should be $ok2)"]
+        problem = self.build_problem(script=script,
+                                     choices=choices,
+                                     choice_names=choice_names,
+                                     credit_type='points')
+
+        # Ensure the expected correctness and choice names
+        self.assert_grade(problem, 'choice_2 + 9 is even ... (should be False)', 'incorrect')
+        self.assert_grade(problem, 'choice_2 + 9 is odd ... (should be True)', 'correct')
+        self.assert_grade(problem, 'choice_infinity may be both ... (should be partial)', 'partially-correct')
+
+
+class TrueFalseResponseTest(ResponseTest):   # pylint: disable=missing-class-docstring
+    xml_factory_class = TrueFalseResponseXMLFactory
+
+    def test_true_false_grade(self):
+        problem = self.build_problem(choices=[False, True, True])
+
+        # Check the results
+        # Mark correct if and only if ALL (and only) correct choices selected
+        self.assert_grade(problem, 'choice_0', 'incorrect')
+        self.assert_grade(problem, 'choice_1', 'incorrect')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_1', 'choice_2'], 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_2'], 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_1'], 'incorrect')
+        self.assert_grade(problem, ['choice_1', 'choice_2'], 'correct')
+
+        # Invalid choices should be marked incorrect (we have no choice 3)
+        self.assert_grade(problem, 'choice_3', 'incorrect')
+        self.assert_grade(problem, 'not_a_choice', 'incorrect')
+
+    def test_named_true_false_grade(self):
+        problem = self.build_problem(choices=[False, True, True],
+                                     choice_names=['foil_1', 'foil_2', 'foil_3'])
+
+        # Check the results
+        # Mark correct if and only if ALL (and only) correct chocies selected
+        self.assert_grade(problem, 'choice_foil_1', 'incorrect')
+        self.assert_grade(problem, 'choice_foil_2', 'incorrect')
+        self.assert_grade(problem, 'choice_foil_3', 'incorrect')
+        self.assert_grade(problem, ['choice_foil_1', 'choice_foil_2', 'choice_foil_3'], 'incorrect')
+        self.assert_grade(problem, ['choice_foil_1', 'choice_foil_3'], 'incorrect')
+        self.assert_grade(problem, ['choice_foil_1', 'choice_foil_2'], 'incorrect')
+        self.assert_grade(problem, ['choice_foil_2', 'choice_foil_3'], 'correct')
+
+        # Invalid choices should be marked incorrect
+        self.assert_grade(problem, 'choice_foil_4', 'incorrect')
+        self.assert_grade(problem, 'not_a_choice', 'incorrect')
+
+    def test_single_correct_response(self):
+        problem = self.build_problem(choices=[True, False])
+        self.assert_grade(problem, 'choice_0', 'correct')
+        self.assert_grade(problem, ['choice_0'], 'correct')
+
+
+class ImageResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = ImageResponseXMLFactory
+
+    def test_rectangle_grade(self):
+        # Define a rectangle with corners (10,10) and (20,20)
+        problem = self.build_problem(rectangle="(10,10)-(20,20)")
+
+        # Anything inside the rectangle (and along the borders) is correct
+        # Everything else is incorrect
+        correct_inputs = ["[12,19]", "[10,10]", "[20,20]",
+                          "[10,15]", "[20,15]", "[15,10]", "[15,20]"]
+        incorrect_inputs = ["[4,6]", "[25,15]", "[15,40]", "[15,4]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_multiple_rectangles_grade(self):
+        # Define two rectangles
+        rectangle_str = "(10,10)-(20,20);(100,100)-(200,200)"
+
+        # Expect that only points inside the rectangles are marked correct
+        problem = self.build_problem(rectangle=rectangle_str)
+        correct_inputs = ["[12,19]", "[120, 130]"]
+        incorrect_inputs = ["[4,6]", "[25,15]", "[15,40]", "[15,4]",
+                            "[50,55]", "[300, 14]", "[120, 400]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_region_grade(self):
+        # Define a triangular region with corners (0,0), (5,10), and (0, 10)
+        region_str = "[ [1,1], [5,10], [0,10] ]"
+
+        # Expect that only points inside the triangle are marked correct
+        problem = self.build_problem(regions=region_str)
+        correct_inputs = ["[2,4]", "[1,3]"]
+        incorrect_inputs = ["[0,0]", "[3,5]", "[5,15]", "[30, 12]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_multiple_regions_grade(self):
+        # Define multiple regions that the user can select
+        region_str = "[[[10,10], [20,10], [20, 30]], [[100,100], [120,100], [120,150]]]"
+
+        # Expect that only points inside the regions are marked correct
+        problem = self.build_problem(regions=region_str)
+        correct_inputs = ["[15,12]", "[110,112]"]
+        incorrect_inputs = ["[0,0]", "[600,300]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_region_and_rectangle_grade(self):
+        rectangle_str = "(100,100)-(200,200)"
+        region_str = "[[10,10], [20,10], [20, 30]]"
+
+        # Expect that only points inside the rectangle or region are marked correct
+        problem = self.build_problem(regions=region_str, rectangle=rectangle_str)
+        correct_inputs = ["[13,12]", "[110,112]"]
+        incorrect_inputs = ["[0,0]", "[600,300]"]
+        self.assert_multiple_grade(problem, correct_inputs, incorrect_inputs)
+
+    def test_show_answer(self):
+        rectangle_str = "(100,100)-(200,200)"
+        region_str = "[[10,10], [20,10], [20, 30]]"
+
+        problem = self.build_problem(regions=region_str, rectangle=rectangle_str)
+        self.assert_answer_format(problem)
+
+
+class SymbolicResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = SymbolicResponseXMLFactory
+
+    def test_grade_single_input_incorrect(self):
+        problem = self.build_problem(math_display=True, expect="2*x+3*y")
+
+        # Incorrect answers
+        incorrect_inputs = [
+            ('0', ''),
+            ('4x+3y', textwrap.dedent("""
+                <math xmlns="http://www.w3.org/1998/Math/MathML">
+                    <mstyle displaystyle="true">
+                    <mn>4</mn><mo>*</mo><mi>x</mi><mo>+</mo><mn>3</mn><mo>*</mo><mi>y</mi>
+                    </mstyle></math>""")),
+        ]
+
+        for (input_str, input_mathml) in incorrect_inputs:
+            self._assert_symbolic_grade(problem, input_str, input_mathml, 'incorrect')
+
+    def test_complex_number_grade_incorrect(self):
+
+        problem = self.build_problem(math_display=True,
+                                     expect="[[cos(theta),i*sin(theta)],[i*sin(theta),cos(theta)]]",
+                                     options=["matrix", "imaginary"])
+
+        wrong_snuggletex = load_fixture('snuggletex_wrong.html')
+        dynamath_input = textwrap.dedent("""
+            <math xmlns="http://www.w3.org/1998/Math/MathML">
+              <mstyle displaystyle="true"><mn>2</mn></mstyle>
+            </math>
+        """)
+
+        self._assert_symbolic_grade(
+            problem, "2", dynamath_input,
+            'incorrect',
+            snuggletex_resp=wrong_snuggletex,
+        )
+
+    def test_multiple_inputs_exception(self):
+
+        # Should not allow multiple inputs, since we specify
+        # only one "expect" value
+        with pytest.raises(Exception):
+            self.build_problem(math_display=True, expect="2*x+3*y", num_inputs=3)
+
+    def _assert_symbolic_grade(
+        self, problem, student_input, dynamath_input, expected_correctness,
+        snuggletex_resp=""
+    ):
+        """
+        Assert that the symbolic response has a certain grade.
+
+        `problem` is the capa problem containing the symbolic response.
+        `student_input` is the text the student entered.
+        `dynamath_input` is the JavaScript rendered MathML from the page.
+        `expected_correctness` is either "correct" or "incorrect"
+        `snuggletex_resp` is the simulated response from the Snuggletex server
+        """
+        input_dict = {'1_2_1': str(student_input),
+                      '1_2_1_dynamath': str(dynamath_input)}
+
+        # Simulate what the Snuggletex server would respond
+        with mock.patch.object(requests, 'post') as mock_post:
+            mock_post.return_value.text = snuggletex_resp
+
+            correct_map = problem.grade_answers(input_dict)
+
+            assert correct_map.get_correctness('1_2_1') == expected_correctness
+
+
+class OptionResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = OptionResponseXMLFactory
+
+    def test_grade(self):
+        problem = self.build_problem(options=["first", "second", "third"],
+                                     correct_option="second")
+
+        # Assert that we get the expected grades
+        self.assert_grade(problem, "first", "incorrect")
+        self.assert_grade(problem, "second", "correct")
+        self.assert_grade(problem, "third", "incorrect")
+
+        # Options not in the list should be marked incorrect
+        self.assert_grade(problem, "invalid_option", "incorrect")
+
+    def test_quote_option(self):
+        # Test that option response properly escapes quotes inside options strings
+        problem = self.build_problem(options=["hasnot", "hasn't", "has'nt"],
+                                     correct_option="hasn't")
+
+        # Assert that correct option with a quote inside is marked correctly
+        self.assert_grade(problem, "hasnot", "incorrect")
+        self.assert_grade(problem, "hasn't", "correct")
+        self.assert_grade(problem, "hasn\'t", "correct")
+        self.assert_grade(problem, "has'nt", "incorrect")
+
+    def test_variable_options(self):
+        """
+        Test that if variable are given in option response then correct map must contain answervariable value.
+        """
+        script = textwrap.dedent("""\
+        a = 1000
+        b = a*2
+        c = a*3
+        """)
+        problem = self.build_problem(
+            options=['$a', '$b', '$c'],
+            correct_option='$a',
+            script=script
+        )
+
+        input_dict = {'1_2_1': '1000'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_correctness('1_2_1') == 'correct'
+        assert correct_map.get_property('1_2_1', 'answervariable') == '$a'
+
+
+class FormulaResponseTest(ResponseTest):
+    """
+    Test the FormulaResponse class
+    """
+    xml_factory_class = FormulaResponseXMLFactory
+
+    def test_grade(self):
+        """
+        Test basic functionality of FormulaResponse
+
+        Specifically, if it can understand equivalence of formulae
+        """
+        # Sample variables x and y in the range [-10, 10]
+        sample_dict = {'x': (-10, 10), 'y': (-10, 10)}
+
+        # The expected solution is numerically equivalent to x+2y
+        problem = self.build_problem(sample_dict=sample_dict,
+                                     num_samples=10,
+                                     tolerance=0.01,
+                                     answer="x+2*y")
+
+        # Expect an equivalent formula to be marked correct
+        # 2x - x + y + y = x + 2y
+        input_formula = "2*x - x + y + y"
+        self.assert_grade(problem, input_formula, "correct")
+
+        # Expect an incorrect formula to be marked incorrect
+        # x + y != x + 2y
+        input_formula = "x + y"
+        self.assert_grade(problem, input_formula, "incorrect")
+
+    def test_hint(self):
+        """
+        Test the hint-giving functionality of FormulaResponse
+        """
+        # Sample variables x and y in the range [-10, 10]
+        sample_dict = {'x': (-10, 10), 'y': (-10, 10)}
+
+        # Give a hint if the user leaves off the coefficient
+        # or leaves out x
+        hints = [('x + 3*y', 'y_coefficient', 'Check the coefficient of y'),
+                 ('2*y', 'missing_x', 'Try including the variable x')]
+
+        # The expected solution is numerically equivalent to x+2y
+        problem = self.build_problem(sample_dict=sample_dict,
+                                     num_samples=10,
+                                     tolerance=0.01,
+                                     answer="x+2*y",
+                                     hints=hints)
+
+        # Expect to receive a hint  if we add an extra y
+        input_dict = {'1_2_1': "x + 2*y + y"}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'Check the coefficient of y'
+
+        # Expect to receive a hint if we leave out x
+        input_dict = {'1_2_1': "2*y"}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'Try including the variable x'
+
+    def test_script(self):
+        """
+        Test if python script can be used to generate answers
+        """
+
+        # Calculate the answer using a script
+        script = "calculated_ans = 'x+x'"
+
+        # Sample x in the range [-10,10]
+        sample_dict = {'x': (-10, 10)}
+
+        # The expected solution is numerically equivalent to 2*x
+        problem = self.build_problem(sample_dict=sample_dict,
+                                     num_samples=10,
+                                     tolerance=0.01,
+                                     answer="$calculated_ans",
+                                     script=script)
+
+        # Expect that the inputs are graded correctly
+        self.assert_grade(problem, '2*x', 'correct')
+        self.assert_grade(problem, '3*x', 'incorrect')
+
+    def test_grade_infinity(self):
+        """
+        Test that a large input on a problem with relative tolerance isn't
+        erroneously marked as correct.
+        """
+
+        sample_dict = {'x': (1, 2)}
+
+        # Test problem
+        problem = self.build_problem(sample_dict=sample_dict,
+                                     num_samples=10,
+                                     tolerance="1%",
+                                     answer="x")
+        # Expect such a large answer to be marked incorrect
+        input_formula = "x*1e999"
+        self.assert_grade(problem, input_formula, "incorrect")
+        # Expect such a large negative answer to be marked incorrect
+        input_formula = "-x*1e999"
+        self.assert_grade(problem, input_formula, "incorrect")
+
+    def test_grade_nan(self):
+        """
+        Test that expressions that evaluate to NaN are not marked as correct.
+        """
+
+        sample_dict = {'x': (1, 2)}
+
+        # Test problem
+        problem = self.build_problem(sample_dict=sample_dict,
+                                     num_samples=10,
+                                     tolerance="1%",
+                                     answer="x")
+        # Expect an incorrect answer (+ nan) to be marked incorrect
+        # Right now this evaluates to 'nan' for a given x (Python implementation-dependent)
+        input_formula = "10*x + 0*1e999"
+        self.assert_grade(problem, input_formula, "incorrect")
+        # Expect an correct answer (+ nan) to be marked incorrect
+        input_formula = "x + 0*1e999"
+        self.assert_grade(problem, input_formula, "incorrect")
+
+    def test_raises_zero_division_err(self):
+        """
+        See if division by zero raises an error.
+        """
+        sample_dict = {'x': (1, 2)}
+        problem = self.build_problem(sample_dict=sample_dict,
+                                     num_samples=10,
+                                     tolerance="1%",
+                                     answer="x")  # Answer doesn't matter
+        input_dict = {'1_2_1': '1/0'}
+        self.assertRaises(StudentInputError, problem.grade_answers, input_dict)
+
+    def test_validate_answer(self):
+        """
+        Makes sure that validate_answer works.
+        """
+        sample_dict = {'x': (1, 2)}
+        problem = self.build_problem(
+            sample_dict=sample_dict,
+            num_samples=10,
+            tolerance="1%",
+            answer="x"
+        )
+        assert list(problem.responders.values())[0].validate_answer('14*x')
+        assert not list(problem.responders.values())[0].validate_answer('3*y+2*x')
+
+
+class StringResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = StringResponseXMLFactory
+
+    def test_backward_compatibility_for_multiple_answers(self):
+        """
+        Remove this test, once support for _or_ separator will be removed.
+        """
+
+        answers = ["Second", "Third", "Fourth"]
+        problem = self.build_problem(answer="_or_".join(answers), case_sensitive=True)
+
+        for answer in answers:
+            # Exact string should be correct
+            self.assert_grade(problem, answer, "correct")
+        # Other strings and the lowercase version of the string are incorrect
+        self.assert_grade(problem, "Other String", "incorrect")
+
+        problem = self.build_problem(answer="_or_".join(answers), case_sensitive=False)
+        for answer in answers:
+            # Exact string should be correct
+            self.assert_grade(problem, answer, "correct")
+            self.assert_grade(problem, answer.lower(), "correct")
+        self.assert_grade(problem, "Other String", "incorrect")
+
+    def test_regexp(self):
+        problem = self.build_problem(answer="Second", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "Second", "correct")
+
+        problem = self.build_problem(answer="sec", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "Second", "incorrect")
+
+        problem = self.build_problem(answer="sec.*", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "Second", "correct")
+
+        problem = self.build_problem(answer="sec.*", case_sensitive=True, regexp=True)
+        self.assert_grade(problem, "Second", "incorrect")
+
+        problem = self.build_problem(answer="Sec.*$", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "Second", "correct")
+
+        problem = self.build_problem(answer="^sec$", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "Second", "incorrect")
+
+        problem = self.build_problem(answer="^Sec(ond)?$", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "Second", "correct")
+
+        problem = self.build_problem(answer="^Sec(ond)?$", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "Sec", "correct")
+
+        problem = self.build_problem(answer="tre+", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "There is a tree", "incorrect")
+
+        problem = self.build_problem(answer=".*tre+", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "There is a tree", "correct")
+
+        # test with case_sensitive not specified
+        problem = self.build_problem(answer=".*tre+", regexp=True)
+        self.assert_grade(problem, "There is a tree", "correct")
+
+        answers = [
+            "Martin Luther King Junior",
+            "Doctor Martin Luther King Junior",
+            "Dr. Martin Luther King Jr.",
+            "Martin Luther King"
+        ]
+
+        problem = self.build_problem(answer=r"\w*\.?.*Luther King\s*.*", case_sensitive=True, regexp=True)
+
+        for answer in answers:
+            self.assert_grade(problem, answer, "correct")
+
+        problem = self.build_problem(answer=r"^(-\|){2,5}$", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "-|-|-|", "correct")
+        self.assert_grade(problem, "-|", "incorrect")
+        self.assert_grade(problem, "-|-|-|-|-|-|", "incorrect")
+
+        regexps = [
+            "^One$",
+            "two",
+            "^thre+",
+            "^4|Four$",
+        ]
+        problem = self.build_problem(
+            answer="just_sample",
+            case_sensitive=False,
+            regexp=True,
+            additional_answers=regexps
+        )
+
+        self.assert_grade(problem, "One", "correct")
+        self.assert_grade(problem, "two", "correct")
+        self.assert_grade(problem, "!!two!!", "correct")
+        self.assert_grade(problem, "threeeee", "correct")
+        self.assert_grade(problem, "three", "correct")
+        self.assert_grade(problem, "4", "correct")
+        self.assert_grade(problem, "Four", "correct")
+        self.assert_grade(problem, "Five", "incorrect")
+        self.assert_grade(problem, "|", "incorrect")
+
+        # test unicode
+        problem = self.build_problem(answer="æ", case_sensitive=False, regexp=True, additional_answers=['ö'])
+        self.assert_grade(problem, "æ", "correct")
+        self.assert_grade(problem, "ö", "correct")
+        self.assert_grade(problem, "î", "incorrect")
+        self.assert_grade(problem, "o", "incorrect")
+
+    def test_backslash_and_unicode_regexps(self):
+        r"""
+        Test some special cases of [unicode] regexps.
+
+        One needs to use either r'' strings or write real `repr` of unicode strings, because of the following
+        (from python docs, http://docs.python.org/2/library/re.html):
+
+        'for example, to match a literal backslash, one might have to write '\\\\' as the pattern string,
+        because the regular expression must be \\,
+        and each backslash must be expressed as \\ inside a regular Python string literal.'
+
+        Example of real use case in Studio:
+            a) user inputs regexp in usual regexp language,
+            b) regexp is saved to xml and is read in python as repr of that string
+            So  a\d in front-end editor will become a\\\\d in xml,  so it will match a1 as student answer.
+        """
+        problem = self.build_problem(answer="5\\\\æ", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "5\\æ", "correct")
+
+        problem = self.build_problem(answer="5\\\\æ", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "5\\æ", "correct")
+
+    def test_backslash(self):
+        problem = self.build_problem(answer="a\\\\c1", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "a\\c1", "correct")
+
+    def test_special_chars(self):
+        problem = self.build_problem(answer="a \\s1", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "a  1", "correct")
+
+    def test_case_sensitive(self):
+        # Test single answer
+        problem_specified = self.build_problem(answer="Second", case_sensitive=True)
+
+        # should also be case_sensitive if case sensitivity is not specified
+        problem_not_specified = self.build_problem(answer="Second")
+        problems = [problem_specified, problem_not_specified]
+
+        for problem in problems:
+            # Exact string should be correct
+            self.assert_grade(problem, "Second", "correct")
+
+            # Other strings and the lowercase version of the string are incorrect
+            self.assert_grade(problem, "Other String", "incorrect")
+            self.assert_grade(problem, "second", "incorrect")
+
+        # Test multiple answers
+        answers = ["Second", "Third", "Fourth"]
+
+        # set up problems
+        problem_specified = self.build_problem(
+            answer="sample_answer", case_sensitive=True, additional_answers=answers
+        )
+        problem_not_specified = self.build_problem(
+            answer="sample_answer", additional_answers=answers
+        )
+        problems = [problem_specified, problem_not_specified]
+        for problem in problems:
+            for answer in answers:
+                # Exact string should be correct
+                self.assert_grade(problem, answer, "correct")
+
+            # Other strings and the lowercase version of the string are incorrect
+            self.assert_grade(problem, "Other String", "incorrect")
+            self.assert_grade(problem, "second", "incorrect")
+
+    def test_bogus_escape_not_raised(self):
+        """
+        We now adding ^ and $ around regexp, so no bogus escape error will be raised.
+        """
+        problem = self.build_problem(answer="\\", case_sensitive=False, regexp=True)
+
+        self.assert_grade(problem, "\\", "incorrect")
+
+        # right way to search for \
+        problem = self.build_problem(answer="\\\\", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, "\\", "correct")
+
+    def test_case_insensitive(self):
+        # Test single answer
+        problem = self.build_problem(answer="Second", case_sensitive=False)
+
+        # Both versions of the string should be allowed, regardless
+        # of capitalization
+        self.assert_grade(problem, "Second", "correct")
+        self.assert_grade(problem, "second", "correct")
+
+        # Other strings are not allowed
+        self.assert_grade(problem, "Other String", "incorrect")
+
+        # Test multiple answers
+        answers = ["Second", "Third", "Fourth"]
+        problem = self.build_problem(answer="sample_answer", case_sensitive=False, additional_answers=answers)
+
+        for answer in answers:
+            # Exact string should be correct
+            self.assert_grade(problem, answer, "correct")
+            self.assert_grade(problem, answer.lower(), "correct")
+
+        # Other strings and the lowercase version of the string are incorrect
+        self.assert_grade(problem, "Other String", "incorrect")
+
+    def test_compatible_non_attribute_additional_answer_xml(self):
+        problem = self.build_problem(answer="Donut", non_attribute_answers=["Sprinkles"])
+        self.assert_grade(problem, "Donut", "correct")
+        self.assert_grade(problem, "Sprinkles", "correct")
+        self.assert_grade(problem, "Meh", "incorrect")
+
+    def test_partial_matching(self):
+        problem = self.build_problem(answer="a2", case_sensitive=False, regexp=True, additional_answers=['.?\\d.?'])
+        self.assert_grade(problem, "a3", "correct")
+        self.assert_grade(problem, "3a", "correct")
+
+    def test_exception(self):
+        problem = self.build_problem(answer="a2", case_sensitive=False, regexp=True, additional_answers=['?\\d?'])
+        with pytest.raises(Exception) as cm:
+            self.assert_grade(problem, "a3", "correct")
+        exception_message = text_type(cm.value)
+        assert 'nothing to repeat' in exception_message
+
+    def test_hints(self):
+
+        hints = [
+            ("wisconsin", "wisc", "The state capital of Wisconsin is Madison"),
+            ("minnesota", "minn", "The state capital of Minnesota is St. Paul"),
+        ]
+        problem = self.build_problem(
+            answer="Michigan",
+            case_sensitive=False,
+            hints=hints,
+        )
+        # We should get a hint for Wisconsin
+        input_dict = {'1_2_1': 'Wisconsin'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'The state capital of Wisconsin is Madison'
+
+        # We should get a hint for Minnesota
+        input_dict = {'1_2_1': 'Minnesota'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'The state capital of Minnesota is St. Paul'
+
+        # We should NOT get a hint for Michigan (the correct answer)
+        input_dict = {'1_2_1': 'Michigan'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == ''
+
+        # We should NOT get a hint for any other string
+        input_dict = {'1_2_1': 'California'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == ''
+
+    def test_hints_regexp_and_answer_regexp(self):
+        different_student_answers = [
+            "May be it is Boston",
+            "Boston, really?",
+            "Boston",
+            "OK, I see, this is Boston",
+        ]
+
+        # if problem has regexp = true, it will accept hints written in regexp
+        hints = [
+            ("wisconsin", "wisc", "The state capital of Wisconsin is Madison"),
+            ("minnesota", "minn", "The state capital of Minnesota is St. Paul"),
+            (".*Boston.*", "bst", "First letter of correct answer is M."),
+            ('^\\d9$', "numbers", "Should not end with 9."),
+        ]
+
+        additional_answers = [
+            '^\\d[0-8]$',
+        ]
+        problem = self.build_problem(
+            answer="Michigan",
+            case_sensitive=False,
+            hints=hints,
+            additional_answers=additional_answers,
+            regexp=True
+        )
+
+        # We should get a hint for Wisconsin
+        input_dict = {'1_2_1': 'Wisconsin'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'The state capital of Wisconsin is Madison'
+
+        # We should get a hint for Minnesota
+        input_dict = {'1_2_1': 'Minnesota'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'The state capital of Minnesota is St. Paul'
+
+        # We should NOT get a hint for Michigan (the correct answer)
+        input_dict = {'1_2_1': 'Michigan'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == ''
+
+        # We should NOT get a hint for any other string
+        input_dict = {'1_2_1': 'California'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == ''
+
+        # We should get the same hint for each answer
+        for answer in different_student_answers:
+            input_dict = {'1_2_1': answer}
+            correct_map = problem.grade_answers(input_dict)
+            assert correct_map.get_hint('1_2_1') == 'First letter of correct answer is M.'
+
+        input_dict = {'1_2_1': '59'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'Should not end with 9.'
+
+        input_dict = {'1_2_1': '57'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == ''
+
+    def test_computed_hints(self):
+        problem = self.build_problem(
+            answer="Michigan",
+            hintfn="gimme_a_hint",
+            script=textwrap.dedent("""
+                def gimme_a_hint(answer_ids, student_answers, new_cmap, old_cmap):
+                    aid = answer_ids[0]
+                    answer = student_answers[aid]
+                    new_cmap.set_hint_and_mode(aid, answer+"??", "always")
+            """)
+        )
+
+        input_dict = {'1_2_1': 'Hello'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_hint('1_2_1') == 'Hello??'
+
+    def test_hint_function_randomization(self):
+        # The hint function should get the seed from the problem.
+        problem = self.build_problem(
+            answer="1",
+            hintfn="gimme_a_random_hint",
+            script=textwrap.dedent("""
+                def gimme_a_random_hint(answer_ids, student_answers, new_cmap, old_cmap):
+                    answer = {code}
+                    new_cmap.set_hint_and_mode(answer_ids[0], answer, "always")
+
+            """.format(code=self._get_random_number_code()))
+        )
+        correct_map = problem.grade_answers({'1_2_1': '2'})
+        hint = correct_map.get_hint('1_2_1')
+        assert hint == self._get_random_number_result(problem.seed)
+
+    def test_empty_answer_graded_as_incorrect(self):
+        """
+        Tests that problem should be graded incorrect if blank space is chosen as answer
+        """
+        problem = self.build_problem(answer=" ", case_sensitive=False, regexp=True)
+        self.assert_grade(problem, " ", "incorrect")
+
+
+class CodeResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = CodeResponseXMLFactory
+
+    def setUp(self):
+        super(CodeResponseTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+
+        grader_payload = json.dumps({"grader": "ps04/grade_square.py"})
+        self.problem = self.build_problem(initial_display="def square(x):",
+                                          answer_display="answer",
+                                          grader_payload=grader_payload,
+                                          num_responses=2)
+
+    @staticmethod
+    def make_queuestate(key, time):
+        """Create queuestate dict"""
+        timestr = datetime.strftime(time, dateformat)
+        return {'key': key, 'time': timestr}
+
+    def test_is_queued(self):
+        """
+        Simple test of whether LoncapaProblem knows when it's been queued
+        """
+
+        answer_ids = sorted(self.problem.get_question_answers())
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for answer_id in answer_ids:
+            cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+        self.problem.correct_map.update(cmap)
+
+        assert self.problem.is_queued() is False
+
+        # Now we queue the LCP
+        cmap = CorrectMap()
+        for i, answer_id in enumerate(answer_ids):
+            queuestate = CodeResponseTest.make_queuestate(i, datetime.now(UTC))
+            cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+        self.problem.correct_map.update(cmap)
+
+        assert self.problem.is_queued() is True
+
+    def test_update_score(self):
+        '''
+        Test whether LoncapaProblem.update_score can deliver queued result to the right subproblem
+        '''
+        answer_ids = sorted(self.problem.get_question_answers())
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        old_cmap = CorrectMap()
+        for i, answer_id in enumerate(answer_ids):
+            queuekey = 1000 + i
+            queuestate = CodeResponseTest.make_queuestate(queuekey, datetime.now(UTC))
+            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+
+        # Message format common to external graders
+        grader_msg = '<span>MESSAGE</span>'   # Must be valid XML
+        correct_score_msg = json.dumps({'correct': True, 'score': 1, 'msg': grader_msg})
+        incorrect_score_msg = json.dumps({'correct': False, 'score': 0, 'msg': grader_msg})
+
+        xserver_msgs = {'correct': correct_score_msg,
+                        'incorrect': incorrect_score_msg, }
+
+        # Incorrect queuekey, state should not be updated
+        for correctness in ['correct', 'incorrect']:
+            self.problem.correct_map = CorrectMap()
+            self.problem.correct_map.update(old_cmap)  # Deep copy
+
+            self.problem.update_score(xserver_msgs[correctness], queuekey=0)
+            assert self.problem.correct_map.get_dict() == old_cmap.get_dict()
+            # Deep comparison
+
+            for answer_id in answer_ids:
+                assert self.problem.correct_map.is_queued(answer_id)
+                # Should be still queued, since message undelivered  # lint-amnesty, pylint: disable=line-too-long
+
+        # Correct queuekey, state should be updated
+        for correctness in ['correct', 'incorrect']:
+            for i, answer_id in enumerate(answer_ids):
+                self.problem.correct_map = CorrectMap()
+                self.problem.correct_map.update(old_cmap)
+
+                new_cmap = CorrectMap()
+                new_cmap.update(old_cmap)
+                npoints = 1 if correctness == 'correct' else 0
+                new_cmap.set(answer_id=answer_id, npoints=npoints, correctness=correctness, msg=grader_msg, queuestate=None)  # lint-amnesty, pylint: disable=line-too-long
+
+                self.problem.update_score(xserver_msgs[correctness], queuekey=1000 + i)
+                assert self.problem.correct_map.get_dict() == new_cmap.get_dict()
+
+                for j, test_id in enumerate(answer_ids):
+                    if j == i:
+                        assert not self.problem.correct_map.is_queued(test_id)
+                        # Should be dequeued, message delivered  # lint-amnesty, pylint: disable=line-too-long
+                    else:
+                        assert self.problem.correct_map.is_queued(test_id)
+                        # Should be queued, message undelivered  # lint-amnesty, pylint: disable=line-too-long
+
+    def test_recentmost_queuetime(self):
+        '''
+        Test whether the LoncapaProblem knows about the time of queue requests
+        '''
+        answer_ids = sorted(self.problem.get_question_answers())
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the unqueued state
+        cmap = CorrectMap()
+        for answer_id in answer_ids:
+            cmap.update(CorrectMap(answer_id=answer_id, queuestate=None))
+        self.problem.correct_map.update(cmap)
+
+        assert self.problem.get_recentmost_queuetime() is None
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        cmap = CorrectMap()
+        for i, answer_id in enumerate(answer_ids):
+            queuekey = 1000 + i
+            latest_timestamp = datetime.now(UTC)
+            queuestate = CodeResponseTest.make_queuestate(queuekey, latest_timestamp)
+            cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
+        self.problem.correct_map.update(cmap)
+
+        # Queue state only tracks up to second
+        latest_timestamp = datetime.strptime(
+            datetime.strftime(latest_timestamp, dateformat), dateformat
+        ).replace(tzinfo=UTC)
+
+        assert self.problem.get_recentmost_queuetime() == latest_timestamp
+
+    def test_convert_files_to_filenames(self):
+        '''
+        Test whether file objects are converted to filenames without altering other structures
+        '''
+        problem_file = os.path.join(os.path.dirname(__file__), "test_files/filename_convert_test.txt")
+        with open(problem_file) as fp:
+            answers_with_file = {'1_2_1': 'String-based answer',
+                                 '1_3_1': ['answer1', 'answer2', 'answer3'],
+                                 '1_4_1': [fp, fp]}
+            answers_converted = convert_files_to_filenames(answers_with_file)
+            assert answers_converted['1_2_1'] == 'String-based answer'
+            assert answers_converted['1_3_1'] == ['answer1', 'answer2', 'answer3']
+            assert answers_converted['1_4_1'] == [fp.name, fp.name]
+
+    def test_parse_score_msg_of_responder(self):
+        """
+        Test whether LoncapaProblem._parse_score_msg correcly parses valid HTML5 html.
+        """
+        valid_grader_msgs = [
+            '<span>MESSAGE</span>',  # Valid XML
+            textwrap.dedent("""
+                <div class='matlabResponse'><div id='mwAudioPlaceHolder'>
+                <audio controls autobuffer autoplay src='data:audio/wav;base64='>Audio is not supported on this browser.</audio>
+                <div>Right click <a href=https://endpoint.mss-mathworks.com/media/filename.wav>here</a> and click \"Save As\" to download the file</div></div>
+                <div style='white-space:pre' class='commandWindowOutput'></div><ul></ul></div>
+            """).replace('\n', ''),  # Valid HTML5 real case Matlab response, invalid XML
+            '<aaa></bbb>'  # Invalid XML, but will be parsed by html5lib to <aaa/>
+        ]
+
+        invalid_grader_msgs = [
+            '<audio',  # invalid XML and HTML5
+            '<p>\b</p>',  # invalid special character
+        ]
+
+        answer_ids = sorted(self.problem.get_question_answers())
+
+        # CodeResponse requires internal CorrectMap state. Build it now in the queued state
+        old_cmap = CorrectMap()
+        for i, answer_id in enumerate(answer_ids):
+            queuekey = 1000 + i
+            queuestate = CodeResponseTest.make_queuestate(queuekey, datetime.now(UTC))
+            old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
+
+        for grader_msg in valid_grader_msgs:
+            correct_score_msg = json.dumps({'correct': True, 'score': 1, 'msg': grader_msg})
+            incorrect_score_msg = json.dumps({'correct': False, 'score': 0, 'msg': grader_msg})
+            xserver_msgs = {'correct': correct_score_msg, 'incorrect': incorrect_score_msg, }
+
+            for i, answer_id in enumerate(answer_ids):
+                self.problem.correct_map = CorrectMap()
+                self.problem.correct_map.update(old_cmap)
+                output = self.problem.update_score(xserver_msgs['correct'], queuekey=1000 + i)
+                assert output[answer_id]['msg'] == grader_msg
+
+        for grader_msg in invalid_grader_msgs:
+            correct_score_msg = json.dumps({'correct': True, 'score': 1, 'msg': grader_msg})
+            incorrect_score_msg = json.dumps({'correct': False, 'score': 0, 'msg': grader_msg})
+            xserver_msgs = {'correct': correct_score_msg, 'incorrect': incorrect_score_msg, }
+
+            for i, answer_id in enumerate(answer_ids):
+                self.problem.correct_map = CorrectMap()
+                self.problem.correct_map.update(old_cmap)
+
+                output = self.problem.update_score(xserver_msgs['correct'], queuekey=1000 + i)
+                assert output[answer_id]['msg'] == 'Invalid grader reply. Please contact the course staff.'
+
+
+class ChoiceResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = ChoiceResponseXMLFactory
+
+    def test_radio_group_grade(self):
+        problem = self.build_problem(choice_type='radio',
+                                     choices=[False, True, False])
+
+        # Check that we get the expected results
+        self.assert_grade(problem, 'choice_0', 'incorrect')
+        self.assert_grade(problem, 'choice_1', 'correct')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+
+        # No choice 3 exists --> mark incorrect
+        self.assert_grade(problem, 'choice_3', 'incorrect')
+
+    def test_checkbox_group_grade(self):
+        problem = self.build_problem(choice_type='checkbox',
+                                     choices=[False, True, True])
+
+        # Check that we get the expected results
+        # (correct if and only if BOTH correct choices chosen)
+        self.assert_grade(problem, ['choice_1', 'choice_2'], 'correct')
+        self.assert_grade(problem, 'choice_1', 'incorrect')
+        self.assert_grade(problem, 'choice_2', 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_1'], 'incorrect')
+        self.assert_grade(problem, ['choice_0', 'choice_2'], 'incorrect')
+
+        # No choice 3 exists --> mark incorrect
+        self.assert_grade(problem, 'choice_3', 'incorrect')
+
+    def test_checkbox_group_valid_grading_schemes(self):
+        # Checkbox-type problems only allow one partial credit scheme.
+        # Change this test if that changes.
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True],
+            credit_type='edc,halves,bongo'
+        )
+        with pytest.raises(LoncapaProblemError):
+            input_dict = {'1_2_1': 'choice_1'}
+            problem.grade_answers(input_dict)
+
+        # 'bongo' is not a valid grading scheme.
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True],
+            credit_type='bongo'
+        )
+        with pytest.raises(LoncapaProblemError):
+            input_dict = {'1_2_1': 'choice_1'}
+            problem.grade_answers(input_dict)
+
+    def test_checkbox_group_partial_credit_grade(self):
+        # First: Every Decision Counts grading style
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True],
+            credit_type='edc'
+        )
+
+        # Check that we get the expected results
+        # (correct if and only if BOTH correct choices chosen)
+        # (partially correct if at least one choice is right)
+        # (incorrect if totally wrong)
+        self.assert_grade(problem, ['choice_0', 'choice_1'], 'incorrect')
+        self.assert_grade(problem, ['choice_2', 'choice_3'], 'correct')
+        self.assert_grade(problem, 'choice_0', 'partially-correct')
+        self.assert_grade(problem, 'choice_2', 'partially-correct')
+        self.assert_grade(problem, ['choice_0', 'choice_1', 'choice_2', 'choice_3'], 'partially-correct')
+
+        # Second: Halves grading style
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True],
+            credit_type='halves'
+        )
+
+        # Check that we get the expected results
+        # (correct if and only if BOTH correct choices chosen)
+        # (partially correct on one error)
+        # (incorrect for more errors, at least with this # of choices.)
+        self.assert_grade(problem, ['choice_0', 'choice_1'], 'incorrect')
+        self.assert_grade(problem, ['choice_2', 'choice_3'], 'correct')
+        self.assert_grade(problem, 'choice_2', 'partially-correct')
+        self.assert_grade(problem, ['choice_1', 'choice_2', 'choice_3'], 'partially-correct')
+        self.assert_grade(problem, ['choice_0', 'choice_1', 'choice_2', 'choice_3'], 'incorrect')
+
+        # Third: Halves grading style with more options
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True, False],
+            credit_type='halves'
+        )
+
+        # Check that we get the expected results
+        # (2 errors allowed with 5+ choices)
+        self.assert_grade(problem, ['choice_0', 'choice_1', 'choice_4'], 'incorrect')
+        self.assert_grade(problem, ['choice_2', 'choice_3'], 'correct')
+        self.assert_grade(problem, 'choice_2', 'partially-correct')
+        self.assert_grade(problem, ['choice_1', 'choice_2', 'choice_3'], 'partially-correct')
+        self.assert_grade(problem, ['choice_0', 'choice_1', 'choice_2', 'choice_3'], 'partially-correct')
+        self.assert_grade(problem, ['choice_0', 'choice_1', 'choice_2', 'choice_3', 'choice_4'], 'incorrect')
+
+    def test_checkbox_group_partial_points_grade(self):
+        # Ensure that we get the expected number of points
+        # Using assertAlmostEqual to avoid floating point issues
+        # First: Every Decision Counts grading style
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True],
+            credit_type='edc'
+        )
+
+        correct_map = problem.grade_answers({'1_2_1': 'choice_2'})
+        assert round(correct_map.get_npoints('1_2_1') - 0.75, 7) >= 0
+
+        # Second: Halves grading style
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True],
+            credit_type='halves'
+        )
+
+        correct_map = problem.grade_answers({'1_2_1': 'choice_2'})
+        assert round(correct_map.get_npoints('1_2_1') - 0.5, 7) >= 0
+
+        # Third: Halves grading style with more options
+        problem = self.build_problem(
+            choice_type='checkbox',
+            choices=[False, False, True, True, False],
+            credit_type='halves'
+        )
+
+        correct_map = problem.grade_answers({'1_2_1': 'choice_2,choice4'})
+        assert round(correct_map.get_npoints('1_2_1') - 0.25, 7) >= 0
+
+    def test_grade_with_no_checkbox_selected(self):
+        """
+        Test that answer marked as incorrect if no checkbox selected.
+        """
+        problem = self.build_problem(
+            choice_type='checkbox', choices=[False, False, False]
+        )
+
+        correct_map = problem.grade_answers({})
+        assert correct_map.get_correctness('1_2_1') == 'incorrect'
+
+    def test_contextualized_choices(self):
+        script = textwrap.dedent("""
+            a = 6
+            b = 4
+            c = a + b
+
+            ok0 = c % 2 == 0 # check remainder modulo 2
+            ok1 = c % 3 == 0 # check remainder modulo 3
+            ok2 = c % 5 == 0 # check remainder modulo 5
+            ok3 = not any((ok0, ok1, ok2))
+        """)
+        choices = ["$ok0", "$ok1", "$ok2", "$ok3"]
+        problem = self.build_problem(script=script,
+                                     choice_type='checkbox',
+                                     choices=choices)
+
+        # Ensure the expected correctness
+        self.assert_grade(problem, ['choice_0', 'choice_2'], 'correct')
+        self.assert_grade(problem, ['choice_1', 'choice_3'], 'incorrect')
+
+
+class NumericalResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = NumericalResponseXMLFactory
+
+    # We blend the line between integration (using evaluator) and exclusively
+    # unit testing the NumericalResponse (mocking out the evaluator)
+    # For simple things its not worth the effort.
+    def test_grade_range_tolerance(self):
+        problem_setup = [
+            # [given_answer, [list of correct responses], [list of incorrect responses]]
+            ['[5, 7)', ['5', '6', '6.999'], ['4.999', '7']],
+            ['[1.6e-5, 1.9e24)', ['0.000016', '1.6*10^-5', '1.59e24'], ['1.59e-5', '1.9e24', '1.9*10^24']],
+            ['[0, 1.6e-5]', ['1.6*10^-5'], ["2"]],
+            ['(1.6e-5, 10]', ["2"], ['1.6*10^-5']],
+        ]
+        for given_answer, correct_responses, incorrect_responses in problem_setup:
+            problem = self.build_problem(answer=given_answer)
+            self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_additional_answer_grading(self):
+        """
+        Test additional answers are graded correct with their associated correcthint.
+        """
+        primary_answer = '100'
+        primary_correcthint = 'primary feedback'
+        additional_answers = {
+            '1': '1. additional feedback',
+            '2': '2. additional feedback',
+            '4': '4. additional feedback',
+            '5': ''
+        }
+        problem = self.build_problem(
+            answer=primary_answer,
+            additional_answers=additional_answers,
+            correcthint=primary_correcthint
+        )
+
+        # Assert primary answer is graded correctly.
+        correct_map = problem.grade_answers({'1_2_1': primary_answer})
+        assert correct_map.get_correctness('1_2_1') == 'correct'
+        assert primary_correcthint in correct_map.get_msg('1_2_1')
+
+        # Assert additional answers are graded correct
+        for answer, correcthint in additional_answers.items():
+            correct_map = problem.grade_answers({'1_2_1': answer})
+            assert correct_map.get_correctness('1_2_1') == 'correct'
+            assert correcthint in correct_map.get_msg('1_2_1')
+
+    def test_additional_answer_get_score(self):
+        """
+        Test `get_score` is working for additional answers.
+        """
+        problem = self.build_problem(answer='100', additional_answers={'1': ''})
+        responder = list(problem.responders.values())[0]
+
+        # Check primary answer.
+        new_cmap = responder.get_score({'1_2_1': '100'})
+        assert new_cmap.get_correctness('1_2_1') == 'correct'
+
+        # Check additional answer.
+        new_cmap = responder.get_score({'1_2_1': '1'})
+        assert new_cmap.get_correctness('1_2_1') == 'correct'
+
+        # Check any wrong answer.
+        new_cmap = responder.get_score({'1_2_1': '2'})
+        assert new_cmap.get_correctness('1_2_1') == 'incorrect'
+
+    def test_grade_range_tolerance_partial_credit(self):
+        problem_setup = [
+            # [given_answer,
+            #   [list of correct responses],
+            #   [list of incorrect responses],
+            #   [list of partially correct responses]]
+            [
+                '[5, 7)',
+                ['5', '6', '6.999'],
+                ['0', '100'],
+                ['4', '8']
+            ],
+            [
+                '[1.6e-5, 1.9e24)',
+                ['0.000016', '1.6*10^-5', '1.59e24'],
+                ['-1e26', '1.9e26', '1.9*10^26'],
+                ['0', '2e24']
+            ],
+            [
+                '[0, 1.6e-5]',
+                ['1.6*10^-5'],
+                ['2'],
+                ['1.9e-5', '-1e-6']
+            ],
+            [
+                '(1.6e-5, 10]',
+                ['2'],
+                ['-20', '30'],
+                ['-1', '12']
+            ],
+        ]
+        for given_answer, correct_responses, incorrect_responses, partial_responses in problem_setup:
+            problem = self.build_problem(answer=given_answer, credit_type='close')
+            self.assert_multiple_partial(problem, correct_responses, incorrect_responses, partial_responses)
+
+    def test_grade_range_tolerance_exceptions(self):
+        # no complex number in range tolerance staff answer
+        problem = self.build_problem(answer='[1j, 5]')
+        input_dict = {'1_2_1': '3'}
+        with pytest.raises(StudentInputError):
+            problem.grade_answers(input_dict)
+
+        # no complex numbers in student ansers to range tolerance problems
+        problem = self.build_problem(answer='(1, 5)')
+        input_dict = {'1_2_1': '1*J'}
+        with pytest.raises(StudentInputError):
+            problem.grade_answers(input_dict)
+
+        # test isnan student input: no exception,
+        # but problem should be graded as incorrect
+        problem = self.build_problem(answer='(1, 5)')
+        input_dict = {'1_2_1': ''}
+        correct_map = problem.grade_answers(input_dict)
+        correctness = correct_map.get_correctness('1_2_1')
+        assert correctness == 'incorrect'
+
+        # test invalid range tolerance answer
+        with pytest.raises(StudentInputError):
+            problem = self.build_problem(answer='(1 5)')
+
+        # test empty boundaries
+        problem = self.build_problem(answer='(1, ]')
+        input_dict = {'1_2_1': '3'}
+        with pytest.raises(StudentInputError):
+            problem.grade_answers(input_dict)
+
+    def test_grade_exact(self):
+        problem = self.build_problem(answer=4)
+        correct_responses = ["4", "4.0", "4.00"]
+        incorrect_responses = ["", "3.9", "4.1", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_grade_partial(self):
+        # First: "list"-style grading scheme.
+        problem = self.build_problem(
+            answer=4,
+            credit_type='list',
+            partial_answers='2,8,-4'
+        )
+        correct_responses = ["4", "4.0"]
+        incorrect_responses = ["1", "3", "4.1", "0", "-2"]
+        partial_responses = ["2", "2.0", "-4", "-4.0", "8", "8.0"]
+        self.assert_multiple_partial(problem, correct_responses, incorrect_responses, partial_responses)
+
+        # Second: "close"-style grading scheme. Default range is twice tolerance.
+        problem = self.build_problem(
+            answer=4,
+            tolerance=0.2,
+            credit_type='close'
+        )
+        correct_responses = ["4", "4.1", "3.9"]
+        incorrect_responses = ["1", "3", "4.5", "0", "-2"]
+        partial_responses = ["4.3", "3.7"]
+        self.assert_multiple_partial(problem, correct_responses, incorrect_responses, partial_responses)
+
+        # Third: "close"-style grading scheme with partial_range set.
+        problem = self.build_problem(
+            answer=4,
+            tolerance=0.2,
+            partial_range=3,
+            credit_type='close'
+        )
+        correct_responses = ["4", "4.1"]
+        incorrect_responses = ["1", "3", "0", "-2"]
+        partial_responses = ["4.5", "3.5"]
+        self.assert_multiple_partial(problem, correct_responses, incorrect_responses, partial_responses)
+
+        # Fourth: both "list"- and "close"-style grading schemes at once.
+        problem = self.build_problem(
+            answer=4,
+            tolerance=0.2,
+            partial_range=3,
+            credit_type='close,list',
+            partial_answers='2,8,-4'
+        )
+        correct_responses = ["4", "4.0"]
+        incorrect_responses = ["1", "3", "0", "-2"]
+        partial_responses = ["2", "2.1", "1.5", "8", "7.5", "8.1", "-4", "-4.15", "-3.5", "4.5", "3.5"]
+        self.assert_multiple_partial(problem, correct_responses, incorrect_responses, partial_responses)
+
+    def test_numerical_valid_grading_schemes(self):
+        # 'bongo' is not a valid grading scheme.
+        problem = self.build_problem(answer=4, tolerance=0.1, credit_type='bongo')
+        input_dict = {'1_2_1': '4'}
+        with pytest.raises(LoncapaProblemError):
+            problem.grade_answers(input_dict)
+
+    def test_grade_decimal_tolerance(self):
+        problem = self.build_problem(answer=4, tolerance=0.1)
+        correct_responses = ["4.0", "4.00", "4.09", "3.91"]
+        incorrect_responses = ["", "4.11", "3.89", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_grade_percent_tolerance(self):
+        # Positive only range
+        problem = self.build_problem(answer=4, tolerance="10%")
+        correct_responses = ["4.0", "4.00", "4.39", "3.61"]
+        incorrect_responses = ["", "4.41", "3.59", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+        # Negative only range
+        problem = self.build_problem(answer=-4, tolerance="10%")
+        correct_responses = ["-4.0", "-4.00", "-4.39", "-3.61"]
+        incorrect_responses = ["", "-4.41", "-3.59", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+        # Mixed negative/positive range
+        problem = self.build_problem(answer=1, tolerance="200%")
+        correct_responses = ["1", "1.00", "2.99", "0.99"]
+        incorrect_responses = ["", "3.01", "-1.01"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_grade_percent_tolerance_with_spaces(self):
+        """
+        Tests that system does not throw an error when tolerance data contains spaces before or after.
+        """
+        problem = self.build_problem(answer=4, tolerance="10% ")
+        correct_responses = ["4.0", "4.00", "4.39", "3.61"]
+        incorrect_responses = ["", "4.41", "3.59", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_floats(self):
+        """
+        Default tolerance for all responsetypes is 1e-3%.
+        """
+        problem_setup = [
+            # [given_answer, [list of correct responses], [list of incorrect responses]]
+            [1, ["1"], ["1.1"]],
+            [2.0, ["2.0"], ["1.0"]],
+            [4, ["4.0", "4.00004"], ["4.00005"]],
+            [0.00016, ["1.6*10^-4"], [""]],
+            [0.000016, ["1.6*10^-5"], ["0.000165"]],
+            [1.9e24, ["1.9*10^24"], ["1.9001*10^24"]],
+            [2e-15, ["2*10^-15"], [""]],
+            [3141592653589793238., ["3141592653589793115."], [""]],
+            [0.1234567, ["0.123456", "0.1234561"], ["0.123451"]],
+            [1e-5, ["1e-5", "1.0e-5"], ["-1e-5", "2*1e-5"]],
+        ]
+        for given_answer, correct_responses, incorrect_responses in problem_setup:
+            problem = self.build_problem(answer=given_answer)
+            self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_grade_with_script(self):
+        script_text = "computed_response = math.sqrt(4)"
+        problem = self.build_problem(answer="$computed_response", script=script_text)
+        correct_responses = ["2", "2.0"]
+        incorrect_responses = ["", "2.01", "1.99", "0"]
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_raises_zero_division_err(self):
+        """See if division by zero is handled correctly."""
+        problem = self.build_problem(answer="1")  # Answer doesn't matter
+        input_dict = {'1_2_1': '1/0'}
+        with pytest.raises(StudentInputError):
+            problem.grade_answers(input_dict)
+
+    def test_staff_inputs_expressions(self):
+        """Test that staff may enter in an expression as the answer."""
+        problem = self.build_problem(answer="1/3", tolerance=1e-3)
+        correct_responses = ["1/3", "0.333333"]
+        incorrect_responses = []
+        self.assert_multiple_grade(problem, correct_responses, incorrect_responses)
+
+    def test_staff_inputs_expressions_legacy(self):
+        """Test that staff may enter in a complex number as the answer."""
+        problem = self.build_problem(answer="1+1j", tolerance=1e-3)
+        self.assert_grade(problem, '1+j', 'correct')
+
+    @mock.patch('xmodule.capa.responsetypes.log')
+    def test_staff_inputs_bad_syntax(self, mock_log):
+        """Test that staff may enter in a complex number as the answer."""
+        staff_ans = "clearly bad syntax )[+1e"
+        problem = self.build_problem(answer=staff_ans, tolerance=1e-3)
+
+        msg = "There was a problem with the staff answer to this problem"
+        with self.assertRaisesRegex(StudentInputError, msg):
+            self.assert_grade(problem, '1+j', 'correct')
+
+        mock_log.debug.assert_called_once_with(
+            "Content error--answer '%s' is not a valid number", staff_ans
+        )
+
+    @mock.patch('xmodule.capa.responsetypes.log')
+    def test_responsetype_i18n(self, mock_log):  # lint-amnesty, pylint: disable=unused-argument
+        """Test that LoncapaSystem has an i18n that works."""
+        staff_ans = "clearly bad syntax )[+1e"
+        problem = self.build_problem(answer=staff_ans, tolerance=1e-3)
+
+        class FakeTranslations(object):
+            """A fake gettext.Translations object."""
+            def ugettext(self, text):
+                """Return the 'translation' of `text`."""
+                if text == "There was a problem with the staff answer to this problem.":
+                    text = "TRANSLATED!"
+                return text
+            gettext = ugettext
+
+        problem.capa_system.i18n = FakeTranslations()
+
+        with self.assertRaisesRegex(StudentInputError, "TRANSLATED!"):
+            self.assert_grade(problem, '1+j', 'correct')
+
+    def test_grade_infinity(self):
+        """
+        Check that infinity doesn't automatically get marked correct.
+
+        This resolves a bug where a problem with relative tolerance would
+        pass with any arbitrarily large student answer.
+        """
+        mapping = {
+            'some big input': float('inf'),
+            'some neg input': -float('inf'),
+            'weird NaN input': float('nan'),
+            '4': 4
+        }
+
+        def evaluator_side_effect(_, __, math_string):
+            """Look up the given response for `math_string`."""
+            return mapping[math_string]
+
+        problem = self.build_problem(answer=4, tolerance='10%')
+
+        with mock.patch('xmodule.capa.responsetypes.evaluator') as mock_eval:
+            mock_eval.side_effect = evaluator_side_effect
+            self.assert_grade(problem, 'some big input', 'incorrect')
+            self.assert_grade(problem, 'some neg input', 'incorrect')
+            self.assert_grade(problem, 'weird NaN input', 'incorrect')
+
+    def test_err_handling(self):
+        """
+        See that `StudentInputError`s are raised when things go wrong.
+        """
+        problem = self.build_problem(answer=4)
+
+        errors = [  # (exception raised, message to student)
+            (calc.UndefinedVariable("Invalid Input: x not permitted in answer as a variable"),
+             r"Invalid Input: x not permitted in answer as a variable"),
+            (ValueError("factorial() mess-up"), "Factorial function evaluated outside its domain"),
+            (ValueError(), "Could not interpret '.*' as a number"),
+            (pyparsing.ParseException("oopsie"), "Invalid math syntax"),
+            (ZeroDivisionError(), "Could not interpret '.*' as a number")
+        ]
+
+        with mock.patch('xmodule.capa.responsetypes.evaluator') as mock_eval:
+            for err, msg_regex in errors:
+
+                def evaluator_side_effect(_, __, math_string):
+                    """Raise an error only for the student input."""
+                    if math_string != '4':
+                        raise err  # lint-amnesty, pylint: disable=cell-var-from-loop
+                mock_eval.side_effect = evaluator_side_effect
+
+                with self.assertRaisesRegex(StudentInputError, msg_regex):
+                    problem.grade_answers({'1_2_1': 'foobar'})
+
+    def test_compare_answer(self):
+        """Tests the answer compare function."""
+        problem = self.build_problem(answer="42")
+        responder = list(problem.responders.values())[0]
+        assert responder.compare_answer('48', '8*6')
+        assert not responder.compare_answer('48', '9*5')
+
+    def test_validate_answer(self):
+        """Tests the answer validation function."""
+        problem = self.build_problem(answer="42")
+        responder = list(problem.responders.values())[0]
+        assert responder.validate_answer('23.5')
+        assert not responder.validate_answer('fish')
+
+
+class CustomResponseTest(ResponseTest):  # pylint: disable=missing-class-docstring
+    xml_factory_class = CustomResponseXMLFactory
+
+    def test_inline_code(self):
+        # For inline code, we directly modify global context variables
+        # 'answers' is a list of answers provided to us
+        # 'correct' is a list we fill in with True/False
+        # 'expect' is given to us (if provided in the XML)
+        inline_script = """correct[0] = 'correct' if (answers['1_2_1'] == expect) else 'incorrect'"""
+        problem = self.build_problem(answer=inline_script, expect="42")
+
+        # Check results
+        self.assert_grade(problem, '42', 'correct')
+        self.assert_grade(problem, '0', 'incorrect')
+
+    def test_inline_message(self):
+        # Inline code can update the global messages list
+        # to pass messages to the CorrectMap for a particular input
+        # The code can also set the global overall_message (str)
+        # to pass a message that applies to the whole response
+        inline_script = textwrap.dedent("""
+            messages[0] = "Test Message"
+            overall_message = "Overall message"
+            """)
+        problem = self.build_problem(answer=inline_script)
+
+        input_dict = {'1_2_1': '0'}
+        correctmap = problem.grade_answers(input_dict)
+
+        # Check that the message for the particular input was received
+        input_msg = correctmap.get_msg('1_2_1')
+        assert input_msg == 'Test Message'
+
+        # Check that the overall message (for the whole response) was received
+        overall_msg = correctmap.get_overall_message()
+        assert overall_msg == 'Overall message'
+
+    def test_inline_randomization(self):
+        # Make sure the seed from the problem gets fed into the script execution.
+        inline_script = "messages[0] = {code}".format(code=self._get_random_number_code())
+        problem = self.build_problem(answer=inline_script)
+
+        input_dict = {'1_2_1': '0'}
+        correctmap = problem.grade_answers(input_dict)
+
+        input_msg = correctmap.get_msg('1_2_1')
+        assert input_msg == self._get_random_number_result(problem.seed)
+
+    def test_function_code_single_input(self):
+        # For function code, we pass in these arguments:
+        #
+        #   'expect' is the expect attribute of the <customresponse>
+        #
+        #   'answer_given' is the answer the student gave (if there is just one input)
+        #       or an ordered list of answers (if there are multiple inputs)
+        #
+        # The function should return a dict of the form
+        # { 'ok': BOOL or STRING, 'msg': STRING } (no 'grade_decimal' key to test that it's optional)
+        #
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                partial_credit = '21'
+                if answer_given == expect:
+                    retval = True
+                elif answer_given == partial_credit:
+                    retval = 'partial'
+                else:
+                    retval = False
+                return {'ok': retval, 'msg': 'Message text'}
+        """)
+
+        problem = self.build_problem(script=script, cfn="check_func", expect="42")
+
+        # Correct answer
+        input_dict = {'1_2_1': '42'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+        npoints = correct_map.get_npoints('1_2_1')
+
+        assert correctness == 'correct'
+        assert msg == 'Message text'
+        assert npoints == 1
+
+        # Partially Credit answer
+        input_dict = {'1_2_1': '21'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+        npoints = correct_map.get_npoints('1_2_1')
+
+        assert correctness == 'partially-correct'
+        assert msg == 'Message text'
+        assert 0 <= npoints <= 1
+
+        # Incorrect answer
+        input_dict = {'1_2_1': '0'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+        npoints = correct_map.get_npoints('1_2_1')
+
+        assert correctness == 'incorrect'
+        assert msg == 'Message text'
+        assert npoints == 0
+
+    def test_function_code_single_input_decimal_score(self):
+        # For function code, we pass in these arguments:
+        #
+        #   'expect' is the expect attribute of the <customresponse>
+        #
+        #   'answer_given' is the answer the student gave (if there is just one input)
+        #       or an ordered list of answers (if there are multiple inputs)
+        #
+        # The function should return a dict of the form
+        # { 'ok': BOOL or STRING, 'msg': STRING, 'grade_decimal': FLOAT }
+        #
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                partial_credit = '21'
+                if answer_given == expect:
+                    retval = True
+                    score = 0.9
+                elif answer_given == partial_credit:
+                    retval = 'partial'
+                    score = 0.5
+                else:
+                    retval = False
+                    score = 0.1
+                return {
+                    'ok': retval,
+                    'msg': 'Message text',
+                    'grade_decimal': score,
+                }
+        """)
+
+        problem = self.build_problem(script=script, cfn="check_func", expect="42")
+
+        # Correct answer
+        input_dict = {'1_2_1': '42'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_npoints('1_2_1') == 0.9
+        assert correct_map.get_correctness('1_2_1') == 'correct'
+
+        # Incorrect answer
+        input_dict = {'1_2_1': '43'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_npoints('1_2_1') == 0.1
+        assert correct_map.get_correctness('1_2_1') == 'incorrect'
+
+        # Partially Correct answer
+        input_dict = {'1_2_1': '21'}
+        correct_map = problem.grade_answers(input_dict)
+        assert correct_map.get_npoints('1_2_1') == 0.5
+        assert correct_map.get_correctness('1_2_1') == 'partially-correct'
+
+    def test_script_context(self):
+        # Ensure that python script variables can be used in the "expect" and "answer" fields,
+
+        script = script = textwrap.dedent("""
+            expected_ans = 42
+
+            def check_func(expect, answer_given):
+                return answer_given == expect
+        """)
+
+        problems = (
+            self.build_problem(script=script, cfn="check_func", expect="$expected_ans"),
+            self.build_problem(script=script, cfn="check_func", answer_attr="$expected_ans")
+        )
+
+        input_dict = {'1_2_1': '42'}
+
+        for problem in problems:
+            correctmap = problem.grade_answers(input_dict)
+
+            # CustomResponse also adds 'expect' to the problem context; check that directly first:
+            assert problem.context['expect'] == '42'
+
+            # Also make sure the problem was graded correctly:
+            correctness = correctmap.get_correctness('1_2_1')
+            assert correctness == 'correct'
+
+    def test_function_code_multiple_input_no_msg(self):
+
+        # Check functions also have the option of returning
+        # a single boolean or string value
+        # If true, mark all the inputs correct
+        # If one is true but not the other, mark all partially correct
+        # If false, mark all the inputs incorrect
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                if answer_given[0] == expect and answer_given[1] == expect:
+                    retval = True
+                elif answer_given[0] == expect or answer_given[1] == expect:
+                    retval = 'partial'
+                else:
+                    retval = False
+                return retval
+        """)
+
+        problem = self.build_problem(script=script, cfn="check_func",
+                                     expect="42", num_inputs=2)
+
+        # Correct answer -- expect both inputs marked correct
+        input_dict = {'1_2_1': '42', '1_2_2': '42'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        assert correctness == 'correct'
+
+        correctness = correct_map.get_correctness('1_2_2')
+        assert correctness == 'correct'
+
+        # One answer incorrect -- expect both inputs marked partially correct
+        input_dict = {'1_2_1': '0', '1_2_2': '42'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        assert correctness == 'partially-correct'
+        assert 0 <= correct_map.get_npoints('1_2_1') <= 1
+
+        correctness = correct_map.get_correctness('1_2_2')
+        assert correctness == 'partially-correct'
+        assert 0 <= correct_map.get_npoints('1_2_2') <= 1
+
+        # Both answers incorrect -- expect both inputs marked incorrect
+        input_dict = {'1_2_1': '0', '1_2_2': '0'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        assert correctness == 'incorrect'
+
+        correctness = correct_map.get_correctness('1_2_2')
+        assert correctness == 'incorrect'
+
+    def test_function_code_multiple_inputs(self):
+
+        # If the <customresponse> has multiple inputs associated with it,
+        # the check function can return a dict of the form:
+        #
+        # {'overall_message': STRING,
+        #  'input_list': [{'ok': BOOL or STRING, 'msg': STRING}, ...] }
+        # (no grade_decimal to test it's optional)
+        #
+        # 'overall_message' is displayed at the end of the response
+        #
+        # 'input_list' contains dictionaries representing the correctness
+        #           and message for each input.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                check1 = (int(answer_given[0]) == 1)
+                check2 = (int(answer_given[1]) == 2)
+                check3 = (int(answer_given[2]) == 3)
+                check4 = 'partial' if answer_given[3] == 'four' else False
+                return {'overall_message': 'Overall message',
+                        'input_list': [
+                            {'ok': check1,  'msg': 'Feedback 1'},
+                            {'ok': check2,  'msg': 'Feedback 2'},
+                            {'ok': check3,  'msg': 'Feedback 3'},
+                            {'ok': check4,  'msg': 'Feedback 4'} ] }
+            """)
+
+        problem = self.build_problem(
+            script=script,
+            cfn="check_func",
+            num_inputs=4
+        )
+
+        # Grade the inputs (one input incorrect)
+        input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3', '1_2_4': 'four'}
+        correct_map = problem.grade_answers(input_dict)
+
+        # Expect that we receive the overall message (for the whole response)
+        assert correct_map.get_overall_message() == 'Overall message'
+
+        # Expect that the inputs were graded individually
+        assert correct_map.get_correctness('1_2_1') == 'incorrect'
+        assert correct_map.get_correctness('1_2_2') == 'correct'
+        assert correct_map.get_correctness('1_2_3') == 'correct'
+        assert correct_map.get_correctness('1_2_4') == 'partially-correct'
+
+        # Expect that the inputs were given correct npoints
+        assert correct_map.get_npoints('1_2_1') == 0
+        assert correct_map.get_npoints('1_2_2') == 1
+        assert correct_map.get_npoints('1_2_3') == 1
+        assert 0 <= correct_map.get_npoints('1_2_4') <= 1
+
+        # Expect that we received messages for each individual input
+        assert correct_map.get_msg('1_2_1') == 'Feedback 1'
+        assert correct_map.get_msg('1_2_2') == 'Feedback 2'
+        assert correct_map.get_msg('1_2_3') == 'Feedback 3'
+        assert correct_map.get_msg('1_2_4') == 'Feedback 4'
+
+    def test_function_code_multiple_inputs_decimal_score(self):
+
+        # If the <customresponse> has multiple inputs associated with it,
+        # the check function can return a dict of the form:
+        #
+        # {'overall_message': STRING,
+        #  'input_list': [{'ok': BOOL or STRING,
+        #                  'msg': STRING, 'grade_decimal': FLOAT}, ...] }
+        #        #
+        # 'input_list' contains dictionaries representing the correctness
+        #           and message for each input.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                check1 = (int(answer_given[0]) == 1)
+                check2 = (int(answer_given[1]) == 2)
+                check3 = (int(answer_given[2]) == 3)
+                check4 = 'partial' if answer_given[3] == 'four' else False
+                score1 = 0.9 if check1 else 0.1
+                score2 = 0.9 if check2 else 0.1
+                score3 = 0.9 if check3 else 0.1
+                score4 = 0.7 if check4 == 'partial' else 0.1
+                return {
+                    'input_list': [
+                        {'ok': check1, 'grade_decimal': score1, 'msg': 'Feedback 1'},
+                        {'ok': check2, 'grade_decimal': score2, 'msg': 'Feedback 2'},
+                        {'ok': check3, 'grade_decimal': score3, 'msg': 'Feedback 3'},
+                        {'ok': check4, 'grade_decimal': score4, 'msg': 'Feedback 4'},
+                    ]
+                }
+            """)
+
+        problem = self.build_problem(script=script, cfn="check_func", num_inputs=4)
+
+        # Grade the inputs (one input incorrect)
+        input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3', '1_2_4': 'four'}
+        correct_map = problem.grade_answers(input_dict)
+
+        # Expect that the inputs were graded individually
+        assert correct_map.get_correctness('1_2_1') == 'incorrect'
+        assert correct_map.get_correctness('1_2_2') == 'correct'
+        assert correct_map.get_correctness('1_2_3') == 'correct'
+        assert correct_map.get_correctness('1_2_4') == 'partially-correct'
+
+        # Expect that the inputs were given correct npoints
+        assert correct_map.get_npoints('1_2_1') == 0.1
+        assert correct_map.get_npoints('1_2_2') == 0.9
+        assert correct_map.get_npoints('1_2_3') == 0.9
+        assert correct_map.get_npoints('1_2_4') == 0.7
+
+    def test_function_code_with_extra_args(self):
+        script = textwrap.dedent("""\
+                    def check_func(expect, answer_given, options, dynamath):
+                        assert options == "xyzzy", "Options was %r" % options
+                        partial_credit = '21'
+                        if answer_given == expect:
+                            retval = True
+                        elif answer_given == partial_credit:
+                            retval = 'partial'
+                        else:
+                            retval = False
+                        return {'ok': retval, 'msg': 'Message text'}
+                    """)
+
+        problem = self.build_problem(
+            script=script,
+            cfn="check_func",
+            expect="42",
+            options="xyzzy",
+            cfn_extra_args="options dynamath"
+        )
+
+        # Correct answer
+        input_dict = {'1_2_1': '42'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+
+        assert correctness == 'correct'
+        assert msg == 'Message text'
+
+        # Partially Correct answer
+        input_dict = {'1_2_1': '21'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+
+        assert correctness == 'partially-correct'
+        assert msg == 'Message text'
+
+        # Incorrect answer
+        input_dict = {'1_2_1': '0'}
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+
+        assert correctness == 'incorrect'
+        assert msg == 'Message text'
+
+    def test_function_code_with_attempt_number(self):
+        script = textwrap.dedent("""\
+                    def gradeit(expect, ans, **kwargs):
+                        attempt = kwargs["attempt"]
+                        message = "This is attempt number {}".format(str(attempt))
+                        return {
+                            'input_list': [
+                                { 'ok': True, 'msg': message},
+                            ]
+                        }
+                    """)
+
+        problem = self.build_problem(
+            script=script,
+            cfn="gradeit",
+            expect="42",
+            cfn_extra_args="attempt"
+        )
+
+        # first attempt
+        input_dict = {'1_2_1': '42'}
+        problem.context['attempt'] = 1
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+
+        assert correctness == 'correct'
+        assert msg == 'This is attempt number 1'
+
+        # second attempt
+        problem.context['attempt'] = 2
+        correct_map = problem.grade_answers(input_dict)
+
+        correctness = correct_map.get_correctness('1_2_1')
+        msg = correct_map.get_msg('1_2_1')
+
+        assert correctness == 'correct'
+        assert msg == 'This is attempt number 2'
+
+    def test_multiple_inputs_return_one_status(self):
+        # When given multiple inputs, the 'answer_given' argument
+        # to the check_func() is a list of inputs
+        #
+        # The sample script below marks the problem as correct
+        # if and only if it receives answer_given=[1,2,3]
+        # (or string values ['1','2','3'])
+        #
+        # Since we return a dict describing the status of one input,
+        # we expect that the same 'ok' value is applied to each
+        # of the inputs.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                check1 = (int(answer_given[0]) == 1)
+                check2 = (int(answer_given[1]) == 2)
+                check3 = (int(answer_given[2]) == 3)
+                if (int(answer_given[0]) == -1) and check2 and check3:
+                    return {'ok': 'partial',
+                            'msg': 'Message text'}
+                else:
+                    return {'ok': (check1 and check2 and check3),
+                            'msg': 'Message text'}
+            """)
+
+        problem = self.build_problem(script=script,
+                                     cfn="check_func", num_inputs=3)
+
+        # Grade the inputs (one input incorrect)
+        input_dict = {'1_2_1': '-999', '1_2_2': '2', '1_2_3': '3'}
+        correct_map = problem.grade_answers(input_dict)
+
+        # Everything marked incorrect
+        assert correct_map.get_correctness('1_2_1') == 'incorrect'
+        assert correct_map.get_correctness('1_2_2') == 'incorrect'
+        assert correct_map.get_correctness('1_2_3') == 'incorrect'
+
+        # Grade the inputs (one input partially correct)
+        input_dict = {'1_2_1': '-1', '1_2_2': '2', '1_2_3': '3'}
+        correct_map = problem.grade_answers(input_dict)
+
+        # Everything marked partially correct
+        assert correct_map.get_correctness('1_2_1') == 'partially-correct'
+        assert correct_map.get_correctness('1_2_2') == 'partially-correct'
+        assert correct_map.get_correctness('1_2_3') == 'partially-correct'
+
+        # Grade the inputs (everything correct)
+        input_dict = {'1_2_1': '1', '1_2_2': '2', '1_2_3': '3'}
+        correct_map = problem.grade_answers(input_dict)
+
+        # Everything marked incorrect
+        assert correct_map.get_correctness('1_2_1') == 'correct'
+        assert correct_map.get_correctness('1_2_2') == 'correct'
+        assert correct_map.get_correctness('1_2_3') == 'correct'
+
+        # Message is interpreted as an "overall message"
+        assert correct_map.get_overall_message() == 'Message text'
+
+    def test_script_exception_function(self):
+
+        # Construct a script that will raise an exception
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                raise Exception("Test")
+            """)
+
+        problem = self.build_problem(script=script, cfn="check_func")
+
+        # Expect that an exception gets raised when we check the answer
+        with pytest.raises(ResponseError):
+            problem.grade_answers({'1_2_1': '42'})
+
+    def test_script_exception_inline(self):
+
+        # Construct a script that will raise an exception
+        script = 'raise Exception("Test")'
+        problem = self.build_problem(answer=script)
+
+        # Expect that an exception gets raised when we check the answer
+        with pytest.raises(ResponseError):
+            problem.grade_answers({'1_2_1': '42'})
+
+    def test_invalid_dict_exception(self):
+
+        # Construct a script that passes back an invalid dict format
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                return {'invalid': 'test'}
+            """)
+
+        problem = self.build_problem(script=script, cfn="check_func")
+
+        # Expect that an exception gets raised when we check the answer
+        with pytest.raises(ResponseError):
+            problem.grade_answers({'1_2_1': '42'})
+
+    def test_setup_randomization(self):
+        # Ensure that the problem setup script gets the random seed from the problem.
+        script = textwrap.dedent("""
+            num = {code}
+            """.format(code=self._get_random_number_code()))
+        problem = self.build_problem(script=script)
+        assert problem.context['num'] == self._get_random_number_result(problem.seed)
+
+    def test_check_function_randomization(self):
+        # The check function should get random-seeded from the problem.
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                return {{'ok': True, 'msg': {code} }}
+        """.format(code=self._get_random_number_code()))
+
+        problem = self.build_problem(script=script, cfn="check_func", expect="42")
+        input_dict = {'1_2_1': '42'}
+        correct_map = problem.grade_answers(input_dict)
+        msg = correct_map.get_msg('1_2_1')
+        assert msg == self._get_random_number_result(problem.seed)
+
+    def test_random_isnt_none(self):
+        # Bug LMS-500 says random.seed(10) fails with:
+        #     File "<string>", line 61, in <module>
+        #     File "/usr/lib/python2.7/random.py", line 116, in seed
+        #       super(Random, self).seed(a)
+        #   TypeError: must be type, not None
+
+        r = random.Random()
+        r.seed(10)
+        num = r.randint(0, 1e9)
+
+        script = textwrap.dedent("""
+            random.seed(10)
+            num = random.randint(0, 1e9)
+            """)
+        problem = self.build_problem(script=script)
+        assert problem.context['num'] == num
+
+    def test_module_imports_inline(self):
+        '''
+        Check that the correct modules are available to custom
+        response scripts
+        '''
+
+        for module_name in ['random', 'numpy', 'math', 'scipy',
+                            'calc', 'eia', 'chemcalc', 'chemtools',
+                            'miller', 'draganddrop']:
+
+            # Create a script that checks that the name is defined
+            # If the name is not defined, then the script
+            # will raise an exception
+            script = textwrap.dedent('''
+            correct[0] = 'correct'
+            assert('%s' in globals())''' % module_name)
+
+            # Create the problem
+            problem = self.build_problem(answer=script)
+
+            # Expect that we can grade an answer without
+            # getting an exception
+            try:
+                problem.grade_answers({'1_2_1': '42'})
+
+            except ResponseError:
+                self.fail("Could not use name '{0}s' in custom response".format(module_name))
+
+    def test_module_imports_function(self):
+        '''
+        Check that the correct modules are available to custom
+        response scripts
+        '''
+
+        for module_name in ['random', 'numpy', 'math', 'scipy',
+                            'calc', 'eia', 'chemcalc', 'chemtools',
+                            'miller', 'draganddrop']:
+
+            # Create a script that checks that the name is defined
+            # If the name is not defined, then the script
+            # will raise an exception
+            script = textwrap.dedent('''
+            def check_func(expect, answer_given):
+                assert('%s' in globals())
+                return True''' % module_name)
+
+            # Create the problem
+            problem = self.build_problem(script=script, cfn="check_func")
+
+            # Expect that we can grade an answer without
+            # getting an exception
+            try:
+                problem.grade_answers({'1_2_1': '42'})
+
+            except ResponseError:
+                self.fail("Could not use name '{0}s' in custom response".format(module_name))
+
+    def test_python_lib_zip_is_available(self):
+        # Prove that we can import code from a zipfile passed down to us.
+
+        # Make a zipfile with one module in it with one function.
+        zipstring = io.BytesIO()
+        zipf = zipfile.ZipFile(zipstring, "w")  # lint-amnesty, pylint: disable=consider-using-with
+        zipf.writestr("my_helper.py", textwrap.dedent("""\
+            def seventeen():
+                return 17
+            """))
+        zipf.close()
+
+        # Use that module in our Python script.
+        script = textwrap.dedent("""
+            import my_helper
+            num = my_helper.seventeen()
+            """)
+        capa_system = test_capa_system()
+        capa_system.get_python_lib_zip = lambda: zipstring.getvalue()  # lint-amnesty, pylint: disable=unnecessary-lambda
+        problem = self.build_problem(script=script, capa_system=capa_system)
+        assert problem.context['num'] == 17
+
+    def test_function_code_multiple_inputs_order(self):
+        # Ensure that order must be correct according to sub-problem position
+        script = textwrap.dedent("""
+            def check_func(expect, answer_given):
+                check1 = (int(answer_given[0]) == 1)
+                check2 = (int(answer_given[1]) == 2)
+                check3 = (int(answer_given[2]) == 3)
+                check4 = (int(answer_given[3]) == 4)
+                check5 = (int(answer_given[4]) == 5)
+                check6 = (int(answer_given[5]) == 6)
+                check7 = (int(answer_given[6]) == 7)
+                check8 = (int(answer_given[7]) == 8)
+                check9 = (int(answer_given[8]) == 9)
+                check10 = (int(answer_given[9]) == 10)
+                check11 = (int(answer_given[10]) == 11)
+                return {'overall_message': 'Overall message',
+                            'input_list': [
+                                { 'ok': check1, 'msg': '1'},
+                                { 'ok': check2, 'msg': '2'},
+                                { 'ok': check3, 'msg': '3'},
+                                { 'ok': check4, 'msg': '4'},
+                                { 'ok': check5, 'msg': '5'},
+                                { 'ok': check6, 'msg': '6'},
+                                { 'ok': check7, 'msg': '7'},
+                                { 'ok': check8, 'msg': '8'},
+                                { 'ok': check9, 'msg': '9'},
+                                { 'ok': check10, 'msg': '10'},
+                                { 'ok': check11, 'msg': '11'},
+                ]}
+            """)
+
+        problem = self.build_problem(script=script, cfn="check_func", num_inputs=11)
+
+        # Grade the inputs showing out of order
+        input_dict = {
+            '1_2_1': '1',
+            '1_2_2': '2',
+            '1_2_3': '3',
+            '1_2_4': '4',
+            '1_2_5': '5',
+            '1_2_6': '6',
+            '1_2_10': '10',
+            '1_2_11': '16',
+            '1_2_7': '7',
+            '1_2_8': '8',
+            '1_2_9': '9'
+        }
+
+        correct_order = [
+            '1_2_1', '1_2_2', '1_2_3', '1_2_4', '1_2_5', '1_2_6', '1_2_7', '1_2_8', '1_2_9', '1_2_10', '1_2_11'
+        ]
+
+        correct_map = problem.grade_answers(input_dict)
+
+        assert list(problem.student_answers.keys()) != correct_order
+
+        # euqal to correct order after sorting at get_score
+        self.assertListEqual(list(problem.responders.values())[0].context['idset'], correct_order)
+
+        assert correct_map.get_correctness('1_2_1') == 'correct'
+        assert correct_map.get_correctness('1_2_9') == 'correct'
+        assert correct_map.get_correctness('1_2_11') == 'incorrect'
+
+        assert correct_map.get_msg('1_2_1') == '1'
+        assert correct_map.get_msg('1_2_9') == '9'
+        assert correct_map.get_msg('1_2_11') == '11'
+
+
+class SchematicResponseTest(ResponseTest):
+    """
+    Class containing setup and tests for Schematic responsetype.
+    """
+    xml_factory_class = SchematicResponseXMLFactory
+
+    def test_grade(self):
+        # Most of the schematic-specific work is handled elsewhere
+        # (in client-side JavaScript)
+        # The <schematicresponse> is responsible only for executing the
+        # Python code in <answer> with *submission* (list)
+        # in the global context.
+
+        # To test that the context is set up correctly,
+        # we create a script that sets *correct* to true
+        # if and only if we find the *submission* (list)
+        script = "correct = ['correct' if 'test' in submission[0] else 'incorrect']"
+        problem = self.build_problem(answer=script)
+
+        # The actual dictionary would contain schematic information
+        # sent from the JavaScript simulation
+        submission_dict = {'test': 'the_answer'}
+        input_dict = {'1_2_1': json.dumps(submission_dict)}
+        correct_map = problem.grade_answers(input_dict)
+
+        # Expect that the problem is graded as true
+        # (That is, our script verifies that the context
+        # is what we expect)
+        assert correct_map.get_correctness('1_2_1') == 'correct'
+
+    def test_check_function_randomization(self):
+        # The check function should get a random seed from the problem.
+        script = "correct = ['correct' if (submission[0]['num'] == {code}) else 'incorrect']".format(code=self._get_random_number_code())  # lint-amnesty, pylint: disable=line-too-long
+        problem = self.build_problem(answer=script)
+
+        submission_dict = {'num': self._get_random_number_result(problem.seed)}
+        input_dict = {'1_2_1': json.dumps(submission_dict)}
+        correct_map = problem.grade_answers(input_dict)
+
+        assert correct_map.get_correctness('1_2_1') == 'correct'
+
+    def test_script_exception(self):
+        # Construct a script that will raise an exception
+        script = "raise Exception('test')"
+        problem = self.build_problem(answer=script)
+
+        # Expect that an exception gets raised when we check the answer
+        with pytest.raises(ResponseError):
+            submission_dict = {'test': 'test'}
+            input_dict = {'1_2_1': json.dumps(submission_dict)}
+            problem.grade_answers(input_dict)
+
+
+class AnnotationResponseTest(ResponseTest):  # lint-amnesty, pylint: disable=missing-class-docstring
+    xml_factory_class = AnnotationResponseXMLFactory
+
+    def test_grade(self):
+        (correct, partially, incorrect) = ('correct', 'partially-correct', 'incorrect')
+
+        answer_id = '1_2_1'
+        options = (('x', correct), ('y', partially), ('z', incorrect))
+        make_answer = lambda option_ids: {answer_id: json.dumps({'options': option_ids})}
+
+        tests = [
+            {'correctness': correct, 'points': 2, 'answers': make_answer([0])},
+            {'correctness': partially, 'points': 1, 'answers': make_answer([1])},
+            {'correctness': incorrect, 'points': 0, 'answers': make_answer([2])},
+            {'correctness': incorrect, 'points': 0, 'answers': make_answer([0, 1, 2])},
+            {'correctness': incorrect, 'points': 0, 'answers': make_answer([])},
+            {'correctness': incorrect, 'points': 0, 'answers': make_answer('')},
+            {'correctness': incorrect, 'points': 0, 'answers': make_answer(None)},
+            {'correctness': incorrect, 'points': 0, 'answers': {answer_id: 'null'}},
+        ]
+
+        for test in tests:
+            expected_correctness = test['correctness']
+            expected_points = test['points']
+            answers = test['answers']
+
+            problem = self.build_problem(options=options)
+            correct_map = problem.grade_answers(answers)
+            actual_correctness = correct_map.get_correctness(answer_id)
+            actual_points = correct_map.get_npoints(answer_id)
+
+            assert expected_correctness == actual_correctness,\
+                ('%s should be marked %s' % (answer_id, expected_correctness))
+            assert expected_points == actual_points, ('%s should have %d points' % (answer_id, expected_points))
+
+
+class ChoiceTextResponseTest(ResponseTest):
+    """
+    Class containing setup and tests for ChoiceText responsetype.
+    """
+
+    xml_factory_class = ChoiceTextResponseXMLFactory
+
+    # `TEST_INPUTS` is a dictionary mapping from
+    # test_name to a representation of inputs for a test problem.
+    TEST_INPUTS = {
+        "1_choice_0_input_correct": [(True, [])],
+        "1_choice_0_input_incorrect": [(False, [])],
+        "1_choice_0_input_invalid_choice": [(False, []), (True, [])],
+        "1_choice_1_input_correct": [(True, ["123"])],
+        "1_input_script_correct": [(True, ["2"])],
+        "1_input_script_incorrect": [(True, ["3.25"])],
+        "1_choice_2_inputs_correct": [(True, ["123", "456"])],
+        "1_choice_2_inputs_tolerance": [(True, ["123 + .5", "456 + 9"])],
+        "1_choice_2_inputs_1_wrong": [(True, ["0", "456"])],
+        "1_choice_2_inputs_both_wrong": [(True, ["0", "0"])],
+        "1_choice_2_inputs_inputs_blank": [(True, ["", ""])],
+        "1_choice_2_inputs_empty": [(False, [])],
+        "1_choice_2_inputs_fail_tolerance": [(True, ["123 + 1.5", "456 + 9"])],
+        "1_choice_1_input_within_tolerance": [(True, ["122.5"])],
+        "1_choice_1_input_answer_incorrect": [(True, ["345"])],
+        "1_choice_1_input_choice_incorrect": [(False, ["123"])],
+        "2_choices_0_inputs_correct": [(False, []), (True, [])],
+        "2_choices_0_inputs_incorrect": [(True, []), (False, [])],
+        "2_choices_0_inputs_blank": [(False, []), (False, [])],
+        "2_choices_1_input_1_correct": [(False, []), (True, ["123"])],
+        "2_choices_1_input_1_incorrect": [(True, []), (False, ["123"])],
+        "2_choices_1_input_input_wrong": [(False, []), (True, ["321"])],
+        "2_choices_1_input_1_blank": [(False, []), (False, [])],
+        "2_choices_1_input_2_correct": [(True, []), (False, ["123"])],
+        "2_choices_1_input_2_incorrect": [(False, []), (True, ["123"])],
+        "2_choices_2_inputs_correct": [(True, ["123"]), (False, [])],
+        "2_choices_2_inputs_wrong_choice": [(False, ["123"]), (True, [])],
+        "2_choices_2_inputs_wrong_input": [(True, ["321"]), (False, [])]
+    }
+
+    # `TEST_SCENARIOS` is a dictionary of the form
+    # {Test_Name" : (Test_Problem_name, correctness)}
+    # correctness represents whether the problem should be graded as
+    # correct or incorrect when the test is run.
+    TEST_SCENARIOS = {
+        "1_choice_0_input_correct": ("1_choice_0_input", "correct"),
+        "1_choice_0_input_incorrect": ("1_choice_0_input", "incorrect"),
+        "1_choice_0_input_invalid_choice": ("1_choice_0_input", "incorrect"),
+        "1_input_script_correct": ("1_input_script", "correct"),
+        "1_input_script_incorrect": ("1_input_script", "incorrect"),
+        "1_choice_2_inputs_correct": ("1_choice_2_inputs", "correct"),
+        "1_choice_2_inputs_tolerance": ("1_choice_2_inputs", "correct"),
+        "1_choice_2_inputs_1_wrong": ("1_choice_2_inputs", "incorrect"),
+        "1_choice_2_inputs_both_wrong": ("1_choice_2_inputs", "incorrect"),
+        "1_choice_2_inputs_inputs_blank": ("1_choice_2_inputs", "incorrect"),
+        "1_choice_2_inputs_empty": ("1_choice_2_inputs", "incorrect"),
+        "1_choice_2_inputs_fail_tolerance": ("1_choice_2_inputs", "incorrect"),
+        "1_choice_1_input_correct": ("1_choice_1_input", "correct"),
+        "1_choice_1_input_within_tolerance": ("1_choice_1_input", "correct"),
+        "1_choice_1_input_answer_incorrect": ("1_choice_1_input", "incorrect"),
+        "1_choice_1_input_choice_incorrect": ("1_choice_1_input", "incorrect"),
+        "2_choices_0_inputs_correct": ("2_choices_0_inputs", "correct"),
+        "2_choices_0_inputs_incorrect": ("2_choices_0_inputs", "incorrect"),
+        "2_choices_0_inputs_blank": ("2_choices_0_inputs", "incorrect"),
+        "2_choices_1_input_1_correct": ("2_choices_1_input_1", "correct"),
+        "2_choices_1_input_1_incorrect": ("2_choices_1_input_1", "incorrect"),
+        "2_choices_1_input_input_wrong": ("2_choices_1_input_1", "incorrect"),
+        "2_choices_1_input_1_blank": ("2_choices_1_input_1", "incorrect"),
+        "2_choices_1_input_2_correct": ("2_choices_1_input_2", "correct"),
+        "2_choices_1_input_2_incorrect": ("2_choices_1_input_2", "incorrect"),
+        "2_choices_2_inputs_correct": ("2_choices_2_inputs", "correct"),
+        "2_choices_2_inputs_wrong_choice": ("2_choices_2_inputs", "incorrect"),
+        "2_choices_2_inputs_wrong_input": ("2_choices_2_inputs", "incorrect")
+    }
+
+    # Dictionary that maps from problem_name to arguments for
+    # _make_problem, that will create the problem.
+    TEST_PROBLEM_ARGS = {
+        "1_choice_0_input": {"choices": ("true", {}), "script": ''},
+        "1_choice_1_input": {
+            "choices": ("true", {"answer": "123", "tolerance": "1"}),
+            "script": ''
+        },
+
+        "1_input_script": {
+            "choices": ("true", {"answer": "$computed_response", "tolerance": "1"}),
+            "script": "computed_response = math.sqrt(4)"
+        },
+
+        "1_choice_2_inputs": {
+            "choices": [
+                (
+                    "true", (
+                        {"answer": "123", "tolerance": "1"},
+                        {"answer": "456", "tolerance": "10"}
+                    )
+                )
+            ],
+            "script": ''
+        },
+        "2_choices_0_inputs": {
+            "choices": [("false", {}), ("true", {})],
+            "script": ''
+
+        },
+        "2_choices_1_input_1": {
+            "choices": [
+                ("false", {}), ("true", {"answer": "123", "tolerance": "0"})
+            ],
+            "script": ''
+        },
+        "2_choices_1_input_2": {
+            "choices": [("true", {}), ("false", {"answer": "123", "tolerance": "0"})],
+            "script": ''
+        },
+        "2_choices_2_inputs": {
+            "choices": [
+                ("true", {"answer": "123", "tolerance": "0"}),
+                ("false", {"answer": "999", "tolerance": "0"})
+            ],
+            "script": ''
+        }
+    }
+
+    def _make_problem(self, choices, in_type='radiotextgroup', script=''):
+        """
+        Convenience method to fill in default values for script and
+        type if needed, then call self.build_problem
+        """
+        return self.build_problem(
+            choices=choices,
+            type=in_type,
+            script=script
+        )
+
+    def _make_answer_dict(self, choice_list):
+        """
+        Convenience method to make generation of answers less tedious,
+        pass in an iterable argument with elements of the form: [bool, [ans,]]
+        Will generate an answer dict for those options
+        """
+
+        answer_dict = {}
+        for index, choice_answers_pair in enumerate(choice_list):
+            # Choice is whether this choice is correct
+            # Answers contains a list of answers to textinpts for the choice
+            choice, answers = choice_answers_pair
+
+            if choice:
+                # Radio/Checkbox inputs in choicetext problems follow
+                # a naming convention that gives them names ending with "bc"
+                choice_id = "1_2_1_choiceinput_{index}bc".format(index=index)
+                choice_value = "choiceinput_{index}".format(index=index)
+                answer_dict[choice_id] = choice_value
+            # Build the names for the numtolerance_inputs and add their answers
+            # to `answer_dict`.
+            for ind, answer in enumerate(answers):
+                # In `answer_id` `index` represents the ordinality of the
+                # choice and `ind` represents the ordinality of the
+                # numtolerance_input inside the parent choice.
+                answer_id = "1_2_1_choiceinput_{index}_numtolerance_input_{ind}".format(
+                    index=index,
+                    ind=ind
+                )
+                answer_dict[answer_id] = answer
+
+        return answer_dict
+
+    def test_invalid_xml(self):
+        """
+        Test that build problem raises errors for invalid options
+        """
+        with pytest.raises(Exception):
+            self.build_problem(type="invalidtextgroup")
+
+    def test_unchecked_input_not_validated(self):
+        """
+        Test that a student can have a non numeric answer in an unselected
+        choice without causing an error to be raised when the problem is
+        checked.
+        """
+
+        two_choice_two_input = self._make_problem(
+            [
+                ("true", {"answer": "123", "tolerance": "1"}),
+                ("false", {})
+            ],
+            "checkboxtextgroup"
+        )
+
+        self.assert_grade(
+            two_choice_two_input,
+            self._make_answer_dict([(True, ["1"]), (False, ["Platypus"])]),
+            "incorrect"
+        )
+
+    def test_interpret_error(self):
+        """
+        Test that student answers that cannot be interpeted as numbers
+        cause the response type to raise an error.
+        """
+        two_choice_two_input = self._make_problem(
+            [
+                ("true", {"answer": "123", "tolerance": "1"}),
+                ("false", {})
+            ],
+            "checkboxtextgroup"
+        )
+
+        with self.assertRaisesRegex(StudentInputError, "Could not interpret"):
+            # Test that error is raised for input in selected correct choice.
+            self.assert_grade(
+                two_choice_two_input,
+                self._make_answer_dict([(True, ["Platypus"])]),
+                "correct"
+            )
+
+        with self.assertRaisesRegex(StudentInputError, "Could not interpret"):
+            # Test that error is raised for input in selected incorrect choice.
+            self.assert_grade(
+                two_choice_two_input,
+                self._make_answer_dict([(True, ["1"]), (True, ["Platypus"])]),
+                "correct"
+            )
+
+    def test_staff_answer_error(self):
+        broken_problem = self._make_problem(
+            [("true", {"answer": "Platypus", "tolerance": "0"}),
+             ("true", {"answer": "edX", "tolerance": "0"})
+             ],
+            "checkboxtextgroup"
+        )
+        with self.assertRaisesRegex(
+            StudentInputError,
+            "The Staff answer could not be interpreted as a number."
+        ):
+            self.assert_grade(
+                broken_problem,
+                self._make_answer_dict(
+                    [(True, ["1"]), (True, ["1"])]
+                ),
+                "correct"
+            )
+
+    def test_radio_grades(self):
+        """
+        Test that confirms correct operation of grading when the inputtag is
+        radiotextgroup.
+        """
+
+        for name, inputs in six.iteritems(self.TEST_INPUTS):
+            # Turn submission into the form expected when grading this problem.
+            submission = self._make_answer_dict(inputs)
+            # Lookup the problem_name, and the whether this test problem
+            # and inputs should be graded as correct or incorrect.
+            problem_name, correctness = self.TEST_SCENARIOS[name]
+            # Load the args needed to build the problem for this test.
+            problem_args = self.TEST_PROBLEM_ARGS[problem_name]
+            test_choices = problem_args["choices"]
+            test_script = problem_args["script"]
+            # Build the actual problem for the test.
+            test_problem = self._make_problem(test_choices, 'radiotextgroup', test_script)
+            # Make sure the actual grade matches the expected grade.
+            self.assert_grade(
+                test_problem,
+                submission,
+                correctness,
+                msg="{0} should be {1}".format(
+                    name,
+                    correctness
+                )
+            )
+
+    def test_checkbox_grades(self):
+        """
+        Test that confirms correct operation of grading when the inputtag is
+        checkboxtextgroup.
+        """
+        # Dictionary from name of test_scenario to (problem_name, correctness)
+        # Correctness is used to test whether the problem was graded properly
+        scenarios = {
+            "2_choices_correct": ("checkbox_two_choices", "correct"),
+            "2_choices_incorrect": ("checkbox_two_choices", "incorrect"),
+
+            "2_choices_2_inputs_correct": (
+                "checkbox_2_choices_2_inputs",
+                "correct"
+            ),
+
+            "2_choices_2_inputs_missing_choice": (
+                "checkbox_2_choices_2_inputs",
+                "incorrect"
+            ),
+
+            "2_choices_2_inputs_wrong_input": (
+                "checkbox_2_choices_2_inputs",
+                "incorrect"
+            )
+        }
+        # Dictionary scenario_name: test_inputs
+        inputs = {
+            "2_choices_correct": [(True, []), (True, [])],
+            "2_choices_incorrect": [(True, []), (False, [])],
+            "2_choices_2_inputs_correct": [(True, ["123"]), (True, ["456"])],
+            "2_choices_2_inputs_missing_choice": [
+                (True, ["123"]), (False, ["456"])
+            ],
+            "2_choices_2_inputs_wrong_input": [
+                (True, ["123"]), (True, ["654"])
+            ]
+        }
+
+        # Two choice zero input problem with both choices being correct.
+        checkbox_two_choices = self._make_problem(
+            [("true", {}), ("true", {})], "checkboxtextgroup"
+        )
+        # Two choice two input problem with both choices correct.
+        checkbox_two_choices_two_inputs = self._make_problem(
+            [("true", {"answer": "123", "tolerance": "0"}),
+             ("true", {"answer": "456", "tolerance": "0"})
+             ],
+            "checkboxtextgroup"
+        )
+
+        # Dictionary problem_name: problem
+        problems = {
+            "checkbox_two_choices": checkbox_two_choices,
+            "checkbox_2_choices_2_inputs": checkbox_two_choices_two_inputs
+        }
+
+        for name, inputs in six.iteritems(inputs):
+            submission = self._make_answer_dict(inputs)
+            # Load the test problem's name and desired correctness
+            problem_name, correctness = scenarios[name]
+            # Load the problem
+            problem = problems[problem_name]
+
+            # Make sure the actual grade matches the expected grade
+            self.assert_grade(
+                problem,
+                submission,
+                correctness,
+                msg="{0} should be {1}".format(name, correctness)
+            )

xmodule/capa/tests/test_shuffle.py

@@ -0,0 +1,308 @@
+"""Tests the capa shuffle and name-masking."""
+
+
+import textwrap
+import unittest
+
+from xmodule.capa.responsetypes import LoncapaProblemError
+from xmodule.capa.tests.helpers import new_loncapa_problem, test_capa_system
+
+
+class CapaShuffleTest(unittest.TestCase):
+    """Capa problem tests for shuffling and choice-name masking."""
+
+    def setUp(self):
+        super(CapaShuffleTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.system = test_capa_system()
+
+    def test_shuffle_4_choices(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false">Apple</choice>
+                <choice correct="false">Banana</choice>
+                <choice correct="false">Chocolate</choice>
+                <choice correct ="true">Donut</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        # shuffling 4 things with seed of 0 yields: B A C D
+        # Check that the choices are shuffled
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'Banana'.*'Apple'.*'Chocolate'.*'Donut'.*\].*</div>")
+        # Check that choice name masking is enabled and that unmasking works
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert response.unmask_order() == ['choice_1', 'choice_0', 'choice_2', 'choice_3']
+        assert the_html == problem.get_html(), 'should be able to call get_html() twice'
+
+    def test_shuffle_custom_names(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false" name="aaa">Apple</choice>
+                <choice correct="false">Banana</choice>
+                <choice correct="false">Chocolate</choice>
+                <choice correct ="true" name="ddd">Donut</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        # B A C D
+        # Check that the custom name= names come through
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert response.has_shuffle()
+        assert response.unmask_order() == ['choice_0', 'choice_aaa', 'choice_1', 'choice_ddd']
+
+    def test_shuffle_different_seed(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false">Apple</choice>
+                <choice correct="false">Banana</choice>
+                <choice correct="false">Chocolate</choice>
+                <choice correct ="true">Donut</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=341)  # yields D A B C
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'Donut'.*'Apple'.*'Banana'.*'Chocolate'.*\].*</div>")
+
+    def test_shuffle_1_choice(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="true">Apple</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'Apple'.*\].*</div>")
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert response.has_shuffle()
+        assert response.unmask_order() == ['choice_0']
+
+    def test_shuffle_6_choices(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false">Apple</choice>
+                <choice correct="false">Banana</choice>
+                <choice correct="false">Chocolate</choice>
+                <choice correct ="true">Zonut</choice>
+                <choice correct ="false">Eggplant</choice>
+                <choice correct ="false">Filet Mignon</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)  # yields: C E A B D F
+        # Donut -> Zonut to show that there is not some hidden alphabetic ordering going on
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'Chocolate'.*'Eggplant'.*'Apple'.*'Banana'.*'Zonut'.*'Filet Mignon'.*\].*</div>")  # lint-amnesty, pylint: disable=line-too-long
+
+    def test_shuffle_false(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="false">
+                <choice correct="false">Apple</choice>
+                <choice correct="false">Banana</choice>
+                <choice correct="false">Chocolate</choice>
+                <choice correct ="true">Donut</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'Apple'.*'Banana'.*'Chocolate'.*'Donut'.*\].*</div>")
+        response = list(problem.responders.values())[0]
+        assert not response.has_mask()
+        assert not response.has_shuffle()
+
+    def test_shuffle_fixed_head_end(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false" fixed="true">Alpha</choice>
+                <choice correct="false" fixed="true">Beta</choice>
+                <choice correct="false">A</choice>
+                <choice correct="false">B</choice>
+                <choice correct="false">C</choice>
+                <choice correct ="true">D</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        the_html = problem.get_html()
+        # Alpha Beta held back from shuffle (head end)
+        self.assertRegex(the_html, r"<div>.*\[.*'Alpha'.*'Beta'.*'B'.*'A'.*'C'.*'D'.*\].*</div>")
+
+    def test_shuffle_fixed_tail_end(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false">A</choice>
+                <choice correct="false">B</choice>
+                <choice correct="false">C</choice>
+                <choice correct ="true">D</choice>
+                <choice correct="false" fixed="true">Alpha</choice>
+                <choice correct="false" fixed="true">Beta</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        the_html = problem.get_html()
+        # Alpha Beta held back from shuffle (tail end)
+        self.assertRegex(the_html, r"<div>.*\[.*'B'.*'A'.*'C'.*'D'.*'Alpha'.*'Beta'.*\].*</div>")
+
+    def test_shuffle_fixed_both_ends(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false" fixed="true">Alpha</choice>
+                <choice correct="false" fixed="true">Beta</choice>
+                <choice correct="false">A</choice>
+                <choice correct="false">B</choice>
+                <choice correct="false">C</choice>
+                <choice correct ="true">D</choice>
+                <choice correct="false" fixed="true">Psi</choice>
+                <choice correct="false" fixed="true">Omega</choice>
+
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        the_html = problem.get_html()
+        self.assertRegex(
+            the_html,
+            r"<div>.*\[.*'Alpha'.*'Beta'.*'B'.*'A'.*'C'.*'D'.*'Psi'.*'Omega'.*\].*</div>"
+        )
+
+    def test_shuffle_fixed_both_ends_thin(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false" fixed="true">Alpha</choice>
+                <choice correct="false">A</choice>
+                <choice correct="true" fixed="true">Omega</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'Alpha'.*'A'.*'Omega'.*\].*</div>")
+
+    def test_shuffle_fixed_all(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false" fixed="true">A</choice>
+                <choice correct="false" fixed="true">B</choice>
+                <choice correct="true" fixed="true">C</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'A'.*'B'.*'C'.*\].*</div>")
+
+    def test_shuffle_island(self):
+        """A fixed 'island' choice not at the head or tail end gets lumped into the tail end."""
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false" fixed="true">A</choice>
+                <choice correct="false">Mid</choice>
+                <choice correct="true" fixed="true">C</choice>
+                <choice correct="False">Mid</choice>
+                <choice correct="false" fixed="true">D</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<div>.*\[.*'A'.*'Mid'.*'Mid'.*'C'.*'D'.*\].*</div>")
+
+    def test_multiple_shuffle_responses(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false">Apple</choice>
+                <choice correct="false">Banana</choice>
+                <choice correct="false">Chocolate</choice>
+                <choice correct ="true">Donut</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            <p>Here is some text</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true">
+                <choice correct="false">A</choice>
+                <choice correct="false">B</choice>
+                <choice correct="false">C</choice>
+                <choice correct ="true">D</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+        problem = new_loncapa_problem(xml_str, seed=0)
+        orig_html = problem.get_html()
+        assert orig_html == problem.get_html(), 'should be able to call get_html() twice'
+        html = orig_html.replace('\n', ' ')  # avoid headaches with .* matching
+        print(html)
+        self.assertRegex(html, r"<div>.*\[.*'Banana'.*'Apple'.*'Chocolate'.*'Donut'.*\].*</div>.*" +
+                         r"<div>.*\[.*'C'.*'A'.*'D'.*'B'.*\].*</div>")
+        # Look at the responses in their authored order
+        responses = sorted(list(problem.responders.values()), key=lambda resp: int(resp.id[resp.id.rindex('_') + 1:]))
+        assert not responses[0].has_mask()
+        assert responses[0].has_shuffle()
+        assert responses[1].has_shuffle()
+        assert responses[0].unmask_order() == ['choice_1', 'choice_0', 'choice_2', 'choice_3']
+        assert responses[1].unmask_order() == ['choice_2', 'choice_0', 'choice_3', 'choice_1']
+
+    def test_shuffle_not_with_answerpool(self):
+        """Raise error if shuffle and answer-pool are both used."""
+        xml_str = textwrap.dedent("""
+            <problem>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice" shuffle="true" answer-pool="4">
+                <choice correct="false" fixed="true">A</choice>
+                <choice correct="false">Mid</choice>
+                <choice correct="true" fixed="true">C</choice>
+                <choice correct="False">Mid</choice>
+                <choice correct="false" fixed="true">D</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+            </problem>
+        """)
+
+        with self.assertRaisesRegex(LoncapaProblemError, "shuffle and answer-pool"):
+            new_loncapa_problem(xml_str)

xmodule/capa/tests/test_targeted_feedback.py

@@ -0,0 +1,610 @@
+"""
+Tests the logic of the "targeted-feedback" attribute for MultipleChoice questions,
+i.e. those with the <multiplechoiceresponse> element
+"""
+
+
+import textwrap
+import unittest
+from xmodule.capa.tests.helpers import load_fixture, new_loncapa_problem, test_capa_system
+
+
+class CapaTargetedFeedbackTest(unittest.TestCase):
+    '''
+    Testing class
+    '''
+
+    def setUp(self):
+        super(CapaTargetedFeedbackTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.system = test_capa_system()
+
+    def test_no_targeted_feedback(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse>
+              <choicegroup type="MultipleChoice">
+                <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+                <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+                <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+                <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedback1">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 1st WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback2">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 2nd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback3">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 3rd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on your correct solution...</p>
+                </div>
+                </targetedfeedback>
+
+            </targetedfeedbackset>
+
+            <solution explanation-id="feedbackC">
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the solution explanation</p>
+                <p>Not much to explain here, sorry!</p>
+            </div>
+            </solution>
+        </problem>
+
+        """)
+
+        problem = new_loncapa_problem(xml_str)
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+
+        self.assertRegex(without_new_lines, r"<div>.*'wrong-1'.*'wrong-2'.*'correct-1'.*'wrong-3'.*</div>")
+        self.assertRegex(without_new_lines, r"feedback1|feedback2|feedback3|feedbackC")
+
+    def test_targeted_feedback_not_finished(self):
+        problem = new_loncapa_problem(load_fixture('targeted_feedback.xml'))
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+
+        self.assertRegex(without_new_lines, r"<div>.*'wrong-1'.*'wrong-2'.*'correct-1'.*'wrong-3'.*</div>")
+        self.assertNotRegex(without_new_lines, r"feedback1|feedback2|feedback3|feedbackC")
+        assert the_html == problem.get_html(), 'Should be able to call get_html() twice'
+
+    def test_targeted_feedback_student_answer1(self):
+        problem = new_loncapa_problem(load_fixture('targeted_feedback.xml'))
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_3'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\\n", "").replace("\n", "")
+        # pylint: disable=line-too-long
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedback3\" role=\"group\" aria-describedby=\"1_2_1-legend\">\s*<span class=\"sr\">Incorrect</span>.*3rd WRONG solution")
+        self.assertNotRegex(without_new_lines, r"feedback1|feedback2|feedbackC")
+        # Check that calling it multiple times yields the same thing
+        the_html2 = problem.get_html()
+        assert the_html == the_html2
+
+    def test_targeted_feedback_student_answer2(self):
+        problem = new_loncapa_problem(load_fixture('targeted_feedback.xml'))
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_0'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\\n", "").replace("\n", "")
+        # pylint: disable=line-too-long
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedback1\" role=\"group\" aria-describedby=\"1_2_1-legend\">\s*<span class=\"sr\">Incorrect</span>.*1st WRONG solution")
+        self.assertRegex(without_new_lines, r"<div>\{.*'1_solution_1'.*\}</div>")
+        self.assertNotRegex(without_new_lines, r"feedback2|feedback3|feedbackC")
+
+    def test_targeted_feedback_correct_answer(self):
+        """ Test the case of targeted feedback for a correct answer. """
+        problem = new_loncapa_problem(load_fixture('targeted_feedback.xml'))
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_2'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\\n", "").replace("\n", "")
+        # pylint: disable=line-too-long
+        self.assertRegex(without_new_lines,
+                         r"<targetedfeedback explanation-id=\"feedbackC\" role=\"group\" aria-describedby=\"1_2_1-legend\">\s*<span class=\"sr\">Correct</span>.*Feedback on your correct solution...")
+        self.assertNotRegex(without_new_lines, r"feedback1|feedback2|feedback3")
+
+    def test_targeted_feedback_id_typos(self):
+        """Cases where the explanation-id's don't match anything."""
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse targeted-feedback="">
+              <choicegroup type="MultipleChoice">
+                <choice correct="false" explanation-id="feedback1TYPO">wrong-1</choice>
+                <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+                <choice correct="true" explanation-id="feedbackCTYPO">correct-1</choice>
+                <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedback1">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 1st WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback2">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 2nd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback3">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 3rd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on your correct solution...</p>
+                </div>
+                </targetedfeedback>
+
+            </targetedfeedbackset>
+
+            <solution explanation-id="feedbackC">
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the solution explanation</p>
+                <p>Not much to explain here, sorry!</p>
+            </div>
+            </solution>
+        </problem>
+        """)
+
+        # explanation-id does not match anything: fall back to empty targetedfeedbackset
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_0'}
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<targetedfeedbackset>\s*</targetedfeedbackset>")
+
+        # New problem with same XML -- try the correct choice.
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_2'}  # correct
+        the_html = problem.get_html()
+        self.assertRegex(the_html, r"<targetedfeedbackset>\s*</targetedfeedbackset>")
+
+    def test_targeted_feedback_no_solution_element(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse targeted-feedback="">
+              <choicegroup type="MultipleChoice">
+                <choice correct="false">wrong-1</choice>
+                <choice correct="false">wrong-2</choice>
+                <choice correct="true"  explanation-id="feedbackC">correct-1</choice>
+                <choice correct="false">wrong-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                </div>
+                </targetedfeedback>
+            </targetedfeedbackset>
+            </problem>
+        """)
+
+        # Solution element not found
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_2'}
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+        # </div> right after </targetedfeedbackset>
+        self.assertRegex(
+            without_new_lines,
+            r"<div>.*<targetedfeedbackset>.*</targetedfeedbackset>\s*</div>"
+        )
+
+    def test_targeted_feedback_show_solution_explanation(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse targeted-feedback="alwaysShowCorrectChoiceExplanation">
+              <choicegroup type="MultipleChoice">
+                <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+                <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+                <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+                <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedback1">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 1st WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback2">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 2nd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback3">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 3rd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on your correct solution...</p>
+                </div>
+                </targetedfeedback>
+
+            </targetedfeedbackset>
+
+            <solution explanation-id="feedbackC">
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the solution explanation</p>
+                <p>Not much to explain here, sorry!</p>
+            </div>
+            </solution>
+        </problem>
+
+        """)
+
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_0'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+        # pylint: disable=line-too-long
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedback1\" role=\"group\" aria-describedby=\"1_2_1-legend\">.*1st WRONG solution")
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedbackC\".*solution explanation")
+        self.assertNotRegex(without_new_lines, r"<div>\{.*'1_solution_1'.*\}</div>")
+        self.assertNotRegex(without_new_lines, r"feedback2|feedback3")
+        # Check that calling it multiple times yields the same thing
+        the_html2 = problem.get_html()
+        assert the_html == the_html2
+
+    def test_targeted_feedback_no_show_solution_explanation(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse targeted-feedback="">
+              <choicegroup type="MultipleChoice">
+                <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+                <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+                <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+                <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedback1">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 1st WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback2">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 2nd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback3">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 3rd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on your correct solution...</p>
+                </div>
+                </targetedfeedback>
+
+            </targetedfeedbackset>
+
+            <solution explanation-id="feedbackC">
+            <div class="detailed-solution">
+                <p>Explanation</p>
+                <p>This is the solution explanation</p>
+                <p>Not much to explain here, sorry!</p>
+            </div>
+            </solution>
+        </problem>
+
+        """)
+
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_0'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+        # pylint: disable=line-too-long
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedback1\" role=\"group\" aria-describedby=\"1_2_1-legend\">.*1st WRONG solution")
+        self.assertNotRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedbackC\".*solution explanation")
+        self.assertRegex(without_new_lines, r"<div>\{.*'1_solution_1'.*\}</div>")
+        self.assertNotRegex(without_new_lines, r"feedback2|feedback3|feedbackC")
+
+    def test_targeted_feedback_with_solutionset_explanation(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse targeted-feedback="alwaysShowCorrectChoiceExplanation">
+              <choicegroup type="MultipleChoice">
+                <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+                <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+                <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+                <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+                <choice correct="true" explanation-id="feedbackC2">correct-2</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedback1">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 1st WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback2">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 2nd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback3">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 3rd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on your correct solution...</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC2">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on the other solution...</p>
+                </div>
+                </targetedfeedback>
+
+            </targetedfeedbackset>
+
+            <solutionset>
+                <solution explanation-id="feedbackC2">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the other solution explanation</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+            </solutionset>
+        </problem>
+
+        """)
+
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_0'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+        # pylint: disable=line-too-long
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedback1\" role=\"group\" aria-describedby=\"1_2_1-legend\">.*1st WRONG solution")
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedbackC2\".*other solution explanation")
+        self.assertNotRegex(without_new_lines, r"<div>\{.*'1_solution_1'.*\}</div>")
+        self.assertNotRegex(without_new_lines, r"feedback2|feedback3")
+
+    def test_targeted_feedback_no_feedback_for_selected_choice1(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse targeted-feedback="alwaysShowCorrectChoiceExplanation">
+              <choicegroup type="MultipleChoice">
+                <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+                <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+                <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+                <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedback1">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 1st WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback3">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 3rd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on your correct solution...</p>
+                </div>
+                </targetedfeedback>
+
+            </targetedfeedbackset>
+
+            <solutionset>
+                <solution explanation-id="feedbackC">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the solution explanation</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+            </solutionset>
+        </problem>
+
+        """)
+
+        # The student choses one with no feedback, but alwaysShowCorrectChoiceExplanation
+        # is in force, so we should see the correct solution feedback.
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_1'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+
+        self.assertRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedbackC\".*solution explanation")
+        self.assertNotRegex(without_new_lines, r"<div>\{.*'1_solution_1'.*\}</div>")
+        self.assertNotRegex(without_new_lines, r"feedback1|feedback3")
+
+    def test_targeted_feedback_no_feedback_for_selected_choice2(self):
+        xml_str = textwrap.dedent("""
+            <problem>
+            <p>What is the correct answer?</p>
+            <multiplechoiceresponse targeted-feedback="">
+              <choicegroup type="MultipleChoice">
+                <choice correct="false" explanation-id="feedback1">wrong-1</choice>
+                <choice correct="false" explanation-id="feedback2">wrong-2</choice>
+                <choice correct="true" explanation-id="feedbackC">correct-1</choice>
+                <choice correct="false" explanation-id="feedback3">wrong-3</choice>
+              </choicegroup>
+            </multiplechoiceresponse>
+
+            <targetedfeedbackset>
+                <targetedfeedback explanation-id="feedback1">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 1st WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedback3">
+                <div class="detailed-targeted-feedback">
+                    <p>Targeted Feedback</p>
+                    <p>This is the 3rd WRONG solution</p>
+                </div>
+                </targetedfeedback>
+
+                <targetedfeedback explanation-id="feedbackC">
+                <div class="detailed-targeted-feedback-correct">
+                    <p>Targeted Feedback</p>
+                    <p>Feedback on your correct solution...</p>
+                </div>
+                </targetedfeedback>
+
+            </targetedfeedbackset>
+
+            <solutionset>
+                <solution explanation-id="feedbackC">
+                <div class="detailed-solution">
+                    <p>Explanation</p>
+                    <p>This is the solution explanation</p>
+                    <p>Not much to explain here, sorry!</p>
+                </div>
+                </solution>
+            </solutionset>
+        </problem>
+
+        """)
+
+        # The student chooses one with no feedback set, so we check that there's no feedback.
+        problem = new_loncapa_problem(xml_str)
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_1'}
+
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+
+        self.assertNotRegex(without_new_lines, r"<targetedfeedback explanation-id=\"feedbackC\".*solution explanation")
+        self.assertRegex(without_new_lines, r"<div>\{.*'1_solution_1'.*\}</div>")
+        self.assertNotRegex(without_new_lines, r"feedback1|feedback3|feedbackC")
+
+    def test_targeted_feedback_multiple_not_answered(self):
+        # Not answered -> empty targeted feedback
+        problem = new_loncapa_problem(load_fixture('targeted_feedback_multiple.xml'))
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+        # Q1 and Q2 have no feedback
+        self.assertRegex(
+            without_new_lines,
+            r'<targetedfeedbackset>\s*</targetedfeedbackset>.*<targetedfeedbackset>\s*</targetedfeedbackset>'
+        )
+
+    def test_targeted_feedback_multiple_answer_1(self):
+        problem = new_loncapa_problem(load_fixture('targeted_feedback_multiple.xml'))
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_0'}  # feedback1
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+        # Q1 has feedback1 and Q2 has nothing
+        self.assertRegex(
+            without_new_lines,
+            r'<targetedfeedbackset.*?>.*?explanation-id="feedback1".*?</targetedfeedbackset>.*' +
+            r'<targetedfeedbackset>\s*</targetedfeedbackset>'
+        )
+
+    def test_targeted_feedback_multiple_answer_2(self):
+        problem = new_loncapa_problem(load_fixture('targeted_feedback_multiple.xml'))
+        problem.done = True
+        problem.student_answers = {'1_2_1': 'choice_0', '1_3_1': 'choice_2'}  # Q1 wrong, Q2 correct
+        the_html = problem.get_html()
+        without_new_lines = the_html.replace("\n", "")
+        # Q1 has feedback1 and Q2 has feedbackC
+        self.assertRegex(
+            without_new_lines,
+            r'<targetedfeedbackset.*?>.*?explanation-id="feedback1".*?</targetedfeedbackset>.*' +
+            r'<targetedfeedbackset.*?>.*explanation-id="feedbackC".*?</targetedfeedbackset>'
+        )

xmodule/capa/tests/test_util.py

@@ -0,0 +1,169 @@
+# coding=utf-8
+"""
+Tests capa util
+"""
+
+
+import unittest
+
+import ddt
+from lxml import etree
+
+from xmodule.capa.tests.helpers import test_capa_system
+from xmodule.capa.util import (
+    compare_with_tolerance,
+    contextualize_text,
+    get_inner_html_from_xpath,
+    remove_markup,
+    sanitize_html
+)
+
+
+@ddt.ddt
+class UtilTest(unittest.TestCase):
+    """Tests for util"""
+
+    def setUp(self):
+        super(UtilTest, self).setUp()  # lint-amnesty, pylint: disable=super-with-arguments
+        self.system = test_capa_system()
+
+    def test_compare_with_tolerance(self):  # lint-amnesty, pylint: disable=too-many-statements
+        # Test default tolerance '0.001%' (it is relative)
+        result = compare_with_tolerance(100.0, 100.0)
+        assert result
+        result = compare_with_tolerance(100.001, 100.0)
+        assert result
+        result = compare_with_tolerance(101.0, 100.0)
+        assert not result
+        # Test absolute percentage tolerance
+        result = compare_with_tolerance(109.9, 100.0, '10%', False)
+        assert result
+        result = compare_with_tolerance(110.1, 100.0, '10%', False)
+        assert not result
+        # Test relative percentage tolerance
+        result = compare_with_tolerance(111.0, 100.0, '10%', True)
+        assert result
+        result = compare_with_tolerance(112.0, 100.0, '10%', True)
+        assert not result
+        # Test absolute tolerance (string)
+        result = compare_with_tolerance(109.9, 100.0, '10.0', False)
+        assert result
+        result = compare_with_tolerance(110.1, 100.0, '10.0', False)
+        assert not result
+        # Test relative tolerance (string)
+        result = compare_with_tolerance(111.0, 100.0, '0.1', True)
+        assert result
+        result = compare_with_tolerance(112.0, 100.0, '0.1', True)
+        assert not result
+        # Test absolute tolerance (float)
+        result = compare_with_tolerance(109.9, 100.0, 10.0, False)
+        assert result
+        result = compare_with_tolerance(110.1, 100.0, 10.0, False)
+        assert not result
+        # Test relative tolerance (float)
+        result = compare_with_tolerance(111.0, 100.0, 0.1, True)
+        assert result
+        result = compare_with_tolerance(112.0, 100.0, 0.1, True)
+        assert not result
+        ##### Infinite values #####
+        infinity = float('Inf')
+        # Test relative tolerance (float)
+        result = compare_with_tolerance(infinity, 100.0, 1.0, True)
+        assert not result
+        result = compare_with_tolerance(100.0, infinity, 1.0, True)
+        assert not result
+        result = compare_with_tolerance(infinity, infinity, 1.0, True)
+        assert result
+        # Test absolute tolerance (float)
+        result = compare_with_tolerance(infinity, 100.0, 1.0, False)
+        assert not result
+        result = compare_with_tolerance(100.0, infinity, 1.0, False)
+        assert not result
+        result = compare_with_tolerance(infinity, infinity, 1.0, False)
+        assert result
+        # Test relative tolerance (string)
+        result = compare_with_tolerance(infinity, 100.0, '1.0', True)
+        assert not result
+        result = compare_with_tolerance(100.0, infinity, '1.0', True)
+        assert not result
+        result = compare_with_tolerance(infinity, infinity, '1.0', True)
+        assert result
+        # Test absolute tolerance (string)
+        result = compare_with_tolerance(infinity, 100.0, '1.0', False)
+        assert not result
+        result = compare_with_tolerance(100.0, infinity, '1.0', False)
+        assert not result
+        result = compare_with_tolerance(infinity, infinity, '1.0', False)
+        assert result
+        # Test absolute tolerance for smaller values
+        result = compare_with_tolerance(100.01, 100.0, 0.01, False)
+        assert result
+        result = compare_with_tolerance(100.001, 100.0, 0.001, False)
+        assert result
+        result = compare_with_tolerance(100.01, 100.0, '0.01%', False)
+        assert result
+        result = compare_with_tolerance(100.002, 100.0, 0.001, False)
+        assert not result
+        result = compare_with_tolerance(0.4, 0.44, 0.01, False)
+        assert not result
+        result = compare_with_tolerance(100.01, 100.0, 0.010, False)
+        assert result
+
+        # Test complex_number instructor_complex
+        result = compare_with_tolerance(0.4, complex(0.44, 0), 0.01, False)
+        assert not result
+        result = compare_with_tolerance(100.01, complex(100.0, 0), 0.010, False)
+        assert result
+        result = compare_with_tolerance(110.1, complex(100.0, 0), '10.0', False)
+        assert not result
+        result = compare_with_tolerance(111.0, complex(100.0, 0), '10%', True)
+        assert result
+
+    def test_sanitize_html(self):
+        """
+        Test for html sanitization with bleach.
+        """
+        allowed_tags = ['div', 'p', 'audio', 'pre', 'span']
+        for tag in allowed_tags:
+            queue_msg = "<{0}>Test message</{0}>".format(tag)
+            assert sanitize_html(queue_msg) == queue_msg
+
+        not_allowed_tag = 'script'
+        queue_msg = "<{0}>Test message</{0}>".format(not_allowed_tag)
+        expected = "&lt;script&gt;Test message&lt;/script&gt;"
+        assert sanitize_html(queue_msg) == expected
+
+    def test_get_inner_html_from_xpath(self):
+        """
+        Test for getting inner html as string from xpath node.
+        """
+        xpath_node = etree.XML('<hint style="smtng">aa<a href="#">bb</a>cc</hint>')
+        assert get_inner_html_from_xpath(xpath_node) == 'aa<a href="#">bb</a>cc'
+
+    def test_remove_markup(self):
+        """
+        Test for markup removal with bleach.
+        """
+        assert remove_markup('The <mark>Truth</mark> is <em>Out There</em> & you need to <strong>find</strong> it') ==\
+               'The Truth is Out There &amp; you need to find it'
+
+    @ddt.data(
+        'When the root level failš the whole hierarchy won’t work anymore.',
+        'あなたあなたあなた'
+    )
+    def test_contextualize_text(self, context_value):
+        """Verify that variable substitution works as intended with non-ascii characters."""
+        key = 'answer0'
+        text = '$answer0'
+        context = {key: context_value}
+        contextual_text = contextualize_text(text, context)
+        assert context_value == contextual_text
+
+    def test_contextualize_text_with_non_ascii_context(self):
+        """Verify that variable substitution works as intended with non-ascii characters."""
+        key = 'あなた$a $b'
+        text = '$' + key
+        context = {'a': 'あなたあなたあなた', 'b': 'あなたhi'}
+        expected_text = '$あなたあなたあなたあなた あなたhi'
+        contextual_text = contextualize_text(text, context)
+        assert expected_text == contextual_text

xmodule/capa/tests/test_xqueue_interface.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+"""
+Tests the xqueue service interface.
+"""
+
+from unittest import TestCase
+from django.conf import settings
+
+from xmodule.capa.xqueue_interface import XQueueInterface, XQueueService
+
+
+class XQueueServiceTest(TestCase):
+    """
+    Tests the XQueue service methods.
+    """
+    @staticmethod
+    def construct_callback(*args, **kwargs):
+        return 'https://lms.url/callback'
+
+    def setUp(self):
+        super().setUp()
+        self.service = XQueueService(
+            url=settings.XQUEUE_INTERFACE['url'],
+            django_auth=settings.XQUEUE_INTERFACE['django_auth'],
+            basic_auth=settings.XQUEUE_INTERFACE['basic_auth'],
+            construct_callback=self.construct_callback,
+            default_queuename='my-very-own-queue',
+            waittime=settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS,
+        )
+
+    def test_interface(self):
+        assert isinstance(self.service.interface, XQueueInterface)
+
+    def test_construct_callback(self):
+        assert self.service.construct_callback() == 'https://lms.url/callback'
+
+    def test_default_queuename(self):
+        assert self.service.default_queuename == 'my-very-own-queue'
+
+    def test_waittime(self):
+        assert self.service.waittime == 5

xmodule/capa/util.py

@@ -0,0 +1,255 @@
+"""
+Utility functions for capa.
+"""
+
+
+import logging
+import re
+from cmath import isinf, isnan
+from decimal import Decimal
+
+import bleach
+import six
+from calc import evaluator
+from lxml import etree
+
+from bleach.css_sanitizer import CSSSanitizer
+from openedx.core.djangolib.markup import HTML
+
+#-----------------------------------------------------------------------------
+#
+# Utility functions used in CAPA responsetypes
+default_tolerance = '0.001%'
+log = logging.getLogger(__name__)
+
+
+def compare_with_tolerance(student_complex, instructor_complex, tolerance=default_tolerance, relative_tolerance=False):
+    """
+    Compare student_complex to instructor_complex with maximum tolerance tolerance.
+
+     - student_complex    :  student result (float complex number)
+     - instructor_complex    :  instructor result (float complex number)
+     - tolerance   :  float, or string (representing a float or a percentage)
+     - relative_tolerance: bool, to explicitly use passed tolerance as relative
+
+     Note: when a tolerance is a percentage (i.e. '10%'), it will compute that
+     percentage of the instructor result and yield a number.
+
+     If relative_tolerance is set to False, it will use that value and the
+     instructor result to define the bounds of valid student result:
+     instructor_complex = 10, tolerance = '10%' will give [9.0, 11.0].
+
+     If relative_tolerance is set to True, it will use that value and both
+     instructor result and student result to define the bounds of valid student
+     result:
+     instructor_complex = 10, student_complex = 20, tolerance = '10%' will give
+     [8.0, 12.0].
+     This is typically used internally to compare float, with a
+     default_tolerance = '0.001%'.
+
+     Default tolerance of 1e-3% is added to compare two floats for
+     near-equality (to handle machine representation errors).
+     Default tolerance is relative, as the acceptable difference between two
+     floats depends on the magnitude of the floats.
+     (http://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/)
+     Examples:
+        In [183]: 0.000016 - 1.6*10**-5
+        Out[183]: -3.3881317890172014e-21
+        In [212]: 1.9e24 - 1.9*10**24
+        Out[212]: 268435456.0
+    """
+    if isinstance(tolerance, str):
+        if tolerance == default_tolerance:
+            relative_tolerance = True
+        if tolerance.endswith('%'):
+            tolerance = evaluator({}, {}, tolerance[:-1]) * 0.01
+            if not relative_tolerance:
+                tolerance = tolerance * abs(instructor_complex)
+        else:
+            tolerance = evaluator({}, {}, tolerance)
+
+    if relative_tolerance:
+        tolerance = tolerance * max(abs(student_complex), abs(instructor_complex))
+
+    if isinf(student_complex) or isinf(instructor_complex):
+        # If an input is infinite, we can end up with `abs(student_complex-instructor_complex)` and
+        # `tolerance` both equal to infinity. Then, below we would have
+        # `inf <= inf` which is a fail. Instead, compare directly.
+        return student_complex == instructor_complex
+
+    # because student_complex and instructor_complex are not necessarily
+    # complex here, we enforce it here:
+    student_complex = complex(student_complex)
+    instructor_complex = complex(instructor_complex)
+
+    # if both the instructor and student input are real,
+    # compare them as Decimals to avoid rounding errors
+    if not (instructor_complex.imag or student_complex.imag):
+        # if either of these are not a number, short circuit and return False
+        if isnan(instructor_complex.real) or isnan(student_complex.real):
+            return False
+        student_decimal = Decimal(str(student_complex.real))
+        instructor_decimal = Decimal(str(instructor_complex.real))
+        tolerance_decimal = Decimal(str(tolerance))
+        return abs(student_decimal - instructor_decimal) <= tolerance_decimal
+
+    else:
+        # v1 and v2 are, in general, complex numbers:
+        # there are some notes about backward compatibility issue: see responsetypes.get_staff_ans()).
+        return abs(student_complex - instructor_complex) <= tolerance
+
+
+def contextualize_text(text, context):  # private
+    """
+    Takes a string with variables. E.g. $a+$b.
+    Does a substitution of those variables from the context
+    """
+    def convert_to_str(value):
+        """The method tries to convert unicode/non-ascii values into string"""
+        try:
+            return str(value)
+        except UnicodeEncodeError:
+            return value.encode('utf8', errors='ignore')
+
+    if not text:
+        return text
+
+    for key in sorted(context, key=len, reverse=True):
+        # TODO (vshnayder): This whole replacement thing is a big hack
+        # right now--context contains not just the vars defined in the
+        # program, but also e.g. a reference to the numpy module.
+        # Should be a separate dict of variables that should be
+        # replaced.
+        context_key = '$' + key
+        if context_key in (text.decode('utf-8') if six.PY3 and isinstance(text, bytes) else text):
+            text = convert_to_str(text)
+            context_value = convert_to_str(context[key])
+            text = text.replace(context_key, context_value)
+
+    return text
+
+
+def convert_files_to_filenames(answers):
+    """
+    Check for File objects in the dict of submitted answers,
+        convert File objects to their filename (string)
+    """
+    new_answers = {}
+    for answer_id in answers.keys():
+        answer = answers[answer_id]
+        # Files are stored as a list, even if one file
+        if is_list_of_files(answer):
+            new_answers[answer_id] = [f.name for f in answer]
+        else:
+            new_answers[answer_id] = answers[answer_id]
+    return new_answers
+
+
+def is_list_of_files(files):
+    return isinstance(files, list) and all(is_file(f) for f in files)
+
+
+def is_file(file_to_test):
+    """
+    Duck typing to check if 'file_to_test' is a File object
+    """
+    return all(hasattr(file_to_test, method) for method in ['read', 'name'])
+
+
+def find_with_default(node, path, default):
+    """
+    Look for a child of node using , and return its text if found.
+    Otherwise returns default.
+
+    Arguments:
+       node: lxml node
+       path: xpath search expression
+       default: value to return if nothing found
+
+    Returns:
+       node.find(path).text if the find succeeds, default otherwise.
+
+    """
+    v = node.find(path)
+    if v is not None:
+        return v.text
+    else:
+        return default
+
+
+def sanitize_html(html_code):
+    """
+    Sanitize html_code for safe embed on LMS pages.
+
+    Used to sanitize XQueue responses from Matlab.
+    """
+    attributes = bleach.ALLOWED_ATTRIBUTES.copy()
+    attributes.update({
+        '*': ['class', 'style', 'id'],
+        'audio': ['controls', 'autobuffer', 'autoplay', 'src'],
+        'img': ['src', 'width', 'height', 'class']
+    })
+    output = bleach.clean(
+        html_code,
+        protocols=bleach.ALLOWED_PROTOCOLS + ['data'],
+        tags=bleach.ALLOWED_TAGS + ['div', 'p', 'audio', 'pre', 'img', 'span'],
+        css_sanitizer=CSSSanitizer(allowed_css_properties=["white-space"]),
+        attributes=attributes
+    )
+    return output
+
+
+def get_inner_html_from_xpath(xpath_node):
+    """
+    Returns inner html as string from xpath node.
+
+    """
+    # returns string from xpath node
+    html = etree.tostring(xpath_node).strip().decode('utf-8')
+    # strips outer tag from html string
+    # xss-lint: disable=python-interpolate-html
+    inner_html = re.sub('(?ms)<%s[^>]*>(.*)</%s>' % (xpath_node.tag, xpath_node.tag), '\\1', html)
+    return inner_html.strip()
+
+
+def remove_markup(html):
+    """
+    Return html with markup stripped and text HTML-escaped.
+
+    >>> bleach.clean("<b>Rock & Roll</b>", tags=[], strip=True)
+    'Rock &amp; Roll'
+    >>> bleach.clean("<b>Rock &amp; Roll</b>", tags=[], strip=True)
+    'Rock &amp; Roll'
+    """
+    return HTML(bleach.clean(html, tags=[], strip=True))
+
+
+def get_course_id_from_capa_module(capa_module):
+    """
+    Extract a stringified course run key from a CAPA module (aka ProblemBlock).
+
+    This is a bit of a hack. Its intended use is to allow us to pass the course id
+    (if available) to `safe_exec`, enabling course-run-specific resource limits
+    in the safe execution environment (codejail).
+
+    Arguments:
+        capa_module (ProblemBlock|None)
+
+    Returns: str|None
+        The stringified course run key of the module.
+        If not available, fall back to None.
+    """
+    if not capa_module:
+        return None
+    try:
+        return str(capa_module.scope_ids.usage_id.course_key)
+    except (AttributeError, TypeError):
+        # AttributeError:
+        #     If the capa module lacks scope ids or has unexpected scope ids, we
+        #     would rather fall back to `None` than let an AttributeError be raised
+        #     here.
+        # TypeError:
+        #     Old Mongo usage keys lack a 'run' specifier, and may
+        #     raise a type error when we try to serialize them into a course
+        #     run key. This is tolerable because such course runs are deprecated.
+        return None

xmodule/capa/xqueue_interface.py

@@ -0,0 +1,200 @@
+"""
+LMS Interface to external queueing system (xqueue)
+"""
+
+import hashlib
+import json
+import logging
+
+import requests
+import six
+
+log = logging.getLogger(__name__)
+dateformat = '%Y%m%d%H%M%S'
+
+XQUEUE_METRIC_NAME = 'edxapp.xqueue'
+
+# Wait time for response from Xqueue.
+XQUEUE_TIMEOUT = 35  # seconds
+CONNECT_TIMEOUT = 3.05  # seconds
+READ_TIMEOUT = 10  # seconds
+
+
+def make_hashkey(seed):
+    """
+    Generate a string key by hashing
+    """
+    h = hashlib.md5()
+    h.update(six.b(str(seed)))
+    return h.hexdigest()
+
+
+def make_xheader(lms_callback_url, lms_key, queue_name):
+    """
+    Generate header for delivery and reply of queue request.
+
+    Xqueue header is a JSON-serialized dict:
+        { 'lms_callback_url': url to which xqueue will return the request (string),
+          'lms_key': secret key used by LMS to protect its state (string),
+          'queue_name': designate a specific queue within xqueue server, e.g. 'MITx-6.00x' (string)
+        }
+    """
+    return json.dumps({
+        'lms_callback_url': lms_callback_url,
+        'lms_key': lms_key,
+        'queue_name': queue_name
+    })
+
+
+def parse_xreply(xreply):
+    """
+    Parse the reply from xqueue. Messages are JSON-serialized dict:
+        { 'return_code': 0 (success), 1 (fail)
+          'content': Message from xqueue (string)
+        }
+    """
+    try:
+        xreply = json.loads(xreply)
+    except ValueError as err:
+        log.error(err)
+        return (1, 'unexpected reply from server')
+
+    return_code = xreply['return_code']
+    content = xreply['content']
+
+    return (return_code, content)
+
+
+class XQueueInterface(object):
+    """
+    Interface to the external grading system
+    """
+
+    def __init__(self, url, django_auth, requests_auth=None):
+        self.url = six.text_type(url)
+        self.auth = django_auth
+        self.session = requests.Session()
+        self.session.auth = requests_auth
+
+    def send_to_queue(self, header, body, files_to_upload=None):
+        """
+        Submit a request to xqueue.
+
+        header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader'
+
+        body: Serialized data for the receipient behind the queueing service. The operation of
+                xqueue is agnostic to the contents of 'body'
+
+        files_to_upload: List of file objects to be uploaded to xqueue along with queue request
+
+        Returns (error_code, msg) where error_code != 0 indicates an error
+        """
+
+        # log the send to xqueue
+        header_info = json.loads(header)
+        queue_name = header_info.get('queue_name', '')  # lint-amnesty, pylint: disable=unused-variable
+
+        # Attempt to send to queue
+        (error, msg) = self._send_to_queue(header, body, files_to_upload)
+
+        # Log in, then try again
+        if error and (msg == 'login_required'):
+            (error, content) = self._login()
+            if error != 0:
+                # when the login fails
+                log.debug("Failed to login to queue: %s", content)
+                return (error, content)
+            if files_to_upload is not None:
+                # Need to rewind file pointers
+                for f in files_to_upload:
+                    f.seek(0)
+            (error, msg) = self._send_to_queue(header, body, files_to_upload)
+
+        return error, msg
+
+    def _login(self):  # lint-amnesty, pylint: disable=missing-function-docstring
+        payload = {
+            'username': self.auth['username'],
+            'password': self.auth['password']
+        }
+        return self._http_post(self.url + '/xqueue/login/', payload)
+
+    def _send_to_queue(self, header, body, files_to_upload):  # lint-amnesty, pylint: disable=missing-function-docstring
+        payload = {
+            'xqueue_header': header,
+            'xqueue_body': body
+        }
+        files = {}
+        if files_to_upload is not None:
+            for f in files_to_upload:
+                files.update({f.name: f})
+
+        return self._http_post(self.url + '/xqueue/submit/', payload, files=files)
+
+    def _http_post(self, url, data, files=None):  # lint-amnesty, pylint: disable=missing-function-docstring
+        try:
+            response = self.session.post(
+                url, data=data, files=files, timeout=(CONNECT_TIMEOUT, READ_TIMEOUT)
+            )
+        except requests.exceptions.ConnectionError as err:
+            log.error(err)
+            return 1, 'cannot connect to server'
+
+        except requests.exceptions.ReadTimeout as err:
+            log.error(err)
+            return 1, 'failed to read from the server'
+
+        if response.status_code not in [200]:
+            return 1, 'unexpected HTTP status code [%d]' % response.status_code
+
+        return parse_xreply(response.text)
+
+
+class XQueueService:
+    """
+    XBlock service providing an interface to the XQueue service.
+
+    Args:
+        construct_callback(callable): function which constructs a fully-qualified callback URL to make xqueue requests.
+        default_queuename(string): course-specific queue name.
+        waittime(int): number of seconds to wait between xqueue requests
+        url(string): base URL for the XQueue service.
+        django_auth(dict): username and password for the XQueue service.
+        basic_auth(array or None): basic authentication credentials, if needed.
+    """
+    def __init__(self, construct_callback, default_queuename, waittime, url, django_auth, basic_auth=None):
+
+        requests_auth = requests.auth.HTTPBasicAuth(*basic_auth) if basic_auth else None
+        self._interface = XQueueInterface(url, django_auth, requests_auth)
+
+        self._construct_callback = construct_callback
+        self._default_queuename = default_queuename.replace(' ', '_')
+        self._waittime = waittime
+
+    @property
+    def interface(self):
+        """
+        Returns the XQueueInterface instance.
+        """
+        return self._interface
+
+    @property
+    def construct_callback(self):
+        """
+        Returns the function to construct the XQueue callback.
+        """
+        return self._construct_callback
+
+    @property
+    def default_queuename(self):
+        """
+        Returns the default queue name for the current course.
+        """
+        return self._default_queuename
+
+    @property
+    def waittime(self):
+        """
+        Returns the number of seconds to wait in between calls to XQueue.
+        """
+        return self._waittime

xmodule/capa_module.py

@@ -26,11 +26,11 @@ from xblock.core import XBlock
 from xblock.fields import Boolean, Dict, Float, Integer, Scope, String, XMLString
 from xblock.scorable import ScorableXBlockMixin, Score
 
-from capa import responsetypes
-from capa.capa_problem import LoncapaProblem, LoncapaSystem
-from capa.inputtypes import Status
-from capa.responsetypes import LoncapaProblemError, ResponseError, StudentInputError
-from capa.util import convert_files_to_filenames, get_inner_html_from_xpath
+from xmodule.capa import responsetypes
+from xmodule.capa.capa_problem import LoncapaProblem, LoncapaSystem
+from xmodule.capa.inputtypes import Status
+from xmodule.capa.responsetypes import LoncapaProblemError, ResponseError, StudentInputError
+from xmodule.capa.util import convert_files_to_filenames, get_inner_html_from_xpath
 from xmodule.contentstore.django import contentstore
 from xmodule.editing_module import EditingMixin
 from xmodule.exceptions import NotFoundError, ProcessingError
@@ -140,7 +140,7 @@ class ProblemBlock(
     An XBlock representing a "problem".
 
     A problem contains zero or more respondable items, such as multiple choice,
-    numeric response, true/false, etc. See common/lib/capa/capa/responsetypes.py
+    numeric response, true/false, etc. See xmodule/capa/responsetypes.py
     for the full ensemble.
 
     The rendering logic of a problem is largely encapsulated within
@@ -1717,7 +1717,7 @@ class ProblemBlock(
         answers_without_files = convert_files_to_filenames(answers)
         event_info['answers'] = answers_without_files
 
-        metric_name = 'capa.check_problem.{}'.format  # lint-amnesty, pylint: disable=unused-variable
+        metric_name = 'xmodule.capa.check_problem.{}'.format  # lint-amnesty, pylint: disable=unused-variable
         # Can override current time
         current_time = datetime.datetime.now(utc)
         if override_time is not False:

xmodule/library_content_module.py

@@ -24,7 +24,7 @@ from xblock.completable import XBlockCompletionMode
 from xblock.core import XBlock
 from xblock.fields import Integer, List, Scope, String, Boolean
 
-from capa.responsetypes import registry
+from xmodule.capa.responsetypes import registry
 from xmodule.mako_module import MakoTemplateBlockBase
 from xmodule.studio_editable import StudioEditableBlock
 from xmodule.util.xmodule_django import add_webpack_to_fragment

xmodule/tests/__init__.py

@@ -26,7 +26,7 @@ from xblock.core import XBlock
 from xblock.field_data import DictFieldData
 from xblock.fields import Reference, ReferenceList, ReferenceValueDict, ScopeIds
 
-from capa.xqueue_interface import XQueueService
+from xmodule.capa.xqueue_interface import XQueueService
 from xmodule.assetstore import AssetMetadata
 from xmodule.contentstore.django import contentstore
 from xmodule.mako_module import MakoDescriptorSystem

xmodule/tests/test_capa_module.py

@@ -28,11 +28,11 @@ from xblock.field_data import DictFieldData
 from xblock.fields import ScopeIds
 from xblock.scorable import Score
 
-from capa import responsetypes
-from capa.correctmap import CorrectMap
-from capa.responsetypes import LoncapaProblemError, ResponseError, StudentInputError
-from capa.xqueue_interface import XQueueInterface
 import xmodule
+from xmodule.capa import responsetypes
+from xmodule.capa.correctmap import CorrectMap
+from xmodule.capa.responsetypes import LoncapaProblemError, ResponseError, StudentInputError
+from xmodule.capa.xqueue_interface import XQueueInterface
 from xmodule.capa_module import ComplexEncoder, ProblemBlock
 from xmodule.tests import DATA_DIR
 
@@ -704,7 +704,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
         # Simulate that all answers are marked correct, no matter
         # what the input is, by patching CorrectMap.is_correct()
         # Also simulate rendering the HTML
-        with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
+        with patch('xmodule.capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
             with patch('xmodule.capa_module.ProblemBlock.get_problem_html') as mock_html:
                 mock_is_correct.return_value = True
                 mock_html.return_value = "Test HTML"
@@ -729,7 +729,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
         module = CapaFactory.create(attempts=0)
 
         # Simulate marking the input incorrect
-        with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
+        with patch('xmodule.capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
             mock_is_correct.return_value = False
 
             # Check the problem
@@ -801,7 +801,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
 
         # Simulate that the problem is queued
         multipatch = patch.multiple(
-            'capa.capa_problem.LoncapaProblem',
+            'xmodule.capa.capa_problem.LoncapaProblem',
             is_queued=DEFAULT,
             get_recentmost_queuetime=DEFAULT
         )
@@ -911,7 +911,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
             module = CapaFactory.create(attempts=1, user_is_staff=False)
 
             # Simulate answering a problem that raises the exception
-            with patch('capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
+            with patch('xmodule.capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
                 mock_grade.side_effect = exception_class('test error')
 
                 get_request_dict = {CapaFactory.input_key(): '3.14'}
@@ -939,7 +939,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
             module = CapaFactory.create(attempts=1, user_is_staff=False)
 
             # Simulate a codejail exception "Exception: Couldn't execute jailed code"
-            with patch('capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
+            with patch('xmodule.capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
                 try:
                     raise ResponseError(
                         'Couldn\'t execute jailed code: stdout: \'\', '
@@ -973,7 +973,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
         module = CapaFactory.create(attempts=1, user_is_staff=False)
 
         # Simulate answering a problem that raises the exception
-        with patch('capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
+        with patch('xmodule.capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
             error_msg = "Superterrible error happened: ☠"
             mock_grade.side_effect = Exception(error_msg)
 
@@ -1008,7 +1008,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
             module = CapaFactory.create(attempts=1, user_is_staff=False)
 
             # Simulate answering a problem that raises the exception
-            with patch('capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
+            with patch('xmodule.capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
                 mock_grade.side_effect = exception_class("ȧƈƈḗƞŧḗḓ ŧḗẋŧ ƒǿř ŧḗşŧīƞɠ")
 
                 get_request_dict = {CapaFactory.input_key(): '3.14'}
@@ -1034,7 +1034,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
             module = CapaFactory.create(attempts=1, user_is_staff=True)
 
             # Simulate answering a problem that raises an exception
-            with patch('capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
+            with patch('xmodule.capa.capa_problem.LoncapaProblem.grade_answers') as mock_grade:
                 mock_grade.side_effect = exception_class('test error')
 
                 get_request_dict = {CapaFactory.input_key(): '3.14'}
@@ -1066,7 +1066,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
                                     correct=is_correct)
 
         # Simulate marking the input correct/incorrect
-        with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
+        with patch('xmodule.capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
             mock_is_correct.return_value = is_correct
 
             # Check the problem
@@ -1137,13 +1137,13 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
 
         # Simulate that all answers are marked correct, no matter
         # what the input is, by patching LoncapaResponse.evaluate_answers()
-        with patch('capa.responsetypes.LoncapaResponse.evaluate_answers') as mock_evaluate_answers:
+        with patch('xmodule.capa.responsetypes.LoncapaResponse.evaluate_answers') as mock_evaluate_answers:
             mock_evaluate_answers.return_value = CorrectMap(
                 answer_id=CapaFactory.answer_key(),
                 correctness='correct',
                 npoints=1,
             )
-            with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
+            with patch('xmodule.capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
                 mock_is_correct.return_value = True
 
                 # Check the problem
@@ -1183,10 +1183,10 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
         # In case of rescore with only_if_higher=True it should update score of module
         # if previous score was lower
 
-        with patch('capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
+        with patch('xmodule.capa.correctmap.CorrectMap.is_correct') as mock_is_correct:
             mock_is_correct.return_value = True
             module.set_score(module.score_from_lcp(module.lcp))
-            with patch('capa.responsetypes.NumericalResponse.get_staff_ans') as get_staff_ans:
+            with patch('xmodule.capa.responsetypes.NumericalResponse.get_staff_ans') as get_staff_ans:
                 get_staff_ans.return_value = 1 + 0j
                 module.rescore(only_if_higher=True)
 
@@ -1205,7 +1205,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
 
         # Simulate that all answers are marked incorrect, no matter
         # what the input is, by patching LoncapaResponse.evaluate_answers()
-        with patch('capa.responsetypes.LoncapaResponse.evaluate_answers') as mock_evaluate_answers:
+        with patch('xmodule.capa.responsetypes.LoncapaResponse.evaluate_answers') as mock_evaluate_answers:
             mock_evaluate_answers.return_value = CorrectMap(CapaFactory.answer_key(), 'incorrect')
             module.rescore(only_if_higher=False)
 
@@ -1229,7 +1229,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
         module = CapaFactory.create(done=True)
 
         # Try to rescore the problem, and get exception
-        with patch('capa.capa_problem.LoncapaProblem.supports_rescoring') as mock_supports_rescoring:
+        with patch('xmodule.capa.capa_problem.LoncapaProblem.supports_rescoring') as mock_supports_rescoring:
             mock_supports_rescoring.return_value = False
             with pytest.raises(NotImplementedError):
                 module.rescore(only_if_higher=False)
@@ -1255,7 +1255,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
 
         # When codejail safe_exec fails upon problem creation, a LoncapaProblemError should be raised.
         with pytest.raises(LoncapaProblemError):
-            with patch('capa.capa_problem.safe_exec') as mock_safe_exec:
+            with patch('xmodule.capa.capa_problem.safe_exec') as mock_safe_exec:
                 mock_safe_exec.side_effect = SafeExecException()
                 factory.create()
 
@@ -1265,7 +1265,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
         module = CapaFactory.create(attempts=1, done=True)
 
         # Simulate answering a problem that raises the exception
-        with patch('capa.capa_problem.LoncapaProblem.get_grade_from_current_answers') as mock_rescore:
+        with patch('xmodule.capa.capa_problem.LoncapaProblem.get_grade_from_current_answers') as mock_rescore:
             mock_rescore.side_effect = exception_class('test error \u03a9')
             with pytest.raises(exception_class):
                 module.rescore(only_if_higher=False)
@@ -1537,7 +1537,7 @@ class ProblemBlockTest(unittest.TestCase):  # lint-amnesty, pylint: disable=miss
         module.should_show_save_button = Mock(return_value=show_save_button)
 
         # Patch the capa problem's HTML rendering
-        with patch('capa.capa_problem.LoncapaProblem.get_html') as mock_html:
+        with patch('xmodule.capa.capa_problem.LoncapaProblem.get_html') as mock_html:
             mock_html.return_value = "<div>Test Problem HTML</div>"
 
             # Render the problem HTML
@@ -3195,11 +3195,11 @@ class ProblemBlockReportGenerationTest(unittest.TestCase):
 
     def setUp(self):  # lint-amnesty, pylint: disable=super-method-not-called
         self.find_question_label_patcher = patch(
-            'capa.capa_problem.LoncapaProblem.find_question_label',
+            'xmodule.capa.capa_problem.LoncapaProblem.find_question_label',
             lambda self, answer_id: answer_id
         )
         self.find_answer_text_patcher = patch(
-            'capa.capa_problem.LoncapaProblem.find_answer_text',
+            'xmodule.capa.capa_problem.LoncapaProblem.find_answer_text',
             lambda self, answer_id, current_answer: current_answer
         )
         self.find_question_label_patcher.start()