461 lines
17 KiB
Python
461 lines
17 KiB
Python
# pylint: disable=missing-docstring
|
|
|
|
# EVERY PROBLEM TYPE MUST HAVE THE FOLLOWING:
|
|
# -Section in Dictionary containing:
|
|
# -factory
|
|
# -kwargs
|
|
# -(optional metadata)
|
|
# -Correct, Incorrect and Unanswered CSS selectors
|
|
# -A way to answer the problem correctly and incorrectly
|
|
# -A way to check the problem was answered correctly, incorrectly and blank
|
|
|
|
from lettuce import world
|
|
import random
|
|
import textwrap
|
|
from common import section_location
|
|
from capa.tests.response_xml_factory import (
|
|
ChoiceResponseXMLFactory,
|
|
ChoiceTextResponseXMLFactory,
|
|
CodeResponseXMLFactory,
|
|
CustomResponseXMLFactory,
|
|
FormulaResponseXMLFactory,
|
|
ImageResponseXMLFactory,
|
|
MultipleChoiceResponseXMLFactory,
|
|
NumericalResponseXMLFactory,
|
|
OptionResponseXMLFactory,
|
|
StringResponseXMLFactory,
|
|
)
|
|
|
|
|
|
# Factories from capa.tests.response_xml_factory that we will use
|
|
# to generate the problem XML, with the keyword args used to configure
|
|
# the output.
|
|
# 'correct', 'incorrect', and 'unanswered' keys are lists of CSS selectors
|
|
# the presence of any in the list is sufficient
|
|
PROBLEM_DICT = {
|
|
'drop down': {
|
|
'factory': OptionResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The correct answer is Option 2',
|
|
'options': ['Option 1', 'Option 2', 'Option 3', 'Option 4'],
|
|
'correct_option': 'Option 2'},
|
|
'correct': ['span.correct'],
|
|
'incorrect': ['span.incorrect'],
|
|
'unanswered': ['span.unanswered']},
|
|
|
|
'multiple choice': {
|
|
'factory': MultipleChoiceResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The correct answer is Choice 3',
|
|
'choices': [False, False, True, False],
|
|
'choice_names': ['choice_0', 'choice_1', 'choice_2', 'choice_3']},
|
|
'correct': ['label.choicegroup_correct', 'span.correct'],
|
|
'incorrect': ['label.choicegroup_incorrect', 'span.incorrect'],
|
|
'unanswered': ['span.unanswered']},
|
|
|
|
'checkbox': {
|
|
'factory': ChoiceResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The correct answer is Choices 1 and 3',
|
|
'choice_type': 'checkbox',
|
|
'choices': [True, False, True, False, False],
|
|
'choice_names': ['Choice 1', 'Choice 2', 'Choice 3', 'Choice 4']},
|
|
'correct': ['span.correct'],
|
|
'incorrect': ['span.incorrect'],
|
|
'unanswered': ['span.unanswered']},
|
|
|
|
'radio': {
|
|
'factory': ChoiceResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The correct answer is Choice 3',
|
|
'choice_type': 'radio',
|
|
'choices': [False, False, True, False],
|
|
'choice_names': ['Choice 1', 'Choice 2', 'Choice 3', 'Choice 4']},
|
|
'correct': ['label.choicegroup_correct', 'span.correct'],
|
|
'incorrect': ['label.choicegroup_incorrect', 'span.incorrect'],
|
|
'unanswered': ['span.unanswered']},
|
|
|
|
'string': {
|
|
'factory': StringResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The answer is "correct string"',
|
|
'case_sensitive': False,
|
|
'answer': 'correct string'},
|
|
'correct': ['div.correct'],
|
|
'incorrect': ['div.incorrect'],
|
|
'unanswered': ['div.unanswered', 'div.unsubmitted']},
|
|
|
|
'numerical': {
|
|
'factory': NumericalResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The answer is pi + 1',
|
|
'answer': '4.14159',
|
|
'tolerance': '0.00001',
|
|
'math_display': True},
|
|
'correct': ['div.correct'],
|
|
'incorrect': ['div.incorrect'],
|
|
'unanswered': ['div.unanswered', 'div.unsubmitted']},
|
|
|
|
'formula': {
|
|
'factory': FormulaResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The solution is [mathjax]x^2+2x+y[/mathjax]',
|
|
'sample_dict': {'x': (-100, 100), 'y': (-100, 100)},
|
|
'num_samples': 10,
|
|
'tolerance': 0.00001,
|
|
'math_display': True,
|
|
'answer': 'x^2+2*x+y'},
|
|
'correct': ['div.correct'],
|
|
'incorrect': ['div.incorrect'],
|
|
'unanswered': ['div.unanswered', 'div.unsubmitted']},
|
|
|
|
'script': {
|
|
'factory': CustomResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'Enter two integers that sum to 10.',
|
|
'cfn': 'test_add_to_ten',
|
|
'expect': '10',
|
|
'num_inputs': 2,
|
|
'script': textwrap.dedent("""
|
|
def test_add_to_ten(expect,ans):
|
|
try:
|
|
a1=int(ans[0])
|
|
a2=int(ans[1])
|
|
except ValueError:
|
|
a1=0
|
|
a2=0
|
|
return (a1+a2)==int(expect)
|
|
""")},
|
|
'correct': ['div.correct'],
|
|
'incorrect': ['div.incorrect'],
|
|
'unanswered': ['div.unanswered', 'div.unsubmitted']},
|
|
|
|
'code': {
|
|
'factory': CodeResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'Submit code to an external grader',
|
|
'initial_display': 'print "Hello world!"',
|
|
'grader_payload': '{"grader": "ps1/Spring2013/test_grader.py"}', },
|
|
'correct': ['span.correct'],
|
|
'incorrect': ['span.incorrect'],
|
|
'unanswered': ['span.unanswered']},
|
|
|
|
'radio_text': {
|
|
'factory': ChoiceTextResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The correct answer is Choice 0 and input 8',
|
|
'type': 'radiotextgroup',
|
|
'choices': [("true", {"answer": "8", "tolerance": "1"}),
|
|
("false", {"answer": "8", "tolerance": "1"})
|
|
]
|
|
},
|
|
'correct': ['section.choicetextgroup_correct'],
|
|
'incorrect': ['section.choicetextgroup_incorrect', 'span.incorrect'],
|
|
'unanswered': ['span.unanswered']},
|
|
|
|
'checkbox_text': {
|
|
'factory': ChoiceTextResponseXMLFactory(),
|
|
'kwargs': {
|
|
'question_text': 'The correct answer is Choice 0 and input 8',
|
|
'type': 'checkboxtextgroup',
|
|
'choices': [("true", {"answer": "8", "tolerance": "1"}),
|
|
("false", {"answer": "8", "tolerance": "1"})
|
|
]
|
|
},
|
|
'correct': ['span.correct'],
|
|
'incorrect': ['span.incorrect'],
|
|
'unanswered': ['span.unanswered']},
|
|
|
|
'image': {
|
|
'factory': ImageResponseXMLFactory(),
|
|
'kwargs': {
|
|
'src': '/static/images/placeholder-image.png',
|
|
'rectangle': '(50,50)-(100,100)'
|
|
},
|
|
'correct': ['span.correct'],
|
|
'incorrect': ['span.incorrect'],
|
|
'unanswered': ['span.unanswered']}
|
|
}
|
|
|
|
|
|
def answer_problem(course, problem_type, correctness):
|
|
# Make sure that the problem has been completely rendered before
|
|
# starting to input an answer.
|
|
world.wait_for_ajax_complete()
|
|
|
|
section_loc = section_location(course)
|
|
|
|
if problem_type == "drop down":
|
|
select_name = "input_{}_2_1".format(
|
|
section_loc.course_key.make_usage_key('problem', 'drop_down').html_id()
|
|
)
|
|
option_text = 'Option 2' if correctness == 'correct' else 'Option 3'
|
|
world.select_option(select_name, option_text)
|
|
|
|
elif problem_type == "multiple choice":
|
|
if correctness == 'correct':
|
|
world.css_check(inputfield(course, 'multiple choice', choice='choice_2'))
|
|
else:
|
|
world.css_check(inputfield(course, 'multiple choice', choice='choice_1'))
|
|
|
|
elif problem_type == "checkbox":
|
|
if correctness == 'correct':
|
|
world.css_check(inputfield(course, 'checkbox', choice='choice_0'))
|
|
world.css_check(inputfield(course, 'checkbox', choice='choice_2'))
|
|
else:
|
|
world.css_check(inputfield(course, 'checkbox', choice='choice_3'))
|
|
|
|
elif problem_type == 'radio':
|
|
if correctness == 'correct':
|
|
world.css_check(inputfield(course, 'radio', choice='choice_2'))
|
|
else:
|
|
world.css_check(inputfield(course, 'radio', choice='choice_1'))
|
|
|
|
elif problem_type == 'string':
|
|
textvalue = 'correct string' if correctness == 'correct' else 'incorrect'
|
|
world.css_fill(inputfield(course, 'string'), textvalue)
|
|
|
|
elif problem_type == 'numerical':
|
|
textvalue = "pi + 1" if correctness == 'correct' else str(random.randint(-2, 2))
|
|
world.css_fill(inputfield(course, 'numerical'), textvalue)
|
|
|
|
elif problem_type == 'formula':
|
|
textvalue = "x^2+2*x+y" if correctness == 'correct' else 'x^2'
|
|
world.css_fill(inputfield(course, 'formula'), textvalue)
|
|
|
|
elif problem_type == 'script':
|
|
# Correct answer is any two integers that sum to 10
|
|
first_addend = random.randint(-100, 100)
|
|
second_addend = 10 - first_addend
|
|
|
|
# If we want an incorrect answer, then change
|
|
# the second addend so they no longer sum to 10
|
|
if correctness == 'incorrect':
|
|
second_addend += random.randint(1, 10)
|
|
|
|
world.css_fill(inputfield(course, 'script', input_num=1), str(first_addend))
|
|
world.css_fill(inputfield(course, 'script', input_num=2), str(second_addend))
|
|
|
|
elif problem_type == 'code':
|
|
# The fake xqueue server is configured to respond
|
|
# correct / incorrect no matter what we submit.
|
|
# Furthermore, since the inline code response uses
|
|
# JavaScript to make the code display nicely, it's difficult
|
|
# to programatically input text
|
|
# (there's not <textarea> we can just fill text into)
|
|
# For this reason, we submit the initial code in the response
|
|
# (configured in the problem XML above)
|
|
pass
|
|
|
|
elif problem_type == 'radio_text' or problem_type == 'checkbox_text':
|
|
|
|
input_value = "8" if correctness == 'correct' else "5"
|
|
choice = "choiceinput_0bc" if correctness == 'correct' else "choiceinput_1bc"
|
|
world.css_fill(
|
|
inputfield(
|
|
course,
|
|
problem_type,
|
|
choice="choiceinput_0_numtolerance_input_0"
|
|
),
|
|
input_value
|
|
)
|
|
world.css_check(inputfield(course, problem_type, choice=choice))
|
|
elif problem_type == 'image':
|
|
offset = 25 if correctness == "correct" else -25
|
|
|
|
def try_click():
|
|
problem_html_loc = section_loc.course_key.make_usage_key('problem', 'image').html_id()
|
|
image_selector = "#imageinput_{}_2_1".format(problem_html_loc)
|
|
input_selector = "#input_{}_2_1".format(problem_html_loc)
|
|
|
|
world.browser.execute_script('$("body").on("click", function(event) {console.log(event);})')
|
|
|
|
initial_input = world.css_value(input_selector)
|
|
world.wait_for_visible(image_selector)
|
|
image = world.css_find(image_selector).first
|
|
(image.action_chains
|
|
.move_to_element(image._element)
|
|
.move_by_offset(offset, offset)
|
|
.click()
|
|
.perform())
|
|
|
|
world.wait_for(lambda _: world.css_value(input_selector) != initial_input)
|
|
|
|
world.retry_on_exception(try_click)
|
|
|
|
|
|
def problem_has_answer(course, problem_type, answer_class):
|
|
if problem_type == "drop down":
|
|
if answer_class == 'blank':
|
|
assert world.is_css_not_present('option[selected="true"]')
|
|
else:
|
|
actual = world.css_value('option[selected="true"]')
|
|
expected = 'Option 2' if answer_class == 'correct' else 'Option 3'
|
|
assert actual == expected
|
|
|
|
elif problem_type == "multiple choice":
|
|
if answer_class == 'correct':
|
|
assert_checked(course, 'multiple choice', ['choice_2'])
|
|
elif answer_class == 'incorrect':
|
|
assert_checked(course, 'multiple choice', ['choice_1'])
|
|
else:
|
|
assert_checked(course, 'multiple choice', [])
|
|
|
|
elif problem_type == "checkbox":
|
|
if answer_class == 'correct':
|
|
assert_checked(course, 'checkbox', ['choice_0', 'choice_2'])
|
|
elif answer_class == 'incorrect':
|
|
assert_checked(course, 'checkbox', ['choice_3'])
|
|
else:
|
|
assert_checked(course, 'checkbox', [])
|
|
|
|
elif problem_type == "radio":
|
|
if answer_class == 'correct':
|
|
assert_checked(course, 'radio', ['choice_2'])
|
|
elif answer_class == 'incorrect':
|
|
assert_checked(course, 'radio', ['choice_1'])
|
|
else:
|
|
assert_checked(course, 'radio', [])
|
|
|
|
elif problem_type == 'string':
|
|
if answer_class == 'blank':
|
|
expected = ''
|
|
else:
|
|
expected = 'correct string' if answer_class == 'correct' else 'incorrect'
|
|
assert_textfield(course, 'string', expected)
|
|
|
|
elif problem_type == 'formula':
|
|
if answer_class == 'blank':
|
|
expected = ''
|
|
else:
|
|
expected = "x^2+2*x+y" if answer_class == 'correct' else 'x^2'
|
|
assert_textfield(course, 'formula', expected)
|
|
|
|
elif problem_type in ("radio_text", "checkbox_text"):
|
|
if answer_class == 'blank':
|
|
expected = ('', '')
|
|
assert_choicetext_values(course, problem_type, (), expected)
|
|
elif answer_class == 'incorrect':
|
|
expected = ('5', '')
|
|
assert_choicetext_values(course, problem_type, ["choiceinput_1bc"], expected)
|
|
else:
|
|
expected = ('8', '')
|
|
assert_choicetext_values(course, problem_type, ["choiceinput_0bc"], expected)
|
|
|
|
else:
|
|
# The other response types use random data,
|
|
# which would be difficult to check
|
|
# We trade input value coverage in the other tests for
|
|
# input type coverage in this test.
|
|
pass
|
|
|
|
|
|
def add_problem_to_course(course, problem_type, extra_meta=None):
|
|
'''
|
|
Add a problem to the course we have created using factories.
|
|
'''
|
|
|
|
assert problem_type in PROBLEM_DICT
|
|
|
|
# Generate the problem XML using capa.tests.response_xml_factory
|
|
factory_dict = PROBLEM_DICT[problem_type]
|
|
problem_xml = factory_dict['factory'].build_xml(**factory_dict['kwargs'])
|
|
metadata = {'rerandomize': 'always'} if 'metadata' not in factory_dict else factory_dict['metadata']
|
|
if extra_meta:
|
|
metadata = dict(metadata, **extra_meta)
|
|
|
|
# Create a problem item using our generated XML
|
|
# We set rerandomize=always in the metadata so that the "Reset" button
|
|
# will appear.
|
|
category_name = "problem"
|
|
return world.ItemFactory.create(
|
|
parent_location=section_location(course),
|
|
category=category_name,
|
|
display_name=str(problem_type),
|
|
data=problem_xml,
|
|
metadata=metadata
|
|
)
|
|
|
|
|
|
def inputfield(course, problem_type, choice=None, input_num=1):
|
|
""" Return the css selector for `problem_type`.
|
|
For example, if problem_type is 'string', return
|
|
the text field for the string problem in the test course.
|
|
|
|
`choice` is the name of the checkbox input in a group
|
|
of checkboxes. """
|
|
|
|
section_loc = section_location(course)
|
|
|
|
ptype = problem_type.replace(" ", "_")
|
|
# this is necessary due to naming requirement for this problem type
|
|
if problem_type in ("radio_text", "checkbox_text"):
|
|
selector_template = "input#{}_2_{input}"
|
|
else:
|
|
selector_template = "input#input_{}_2_{input}"
|
|
|
|
sel = selector_template.format(
|
|
section_loc.course_key.make_usage_key('problem', ptype).html_id(),
|
|
input=input_num,
|
|
)
|
|
|
|
if choice is not None:
|
|
base = "_choice_" if problem_type == "multiple choice" else "_"
|
|
sel = sel + base + str(choice)
|
|
|
|
# If the input element doesn't exist, fail immediately
|
|
assert world.is_css_present(sel)
|
|
|
|
# Retrieve the input element
|
|
return sel
|
|
|
|
|
|
def assert_checked(course, problem_type, choices):
|
|
'''
|
|
Assert that choice names given in *choices* are the only
|
|
ones checked.
|
|
|
|
Works for both radio and checkbox problems
|
|
'''
|
|
|
|
all_choices = ['choice_0', 'choice_1', 'choice_2', 'choice_3']
|
|
for this_choice in all_choices:
|
|
def check_problem():
|
|
element = world.css_find(inputfield(course, problem_type, choice=this_choice))
|
|
if this_choice in choices:
|
|
assert element.checked
|
|
else:
|
|
assert not element.checked
|
|
world.retry_on_exception(check_problem)
|
|
|
|
|
|
def assert_textfield(course, problem_type, expected_text, input_num=1):
|
|
element_value = world.css_value(inputfield(course, problem_type, input_num=input_num))
|
|
assert element_value == expected_text
|
|
|
|
|
|
def assert_choicetext_values(course, problem_type, choices, expected_values):
|
|
"""
|
|
Asserts that only the given choices are checked, and given
|
|
text fields have a desired value
|
|
"""
|
|
# Names of the radio buttons or checkboxes
|
|
all_choices = ['choiceinput_0bc', 'choiceinput_1bc']
|
|
# Names of the numtolerance_inputs
|
|
all_inputs = [
|
|
"choiceinput_0_numtolerance_input_0",
|
|
"choiceinput_1_numtolerance_input_0"
|
|
]
|
|
for this_choice in all_choices:
|
|
element = world.css_find(inputfield(course, problem_type, choice=this_choice))
|
|
|
|
if this_choice in choices:
|
|
assert element.checked
|
|
else:
|
|
assert not element.checked
|
|
|
|
for (name, expected) in zip(all_inputs, expected_values):
|
|
element = world.css_find(inputfield(course, problem_type, name))
|
|
# Remove any trailing spaces that may have been added
|
|
assert element.value.strip() == expected
|