diff --git a/common/test/acceptance/pages/lms/problem.py b/common/test/acceptance/pages/lms/problem.py index 068cf35a1c..5018bff3c7 100644 --- a/common/test/acceptance/pages/lms/problem.py +++ b/common/test/acceptance/pages/lms/problem.py @@ -71,11 +71,20 @@ class ProblemPage(PageObject): description="MathJax rendered in hint" ) - def fill_answer(self, text): + def fill_answer(self, text, input_num=None): """ Fill in the answer to the problem. + + args: + text: String to fill the input with. + + kwargs: + input_num: If provided, fills only the input_numth field. Else, all + input fields will be filled. """ - self.q(css='div.problem div.capa_inputtype.textline input').fill(text) + fields = self.q(css='div.problem div.capa_inputtype.textline input') + fields = fields.nth(input_num) if input_num is not None else fields + fields.fill(text) def fill_answer_numerical(self, text): """ diff --git a/common/test/acceptance/tests/lms/test_lms_problems.py b/common/test/acceptance/tests/lms/test_lms_problems.py index 8a353ea735..0e49fff610 100644 --- a/common/test/acceptance/tests/lms/test_lms_problems.py +++ b/common/test/acceptance/tests/lms/test_lms_problems.py @@ -19,13 +19,14 @@ class ProblemsTest(UniqueCourseTest): """ Base class for tests of problems in the LMS. """ - USERNAME = "joe_student" - EMAIL = "joe@example.com" - PASSWORD = "keep it secret; keep it safe." def setUp(self): super(ProblemsTest, self).setUp() + self.username = "test_student_{uuid}".format(uuid=self.unique_id[0:8]) + self.email = "{username}@example.com".format(username=self.username) + self.password = "keep it secret; keep it safe." + self.xqueue_grade_response = None self.courseware_page = CoursewarePage(self.browser, self.course_id) @@ -46,9 +47,9 @@ class ProblemsTest(UniqueCourseTest): # Auto-auth register for the course. AutoAuthPage( self.browser, - username=self.USERNAME, - email=self.EMAIL, - password=self.PASSWORD, + username=self.username, + email=self.email, + password=self.password, course_id=self.course_id, staff=False ).visit() @@ -382,7 +383,7 @@ class LogoutDuringAnswering(ProblemsTest): login_page = CombinedLoginAndRegisterPage(self.browser) login_page.wait_for_page() - login_page.login(self.EMAIL, self.PASSWORD) + login_page.login(self.email, self.password) problem_page.wait_for_page() self.assertEqual(problem_page.problem_name, 'TEST PROBLEM') diff --git a/common/test/acceptance/tests/lms/test_problem_types.py b/common/test/acceptance/tests/lms/test_problem_types.py new file mode 100644 index 0000000000..23d39f0fbb --- /dev/null +++ b/common/test/acceptance/tests/lms/test_problem_types.py @@ -0,0 +1,766 @@ +""" +Bok choy acceptance and a11y tests for problem types in the LMS + +See also lettuce tests in lms/djangoapps/courseware/features/problems.feature +""" +import random +import textwrap + +from abc import ABCMeta, abstractmethod +from nose.plugins.attrib import attr +from selenium.webdriver import ActionChains + +from capa.tests.response_xml_factory import ( + ChoiceResponseXMLFactory, + ChoiceTextResponseXMLFactory, + CodeResponseXMLFactory, + CustomResponseXMLFactory, + FormulaResponseXMLFactory, + ImageResponseXMLFactory, + MultipleChoiceResponseXMLFactory, + NumericalResponseXMLFactory, + OptionResponseXMLFactory, + StringResponseXMLFactory, +) + +from common.test.acceptance.fixtures.course import XBlockFixtureDesc +from common.test.acceptance.pages.lms.problem import ProblemPage +from common.test.acceptance.tests.helpers import select_option_by_text +from common.test.acceptance.tests.lms.test_lms_problems import ProblemsTest +from common.test.acceptance.tests.helpers import EventsTestMixin + + +class ProblemTypeTestBaseMeta(ABCMeta): + """ + MetaClass for ProblemTypeTestBase to ensure that the required attributes + are defined in the inheriting classes. + """ + def __call__(cls, *args, **kwargs): + obj = type.__call__(cls, *args, **kwargs) + + required_attrs = [ + 'problem_name', + 'problem_type', + 'factory', + 'factory_kwargs', + 'status_indicators', + ] + + for required_attr in required_attrs: + msg = ('{} is a required attribute for {}').format( + required_attr, str(cls) + ) + + try: + if obj.__getattribute__(required_attr) is None: + raise NotImplementedError(msg) + except AttributeError: + raise NotImplementedError(msg) + + return obj + + +class ProblemTypeTestBase(ProblemsTest, EventsTestMixin): + """ + Base class for testing assesment problem types in bok choy. + + This inherits from ProblemsTest, which has capabilities for testing problem + features that are not problem type specific (checking, hinting, etc.). + + The following attributes must be explicitly defined when inheriting from + this class: + problem_name (str) + problem_type (str) + factory (ResponseXMLFactory subclass instance) + + Additionally, the default values for factory_kwargs and status_indicators + may need to be overridden for some problem types. + """ + __metaclass__ = ProblemTypeTestBaseMeta + + problem_name = None + problem_type = None + factory = None + factory_kwargs = {} + status_indicators = { + 'correct': ['span.correct'], + 'incorrect': ['span.incorrect'], + 'unanswered': ['span.unanswered'], + } + + def setUp(self): + """ + Visits courseware_page and defines self.problem_page. + """ + super(ProblemTypeTestBase, self).setUp() + self.courseware_page.visit() + self.problem_page = ProblemPage(self.browser) + + def get_problem(self): + """ + Creates a {problem_type} problem + """ + # Generate the problem XML using capa.tests.response_xml_factory + return XBlockFixtureDesc( + 'problem', + self.problem_name, + data=self.factory.build_xml(**self.factory_kwargs), + metadata={'rerandomize': 'always'} + ) + + def wait_for_status(self, status): + """ + Waits for the expected status indicator. + + Args: + status: one of ("correct", "incorrect", "unanswered) + """ + msg = "Wait for status to be {}".format(status) + selector = ', '.join(self.status_indicators[status]) + self.problem_page.wait_for_element_visibility(selector, msg) + + @abstractmethod + def answer_problem(self, correct): + """ + Args: + `correct` (bool): Inputs correct answer if True, else inputs + incorrect answer. + """ + raise NotImplementedError() + + +class ProblemTypeTestMixin(object): + """ + Test cases shared amongst problem types. + """ + @attr('shard_2') + def test_answer_correctly(self): + """ + Scenario: I can answer a problem correctly + Given External graders respond "correct" + And I am viewing a "" problem + When I answer a "" problem "correctly" + Then my "" answer is marked "correct" + And The "" problem displays a "correct" answer + And a "problem_check" server event is emitted + And a "problem_check" browser event is emitted + """ + # Make sure we're looking at the right problem + self.assertEqual(self.problem_page.problem_name, self.problem_name) + + # Answer the problem correctly + self.answer_problem(correct=True) + self.problem_page.click_check() + self.wait_for_status('correct') + + # Check for corresponding tracking event + expected_events = [ + { + 'event_source': 'server', + 'event_type': 'problem_check', + 'username': self.username, + }, { + 'event_source': 'browser', + 'event_type': 'problem_check', + 'username': self.username, + }, + ] + + for event in expected_events: + self.wait_for_events(event_filter=event, number_of_matches=1) + + @attr('shard_2') + def test_answer_incorrectly(self): + """ + Scenario: I can answer a problem incorrectly + Given External graders respond "incorrect" + And I am viewing a "" problem + When I answer a "" problem "incorrectly" + Then my "" answer is marked "incorrect" + And The "" problem displays a "incorrect" answer + """ + self.problem_page.wait_for( + lambda: self.problem_page.problem_name == self.problem_name, + "Make sure the correct problem is on the page" + ) + + # Answer the problem incorrectly + self.answer_problem(correct=False) + self.problem_page.click_check() + self.wait_for_status('incorrect') + + @attr('shard_2') + def test_submit_blank_answer(self): + """ + Scenario: I can submit a blank answer + Given I am viewing a "" problem + When I check a problem + Then my "" answer is marked "incorrect" + And The "" problem displays a "blank" answer + """ + self.problem_page.wait_for( + lambda: self.problem_page.problem_name == self.problem_name, + "Make sure the correct problem is on the page" + ) + + # Leave the problem unchanged and click check. + self.problem_page.click_check() + self.wait_for_status('incorrect') + + @attr('a11y') + def test_problem_type_a11y(self): + """ + Run accessibility audit for the problem type. + """ + self.problem_page.wait_for( + lambda: self.problem_page.problem_name == self.problem_name, + "Make sure the correct problem is on the page" + ) + + # Set the scope to the problem container + self.problem_page.a11y_audit.config.set_scope( + include=['div#seq_content']) + + # Run the accessibility audit. + self.problem_page.a11y_audit.check_for_accessibility_errors() + + +class CheckboxProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Checkbox Problem Type + """ + problem_name = 'CHECKBOX TEST PROBLEM' + problem_type = 'checkbox' + + factory = ChoiceResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'The correct answer is Choice 0 and Choice 2', + 'choice_type': 'checkbox', + 'choices': [True, False, True, False], + 'choice_names': ['Choice 0', 'Choice 1', 'Choice 2', 'Choice 3'] + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for CheckboxProblemTypeTest + """ + super(CheckboxProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'aria-allowed-attr', # TODO: AC-251 + 'aria-valid-attr', # TODO: AC-251 + 'aria-roles', # TODO: AC-251 + 'checkboxgroup', # TODO: AC-251 + ] + }) + + def answer_problem(self, correct): + """ + Answer checkbox problem. + """ + if correct: + self.problem_page.click_choice("choice_0") + self.problem_page.click_choice("choice_2") + else: + self.problem_page.click_choice("choice_1") + + +class MultipleChoiceProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Multiple Choice Problem Type + """ + problem_name = 'MULTIPLE CHOICE TEST PROBLEM' + problem_type = 'multiple choice' + + factory = MultipleChoiceResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'The correct answer is Choice 2', + 'choices': [False, False, True, False], + 'choice_names': ['choice_0', 'choice_1', 'choice_2', 'choice_3'], + } + status_indicators = { + 'correct': ['label.choicegroup_correct'], + 'incorrect': ['label.choicegroup_incorrect', 'span.incorrect'], + 'unanswered': ['span.unanswered'], + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for MultipleChoiceProblemTypeTest + """ + super(MultipleChoiceProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'aria-valid-attr', # TODO: AC-251 + 'radiogroup', # TODO: AC-251 + ] + }) + + def answer_problem(self, correct): + """ + Answer multiple choice problem. + """ + if correct: + self.problem_page.click_choice("choice_choice_2") + else: + self.problem_page.click_choice("choice_choice_1") + + +class RadioProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Radio Problem Type + """ + problem_name = 'RADIO TEST PROBLEM' + problem_type = 'radio' + + factory = ChoiceResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'The correct answer is Choice 2', + 'choice_type': 'radio', + 'choices': [False, False, True, False], + 'choice_names': ['Choice 0', 'Choice 1', 'Choice 2', 'Choice 3'], + } + status_indicators = { + 'correct': ['label.choicegroup_correct'], + 'incorrect': ['label.choicegroup_incorrect', 'span.incorrect'], + 'unanswered': ['span.unanswered'], + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for RadioProblemTypeTest + """ + super(RadioProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'aria-valid-attr', # TODO: AC-292 + 'radiogroup', # TODO: AC-292 + ] + }) + + def answer_problem(self, correct): + """ + Answer radio problem. + """ + if correct: + self.problem_page.click_choice("choice_2") + else: + self.problem_page.click_choice("choice_1") + + +class DropDownProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Drop Down Problem Type + """ + problem_name = 'DROP DOWN TEST PROBLEM' + problem_type = 'drop down' + + factory = OptionResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'The correct answer is Option 2', + 'options': ['Option 1', 'Option 2', 'Option 3', 'Option 4'], + 'correct_option': 'Option 2' + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for DropDownProblemTypeTest + """ + super(DropDownProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'label', # TODO: AC-291 + ] + }) + + def answer_problem(self, correct): + """ + Answer drop down problem. + """ + answer = 'Option 2' if correct else 'Option 3' + selector_element = self.problem_page.q( + css='.problem .option-input select') + select_option_by_text(selector_element, answer) + + +class StringProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for String Problem Type + """ + problem_name = 'STRING TEST PROBLEM' + problem_type = 'string' + + factory = StringResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'The answer is "correct string"', + 'case_sensitive': False, + 'answer': 'correct string', + } + + status_indicators = { + 'correct': ['div.correct'], + 'incorrect': ['div.incorrect'], + 'unanswered': ['div.unanswered', 'div.unsubmitted'], + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for StringProblemTypeTest + """ + super(StringProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'label', # TODO: AC-290 + ] + }) + + def answer_problem(self, correct): + """ + Answer string problem. + """ + textvalue = 'correct string' if correct else 'incorrect string' + self.problem_page.fill_answer(textvalue) + + +class NumericalProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Numerical Problem Type + """ + problem_name = 'NUMERICAL TEST PROBLEM' + problem_type = 'numerical' + + factory = NumericalResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'The answer is pi + 1', + 'answer': '4.14159', + 'tolerance': '0.00001', + 'math_display': True, + } + + status_indicators = { + 'correct': ['div.correct'], + 'incorrect': ['div.incorrect'], + 'unanswered': ['div.unanswered', 'div.unsubmitted'], + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for NumericalProblemTypeTest + """ + super(NumericalProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'label', # TODO: AC-289 + ] + }) + + def answer_problem(self, correct): + """ + Answer numerical problem. + """ + textvalue = "pi + 1" if correct else str(random.randint(-2, 2)) + self.problem_page.fill_answer(textvalue) + + +class FormulaProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Formula Problem Type + """ + problem_name = 'FORMULA TEST PROBLEM' + problem_type = 'formula' + + factory = FormulaResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'The solution is [mathjax]x^2+2x+y[/mathjax]', + 'sample_dict': {'x': (-100, 100), 'y': (-100, 100)}, + 'num_samples': 10, + 'tolerance': 0.00001, + 'math_display': True, + 'answer': 'x^2+2*x+y', + } + + status_indicators = { + 'correct': ['div.correct'], + 'incorrect': ['div.incorrect'], + 'unanswered': ['div.unanswered', 'div.unsubmitted'], + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for FormulaProblemTypeTest + """ + super(FormulaProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'label', # TODO: AC-288 + ] + }) + + def answer_problem(self, correct): + """ + Answer formula problem. + """ + textvalue = "x^2+2*x+y" if correct else 'x^2' + self.problem_page.fill_answer(textvalue) + + +class ScriptProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Script Problem Type + """ + problem_name = 'SCRIPT TEST PROBLEM' + problem_type = 'script' + + factory = CustomResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'Enter two integers that sum to 10.', + 'cfn': 'test_add_to_ten', + 'expect': '10', + 'num_inputs': 2, + 'script': textwrap.dedent(""" + def test_add_to_ten(expect,ans): + try: + a1=int(ans[0]) + a2=int(ans[1]) + except ValueError: + a1=0 + a2=0 + return (a1+a2)==int(expect) + """), + } + status_indicators = { + 'correct': ['div.correct'], + 'incorrect': ['div.incorrect'], + 'unanswered': ['div.unanswered', 'div.unsubmitted'], + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for ScriptProblemTypeTest + """ + super(ScriptProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'label', # TODO: AC-287 + ] + }) + + def answer_problem(self, correct): + """ + Answer script problem. + """ + # Correct answer is any two integers that sum to 10 + first_addend = random.randint(-100, 100) + second_addend = 10 - first_addend + + # If we want an incorrect answer, then change + # the second addend so they no longer sum to 10 + if not correct: + second_addend += random.randint(1, 10) + + self.problem_page.fill_answer(first_addend, input_num=0) + self.problem_page.fill_answer(second_addend, input_num=1) + + +class CodeProblemTypeTest(ProblemTypeTestBase, ProblemTypeTestMixin): + """ + TestCase Class for Code Problem Type + """ + problem_name = 'CODE TEST PROBLEM' + problem_type = 'code' + + factory = CodeResponseXMLFactory() + + factory_kwargs = { + 'question_text': 'Submit code to an external grader', + 'initial_display': 'print "Hello world!"', + 'grader_payload': '{"grader": "ps1/Spring2013/test_grader.py"}', + } + + status_indicators = { + 'correct': ['.grader-status .correct ~ .debug'], + 'incorrect': ['.grader-status .incorrect ~ .debug'], + 'unanswered': ['.grader-status .unanswered ~ .debug'], + } + + def setUp(self, *args, **kwargs): + """ + Additional setup for CodeProblemTypeTest + """ + super(CodeProblemTypeTest, self).setUp(*args, **kwargs) + self.problem_page.a11y_audit.config.set_rules({ + 'ignore': [ + 'color-contrast', # TODO: AC-286 + 'label', # TODO: AC-286 + ] + }) + + def answer_problem(self, correct): + """ + Answer code problem. + """ + # The fake xqueue server is configured to respond + # correct / incorrect no matter what we submit. + # Furthermore, since the inline code response uses + # JavaScript to make the code display nicely, it's difficult + # to programatically input text + # (there's not