Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added stub XQueue server to bok_choy test suite.

Browse Source 
Added feedback check for AI-assessment test

Added peer assessment feedback test
This commit is contained in:
Will Daly
2014-01-16 11:02:36 -05:00
parent a6b97b402f
commit b73c6f63fc
22 changed files with 706 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
common/test/acceptance/edxapp_pages/lms/open_response.py
View File

@@ -224,15 +224,12 @@ class OpenResponsePage(PageObject):
if assessment_type == 'self':
return EmptyPromise(lambda: self.has_rubric, "Rubric has appeared")
elif assessment_type == 'ai':
elif assessment_type == 'ai' or assessment_type == "peer":
return EmptyPromise(
lambda: self.grader_status != 'Unanswered',
"Problem status is no longer 'unanswered'"
)
elif assessment_type == 'peer':
return EmptyPromise(lambda: False, "Peer assessment not yet implemented")
else:
self.warning("Unrecognized assessment type '{0}'".format(assessment_type))
return EmptyPromise(lambda: True, "Unrecognized assessment type")

2
common/test/acceptance/edxapp_pages/lms/progress.py
View File

@@ -27,7 +27,7 @@ class ProgressPage(PageObject):
for the section.
Example:
section_scores('Week 1', 'Lesson 1', 2) --> [(2, 4), (0, 1)]
scores('Week 1', 'Lesson 1') --> [(2, 4), (0, 1)]
Returns `None` if no such chapter and section can be found.
"""

3
common/test/acceptance/fixtures/__init__.py
View File

@@ -2,3 +2,6 @@ import os
# Get the URL of the instance under test
STUDIO_BASE_URL = os.environ.get('studio_url', 'http://localhost:8031')
# Get the URL of the XQueue stub used in the test
XQUEUE_STUB_URL = os.environ.get('xqueue_url', 'http://localhost:8040')

42
common/test/acceptance/fixtures/xqueue.py Normal file
View File

@@ -0,0 +1,42 @@
"""
Fixture to configure XQueue response.
"""
import requests
import json
from bok_choy.web_app_fixture import WebAppFixture, WebAppFixtureError
from . import XQUEUE_STUB_URL
class XQueueResponseFixture(WebAppFixture):
"""
Configure the XQueue stub's response to submissions.
"""
def __init__(self, queue_name, response_dict):
"""
Configure XQueue stub to POST `response_dict` (a dictionary)
back to the LMS when it receives a submission to a queue
named `queue_name`.
Remember that there is one XQueue stub shared by all the tests;
if possible, you should have tests use unique queue names
to avoid conflict between tests running in parallel.
"""
self._queue_name = queue_name
self._response_dict = response_dict
def install(self):
"""
Configure the stub via HTTP.
"""
url = XQUEUE_STUB_URL + "/set_config"
# Configure the stub to respond to submissions to our queue
payload = {self._queue_name: json.dumps(self._response_dict)}
response = requests.put(url, data=payload)
if not response.ok:
raise WebFixtureError(
"Could not configure XQueue stub for queue '{1}'. Status code: {2}".format(
self._queue_name, self._response_dict))

30
common/test/acceptance/tests/data/ora_peer_problem.xml Normal file
View File

@@ -0,0 +1,30 @@
<combinedopenended max_score="1" accept_file_upload="False" markdown="null" max_attempts="10000" skip_spelling_checks="False" version="1">
<rubric>
<rubric>
<category>
<description>Writing Applications</description>
<option> The essay loses focus, has little information or supporting details, and the organization makes it difficult to follow.</option>
<option> The essay presents a mostly unified theme, includes sufficient information to convey the theme, and is generally organized well.</option>
</category>
<category>
<description> Language Conventions </description>
<option> The essay demonstrates a reasonable command of proper spelling and grammar. </option>
<option> The essay demonstrates superior command of proper spelling and grammar.</option>
</category>
</rubric>
</rubric>
<prompt>
<h4>Censorship in the Libraries</h4>
<p>"All of us can think of a book that we hope none of our children or any other children have taken off the shelf. But if I have the right to remove that book from the shelf -- that work I abhor -- then you also have exactly the same right and so does everyone else. And then we have no books left on the shelf for any of us." --Katherine Paterson, Author</p>
<p>Write a persuasive essay to a newspaper reflecting your views on censorship in libraries. Do you believe that certain materials, such as books, music, movies, magazines, etc., should be removed from the shelves if they are found offensive? Support your position with convincing arguments from your own experience, observations, and/or reading.</p>
</prompt>
<task>
<openended>
<openendedparam>
<initial_display>Enter essay here.</initial_display>
<answer_display>This is the answer.</answer_display>
<grader_payload>{"grader_settings" : "peer_grading.conf", "problem_id" : "700x/Demo"}</grader_payload>
</openendedparam>
</openended>
</task>
</combinedopenended>

1
common/test/acceptance/tests/data/ora_rubric.xml Normal file
View File

@@ -0,0 +1 @@
<rubric><category><description>Writing Applications</description><score>0</score><option points='0'> The essay loses focus, has little information or supporting details, and the organization makes it difficult to follow.</option><option points='1'> The essay presents a mostly unified theme, includes sufficient information to convey the theme, and is generally organized well.</option></category><category><description> Language Conventions </description><score>1</score><option points='0'> The essay demonstrates a reasonable command of proper spelling and grammar. </option><option points='1'> The essay demonstrates superior command of proper spelling and grammar.</option></category></rubric>

32
common/test/acceptance/tests/helpers.py
View File

@@ -1,7 +1,8 @@
"""
Test helper functions.
Test helper functions and base classes.
"""
from path import path
from bok_choy.web_app_test import WebAppTest
def load_data_str(rel_path):
@@ -12,3 +13,32 @@ def load_data_str(rel_path):
full_path = path(__file__).abspath().dirname() / "data" / rel_path #pylint: disable=E1120
with open(full_path) as data_file:
return data_file.read()
class UniqueCourseTest(WebAppTest):
"""
Test that provides a unique course ID.
"""
COURSE_ID_SEPARATOR = "/"
def __init__(self, *args, **kwargs):
"""
Create a unique course ID.
"""
self.course_info = {
'org': 'test_org',
'number': self.unique_id,
'run': 'test_run',
'display_name': 'Test Course' + self.unique_id
}
super(UniqueCourseTest, self).__init__(*args, **kwargs)
@property
def course_id(self):
return self.COURSE_ID_SEPARATOR.join([
self.course_info['org'],
self.course_info['number'],
self.course_info['run']
])

266
common/test/acceptance/tests/test_ora.py
View File

@@ -2,79 +2,167 @@
Tests for ORA (Open Response Assessment) through the LMS UI.
"""
from bok_choy.web_app_test import WebAppTest
import json
from bok_choy.promise import fulfill, Promise
from ..edxapp_pages.studio.auto_auth import AutoAuthPage
from ..edxapp_pages.lms.course_info import CourseInfoPage
from ..edxapp_pages.lms.tab_nav import TabNavPage
from ..edxapp_pages.lms.course_nav import CourseNavPage
from ..edxapp_pages.lms.open_response import OpenResponsePage
from ..edxapp_pages.lms.progress import ProgressPage
from ..fixtures.course import XBlockFixtureDesc, CourseFixture
from ..fixtures.xqueue import XQueueResponseFixture
from .helpers import load_data_str
from .helpers import load_data_str, UniqueCourseTest
class OpenResponseTest(WebAppTest):
class OpenResponseTest(UniqueCourseTest):
"""
Tests that interact with ORA (Open Response Assessment) through the LMS UI.
This base class sets up a course with open response problems and defines
some helper functions used in the ORA tests.
"""
page_object_classes = [
AutoAuthPage, CourseInfoPage, TabNavPage,
CourseNavPage, OpenResponsePage, ProgressPage
]
# Grade response (dict) to return from the XQueue stub
# in response to our unique submission text.
XQUEUE_GRADE_RESPONSE = None
def setUp(self):
"""
Always start in the subsection with open response problems.
"""
# Create a unique course ID
self.course_info = {
'org': 'test_org',
'number': self.unique_id,
'run': 'test_run',
'display_name': 'Test Course' + self.unique_id
}
# Create a unique submission
self.submission = "Test submission " + self.unique_id
# Ensure fixtures are installed
super(OpenResponseTest, self).setUp()
# Log in and navigate to the essay problems
course_id = '{org}/{number}/{run}'.format(**self.course_info)
self.ui.visit('studio.auto_auth', course_id=course_id)
self.ui.visit('lms.course_info', course_id=course_id)
self.ui.visit('studio.auto_auth', course_id=self.course_id)
self.ui.visit('lms.course_info', course_id=self.course_id)
self.ui['lms.tab_nav'].go_to_tab('Courseware')
self.ui['lms.course_nav'].go_to_section(
'Example Week 2: Get Interactive', 'Homework - Essays'
)
@property
def page_object_classes(self):
return [AutoAuthPage, CourseInfoPage, TabNavPage, CourseNavPage, OpenResponsePage]
@property
def fixtures(self):
"""
Create a test course with open response problems.
Configure the XQueue stub to respond to submissions to the open-ended queue.
"""
# Configure the test course
course_fix = CourseFixture(
self.course_info['org'], self.course_info['number'],
self.course_info['run'], self.course_info['display_name']
)
course_fix.add_children(
XBlockFixtureDesc('chapter', 'Test Section').add_children(
XBlockFixtureDesc('sequential', 'Test Subsection').add_children(
XBlockFixtureDesc('combinedopenended', 'Self-Assessed', data=load_data_str('ora_self_problem.xml')),
XBlockFixtureDesc('combinedopenended', 'AI-Assessed', data=load_data_str('ora_ai_problem.xml'))
XBlockFixtureDesc('combinedopenended', 'Self-Assessed',
data=load_data_str('ora_self_problem.xml'), metadata={'graded': True}),
XBlockFixtureDesc('combinedopenended', 'AI-Assessed',
data=load_data_str('ora_ai_problem.xml'), metadata={'graded': True}),
XBlockFixtureDesc('combinedopenended', 'Peer-Assessed',
data=load_data_str('ora_peer_problem.xml'), metadata={'graded': True}),
)
)
)
return [course_fix]
# Configure the XQueue stub's response for the text we will submit
if self.XQUEUE_GRADE_RESPONSE is not None:
xqueue_fix = XQueueResponseFixture(self.submission, self.XQUEUE_GRADE_RESPONSE)
return [course_fix, xqueue_fix]
else:
return [course_fix]
def submit_essay(self, expected_assessment_type, expected_prompt):
"""
Submit an essay and verify that the problem uses
the `expected_assessment_type` ("self", "ai", or "peer") and
shows the `expected_prompt` (a string).
"""
# Check the assessment type and prompt
self.assertEqual(self.ui['lms.open_response'].assessment_type, expected_assessment_type)
self.assertIn(expected_prompt, self.ui['lms.open_response'].prompt)
# Enter a submission, which will trigger a pre-defined response from the XQueue stub.
self.ui['lms.open_response'].set_response(self.submission)
# Save the response and expect some UI feedback
self.ui['lms.open_response'].save_response()
self.assertEqual(
self.ui['lms.open_response'].alert_message,
"Answer saved, but not yet submitted."
)
# Submit the response
self.ui['lms.open_response'].submit_response()
def get_asynch_feedback(self, assessment_type):
"""
Wait for and retrieve asynchronous feedback
(e.g. from AI, instructor, or peer grading)
`assessment_type` is either "ai" or "peer".
"""
feedback_promise = Promise(
self._check_feedback_func(assessment_type),
'Got feedback for {0} problem'.format(assessment_type)
)
return fulfill(feedback_promise)
def _check_feedback_func(self, assessment_type):
"""
Navigate away from, then return to, the peer problem to
receive updated feedback.
The returned function will return a tuple `(is_success, rubric_feedback)`,
`is_success` is True iff we have received feedback for the problem;
`rubric_feedback` is a list of "correct" or "incorrect" strings.
"""
if assessment_type == 'ai':
section_name = 'AI-Assessed'
elif assessment_type == 'peer':
section_name = 'Peer-Assessed'
else:
raise ValueError('Assessment type not recognized. Must be either "ai" or "peer"')
def _inner_check():
self.ui['lms.course_nav'].go_to_sequential('Self-Assessed')
self.ui['lms.course_nav'].go_to_sequential(section_name)
feedback = self.ui['lms.open_response'].rubric_feedback
# Successful if `feedback` is a non-empty list
return (bool(feedback), feedback)
return _inner_check
class SelfAssessmentTest(OpenResponseTest):
"""
Test ORA self-assessment.
"""
def test_self_assessment(self):
"""
Test that the user can self-assess an essay.
Given I am viewing a self-assessment problem
When I submit an essay and complete a self-assessment rubric
Then I see a scored rubric
And I see my score in the progress page.
"""
# Navigate to the self-assessment problem and submit an essay
self.ui['lms.course_nav'].go_to_sequential('Self-Assessed')
self._submit_essay('self', 'Censorship in the Libraries')
self.submit_essay('self', 'Censorship in the Libraries')
# Check the rubric categories
self.assertEqual(
@@ -91,14 +179,42 @@ class OpenResponseTest(WebAppTest):
['incorrect', 'correct']
)
# Verify the progress page
self.ui.visit('lms.progress', course_id=self.course_id)
scores = self.ui['lms.progress'].scores('Test Section', 'Test Subsection')
# The first score is self-assessment, which we've answered, so it's 1/2
# The other scores are AI- and peer-assessment, which we haven't answered so those are 0/2
self.assertEqual(scores, [(1, 2), (0, 2), (0, 2)])
class AIAssessmentTest(OpenResponseTest):
"""
Test ORA AI-assessment.
"""
XQUEUE_GRADE_RESPONSE = {
'score': 1,
'feedback': {"spelling": "Ok.", "grammar": "Ok.", "markup_text": "NA"},
'grader_type': 'BC',
'success': True,
'grader_id': 1,
'submission_id': 1,
'rubric_scores_complete': True,
'rubric_xml': load_data_str('ora_rubric.xml')
}
def test_ai_assessment(self):
"""
Test that a user can submit an essay and receive AI feedback.
Given I am viewing an AI-assessment problem that has a trained ML model
When I submit an essay and wait for a response
Then I see a scored rubric
And I see my score in the progress page.
"""
# Navigate to the AI-assessment problem and submit an essay
self.ui['lms.course_nav'].go_to_sequential('AI-Assessed')
self._submit_essay('ai', 'Censorship in the Libraries')
self.submit_essay('ai', 'Censorship in the Libraries')
# Expect UI feedback that the response was submitted
self.assertEqual(
@@ -106,27 +222,85 @@ class OpenResponseTest(WebAppTest):
"Your response has been submitted. Please check back later for your grade."
)
def _submit_essay(self, expected_assessment_type, expected_prompt):
# Refresh the page to get the updated feedback
# then verify that we get the feedback sent by our stub XQueue implementation
self.assertEqual(self.get_asynch_feedback('ai'), ['incorrect', 'correct'])
# Verify the progress page
self.ui.visit('lms.progress', course_id=self.course_id)
scores = self.ui['lms.progress'].scores('Test Section', 'Test Subsection')
# First score is the self-assessment score, which we haven't answered, so it's 0/2
# Second score is the AI-assessment score, which we have answered, so it's 1/2
# Third score is peer-assessment, which we haven't answered, so it's 0/2
self.assertEqual(scores, [(0, 2), (1, 2), (0, 2)])
class InstructorAssessmentTest(AIAssessmentTest):
"""
Test an AI-assessment that has been graded by an instructor.
This runs the exact same test as the AI-assessment test, except
that the feedback comes from an instructor instead of the machine grader.
From the student's perspective, it should look the same.
"""
XQUEUE_GRADE_RESPONSE = {
'score': 1,
'feedback': {"feedback": "Good job!"},
'grader_type': 'IN',
'success': True,
'grader_id': 1,
'submission_id': 1,
'rubric_scores_complete': True,
'rubric_xml': load_data_str('ora_rubric.xml')
}
class PeerFeedbackTest(OpenResponseTest):
"""
Test ORA peer-assessment. Note that this tests only *receiving* feedback,
not *giving* feedback -- those tests are located in another module.
"""
# Unlike other assessment types, peer assessment has multiple scores
XQUEUE_GRADE_RESPONSE = {
'score': [2, 2, 2],
'feedback': [json.dumps({"feedback": ""})] * 3,
'grader_type': 'PE',
'success': True,
'grader_id': [1, 2, 3],
'submission_id': 1,
'rubric_scores_complete': [True, True, True],
'rubric_xml': [load_data_str('ora_rubric.xml')] * 3
}
def test_peer_assessment(self):
"""
Submit an essay and verify that the problem uses
the `expected_assessment_type` ("self", "ai", or "peer") and
shows the `expected_prompt` (a string).
Given I have submitted an essay for peer-assessment
And enough other students have scored my essay
Then I can view the scores and written feedback
And I see my score in the progress page.
"""
# Navigate to the peer-assessment problem and submit an essay
self.ui['lms.course_nav'].go_to_sequential('Peer-Assessed')
self.submit_essay('peer', 'Censorship in the Libraries')
# Check the assessment type and prompt
self.assertEqual(self.ui['lms.open_response'].assessment_type, expected_assessment_type)
self.assertIn(expected_prompt, self.ui['lms.open_response'].prompt)
# Enter a response
essay = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut vehicula."
self.ui['lms.open_response'].set_response(essay)
# Save the response and expect some UI feedback
self.ui['lms.open_response'].save_response()
# Expect UI feedback that the response was submitted
self.assertEqual(
self.ui['lms.open_response'].alert_message,
"Answer saved, but not yet submitted."
self.ui['lms.open_response'].grader_status,
"Your response has been submitted. Please check back later for your grade."
)
# Submit the response
self.ui['lms.open_response'].submit_response()
# Refresh the page to get feedback from the stub XQueue grader.
# We receive feedback from all three peers, each of which
# provide 2 scores (one for each rubric item)
self.assertEqual(self.get_asynch_feedback('peer'), ['incorrect', 'correct'] * 3)
# Verify the progress page
self.ui.visit('lms.progress', course_id=self.course_id)
scores = self.ui['lms.progress'].scores('Test Section', 'Test Subsection')
# First score is the self-assessment score, which we haven't answered, so it's 0/2
# Second score is the AI-assessment score, which we haven't answered, so it's 0/2
# Third score is peer-assessment, which we have answered, so it's 2/2
self.assertEqual(scores, [(0, 2), (0, 2), (2, 2)])

26
common/test/acceptance/tests/test_studio.py
View File

@@ -22,6 +22,8 @@ from ..edxapp_pages.studio.signup import SignupPage
from ..edxapp_pages.studio.textbooks import TextbooksPage
from ..fixtures.course import CourseFixture
from .helpers import UniqueCourseTest
class LoggedOutTest(WebAppTest):
"""
@@ -59,26 +61,13 @@ class LoggedInPagesTest(WebAppTest):
self.ui.visit('studio.dashboard')
class CoursePagesTest(WebAppTest):
class CoursePagesTest(UniqueCourseTest):
"""
Tests that verify the pages in Studio that you can get to when logged
in and have a course.
"""
def setUp(self):
"""
Create a unique identifier for the course used in this test.
"""
# Define a unique course identifier
self.course_info = {
'org': 'test_org',
'number': '101',
'run': 'test_' + self.unique_id,
'display_name': 'Test Course ' + self.unique_id
}
# Ensure that the superclass sets up
super(CoursePagesTest, self).setUp()
COURSE_ID_SEPARATOR = "."
@property
def page_object_classes(self):
@@ -90,14 +79,13 @@ class CoursePagesTest(WebAppTest):
@property
def fixtures(self):
super_fixtures = super(CoursePagesTest, self).fixtures
course_fix = CourseFixture(
self.course_info['org'],
self.course_info['number'],
self.course_info['run'],
self.course_info['display_name']
)
return set(super_fixtures + [course_fix])
return [course_fix]
def test_page_existence(self):
"""
@@ -113,6 +101,6 @@ class CoursePagesTest(WebAppTest):
# Log in
self.ui.visit('studio.auto_auth', staff=True)
course_id = '{org}.{number}.{run}'.format(**self.course_info)
# Verify that each page is available
for page in pages:
self.ui.visit('studio.{0}'.format(page), course_id=course_id)
self.ui.visit('studio.{0}'.format(page), course_id=self.course_id)