Merge pull request #2431 from edx/will/ora-acceptance-tests-rebase

Peer-grading acceptance tests

common/djangoapps/terrain/stubs/data/ora_graded_rubric.xml

@@ -0,0 +1 @@
+<rubric><category><description>Writing Applications</description><score>0</score><option points='0'> The essay loses focus, has little information or supporting details, and the organization makes it difficult to follow.</option><option points='1'> The essay presents a mostly unified theme, includes sufficient information to convey the theme, and is generally organized well.</option></category><category><description> Language Conventions </description><score>1</score><option points='0'> The essay demonstrates a reasonable command of proper spelling and grammar. </option><option points='1'> The essay demonstrates superior command of proper spelling and grammar.</option></category></rubric>

common/djangoapps/terrain/stubs/data/ora_rubric.xml

@@ -0,0 +1 @@
+<rubric><category><description>Writing Applications</description><option points='0'> The essay loses focus, has little information or supporting details, and the organization makes it difficult to follow.</option><option points='1'> The essay presents a mostly unified theme, includes sufficient information to convey the theme, and is generally organized well.</option></category><category><description> Language Conventions </description><option points='0'> The essay demonstrates a reasonable command of proper spelling and grammar. </option><option points='1'> The essay demonstrates superior command of proper spelling and grammar.</option></category></rubric>

common/djangoapps/terrain/stubs/http.py

@@ -6,12 +6,60 @@ from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler
 import urlparse
 import threading
 import json
+from functools import wraps
 from lazy import lazy
 
 from logging import getLogger
 LOGGER = getLogger(__name__)
 
 
+def require_params(method, *required_keys):
+    """
+    Decorator to ensure that the method has all the required parameters.
+
+    Example:
+
+        @require_params('GET', 'id', 'state')
+        def handle_request(self):
+            # ....
+
+    would send a 400 response if no GET parameters were specified
+    for 'id' or 'state' (or if those parameters had empty values).
+
+    The wrapped function should be a method of a `StubHttpRequestHandler`
+    subclass.
+
+    Currently, "GET" and "POST" are the only supported methods.
+    """
+    def decorator(func):
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+
+            # Read either GET querystring params or POST dict params
+            if method == "GET":
+                params = self.get_params
+            elif method == "POST":
+                params = self.post_dict
+            else:
+                raise ValueError("Unsupported method '{method}'".format(method=method))
+
+            # Check for required values
+            missing = []
+            for key in required_keys:
+                if params.get(key) is None:
+                    missing.append(key)
+
+            if len(missing) > 0:
+                msg = "Missing required key(s) {keys}".format(keys=",".join(missing))
+                self.send_response(400, content=msg, headers={'Content-type': 'text/plain'})
+
+            # If nothing is missing, execute the function as usual
+            else:
+                return func(self, *args, **kwargs)
+        return wrapper
+    return decorator
+
+
 class StubHttpRequestHandler(BaseHTTPRequestHandler, object):
     """
     Handler for the stub HTTP service.
@@ -70,7 +118,26 @@ class StubHttpRequestHandler(BaseHTTPRequestHandler, object):
         """
         Return the GET parameters (querystring in the URL).
         """
-        return urlparse.parse_qs(self.path)
+        query = urlparse.urlparse(self.path).query
+
+        # By default, `parse_qs` returns a list of values for each param
+        # For convenience, we replace lists of 1 element with just the element
+        return {
+            k:v[0] if len(v) == 1 else v
+            for k,v in urlparse.parse_qs(query).items()
+        }
+
+    @lazy
+    def path_only(self):
+        """
+        Return the URL path without GET parameters.
+        Removes the trailing slash if there is one.
+        """
+        path = urlparse.urlparse(self.path).path
+        if path.endswith('/'):
+            return path[:-1]
+        else:
+            return path
 
     def do_PUT(self):
         """

common/djangoapps/terrain/stubs/ora.py

@@ -0,0 +1,538 @@
+"""
+Stub implementation of ORA service.
+
+This is an extremely simple version of the service, with most
+business logic removed.  In particular, the stub:
+
+1) Provides an infinite number of peer and calibration essays,
+   with dummy data.
+
+2) Simulates a set number of pending submissions for each student;
+   grades submitted by one student are not used for any other student.
+
+3) Ignores the scores/feedback students submit.
+
+4) Ignores problem location: an essay graded for *any* problem is graded
+   for *every* problem.
+
+Basically, the stub tracks only the *number* of peer/calibration essays
+submitted by each student.
+"""
+
+import json
+import pkg_resources
+from .http import StubHttpRequestHandler, StubHttpService, require_params
+
+
+class StudentState(object):
+    """
+    Store state about the student that the stub
+    ORA implementation needs to keep track of.
+    """
+    INITIAL_ESSAYS_AVAILABLE = 3
+    NUM_ESSAYS_REQUIRED = 1
+    NUM_CALIBRATION_REQUIRED = 1
+
+    def __init__(self):
+        self.num_graded = 0
+        self.num_calibrated = 0
+
+    def grade_peer_essay(self):
+        self.num_graded += 1
+
+    def grade_calibration_essay(self):
+        self.num_calibrated += 1
+
+    @property
+    def num_pending(self):
+        return max(self.INITIAL_ESSAYS_AVAILABLE- self.num_graded, 0)
+
+    @property
+    def num_required(self):
+        return max(self.NUM_ESSAYS_REQUIRED - self.num_graded, 0)
+
+    @property
+    def is_calibrated(self):
+        return self.num_calibrated >= self.NUM_CALIBRATION_REQUIRED
+
+
+class StubOraHandler(StubHttpRequestHandler):
+    """
+    Handler for ORA requests.
+    """
+
+    GET_URL_HANDLERS = {
+        '/peer_grading/get_next_submission': '_get_next_submission',
+        '/peer_grading/is_student_calibrated': '_is_student_calibrated',
+        '/peer_grading/show_calibration_essay': '_show_calibration_essay',
+        '/peer_grading/get_notifications': '_get_notifications',
+        '/peer_grading/get_data_for_location': '_get_data_for_location',
+        '/peer_grading/get_problem_list': '_get_problem_list',
+    }
+
+    POST_URL_HANDLERS = {
+        '/peer_grading/save_grade': '_save_grade',
+        '/peer_grading/save_calibration_essay': '_save_calibration_essay',
+
+        # Test-specific, used by the XQueue stub to register a new submission,
+        # which we use to discover valid problem locations in the LMS
+        '/test/register_submission': '_register_submission'
+    }
+
+    def do_GET(self):
+        """
+        Handle GET methods to the ORA API stub.
+        """
+        self._send_handler_response('GET')
+
+    def do_POST(self):
+        """
+        Handle POST methods to the ORA API stub.
+        """
+        self._send_handler_response('POST')
+
+    def _send_handler_response(self, method):
+        """
+        Delegate response to handler methods.
+        If no handler defined, send a 404 response.
+        """
+        # Choose the list of handlers based on the HTTP method
+        if method == 'GET':
+            handler_list = self.GET_URL_HANDLERS
+        elif method == 'POST':
+            handler_list = self.POST_URL_HANDLERS
+        else:
+            self.log_error('Unrecognized method "{method}"'.format(method=method))
+            return
+
+        # Check the path (without querystring params) against our list of handlers
+        handler_name = handler_list.get(self.path_only)
+
+        if handler_name is not None:
+            handler = getattr(self, handler_name, None)
+        else:
+            handler = None
+
+        # Delegate to the handler to send a response
+        if handler is not None:
+            handler()
+
+        # If we don't have a handler for this URL and/or HTTP method,
+        # respond with a 404.  This is the same behavior as the ORA API.
+        else:
+            self.send_response(404)
+
+    @require_params('GET', 'student_id', 'problem_id')
+    def _is_student_calibrated(self):
+        """
+        Query whether the student has completed enough calibration
+        essays to begin peer grading.
+
+        Method: GET
+
+        Params:
+            - student_id
+            - problem_id
+
+        Result (JSON):
+            - success (bool)
+            - total_calibrated_on_so_far (int)
+            - calibrated (bool)
+        """
+        student = self._student('GET')
+        if student is None:
+            self._error_response()
+
+        else:
+            self._success_response({
+                'total_calibrated_on_so_far': student.num_calibrated,
+                'calibrated': student.is_calibrated
+            })
+
+    @require_params('GET', 'student_id', 'problem_id')
+    def _show_calibration_essay(self):
+        """
+        Retrieve a calibration essay for the student to grade.
+
+        Method: GET
+
+        Params:
+            - student_id
+            - problem_id
+
+        Result (JSON):
+            - success (bool)
+            - submission_id (str)
+            - submission_key (str)
+            - student_response (str)
+            - prompt (str)
+            - rubric (str)
+            - max_score (int)
+        """
+        self._success_response({
+            'submission_id': self.server.DUMMY_DATA['submission_id'],
+            'submission_key': self.server.DUMMY_DATA['submission_key'],
+            'student_response': self.server.DUMMY_DATA['student_response'],
+            'prompt': self.server.DUMMY_DATA['prompt'],
+            'rubric': self.server.DUMMY_DATA['rubric'],
+            'max_score': self.server.DUMMY_DATA['max_score']
+        })
+
+    @require_params('GET', 'student_id', 'course_id')
+    def _get_notifications(self):
+        """
+        Query counts of submitted, required, graded, and available peer essays
+        for a particular student.
+
+        Method: GET
+
+        Params:
+            - student_id
+            - course_id
+
+        Result (JSON):
+            - success (bool)
+            - student_sub_count (int)
+            - count_required (int)
+            - count_graded (int)
+            - count_available (int)
+        """
+        student = self._student('GET')
+        if student is None:
+            self._error_response()
+
+        else:
+            self._success_response({
+                'student_sub_count': self.server.DUMMY_DATA['student_sub_count'],
+                'count_required': student.num_required,
+                'count_graded': student.num_graded,
+                'count_available': student.num_pending
+            })
+
+    @require_params('GET', 'student_id', 'location')
+    def _get_data_for_location(self):
+        """
+        Query counts of submitted, required, graded, and available peer essays
+        for a problem location.
+
+        This will send an error response if the problem has not
+        been registered at the given `location`.  This allows us
+        to ignore problems that are self- or ai-graded.
+
+        Method: GET
+
+        Params:
+            - student_id
+            - location
+
+        Result (JSON):
+            - success (bool)
+            - student_sub_count (int)
+            - count_required (int)
+            - count_graded (int)
+            - count_available (int)
+        """
+        student = self._student('GET')
+        location = self.get_params.get('location')
+
+        # Do not return data if we're missing the student param
+        # or the problem has not yet been registered.
+        if student is None or location not in self.server.problems:
+            self._error_response()
+
+        else:
+            self._success_response({
+                'student_sub_count': self.server.DUMMY_DATA['student_sub_count'],
+                'count_required': student.num_required,
+                'count_graded': student.num_graded,
+                'count_available': student.num_pending
+            })
+
+    @require_params('GET', 'grader_id', 'location')
+    def _get_next_submission(self):
+        """
+        Retrieve the next submission for the student to peer-grade.
+
+        Method: GET
+
+        Params:
+            - grader_id
+            - location
+
+        Result (JSON):
+            - success (bool)
+            - submission_id (str)
+            - submission_key (str)
+            - student_response (str)
+            - prompt (str, HTML)
+            - rubric (str, XML)
+            - max_score (int)
+        """
+        self._success_response({
+            'submission_id': self.server.DUMMY_DATA['submission_id'],
+            'submission_key': self.server.DUMMY_DATA['submission_key'],
+            'student_response': self.server.DUMMY_DATA['student_response'],
+            'prompt': self.server.DUMMY_DATA['prompt'],
+            'rubric': self.server.DUMMY_DATA['rubric'],
+            'max_score': self.server.DUMMY_DATA['max_score']
+        })
+
+    @require_params('GET', 'course_id')
+    def _get_problem_list(self):
+        """
+        Retrieve the list of problems available for peer grading.
+
+        Method: GET
+
+        Params:
+            - course_id
+
+        Result (JSON):
+            - success (bool)
+            - problem_list (list)
+
+        where `problem_list` is a list of dictionaries with keys:
+            - location (str)
+            - problem_name (str)
+            - num_graded (int)
+            - num_pending (int)
+            - num_required (int)
+        """
+        self._success_response({'problem_list': self.server.problem_list})
+
+
+    @require_params('POST', 'grader_id', 'location', 'submission_id', 'score', 'feedback', 'submission_key')
+    def _save_grade(self):
+        """
+        Save a score and feedback for an essay the student has graded.
+
+        Method: POST
+
+        Params:
+            - grader_id
+            - location
+            - submission_id
+            - score
+            - feedback
+            - submission_key
+
+        Result (JSON):
+            - success (bool)
+        """
+        student = self._student('POST', key='grader_id')
+        if student is None:
+            self._error_response()
+
+        else:
+            # Update the number of essays the student has graded
+            student.grade_peer_essay()
+            return self._success_response({})
+
+    @require_params('POST', 'student_id', 'location', 'calibration_essay_id', 'score', 'feedback', 'submission_key')
+    def _save_calibration_essay(self):
+        """
+        Save a score and feedback for a calibration essay the student has graded.
+        Returns the scores/feedback that the instructor gave for the essay.
+
+        Method: POST
+
+        Params:
+            - student_id
+            - location
+            - calibration_essay_id
+            - score
+            - feedback
+            - submission_key
+
+        Result (JSON):
+            - success (bool)
+            - message (str)
+            - actual_score (int)
+            - actual_rubric (str, XML)
+            - actual_feedback (str)
+        """
+        student = self._student('POST')
+        if student is None:
+            self._error_response()
+
+        else:
+
+            # Increment the student calibration count
+            student.grade_calibration_essay()
+
+            self._success_response({
+                'message': self.server.DUMMY_DATA['message'],
+                'actual_score': self.server.DUMMY_DATA['actual_score'],
+                'actual_rubric': self.server.DUMMY_DATA['actual_rubric'],
+                'actual_feedback': self.server.DUMMY_DATA['actual_feedback']
+            })
+
+    @require_params('POST', 'grader_payload')
+    def _register_submission(self):
+        """
+        Test-specific method to register a new submission.
+        This is used by `get_problem_list` to return valid locations in the LMS courseware.
+        In tests, this end-point gets called by the XQueue stub when it receives new submissions,
+        much like ORA discovers locations when students submit peer-graded problems to the XQueue.
+
+        Since the LMS sends *all* open-ended problems to the XQueue (including self- and ai-graded),
+        we have to ignore everything except peer-graded problems.  We do so by looking
+        for the text 'peer' in the problem's name.  This is a little bit of a hack,
+        but it makes the implementation much simpler.
+
+        Method: POST
+
+        Params:
+            - grader_payload (JSON dict)
+
+        Result: Empty
+
+        The only keys we use in `grader_payload` are 'location' and 'problem_id'.
+        """
+        # Since this is a required param, we know it is in the post dict
+        try:
+            payload = json.loads(self.post_dict['grader_payload'])
+
+        except ValueError:
+            self.log_message(
+                "Could not decode grader payload as JSON: '{0}'".format(
+                    self.post_dict['grader_payload']))
+            self.send_response(400)
+
+        else:
+
+            location = payload.get('location')
+            name = payload.get('problem_id')
+
+            if location is not None and name is not None:
+
+                if "peer" in name.lower():
+                    self.server.register_problem(location, name)
+                    self.send_response(200)
+
+                else:
+                    self.log_message(
+                        "Problem '{0}' does not have 'peer' in its name.  Ignoring...".format(name)
+                    )
+                    self.send_response(200)
+            else:
+                self.log_message(
+                    "Grader payload should contain 'location' and 'problem_id' keys: {0}".format(payload)
+                )
+                self.send_response(400)
+
+
+    def _student(self, method, key='student_id'):
+        """
+        Return the `StudentState` instance for the student ID given
+        in the request parameters.
+
+        `method` is the HTTP request method (either "GET" or "POST")
+        and `key` is the parameter key.
+        """
+        if method == 'GET':
+            student_id = self.get_params.get(key)
+        elif method == 'POST':
+            student_id = self.post_dict.get(key)
+        else:
+            self.log_error("Unrecognized method '{method}'".format(method=method))
+            return None
+
+        if student_id is None:
+            self.log_error("Could not get student ID from parameters")
+            return None
+
+        return self.server.student_state(student_id)
+
+    def _success_response(self, response_dict):
+        """
+        Send a success response.
+        `response_dict` is a Python dictionary to JSON-encode.
+        """
+        response_dict['success'] = True
+        response_dict['version'] = 1
+        self.send_response(
+            200, content=json.dumps(response_dict),
+            headers={'Content-type': 'application/json'}
+        )
+
+    def _error_response(self):
+        """
+        Send an error response.
+        """
+        response_dict = {'success': False, 'version': 1}
+        self.send_response(
+            400, content=json.dumps(response_dict),
+            headers={'Content-type': 'application/json'}
+        )
+
+
+class StubOraService(StubHttpService):
+    """
+    Stub ORA service.
+    """
+    HANDLER_CLASS = StubOraHandler
+
+    DUMMY_DATA = {
+        'submission_id': 1,
+        'submission_key': 'test key',
+        'student_response': 'Test response',
+        'prompt': 'Test prompt',
+        'rubric': pkg_resources.resource_string(__name__, "data/ora_rubric.xml"),
+        'max_score': 2,
+        'message': 'Successfully saved calibration record.',
+        'actual_score': 2,
+        'actual_rubric': pkg_resources.resource_string(__name__, "data/ora_graded_rubric.xml"),
+        'actual_feedback': 'Great job!',
+        'student_sub_count': 1,
+        'problem_name': 'test problem',
+        'problem_list_num_graded': 1,
+        'problem_list_num_pending': 1,
+        'problem_list_num_required': 0,
+    }
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize student submission state.
+        """
+        super(StubOraService, self).__init__(*args, **kwargs)
+
+        # Create a dict to map student ID's to their state
+        self._students = dict()
+
+        # By default, no problems are available for peer grading
+        # You can add to this list using the `register_location` HTTP end-point
+        # This is a dict mapping problem locations to problem names
+        self.problems = dict()
+
+    def student_state(self, student_id):
+        """
+        Return the `StudentState` (named tuple) for the student
+        with ID `student_id`.  The student state can be modified by the caller.
+        """
+        # Create the student state if it does not already exist
+        if student_id not in self._students:
+            student = StudentState()
+            self._students[student_id] = student
+
+        # Retrieve the student state
+        return self._students[student_id]
+
+    @property
+    def problem_list(self):
+        """
+        Return a list of problems available for peer grading.
+        """
+        return [{
+            'location': location, 'problem_name': name,
+            'num_graded': self.DUMMY_DATA['problem_list_num_graded'],
+            'num_pending': self.DUMMY_DATA['problem_list_num_pending'],
+            'num_required': self.DUMMY_DATA['problem_list_num_required']
+            } for location, name in self.problems.items()
+        ]
+
+    def register_problem(self, location, name):
+        """
+        Register a new problem with `location` and `name` for peer grading.
+        """
+        self.problems[location] = name

common/djangoapps/terrain/stubs/start.py

@@ -6,13 +6,15 @@ import time
 import logging
 from .xqueue import StubXQueueService
 from .youtube import StubYouTubeService
+from .ora import StubOraService
 
 
-USAGE = "USAGE: python -m stubs.start SERVICE_NAME PORT_NUM"
+USAGE = "USAGE: python -m stubs.start SERVICE_NAME PORT_NUM [CONFIG_KEY=CONFIG_VAL, ...]"
 
 SERVICES = {
     'xqueue': StubXQueueService,
-    'youtube': StubYouTubeService
+    'youtube': StubYouTubeService,
+    'ora': StubOraService
 }
 
 # Log to stdout, including debug messages
@@ -21,7 +23,7 @@ logging.basicConfig(level=logging.DEBUG, format="%(levelname)s %(message)s")
 
 def get_args():
     """
-    Parse arguments, returning tuple of `(service_name, port_num)`.
+    Parse arguments, returning tuple of `(service_name, port_num, config_dict)`.
     Exits with a message if arguments are invalid.
     """
     if len(sys.argv) < 3:
@@ -30,6 +32,7 @@ def get_args():
 
     service_name = sys.argv[1]
     port_num = sys.argv[2]
+    config_dict = _parse_config_args(sys.argv[3:])
 
     if service_name not in SERVICES:
         print "Unrecognized service '{0}'.  Valid choices are: {1}".format(
@@ -45,17 +48,40 @@ def get_args():
         print "Port '{0}' must be a positive integer".format(port_num)
         sys.exit(1)
 
-    return service_name, port_num
+    return service_name, port_num, config_dict
+
+
+def _parse_config_args(args):
+    """
+    Parse stub configuration arguments, which are strings of the form "KEY=VAL".
+    `args` is a list of arguments from the command line.
+    Any argument that does not match the "KEY=VAL" format will be logged and skipped.
+
+    Returns a dictionary with the configuration keys and values.
+    """
+    config_dict = dict()
+    for config_str in args:
+        try:
+            components = config_str.split('=')
+            if len(components) >= 2:
+                config_dict[components[0]] = "=".join(components[1:])
+
+        except:
+            print "Warning: could not interpret config value '{0}'".format(config_str)
+            pass
+
+    return config_dict
 
 
 def main():
     """
     Start a server; shut down on keyboard interrupt signal.
     """
-    service_name, port_num = get_args()
+    service_name, port_num, config_dict = get_args()
     print "Starting stub service '{0}' on port {1}...".format(service_name, port_num)
 
     server = SERVICES[service_name](port_num=port_num)
+    server.config.update(config_dict)
 
     try:
         while True:

common/djangoapps/terrain/stubs/tests/test_http.py

@@ -5,7 +5,7 @@ Unit tests for stub HTTP server base class.
 import unittest
 import requests
 import json
-from terrain.stubs.http import StubHttpService
+from terrain.stubs.http import StubHttpService, StubHttpRequestHandler, require_params
 
 
 class StubHttpServiceTest(unittest.TestCase):
@@ -62,3 +62,56 @@ class StubHttpServiceTest(unittest.TestCase):
             data="{}"
         )
         self.assertEqual(response.status_code, 404)
+
+
+class RequireRequestHandler(StubHttpRequestHandler):
+    @require_params('GET', 'test_param')
+    def do_GET(self):
+        self.send_response(200)
+
+    @require_params('POST', 'test_param')
+    def do_POST(self):
+        self.send_response(200)
+
+
+class RequireHttpService(StubHttpService):
+    HANDLER_CLASS = RequireRequestHandler
+
+
+class RequireParamTest(unittest.TestCase):
+    """
+    Test the decorator for requiring parameters.
+    """
+
+    def setUp(self):
+        self.server = RequireHttpService()
+        self.addCleanup(self.server.shutdown)
+        self.url = "http://127.0.0.1:{port}".format(port=self.server.port)
+
+    def test_require_get_param(self):
+
+        # Expect success when we provide the required param
+        response = requests.get(self.url, params={"test_param": 2})
+        self.assertEqual(response.status_code, 200)
+
+        # Expect failure when we do not proivde the param
+        response = requests.get(self.url)
+        self.assertEqual(response.status_code, 400)
+
+        # Expect failure when we provide an empty param
+        response = requests.get(self.url + "?test_param=")
+        self.assertEqual(response.status_code, 400)
+
+    def test_require_post_param(self):
+
+        # Expect success when we provide the required param
+        response = requests.post(self.url, data={"test_param": 2})
+        self.assertEqual(response.status_code, 200)
+
+        # Expect failure when we do not proivde the param
+        response = requests.post(self.url)
+        self.assertEqual(response.status_code, 400)
+
+        # Expect failure when we provide an empty param
+        response = requests.post(self.url, data={"test_param": None})
+        self.assertEqual(response.status_code, 400)

common/djangoapps/terrain/stubs/tests/test_ora.py

@@ -0,0 +1,280 @@
+"""
+Unit tests for stub ORA implementation.
+"""
+
+import unittest
+import requests
+import json
+from ..ora import StubOraService, StubOraHandler, StudentState
+
+
+class StubOraServiceTest(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Start the stub server.
+        """
+        self.server = StubOraService()
+        self.addCleanup(self.server.shutdown)
+
+    def test_calibration(self):
+
+        # Ensure that we use the same student ID throughout
+        student_id = '1234'
+
+        # Initially, student should not be calibrated
+        response = requests.get(
+            self._peer_url('is_student_calibrated'),
+            params={'student_id': student_id, 'problem_id': '5678'}
+        )
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'total_calibrated_on_so_far': 0,
+            'calibrated': False
+        })
+
+        # Retrieve a calibration essay
+        response = requests.get(
+            self._peer_url('show_calibration_essay'),
+            params={'student_id': student_id, 'problem_id': '5678'}
+        )
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'submission_id': self.server.DUMMY_DATA['submission_id'],
+            'submission_key': self.server.DUMMY_DATA['submission_key'],
+            'student_response': self.server.DUMMY_DATA['student_response'],
+            'prompt': self.server.DUMMY_DATA['prompt'],
+            'rubric': self.server.DUMMY_DATA['rubric'],
+            'max_score': self.server.DUMMY_DATA['max_score']
+        })
+
+        # Grade the calibration essay
+        response = requests.post(
+            self._peer_url('save_calibration_essay'),
+            data={
+                'student_id': student_id,
+                'location': 'test location',
+                'calibration_essay_id': 1,
+                'score': 2,
+                'submission_key': 'key',
+                'feedback': 'Good job!'
+            }
+        )
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'message': self.server.DUMMY_DATA['message'],
+            'actual_score': self.server.DUMMY_DATA['actual_score'],
+            'actual_rubric': self.server.DUMMY_DATA['actual_rubric'],
+            'actual_feedback': self.server.DUMMY_DATA['actual_feedback']
+        })
+
+        # Now the student should be calibrated
+        response = requests.get(
+            self._peer_url('is_student_calibrated'),
+            params={'student_id': student_id, 'problem_id': '5678'}
+        )
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'total_calibrated_on_so_far': 1,
+            'calibrated': True
+        })
+
+        # But a student with a different ID should NOT be calibrated.
+        response = requests.get(
+            self._peer_url('is_student_calibrated'),
+            params={'student_id': 'another', 'problem_id': '5678'}
+        )
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'total_calibrated_on_so_far': 0,
+            'calibrated': False
+        })
+
+    def test_grade_peers(self):
+
+        # Ensure a consistent student ID
+        student_id = '1234'
+
+        # Check initial number of submissions
+        # Should be none graded and 1 required
+        self._assert_num_graded(student_id, None, 0, 1)
+
+        # Register a problem that DOES have "peer" in the name
+        self._register_problem('test_location', 'Peer Assessed Problem')
+
+        # Retrieve the next submission
+        response = requests.get(
+            self._peer_url('get_next_submission'),
+            params={'grader_id': student_id, 'location': 'test_location'}
+        )
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'submission_id': self.server.DUMMY_DATA['submission_id'],
+            'submission_key': self.server.DUMMY_DATA['submission_key'],
+            'student_response': self.server.DUMMY_DATA['student_response'],
+            'prompt': self.server.DUMMY_DATA['prompt'],
+            'rubric': self.server.DUMMY_DATA['rubric'],
+            'max_score': self.server.DUMMY_DATA['max_score']
+        })
+
+        # Grade the submission
+        response = requests.post(
+            self._peer_url('save_grade'),
+            data={
+                'location': 'test_location',
+                'grader_id': student_id,
+                'submission_id': 1,
+                'score': 2,
+                'feedback': 'Good job!',
+                'submission_key': 'key'
+            }
+        )
+        self._assert_response(response, {'version': 1, 'success': True})
+
+        # Check final number of submissions
+        # Shoud be one graded  and none required
+        self._assert_num_graded(student_id, 'test_location', 1, 0)
+
+        # Grade the next submission the submission
+        response = requests.post(
+            self._peer_url('save_grade'),
+            data={
+                'location': 'test_location',
+                'grader_id': student_id,
+                'submission_id': 1,
+                'score': 2,
+                'feedback': 'Good job!',
+                'submission_key': 'key'
+            }
+        )
+        self._assert_response(response, {'version': 1, 'success': True})
+
+        # Check final number of submissions
+        # Shoud be two graded  and none required
+        self._assert_num_graded(student_id, 'test_location', 2, 0)
+
+    def test_problem_list(self):
+
+        self._register_problem('test_location', 'Peer Grading Problem')
+
+        # The problem list returns dummy counts which are not updated
+        # The location we use is ignored by the LMS, and we ignore it in the stub,
+        # so we use a dummy value there too.
+        response = requests.get(
+            self._peer_url('get_problem_list'),
+            params={'course_id': 'test course'}
+        )
+
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'problem_list': [{
+                'location': 'test_location',
+                'problem_name': 'Peer Grading Problem',
+                'num_graded': self.server.DUMMY_DATA['problem_list_num_graded'],
+                'num_pending': self.server.DUMMY_DATA['problem_list_num_pending'],
+                'num_required': self.server.DUMMY_DATA['problem_list_num_required']
+            }]
+        })
+
+    def test_ignore_non_peer_problem(self):
+
+        # Register a problem that does NOT have "peer" in the name
+        self._register_problem('test_location', 'Self Assessed Problem')
+
+        # Expect that the problem list is empty
+        response = requests.get(
+            self._peer_url('get_problem_list'),
+            params={'course_id': 'test course'}
+        )
+
+        self._assert_response(response,
+            {'version': 1, 'success': True, 'problem_list': []}
+        )
+
+        # Expect that no data is available for the problem location
+        response = requests.get(
+            self._peer_url('get_data_for_location'),
+            params={'location': 'test_location', 'student_id': 'test'}
+        )
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(response.json(), {'version': 1, 'success': False})
+
+    def test_empty_problem_list(self):
+
+        # Without configuring any problem location, should return an empty list
+        response = requests.get(
+            self._peer_url('get_problem_list'),
+            params={'course_id': 'test course'}
+        )
+        self._assert_response(response, {'version': 1, 'success': True, 'problem_list': []})
+
+    def _peer_url(self, path):
+        """
+        Construt a URL to the stub ORA peer-grading service.
+        """
+        return "http://127.0.0.1:{port}/peer_grading/{path}/".format(
+            port=self.server.port, path=path
+        )
+
+    def _register_problem(self, location, name):
+        """
+        Configure the stub to use a particular problem location
+        The actual implementation discovers problem locations by submission
+        to the XQueue; we do something similar by having the XQueue stub
+        register submitted locations with the ORA stub.
+        """
+        grader_payload = json.dumps({'location': location, 'problem_id': name})
+        url = "http://127.0.0.1:{port}/test/register_submission".format(port=self.server.port)
+        response = requests.post(url, data={'grader_payload': grader_payload})
+        self.assertTrue(response.ok)
+
+    def _assert_response(self, response, expected_json):
+        """
+        Assert that the `response` was successful and contained
+        `expected_json` (dict) as its content.
+        """
+        self.assertTrue(response.ok)
+        self.assertEqual(response.json(), expected_json)
+
+    def _assert_num_graded(self, student_id, location, num_graded, num_required):
+        """
+        ORA provides two distinct ways to get the submitted/graded counts.
+        Here we check both of them to ensure that the number that we've graded
+        is consistently `num_graded`.
+        """
+
+        # Unlike the actual ORA service,
+        # we keep track of counts on a per-student basis.
+        # This means that every user starts with N essays to grade,
+        # and as they grade essays, that number decreases.
+        # We do NOT simulate students adding more essays to the queue,
+        # and essays that the current student submits are NOT graded
+        # by other students.
+        num_pending = StudentState.INITIAL_ESSAYS_AVAILABLE - num_graded
+
+        # Notifications
+        response = requests.get(
+            self._peer_url('get_notifications'),
+            params={'student_id': student_id, 'course_id': 'test course'}
+        )
+        self._assert_response(response, {
+            'version': 1, 'success': True,
+            'count_required': num_required,
+            'student_sub_count': self.server.DUMMY_DATA['student_sub_count'],
+            'count_graded': num_graded,
+            'count_available': num_pending
+        })
+
+        # Location data
+        if location is not None:
+            response = requests.get(
+                self._peer_url('get_data_for_location'),
+                params={'location': location, 'student_id': student_id}
+            )
+            self._assert_response(response, {
+                'version': 1, 'success': True,
+                'count_required': num_required,
+                'student_sub_count': self.server.DUMMY_DATA['student_sub_count'],
+                'count_graded': num_graded,
+                'count_available': num_pending
+            })

common/djangoapps/terrain/stubs/tests/test_xqueue_stub.py

@@ -8,7 +8,7 @@ import json
 import requests
 import time
 import copy
-from terrain.stubs.xqueue import StubXQueueService, StubXQueueHandler
+from ..xqueue import StubXQueueService, StubXQueueHandler
 
 
 class StubXQueueServiceTest(unittest.TestCase):
@@ -95,7 +95,7 @@ class StubXQueueServiceTest(unittest.TestCase):
 
             # Post a submission to the XQueue stub
             callback_url = 'http://127.0.0.1:8000/test_callback'
-            expected_header = self._post_submission(
+            self._post_submission(
                 callback_url, 'test_queuekey', 'test_queue',
                 json.dumps({'submission': 'test_1 and test_2'})
             )
@@ -108,6 +108,23 @@ class StubXQueueServiceTest(unittest.TestCase):
             self.assertFalse(post.called)
             self.assertTrue(logger.error.called)
 
+    @mock.patch('terrain.stubs.xqueue.post')
+    def test_register_submission_url(self, post):
+
+        # Configure the XQueue stub to notify another service
+        # when it receives a submission.
+        register_url = 'http://127.0.0.1:8000/register_submission'
+        self.server.config['register_submission_url'] = register_url
+
+        callback_url = 'http://127.0.0.1:8000/test_callback'
+        submission = json.dumps({'grader_payload': 'test payload'})
+        self._post_submission(
+            callback_url, 'test_queuekey', 'test_queue', submission
+        )
+
+        # Check that a notification was sent
+        post.assert_any_call(register_url, data={'grader_payload': u'test payload'})
+
     def _post_submission(self, callback_url, lms_key, queue_name, xqueue_body):
         """
         Post a submission to the stub XQueue implementation.

common/djangoapps/terrain/stubs/tests/test_youtube_stub.py

@@ -4,7 +4,7 @@ Unit test for stub YouTube implementation.
 
 import unittest
 import requests
-from terrain.stubs.youtube import StubYouTubeService
+from ..youtube import StubYouTubeService
 
 
 class StubYouTubeServiceTest(unittest.TestCase):

common/djangoapps/terrain/stubs/xqueue.py

@@ -4,11 +4,12 @@ Stub implementation of XQueue for acceptance tests.
 Configuration values:
     "default" (dict): Default response to be sent to LMS as a grade for a submission
     "<submission>" (dict): Grade response to return for submissions containing the text <submission>
+    "register_submission_url" (str): URL to send grader payloads when we receive a submission
 
 If no grade response is configured, a default response will be returned.
 """
 
-from .http import StubHttpRequestHandler, StubHttpService
+from .http import StubHttpRequestHandler, StubHttpService, require_params
 import json
 import copy
 from requests import post
@@ -23,6 +24,7 @@ class StubXQueueHandler(StubHttpRequestHandler):
     DEFAULT_RESPONSE_DELAY = 2
     DEFAULT_GRADE_RESPONSE = {'correct': True, 'score': 1, 'msg': ''}
 
+    @require_params('POST', 'xqueue_body', 'xqueue_header')
     def do_POST(self):
         """
         Handle a POST request from the client
@@ -35,6 +37,10 @@ class StubXQueueHandler(StubHttpRequestHandler):
 
         # Respond only to grading requests
         if self._is_grade_request():
+
+            # If configured, send the grader payload to other services.
+            self._register_submission(self.post_dict['xqueue_body'])
+
             try:
                 xqueue_header = json.loads(self.post_dict['xqueue_header'])
                 callback_url = xqueue_header['lms_callback_url']
@@ -118,7 +124,7 @@ class StubXQueueHandler(StubHttpRequestHandler):
         # There is a danger here that a submission will match multiple response patterns.
         # Rather than fail silently (which could cause unpredictable behavior in tests)
         # we abort and log a debugging message.
-        for pattern, response in self.server.config.iteritems():
+        for pattern, response in self.server.queue_responses:
 
             if pattern in xqueue_body_json:
                 if grade_response is None:
@@ -150,7 +156,44 @@ class StubXQueueHandler(StubHttpRequestHandler):
         post(postback_url, data=data)
         self.log_message("XQueue: sent grading response {0} to {1}".format(data, postback_url))
 
+    def _register_submission(self, xqueue_body_json):
+        """
+        If configured, send the submission's grader payload to another service.
+        """
+        url = self.server.config.get('register_submission_url')
+
+        # If not configured, do not need to send anything
+        if url is not None:
+
+            try:
+                xqueue_body = json.loads(xqueue_body_json)
+            except ValueError:
+                self.log_error(
+                    "Could not decode XQueue body as JSON: '{0}'".format(xqueue_body_json))
+
+            else:
+
+                # Retrieve the grader payload, which should be a JSON-encoded dict.
+                # We pass the payload directly to the service we are notifying, without
+                # inspecting the contents.
+                grader_payload = xqueue_body.get('grader_payload')
+
+                if grader_payload is not None:
+                    response = post(url, data={'grader_payload': grader_payload})
+                    if not response.ok:
+                        self.log_error(
+                            "Could register submission at URL '{0}'.  Status was {1}".format(
+                                url, response.status_code))
+
+                else:
+                    self.log_message(
+                        "XQueue body is missing 'grader_payload' key: '{0}'".format(xqueue_body)
+                    )
+
     def _is_grade_request(self):
+        """
+        Return a boolean indicating whether the requested URL indicates a submission.
+        """
         return 'xqueue/submit' in self.path
 
 
@@ -160,3 +203,19 @@ class StubXQueueService(StubHttpService):
     """
 
     HANDLER_CLASS = StubXQueueHandler
+    NON_QUEUE_CONFIG_KEYS = ['default', 'register_submission_url']
+
+    @property
+    def queue_responses(self):
+        """
+        Returns a list of (pattern, response) tuples, where `pattern` is a pattern
+        to match in the XQueue body, and `response` is a dictionary to return
+        as the response from the grader.
+
+        Every configuration key is a queue name,
+        except for 'default' and 'register_submission_url' which have special meaning
+        """
+        return {
+            key:val for key, val in self.config.iteritems()
+            if key not in self.NON_QUEUE_CONFIG_KEYS
+        }.items()

common/djangoapps/terrain/stubs/youtube.py

@@ -75,7 +75,7 @@ class StubYouTubeHandler(StubHttpRequestHandler):
         time.sleep(self.server.config.get('time_to_response', self.DEFAULT_DELAY_SEC))
 
         # Construct the response content
-        callback = self.get_params['callback'][0]
+        callback = self.get_params['callback']
         data = OrderedDict({
             'data': OrderedDict({
                 'id': youtube_id,