From 64e1eb9903933b234220ada66889b920a2682c22 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Fri, 15 Mar 2013 11:35:27 -0700 Subject: [PATCH 01/24] Fix randomness bug in cohort placement capa re-seeds the global RNG all the time, resulting in non-random cohort placement. Switch to using a local random module. (Thanks Cale for helping me figure this out!) --- common/djangoapps/course_groups/cohorts.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/common/djangoapps/course_groups/cohorts.py b/common/djangoapps/course_groups/cohorts.py index c362ed4e89..ded20e1613 100644 --- a/common/djangoapps/course_groups/cohorts.py +++ b/common/djangoapps/course_groups/cohorts.py @@ -15,6 +15,12 @@ from .models import CourseUserGroup log = logging.getLogger(__name__) +# tl;dr: global state is bad. capa reseeds random every time a problem is loaded. Even +# if and when that's fixed, it's a good idea to have a local generator to avoid any other +# code that messes with the global random module. +local_random = random.Random() + + def is_course_cohorted(course_id): """ Given a course id, return a boolean for whether or not the course is @@ -129,13 +135,7 @@ def get_cohort(user, course_id): return None # Put user in a random group, creating it if needed - choice = random.randrange(0, n) - group_name = choices[choice] - - # Victor: we are seeing very strange behavior on prod, where almost all users - # end up in the same group. Log at INFO to try to figure out what's going on. - log.info("DEBUG: adding user {0} to cohort {1}. choice={2}".format( - user, group_name,choice)) + group_name = local_random.choice(choices) group, created = CourseUserGroup.objects.get_or_create( course_id=course_id, From d91008b73a14e5b1599a2bc965e39d75d1785138 Mon Sep 17 00:00:00 2001 From: Victor Shnayder Date: Fri, 15 Mar 2013 19:36:14 -0700 Subject: [PATCH 02/24] Prevent random.Random() from running at import time. --- common/djangoapps/course_groups/cohorts.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/common/djangoapps/course_groups/cohorts.py b/common/djangoapps/course_groups/cohorts.py index ded20e1613..7924012bfe 100644 --- a/common/djangoapps/course_groups/cohorts.py +++ b/common/djangoapps/course_groups/cohorts.py @@ -18,8 +18,20 @@ log = logging.getLogger(__name__) # tl;dr: global state is bad. capa reseeds random every time a problem is loaded. Even # if and when that's fixed, it's a good idea to have a local generator to avoid any other # code that messes with the global random module. -local_random = random.Random() +_local_random = None +def local_random(): + """ + Get the local random number generator. In a function so that we don't run + random.Random() at import time. + """ + # ironic, isn't it? + global _local_random + + if _local_random is None: + _local_random = random.Random() + + return _local_random def is_course_cohorted(course_id): """ @@ -135,7 +147,7 @@ def get_cohort(user, course_id): return None # Put user in a random group, creating it if needed - group_name = local_random.choice(choices) + group_name = local_random().choice(choices) group, created = CourseUserGroup.objects.get_or_create( course_id=course_id, From e59b650fab1b66dcff9360233eb008f6e0e108e8 Mon Sep 17 00:00:00 2001 From: Chris Dodge Date: Mon, 18 Mar 2013 15:02:49 -0400 Subject: [PATCH 03/24] add an array of courses to not refresh the metadata cache on writes. This is needed for imports where we are doing bulk updates otherwise, the DB gets thrashed and the import takes too long, timing out the browser --- common/lib/xmodule/xmodule/modulestore/mongo.py | 14 ++++++++++---- .../xmodule/xmodule/modulestore/xml_importer.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo.py b/common/lib/xmodule/xmodule/modulestore/mongo.py index c5e5bbfdf8..4ec7a699e0 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo.py @@ -246,6 +246,7 @@ class MongoModuleStore(ModuleStoreBase): self.fs_root = path(fs_root) self.error_tracker = error_tracker self.render_template = render_template + self.ignore_write_events_on_courses = [] def get_metadata_inheritance_tree(self, location): ''' @@ -329,6 +330,11 @@ class MongoModuleStore(ModuleStoreBase): return tree + def refresh_cached_metadata_inheritance_tree(self, location): + pseudo_course_id = '/'.join([location.org, location.course]) + if pseudo_course_id not in self.ignore_write_events_on_courses: + self.get_cached_metadata_inheritance_tree(location, force_refresh = True) + def clear_cached_metadata_inheritance_tree(self, location): key_name = '{0}/{1}'.format(location.org, location.course) if self.metadata_inheritance_cache is not None: @@ -519,7 +525,7 @@ class MongoModuleStore(ModuleStoreBase): raise DuplicateItemError(location) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(Location(location)) def get_course_for_item(self, location, depth=0): ''' @@ -586,7 +592,7 @@ class MongoModuleStore(ModuleStoreBase): self._update_single_item(location, {'definition.children': children}) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(Location(location)) def update_metadata(self, location, metadata): """ @@ -612,7 +618,7 @@ class MongoModuleStore(ModuleStoreBase): self._update_single_item(location, {'metadata': metadata}) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(loc, force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(loc) def delete_item(self, location): """ @@ -632,7 +638,7 @@ class MongoModuleStore(ModuleStoreBase): self.collection.remove({'_id': Location(location).dict()}) # recompute (and update) the metadata inheritance tree which is cached - self.get_cached_metadata_inheritance_tree(Location(location), force_refresh = True) + self.refresh_cached_metadata_inheritance_tree(Location(location)) def get_parent_locations(self, location, course_id): diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index fa232596f2..5955b0ebe7 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -201,6 +201,16 @@ def import_from_xml(store, data_dir, course_dirs=None, course_items = [] for course_id in module_store.modules.keys(): + if target_location_namespace is not None: + pseudo_course_id = '/'.join([target_location_namespace.org, target_location_namespace.course]) + else: + course_id_components = course_id.split('/') + pseudo_course_id = '/'.join([course_id_components[0], course_id_components[1]]) + + # turn off all write signalling while importing as this is a high volume operation + if pseudo_course_id not in store.ignore_write_events_on_courses: + store.ignore_write_events_on_courses.append(pseudo_course_id) + course_data_path = None course_location = None @@ -296,6 +306,12 @@ def import_from_xml(store, data_dir, course_dirs=None, # inherited metadata everywhere. store.update_metadata(module.location, dict(own_metadata(module))) + # turn back on all write signalling + if pseudo_course_id in store.ignore_write_events_on_courses: + store.ignore_write_events_on_courses.remove(pseudo_course_id) + store.refresh_cached_metadata_inheritance_tree(target_location_namespace if + target_location_namespace is not None else course_location) + return module_store, course_items def remap_namespace(module, target_location_namespace): From 2087b8157310f7a97af1ba29f9d3a74f67181eb4 Mon Sep 17 00:00:00 2001 From: Will Daly Date: Tue, 19 Mar 2013 09:16:54 -0400 Subject: [PATCH 04/24] Created MockXQueueServer which listens on a local port and simulates an XQueue. --- .../djangoapps/terrain/mock_xqueue_server.py | 227 ++++++++++++++++++ 1 file changed, 227 insertions(+) create mode 100644 common/djangoapps/terrain/mock_xqueue_server.py diff --git a/common/djangoapps/terrain/mock_xqueue_server.py b/common/djangoapps/terrain/mock_xqueue_server.py new file mode 100644 index 0000000000..50d77a2f19 --- /dev/null +++ b/common/djangoapps/terrain/mock_xqueue_server.py @@ -0,0 +1,227 @@ +from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler +import json +import urllib +import urlparse + + +class MockXQueueRequestHandler(BaseHTTPRequestHandler): + ''' + A handler for XQueue POST requests. + ''' + + protocol = "HTTP/1.0" + + def do_HEAD(self): + self._send_head() + + def do_POST(self): + ''' + Handle a POST request from the client, interpreted + as either a login request or a submission for grading request. + + Sends back an immediate success/failure response. + If grading is required, it then POSTS back to the client + with grading results, as configured in MockXQueueServer. + ''' + self._send_head() + + # Retrieve the POST data + post_dict = self._post_dict() + + # Send a response indicating success/failure + success = self._send_immediate_response(post_dict) + + # If the client submitted a valid submission request, + # we need to post back to the callback url + # with the grading result + if success and self._is_grade_request(): + self._send_grade_response(post_dict['lms_callback_url'], + post_dict['lms_key']) + + def _send_head(self): + ''' + Send the response code and MIME headers + ''' + if self._is_login_request() or self._is_grade_request(): + self.send_response(200) + else: + self.send_response(500) + + self.send_header('Content-type', 'text/plain') + self.end_headers() + + def _post_dict(self): + ''' + Retrieve the POST parameters from the client as a dictionary + ''' + + try: + length = int(self.headers.getheader('content-length')) + + post_dict = urlparse.parse_qs(self.rfile.read(length)) + + # The POST dict will contain a list of values + # for each key. + # None of our parameters are lists, however, + # so we map [val] --> val + # If the list contains multiple entries, + # we pick the first one + post_dict = dict(map(lambda (key, list_val): (key, list_val[0]), + post_dict.items())) + + except: + # We return an empty dict here, on the assumption + # that when we later check that the request has + # the correct fields, it won't find them, + # and will therefore send an error response + return {} + + return post_dict + + def _send_immediate_response(self, post_dict): + ''' + Check the post_dict for the appropriate fields + for this request (login or grade submission) + If it finds them, inform the client of success. + Otherwise, inform the client of failure + ''' + + # Allow any user to log in, as long as the POST + # dict has a username and password + if self._is_login_request(): + success = 'username' in post_dict and 'password' in post_dict + + elif self._is_grade_request(): + success = ('lms_callback_url' in post_dict and + 'lms_key' in post_dict and + 'queue_name' in post_dict) + else: + success = False + + # Send the response indicating success/failure + response_str = json.dumps({'return_code': 0 if success else 1, + 'content': '' if success else 'Error'}) + + self.wfile.write(response_str) + + return success + + def _send_grade_response(self, postback_url, queuekey): + ''' + POST the grade response back to the client + using the response provided by the server configuration + ''' + response_dict = {'queuekey': queuekey, + 'xqueue_body': self.server.grade_response} + + MockXQueueRequestHandler.post_to_url(postback_url, response_dict) + + def _is_login_request(self): + return 'xqueue/login' in self.path + + def _is_grade_request(self): + return 'xqueue/submit' in self.path + + @staticmethod + def post_to_url(url, param_dict): + ''' + POST *param_dict* to *url* + We make this a separate function so we can easily patch + it during testing. + ''' + urllib.urlopen(url, urllib.urlencode(param_dict)) + + +class MockXQueueServer(HTTPServer): + ''' + A mock XQueue grading server that responds + to POST requests to localhost. + ''' + + def __init__(self, port_num, grade_response_dict): + ''' + Initialize the mock XQueue server instance. + + *port_num* is the localhost port to listen to + + *grade_response_dict* is a dictionary that will be JSON-serialized + and sent in response to XQueue grading requests. + ''' + + self.grade_response = grade_response_dict + + handler = MockXQueueRequestHandler + address = ('', port_num) + HTTPServer.__init__(self, address, handler) + + @property + def grade_response(self): + return self._grade_response + + @grade_response.setter + def grade_response(self, grade_response_dict): + self._grade_response = grade_response_dict + + +# ---------------------------- +# Tests + +import mock +import threading +import unittest + + +class MockXQueueServerTest(unittest.TestCase): + + def setUp(self): + + # Create the server + server_port = 8034 + self.server_url = 'http://127.0.0.1:%d' % server_port + self.server = MockXQueueServer(server_port, + {'correct': True, 'score': 1, 'msg': ''}) + + # Start the server in a separate daemon thread + server_thread = threading.Thread(target=self.server.serve_forever) + server_thread.daemon = True + server_thread.start() + + def tearDown(self): + + # Stop the server, freeing up the port + self.server.shutdown() + self.server.socket.close() + + def test_login_request(self): + + # Send a login request + login_request = {'username': 'Test', 'password': 'Test'} + response_handle = urllib.urlopen(self.server_url + '/xqueue/login', + urllib.urlencode(login_request)) + response_dict = json.loads(response_handle.read()) + self.assertEqual(response_dict['return_code'], 0) + + def test_grade_request(self): + + # Patch post_to_url() so we can intercept + # outgoing POST requests from the server + MockXQueueRequestHandler.post_to_url = mock.Mock() + + # Send a grade request + callback_url = 'http://127.0.0.1:8000/test_callback' + grade_request = {'lms_callback_url': callback_url, + 'lms_key': 'test_queuekey', + 'queue_name': 'test_queue'} + response_handle = urllib.urlopen(self.server_url + '/xqueue/submit', + urllib.urlencode(grade_request)) + response_dict = json.loads(response_handle.read()) + + # Expect that the response is success + self.assertEqual(response_dict['return_code'], 0) + + # Expect that the server tries to post back the grading info + expected_callback_dict = {'queuekey': 'test_queuekey', + 'xqueue_body': {'correct': True, + 'score': 1, 'msg': ''}} + MockXQueueRequestHandler.post_to_url.assert_called_with(callback_url, + expected_callback_dict) From 20ccf5937bec59b0712b0c6a8cd3cdaa5f31b93b Mon Sep 17 00:00:00 2001 From: Will Daly Date: Tue, 19 Mar 2013 15:37:53 -0400 Subject: [PATCH 05/24] Implemented CodeResponse lettuce tests --- .../djangoapps/terrain/mock_xqueue_server.py | 169 ++++++++++++------ common/djangoapps/terrain/steps.py | 2 + common/djangoapps/terrain/xqueue_setup.py | 31 ++++ .../courseware/features/problems.feature | 12 +- .../courseware/features/problems.feature.bak | 86 +++++++++ .../courseware/features/problems.py | 56 +++++- lms/envs/acceptance.py | 12 ++ 7 files changed, 298 insertions(+), 70 deletions(-) create mode 100644 common/djangoapps/terrain/xqueue_setup.py create mode 100644 lms/djangoapps/courseware/features/problems.feature.bak diff --git a/common/djangoapps/terrain/mock_xqueue_server.py b/common/djangoapps/terrain/mock_xqueue_server.py index 50d77a2f19..49fee5167a 100644 --- a/common/djangoapps/terrain/mock_xqueue_server.py +++ b/common/djangoapps/terrain/mock_xqueue_server.py @@ -2,7 +2,10 @@ from BaseHTTPServer import HTTPServer, BaseHTTPRequestHandler import json import urllib import urlparse +import time +from logging import getLogger +logger = getLogger(__name__) class MockXQueueRequestHandler(BaseHTTPRequestHandler): ''' @@ -16,11 +19,10 @@ class MockXQueueRequestHandler(BaseHTTPRequestHandler): def do_POST(self): ''' - Handle a POST request from the client, interpreted - as either a login request or a submission for grading request. + Handle a POST request from the client Sends back an immediate success/failure response. - If grading is required, it then POSTS back to the client + It then POSTS back to the client with grading results, as configured in MockXQueueServer. ''' self._send_head() @@ -28,21 +30,60 @@ class MockXQueueRequestHandler(BaseHTTPRequestHandler): # Retrieve the POST data post_dict = self._post_dict() - # Send a response indicating success/failure - success = self._send_immediate_response(post_dict) + # Log the request + logger.debug("XQueue received POST request %s to path %s" % + (str(post_dict), self.path)) + + # Respond only to grading requests + if self._is_grade_request(): + try: + xqueue_header = json.loads(post_dict['xqueue_header']) + xqueue_body = json.loads(post_dict['xqueue_body']) + + callback_url = xqueue_header['lms_callback_url'] + + except KeyError: + # If the message doesn't have a header or body, + # then it's malformed. + # Respond with failure + error_msg = "XQueue received invalid grade request" + self._send_immediate_response(False, message=error_msg) + + except ValueError: + # If we could not decode the body or header, + # respond with failure + + error_msg = "XQueue could not decode grade request" + self._send_immediate_response(False, message=error_msg) + + else: + # Send an immediate response of success + # The grade request is formed correctly + self._send_immediate_response(True) + + # Wait a bit before POSTing back to the callback url with the + # grade result configured by the server + # Otherwise, the problem will not realize it's + # queued and it will keep waiting for a response + # indefinitely + delayed_grade_func = lambda: self._send_grade_response(callback_url, + xqueue_header) + + timer = threading.Timer(2, delayed_grade_func) + timer.start() + + # If we get a request that's not to the grading submission + # URL, return an error + else: + error_message = "Invalid request URL" + self._send_immediate_response(False, message=error_message) - # If the client submitted a valid submission request, - # we need to post back to the callback url - # with the grading result - if success and self._is_grade_request(): - self._send_grade_response(post_dict['lms_callback_url'], - post_dict['lms_key']) def _send_head(self): ''' Send the response code and MIME headers ''' - if self._is_login_request() or self._is_grade_request(): + if self._is_grade_request(): self.send_response(200) else: self.send_response(500) @@ -78,47 +119,34 @@ class MockXQueueRequestHandler(BaseHTTPRequestHandler): return post_dict - def _send_immediate_response(self, post_dict): + def _send_immediate_response(self, success, message=""): ''' - Check the post_dict for the appropriate fields - for this request (login or grade submission) - If it finds them, inform the client of success. - Otherwise, inform the client of failure + Send an immediate success/failure message + back to the client ''' - # Allow any user to log in, as long as the POST - # dict has a username and password - if self._is_login_request(): - success = 'username' in post_dict and 'password' in post_dict - - elif self._is_grade_request(): - success = ('lms_callback_url' in post_dict and - 'lms_key' in post_dict and - 'queue_name' in post_dict) - else: - success = False - # Send the response indicating success/failure response_str = json.dumps({'return_code': 0 if success else 1, - 'content': '' if success else 'Error'}) + 'content': message}) + + # Log the response + logger.debug("XQueue: sent response %s" % response_str) self.wfile.write(response_str) - return success - - def _send_grade_response(self, postback_url, queuekey): + def _send_grade_response(self, postback_url, xqueue_header): ''' POST the grade response back to the client using the response provided by the server configuration ''' - response_dict = {'queuekey': queuekey, - 'xqueue_body': self.server.grade_response} + response_dict = {'xqueue_header': json.dumps(xqueue_header), + 'xqueue_body': json.dumps(self.server.grade_response())} + + # Log the response + logger.debug("XQueue: sent grading response %s" % str(response_dict)) MockXQueueRequestHandler.post_to_url(postback_url, response_dict) - def _is_login_request(self): - return 'xqueue/login' in self.path - def _is_grade_request(self): return 'xqueue/submit' in self.path @@ -138,7 +166,8 @@ class MockXQueueServer(HTTPServer): to POST requests to localhost. ''' - def __init__(self, port_num, grade_response_dict): + def __init__(self, port_num, + grade_response_dict={'correct':True, 'score': 1, 'msg': ''}): ''' Initialize the mock XQueue server instance. @@ -148,18 +177,36 @@ class MockXQueueServer(HTTPServer): and sent in response to XQueue grading requests. ''' - self.grade_response = grade_response_dict + self.set_grade_response(grade_response_dict) handler = MockXQueueRequestHandler address = ('', port_num) HTTPServer.__init__(self, address, handler) - @property + def shutdown(self): + ''' + Stop the server and free up the port + ''' + # First call superclass shutdown() + HTTPServer.shutdown(self) + + # We also need to manually close the socket + self.socket.close() + def grade_response(self): return self._grade_response - @grade_response.setter - def grade_response(self, grade_response_dict): + def set_grade_response(self, grade_response_dict): + + # Check that the grade response has the right keys + assert('correct' in grade_response_dict and + 'score' in grade_response_dict and + 'msg' in grade_response_dict) + + # Wrap the message in
tags to ensure that it is valid XML + grade_response_dict['msg'] = "
%s
" % grade_response_dict['msg'] + + # Save the response dictionary self._grade_response = grade_response_dict @@ -190,16 +237,6 @@ class MockXQueueServerTest(unittest.TestCase): # Stop the server, freeing up the port self.server.shutdown() - self.server.socket.close() - - def test_login_request(self): - - # Send a login request - login_request = {'username': 'Test', 'password': 'Test'} - response_handle = urllib.urlopen(self.server_url + '/xqueue/login', - urllib.urlencode(login_request)) - response_dict = json.loads(response_handle.read()) - self.assertEqual(response_dict['return_code'], 0) def test_grade_request(self): @@ -209,19 +246,33 @@ class MockXQueueServerTest(unittest.TestCase): # Send a grade request callback_url = 'http://127.0.0.1:8000/test_callback' - grade_request = {'lms_callback_url': callback_url, - 'lms_key': 'test_queuekey', - 'queue_name': 'test_queue'} + + grade_header = json.dumps({'lms_callback_url': callback_url, + 'lms_key': 'test_queuekey', + 'queue_name': 'test_queue'}) + + grade_body = json.dumps({'student_info': 'test', + 'grader_payload': 'test', + 'student_response': 'test'}) + + grade_request = {'xqueue_header': grade_header, + 'xqueue_body': grade_body} + response_handle = urllib.urlopen(self.server_url + '/xqueue/submit', urllib.urlencode(grade_request)) + response_dict = json.loads(response_handle.read()) # Expect that the response is success self.assertEqual(response_dict['return_code'], 0) + # Wait a bit before checking that the server posted back + time.sleep(3) + # Expect that the server tries to post back the grading info - expected_callback_dict = {'queuekey': 'test_queuekey', - 'xqueue_body': {'correct': True, - 'score': 1, 'msg': ''}} + xqueue_body = json.dumps({'correct': True, 'score': 1, + 'msg': '
'}) + expected_callback_dict = {'xqueue_header': grade_header, + 'xqueue_body': xqueue_body } MockXQueueRequestHandler.post_to_url.assert_called_with(callback_url, expected_callback_dict) diff --git a/common/djangoapps/terrain/steps.py b/common/djangoapps/terrain/steps.py index 52eeb23c4a..b58c9fa7f2 100644 --- a/common/djangoapps/terrain/steps.py +++ b/common/djangoapps/terrain/steps.py @@ -10,6 +10,8 @@ import time import re import os.path +from .xqueue_setup import * + from logging import getLogger logger = getLogger(__name__) diff --git a/common/djangoapps/terrain/xqueue_setup.py b/common/djangoapps/terrain/xqueue_setup.py new file mode 100644 index 0000000000..003e4ebfa3 --- /dev/null +++ b/common/djangoapps/terrain/xqueue_setup.py @@ -0,0 +1,31 @@ +from mock_xqueue_server import MockXQueueServer +from lettuce import before, after, world +from django.conf import settings +import threading + +@before.all +def setup_mock_xqueue_server(): + + # Retrieve the local port from settings + server_port = settings.XQUEUE_PORT + + # Create the mock server instance + server = MockXQueueServer(server_port) + + # Start the server running in a separate daemon thread + # Because the thread is a daemon, it will terminate + # when the main thread terminates. + server_thread = threading.Thread(target=server.serve_forever) + server_thread.daemon = True + server_thread.start() + + # Store the server instance in lettuce's world + # so that other steps can access it + # (and we can shut it down later) + world.xqueue_server = server + +@after.all +def teardown_mock_xqueue_server(total): + + # Stop the xqueue server and free up the port + world.xqueue_server.shutdown() diff --git a/lms/djangoapps/courseware/features/problems.feature b/lms/djangoapps/courseware/features/problems.feature index a7fbac49c7..8ae03efb92 100644 --- a/lms/djangoapps/courseware/features/problems.feature +++ b/lms/djangoapps/courseware/features/problems.feature @@ -1,10 +1,11 @@ -Feature: Answer choice problems +Feature: Answer problems As a student in an edX course In order to test my understanding of the material - I want to answer choice based problems + I want to answer problems Scenario: I can answer a problem correctly - Given I am viewing a "" problem + Given External graders respond "correct" + And I am viewing a "" problem When I answer a "" problem "correctly" Then My "" answer is marked "correct" @@ -17,9 +18,11 @@ Feature: Answer choice problems | numerical | | formula | | script | + | code | Scenario: I can answer a problem incorrectly - Given I am viewing a "" problem + Given External graders respond "incorrect" + And I am viewing a "" problem When I answer a "" problem "incorrectly" Then My "" answer is marked "incorrect" @@ -32,6 +35,7 @@ Feature: Answer choice problems | numerical | | formula | | script | + | code | Scenario: I can submit a blank answer Given I am viewing a "" problem diff --git a/lms/djangoapps/courseware/features/problems.feature.bak b/lms/djangoapps/courseware/features/problems.feature.bak new file mode 100644 index 0000000000..1bb6eb087b --- /dev/null +++ b/lms/djangoapps/courseware/features/problems.feature.bak @@ -0,0 +1,86 @@ +Feature: Answer problems + As a student in an edX course + In order to test my understanding of the material + I want to answer problems + + Scenario: I can answer a problem correctly + Given I am viewing a "" problem + When I answer a "" problem "correctly" + Then My "" answer is marked "correct" + + Examples: + | ProblemType | + | drop down | + | multiple choice | + | checkbox | + | string | + | numerical | + | formula | + | script | + + Scenario: I can answer a problem incorrectly + Given I am viewing a "" problem + When I answer a "" problem "incorrectly" + Then My "" answer is marked "incorrect" + + Examples: + | ProblemType | + | drop down | + | multiple choice | + | checkbox | + | string | + | numerical | + | formula | + | script | + + Scenario: I can submit a blank answer + Given I am viewing a "" problem + When I check a problem + Then My "" answer is marked "incorrect" + + Examples: + | ProblemType | + | drop down | + | multiple choice | + | checkbox | + | string | + | numerical | + | formula | + | script | + + + Scenario: I can reset a problem + Given I am viewing a "" problem + And I answer a "" problem "ly" + When I reset the problem + Then My "" answer is marked "unanswered" + + Examples: + | ProblemType | Correctness | + | drop down | correct | + | drop down | incorrect | + | multiple choice | correct | + | multiple choice | incorrect | + | checkbox | correct | + | checkbox | incorrect | + | string | correct | + | string | incorrect | + | numerical | correct | + | numerical | incorrect | + | formula | correct | + | formula | incorrect | + | script | correct | + | script | incorrect | + + + Scenario: I can answer a code-based problem + Given I am viewing a "code" problem + And External graders respond "" with message "Test Message" + When I answer a "code" problem "" + Then My "code" answer is marked "" + And I should see "Test Message" somewhere in the page + + Examples: + | Correctness | + | correct | + | incorrect | diff --git a/lms/djangoapps/courseware/features/problems.py b/lms/djangoapps/courseware/features/problems.py index a6575c3d22..96c2f446d3 100644 --- a/lms/djangoapps/courseware/features/problems.py +++ b/lms/djangoapps/courseware/features/problems.py @@ -1,14 +1,17 @@ from lettuce import world, step from lettuce.django import django_url from selenium.webdriver.support.ui import Select +from selenium.common.exceptions import WebDriverException import random import textwrap +import time from common import i_am_registered_for_the_course, TEST_SECTION_NAME, section_location from terrain.factories import ItemFactory from capa.tests.response_xml_factory import OptionResponseXMLFactory, \ ChoiceResponseXMLFactory, MultipleChoiceResponseXMLFactory, \ StringResponseXMLFactory, NumericalResponseXMLFactory, \ - FormulaResponseXMLFactory, CustomResponseXMLFactory + FormulaResponseXMLFactory, CustomResponseXMLFactory, \ + CodeResponseXMLFactory # Factories from capa.tests.response_xml_factory that we will use # to generate the problem XML, with the keyword args used to configure @@ -77,7 +80,13 @@ PROBLEM_FACTORY_DICT = { a1=0 a2=0 return (a1+a2)==int(expect) - """)}}, + """) }}, + 'code': { + 'factory': CodeResponseXMLFactory(), + 'kwargs': { + 'question_text': 'Submit code to an external grader', + 'initial_display': 'print "Hello world!"', + 'grader_payload': '{"grader": "ps1/Spring2013/test_grader.py"}', }}, } @@ -115,6 +124,18 @@ def view_problem(step, problem_type): world.browser.visit(url) +@step(u'External graders respond "([^"]*)"') +def set_external_grader_response(step, correctness): + assert(correctness in ['correct', 'incorrect']) + + response_dict = {'correct': True if correctness == 'correct' else False, + 'score': 1 if correctness == 'correct' else 0, + 'msg': 'Your problem was graded %s' % correctness} + + # Set the fake xqueue server to always respond + # correct/incorrect when asked to grade a problem + world.xqueue_server.set_grade_response(response_dict) + @step(u'I answer a "([^"]*)" problem "([^"]*)ly"') def answer_problem(step, problem_type, correctness): @@ -169,13 +190,32 @@ def answer_problem(step, problem_type, correctness): inputfield('script', input_num=1).fill(str(first_addend)) inputfield('script', input_num=2).fill(str(second_addend)) + elif problem_type == 'code': + # The fake xqueue server is configured to respond + # correct / incorrect no matter what we submit. + # Furthermore, since the inline code response uses + # JavaScript to make the code display nicely, it's difficult + # to programatically input text + # (there's not