From 838a8624c9e392cfc0c991ebd7949721a8d8741b Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 16 Jul 2012 11:28:26 -0400 Subject: [PATCH 01/27] Add docstring to 'score_update' path --- common/lib/capa/capa/capa_problem.py | 8 +++++++- common/lib/capa/capa/responsetypes.py | 11 ++++++++--- common/lib/xmodule/xmodule/capa_module.py | 9 +++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 344843ba10..152ede5b99 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -180,9 +180,15 @@ class LoncapaProblem(object): 'total': self.get_max_score()} def update_score(self, score_msg): + ''' + Deliver grading response (e.g. from async code checking) to + the specific ResponseType + + Returns an updated CorrectMap + ''' newcmap = CorrectMap() for responder in self.responders.values(): - if hasattr(responder,'update_score'): # Is this the best way to implement 'update_score' for CodeResponse? + if hasattr(responder,'update_score'): # TODO: Is this the best way to target 'update_score' of CodeResponse? results = responder.update_score(score_msg) newcmap.update(results) self.correct_map = newcmap diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index b645a2faa7..c63e8ec9fb 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -696,7 +696,9 @@ class SymbolicResponse(CustomResponse): class CodeResponse(LoncapaResponse): ''' - Grade student code using an external server + Grade student code using an external server. Unlike ExternalResponse, CodeResponse: + 1) Goes through a queueing system (xqueue) + 2) Does not do external request for 'get_answers' ''' response_tag = 'coderesponse' @@ -704,9 +706,10 @@ class CodeResponse(LoncapaResponse): def setup_response(self): xml = self.xml - self.url = xml.get('url') or "http://ec2-50-16-59-149.compute-1.amazonaws.com/xqueue/submit/" # FIXME -- hardcoded url + self.url = xml.get('url', "http://ec2-50-16-59-149.compute-1.amazonaws.com/xqueue/submit/") # FIXME -- hardcoded url answer = xml.find('answer') + if answer is not None: answer_src = answer.get('src') if answer_src is not None: @@ -791,7 +794,9 @@ class CodeResponse(LoncapaResponse): # Prepare payload xmlstr = etree.tostring(self.xml, pretty_print=True) header = { 'return_url': self.system.xqueue_callback_url } - header.update({'timestamp': time.time()}) +# header.update({'timestamp': time.time()}) + random.seed() + header.update({'key': random.randint(0,2**32-1)}) payload = {'xqueue_header': json.dumps(header), # 'xqueue_header' should eventually be derived from xqueue.queue_common.HEADER_TAG or something similar 'xml': xmlstr, 'edX_cmd': 'get_score', diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 90522f7ca5..4e41b73b94 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -323,6 +323,15 @@ class CapaModule(XModule): raise self.system.exception404 def update_score(self, get): + """ + Delivers grading response (e.g. from asynchronous code checking) to + the capa problem, so its score can be updated + + 'get' must have a field 'response' which is a string that contains the + grader's response + + No ajax return is needed. Return empty dict. + """ score_msg = get['response'] self.lcp.update_score(score_msg) From 093ac9d1012dd676a0516b835dd986adea9a82f4 Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 16 Jul 2012 16:19:35 -0400 Subject: [PATCH 02/27] CorrectMap in LMS keeps track of problems as being queued --- common/lib/capa/capa/capa_problem.py | 17 ++++++-- common/lib/capa/capa/correctmap.py | 12 +++++- common/lib/capa/capa/responsetypes.py | 45 +++++++++++++--------- common/lib/xmodule/xmodule/capa_module.py | 12 +++--- lms/djangoapps/courseware/module_render.py | 11 ++++-- 5 files changed, 65 insertions(+), 32 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 152ede5b99..00cfc8120f 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -179,21 +179,32 @@ class LoncapaProblem(object): return {'score': correct, 'total': self.get_max_score()} - def update_score(self, score_msg): + def update_score(self, score_msg, queuekey): ''' Deliver grading response (e.g. from async code checking) to - the specific ResponseType + the specific ResponseType that requested grading Returns an updated CorrectMap ''' + oldcmap = self.correct_map newcmap = CorrectMap() for responder in self.responders.values(): if hasattr(responder,'update_score'): # TODO: Is this the best way to target 'update_score' of CodeResponse? - results = responder.update_score(score_msg) + results = responder.update_score(score_msg, oldcmap, queuekey) newcmap.update(results) self.correct_map = newcmap return newcmap + def is_queued(self): + ''' + Returns True if any part of the problem has been submitted to an external queue + ''' + queued = False + for answer_id in self.correct_map: + if self.correct_map.is_queued(answer_id): + queued = True + return queued + def grade_answers(self, answers): ''' Grade student responses. Called by capa_module.check_problem. diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py index 786b2f5e2d..c0426caffb 100644 --- a/common/lib/capa/capa/correctmap.py +++ b/common/lib/capa/capa/correctmap.py @@ -14,6 +14,7 @@ class CorrectMap(object): - msg : string (may have HTML) giving extra message response (displayed below textline or textbox) - hint : string (may have HTML) giving optional hint (displayed below textline or textbox, above msg) - hintmode : one of (None,'on_request','always') criteria for displaying hint + - queuekey : a random integer for xqueue_callback verification Behaves as a dict. ''' @@ -29,13 +30,14 @@ class CorrectMap(object): def __iter__(self): return self.cmap.__iter__() - def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None): + def set(self, answer_id=None, correctness=None, npoints=None, msg='', hint='', hintmode=None, queuekey=None): if answer_id is not None: self.cmap[answer_id] = {'correctness': correctness, 'npoints': npoints, 'msg': msg, 'hint' : hint, 'hintmode' : hintmode, + 'queuekey' : queuekey, } def __repr__(self): @@ -63,6 +65,14 @@ class CorrectMap(object): if answer_id in self.cmap: return self.cmap[answer_id]['correctness'] == 'correct' return None + def is_queued(self,answer_id): + if answer_id in self.cmap: return self.cmap[answer_id]['queuekey'] is not None + return None + + def is_right_queuekey(self, answer_id, test_key): + if answer_id in self.cmap: return self.cmap[answer_id]['queuekey'] == test_key + return None + def get_npoints(self,answer_id): if self.is_correct(answer_id): npoints = self.cmap[answer_id].get('npoints',1) # default to 1 point if correct diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index c63e8ec9fb..8c31972f70 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -18,7 +18,6 @@ import re import requests import traceback import abc -import time # specific library imports from calc import evaluator, UndefinedVariable @@ -709,7 +708,6 @@ class CodeResponse(LoncapaResponse): self.url = xml.get('url', "http://ec2-50-16-59-149.compute-1.amazonaws.com/xqueue/submit/") # FIXME -- hardcoded url answer = xml.find('answer') - if answer is not None: answer_src = answer.get('src') if answer_src is not None: @@ -727,7 +725,7 @@ class CodeResponse(LoncapaResponse): def get_score(self, student_answers): idset = sorted(self.answer_ids) - + try: submission = [student_answers[k] for k in idset] except Exception as err: @@ -737,12 +735,16 @@ class CodeResponse(LoncapaResponse): self.context.update({'submission': submission}) extra_payload = {'edX_student_response': json.dumps(submission)} - # Should do something -- like update the problem state -- based on the queue response - r = self._send_to_queue(extra_payload) - - return CorrectMap() + r, queuekey = self._send_to_queue(extra_payload) # TODO: Perform checks on the xqueue response - def update_score(self, score_msg): + # Non-null CorrectMap['queuekey'] indicates that the problem has been submitted + cmap = CorrectMap() + for answer_id in idset: + cmap.set(answer_id, queuekey=queuekey) + + return cmap + + def update_score(self, score_msg, oldcmap, queuekey): # Parse 'score_msg' as XML try: rxml = etree.fromstring(score_msg) @@ -752,7 +754,6 @@ class CodeResponse(LoncapaResponse): # The following process is lifted directly from ExternalResponse idset = sorted(self.answer_ids) - cmap = CorrectMap() ad = rxml.find('awarddetail').text admap = {'EXACT_ANS':'correct', # TODO: handle other loncapa responses 'WRONG_FORMAT': 'incorrect', @@ -761,13 +762,17 @@ class CodeResponse(LoncapaResponse): if ad in admap: self.context['correct'][0] = admap[ad] - # create CorrectMap - for key in idset: - idx = idset.index(key) - msg = rxml.find('message').text.replace(' ',' ') if idx==0 else None - cmap.set(key, self.context['correct'][idx], msg=msg) + # Replace 'oldcmap' with new grading results if queuekey matches + # If queuekey does not match, we keep waiting for the score_msg that will match + for answer_id in idset: + if oldcmap.is_right_queuekey(answer_id, queuekey): + idx = idset.index(answer_id) + msg = rxml.find('message').text.replace(' ',' ') if idx==0 else None + oldcmap.set(answer_id, self.context['correct'][idx], msg=msg) + else: # Queuekey does not match + log.debug('CodeResponse: queuekey %d does not match for answer_id=%s.' % (queuekey, answer_id)) - return cmap + return oldcmap # CodeResponse differentiates from ExternalResponse in the behavior of 'get_answers'. CodeResponse.get_answers # does NOT require a queue submission, and the answer is computed (extracted from problem XML) locally. @@ -794,10 +799,12 @@ class CodeResponse(LoncapaResponse): # Prepare payload xmlstr = etree.tostring(self.xml, pretty_print=True) header = { 'return_url': self.system.xqueue_callback_url } -# header.update({'timestamp': time.time()}) + random.seed() - header.update({'key': random.randint(0,2**32-1)}) - payload = {'xqueue_header': json.dumps(header), # 'xqueue_header' should eventually be derived from xqueue.queue_common.HEADER_TAG or something similar + queuekey = random.randint(0,2**32-1) + header.update({'queuekey': queuekey}) + + payload = {'xqueue_header': json.dumps(header), # TODO: 'xqueue_header' should eventually be derived from config file 'xml': xmlstr, 'edX_cmd': 'get_score', 'edX_tests': self.tests, @@ -813,7 +820,7 @@ class CodeResponse(LoncapaResponse): log.error(msg) raise Exception(msg) - return r + return r, queuekey #----------------------------------------------------------------------------- diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 4e41b73b94..ba72ad1446 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -332,8 +332,9 @@ class CapaModule(XModule): No ajax return is needed. Return empty dict. """ + queuekey = get['queuekey'] score_msg = get['response'] - self.lcp.update_score(score_msg) + self.lcp.update_score(score_msg, queuekey) return dict() # No AJAX return is needed @@ -433,10 +434,11 @@ class CapaModule(XModule): if not correct_map.is_correct(answer_id): success = 'incorrect' - # log this in the track_function - event_info['correct_map'] = correct_map.get_dict() - event_info['success'] = success - self.system.track_function('save_problem_check', event_info) + # log this in the track_function, ONLY if a full grading has been performed (e.g. not queueing) + if not self.lcp.is_queued(): + event_info['correct_map'] = correct_map.get_dict() + event_info['success'] = success + self.system.track_function('save_problem_check', event_info) # render problem into HTML html = self.get_problem_html(encapsulate=False) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index a0281626ba..b2810e8759 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -33,6 +33,8 @@ class I4xSystem(object): Create a closure around the system environment. ajax_url - the url where ajax calls to the encapsulating module go. + xqueue_callback_url - the url where external queueing system (e.g. for grading) + returns its response track_function - function of (event_type, event), intended for logging or otherwise tracking the event. TODO: Not used, and has inconsistent args in different @@ -324,7 +326,7 @@ def add_histogram(module): module.get_html = get_html return module -# THK: TEMPORARY BYPASS OF AUTH! +# TODO: TEMPORARY BYPASS OF AUTH! from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.models import User @csrf_exempt @@ -336,9 +338,6 @@ def xqueue_callback(request, username, id, dispatch): except Exception as err: msg = "Error in xqueue_callback %s: Invalid return format" % err raise Exception(msg) - - # Should proceed only when the request timestamp is more recent than problem timestamp - timestamp = header['timestamp'] # Retrieve target StudentModule user = User.objects.get(username=username) @@ -354,6 +353,10 @@ def xqueue_callback(request, username, id, dispatch): oldgrade = instance_module.grade old_instance_state = instance_module.state + # Transfer 'queuekey' from xqueue response header to 'get'. This is required to + # use the interface defined by 'handle_ajax' + get.update({'queuekey': header['queuekey']}) + # We go through the "AJAX" path # So far, the only dispatch from xqueue will be 'score_update' try: From 0958485b46fa9ff6d3f9a61a59e3d036d75ac245 Mon Sep 17 00:00:00 2001 From: kimth Date: Tue, 17 Jul 2012 10:31:21 -0400 Subject: [PATCH 03/27] update_score: edit existing CorrectMap rather than starting with empty CorrectMap --- common/lib/capa/capa/capa_problem.py | 12 ++++++------ common/lib/capa/capa/responsetypes.py | 9 +++++---- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 00cfc8120f..9f7b7e4daa 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -22,6 +22,7 @@ import random import re import scipy import struct +import json from lxml import etree from xml.sax.saxutils import unescape @@ -186,14 +187,13 @@ class LoncapaProblem(object): Returns an updated CorrectMap ''' - oldcmap = self.correct_map - newcmap = CorrectMap() + cmap = self.correct_map for responder in self.responders.values(): if hasattr(responder,'update_score'): # TODO: Is this the best way to target 'update_score' of CodeResponse? - results = responder.update_score(score_msg, oldcmap, queuekey) - newcmap.update(results) - self.correct_map = newcmap - return newcmap + # Each LoncapaResponse will update the specific entries of 'cmap' that it's responsible for + cmap = responder.update_score(score_msg, cmap, queuekey) + self.correct_map = cmap + return cmap def is_queued(self): ''' diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 8c31972f70..28758b09b8 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -695,8 +695,9 @@ class SymbolicResponse(CustomResponse): class CodeResponse(LoncapaResponse): ''' - Grade student code using an external server. Unlike ExternalResponse, CodeResponse: - 1) Goes through a queueing system (xqueue) + Grade student code using an external server, called 'xqueue' + In contrast to ExternalResponse, CodeResponse has following behavior: + 1) Goes through a queueing system 2) Does not do external request for 'get_answers' ''' @@ -740,7 +741,7 @@ class CodeResponse(LoncapaResponse): # Non-null CorrectMap['queuekey'] indicates that the problem has been submitted cmap = CorrectMap() for answer_id in idset: - cmap.set(answer_id, queuekey=queuekey) + cmap.set(answer_id, queuekey=queuekey, msg='Submitted to queue') return cmap @@ -804,7 +805,7 @@ class CodeResponse(LoncapaResponse): queuekey = random.randint(0,2**32-1) header.update({'queuekey': queuekey}) - payload = {'xqueue_header': json.dumps(header), # TODO: 'xqueue_header' should eventually be derived from config file + payload = {'xqueue_header': json.dumps(header), # TODO: 'xqueue_header' should eventually be derived from a config file 'xml': xmlstr, 'edX_cmd': 'get_score', 'edX_tests': self.tests, From 269468ba6ef7cedd9130bd05e435c29dbccb3dac Mon Sep 17 00:00:00 2001 From: kimth Date: Tue, 17 Jul 2012 10:47:10 -0400 Subject: [PATCH 04/27] CorrectMap.is_queued and .is_right_queuekey defaults to False if answer_id not found --- common/lib/capa/capa/correctmap.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/common/lib/capa/capa/correctmap.py b/common/lib/capa/capa/correctmap.py index c0426caffb..11c5bb75f1 100644 --- a/common/lib/capa/capa/correctmap.py +++ b/common/lib/capa/capa/correctmap.py @@ -66,12 +66,10 @@ class CorrectMap(object): return None def is_queued(self,answer_id): - if answer_id in self.cmap: return self.cmap[answer_id]['queuekey'] is not None - return None + return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] is not None def is_right_queuekey(self, answer_id, test_key): - if answer_id in self.cmap: return self.cmap[answer_id]['queuekey'] == test_key - return None + return answer_id in self.cmap and self.cmap[answer_id]['queuekey'] == test_key def get_npoints(self,answer_id): if self.is_correct(answer_id): From 803b0c4fd087324570c6953d5b057493a660998b Mon Sep 17 00:00:00 2001 From: kimth Date: Tue, 17 Jul 2012 10:59:27 -0400 Subject: [PATCH 05/27] Callback uses userid rather than username --- lms/djangoapps/courseware/module_render.py | 6 +++--- lms/urls.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index b2810e8759..25f3aabab6 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -210,7 +210,7 @@ def get_module(user, request, location, student_module_cache, position=None): # Setup system context for module instance ajax_url = settings.MITX_ROOT_URL + '/modx/' + descriptor.location.url() + '/' - xqueue_callback_url = settings.MITX_ROOT_URL + '/xqueue/' + user.username + '/' + descriptor.location.url() + '/' + xqueue_callback_url = settings.MITX_ROOT_URL + '/xqueue/' + str(user.id) + '/' + descriptor.location.url() + '/' def _get_module(location): (module, _, _, _) = get_module(user, request, location, student_module_cache, position) @@ -330,7 +330,7 @@ def add_histogram(module): from django.views.decorators.csrf import csrf_exempt from django.contrib.auth.models import User @csrf_exempt -def xqueue_callback(request, username, id, dispatch): +def xqueue_callback(request, userid, id, dispatch): # Parse xqueue response get = request.POST.copy() try: @@ -340,7 +340,7 @@ def xqueue_callback(request, username, id, dispatch): raise Exception(msg) # Retrieve target StudentModule - user = User.objects.get(username=username) + user = User.objects.get(id=userid) student_module_cache = StudentModuleCache(user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) diff --git a/lms/urls.py b/lms/urls.py index 90ccca233a..2eb23d19e5 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -55,7 +55,7 @@ if settings.COURSEWARE_ENABLED: url(r'^masquerade/', include('masquerade.urls')), url(r'^jumpto/(?P[^/]+)/$', 'courseware.views.jump_to'), url(r'^modx/(?P.*?)/(?P[^/]*)$', 'courseware.module_render.modx_dispatch'), #reset_problem'), - url(r'^xqueue/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', 'courseware.module_render.xqueue_callback'), + url(r'^xqueue/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', 'courseware.module_render.xqueue_callback'), url(r'^change_setting$', 'student.views.change_setting'), url(r'^s/(?P