From fac76593159f5b2345df71b7a46e1e417256d51a Mon Sep 17 00:00:00 2001 From: kimth Date: Wed, 1 Aug 2012 10:23:30 -0400 Subject: [PATCH 01/36] Use xqueue cname --- common/lib/capa/capa/responsetypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 75ed92ab3e..812ffbc72c 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -810,7 +810,7 @@ class CodeResponse(LoncapaResponse): def setup_response(self): xml = self.xml - self.url = xml.get('url', "http://107.20.215.194/xqueue/submit/") # FIXME -- hardcoded url + self.url = xml.get('url', "http://xqueue.edx.org/xqueue/submit/") # FIXME -- hardcoded url self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename) answer = xml.find('answer') From 8a983b3ced91f69c4d619949203c3c2786ea47e9 Mon Sep 17 00:00:00 2001 From: kimth Date: Wed, 1 Aug 2012 10:39:09 -0400 Subject: [PATCH 02/36] Queuekey does not need to be integer --- common/lib/capa/capa/responsetypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 812ffbc72c..bb62cdfa1c 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -888,7 +888,7 @@ class CodeResponse(LoncapaResponse): msg = rxml.find('message').text.replace(' ', ' ') oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None) # Queuekey is consumed else: - log.debug('CodeResponse: queuekey %d does not match for answer_id=%s.' % (queuekey, self.answer_id)) + log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id)) return oldcmap @@ -913,7 +913,7 @@ class CodeResponse(LoncapaResponse): h = hashlib.md5() h.update(str(self.system.seed)) h.update(str(time.time())) - queuekey = int(h.hexdigest(), 16) + queuekey = h.hexdigest() header.update({'lms_key': queuekey}) body = {'xml': xmlstr, From 541f5ecdd28a1fdf9fc3e264886153f0e10c007d Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 14:32:40 -0400 Subject: [PATCH 03/36] Filesubmission frontend --- common/lib/capa/capa/capa_problem.py | 2 +- common/lib/capa/capa/inputtypes.py | 13 +++++++ common/lib/capa/capa/responsetypes.py | 2 +- .../capa/capa/templates/filesubmission.html | 3 ++ .../xmodule/js/src/capa/display.coffee | 34 ++++++++++++++++++- lms/djangoapps/courseware/module_render.py | 10 ++++++ 6 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 common/lib/capa/capa/templates/filesubmission.html diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index ed99c71635..55b9096021 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -39,7 +39,7 @@ import responsetypes # dict of tagname, Response Class -- this should come from auto-registering response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__]) -entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup'] +entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup', 'filesubmission'] solution_types = ['solution'] # extra things displayed after "show answers" is pressed response_properties = ["responseparam", "answer"] # these get captured as student responses diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index 8b3867be5b..32ee479414 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -299,6 +299,19 @@ def textline_dynamath(element, value, status, render_template, msg=''): return etree.XML(html) +#----------------------------------------------------------------------------- +@register_render_function +def filesubmission(element, value, status, render_template, msg=''): + ''' + Upload a single file (e.g. for programming assignments) + ''' + eid = element.get('id') + + context = {'id': eid, } + html = render_template("filesubmission.html", context) + return etree.XML(html) + + #----------------------------------------------------------------------------- ## TODO: Make a wrapper for @register_render_function diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index bb62cdfa1c..578352eb44 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -805,7 +805,7 @@ class CodeResponse(LoncapaResponse): ''' response_tag = 'coderesponse' - allowed_inputfields = ['textline', 'textbox'] + allowed_inputfields = ['textbox', 'filesubmission'] max_inputfields = 1 def setup_response(self): diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html new file mode 100644 index 0000000000..f9073799d4 --- /dev/null +++ b/common/lib/capa/capa/templates/filesubmission.html @@ -0,0 +1,3 @@ +
+
+
diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 4ee8257e36..d23c096eec 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -13,7 +13,7 @@ class @Problem MathJax.Hub.Queue ["Typeset", MathJax.Hub] window.update_schematics() @$('section.action input:button').click @refreshAnswers - @$('section.action input.check').click @check + @$('section.action input.check').click @check_fd @$('section.action input.reset').click @reset @$('section.action input.show').click @show @$('section.action input.save').click @save @@ -45,6 +45,38 @@ class @Problem $('head')[0].appendChild(s[0]) $(placeholder).remove() + check_fd: => + Logger.log 'problem_check', @answers + + if not window.FormData + alert "Sorry, your browser does not support file uploads. If you can, please use Chrome or Safari which have been verified to support this feature." + return + + fd = new FormData() + + # For each file input, allow a single file submission, + # routed to Django 'request.FILES' + @$('input:file').each (index, element) -> + fd.append(element.id, element.files[0]) + + # Simple (non-file) answers, + # routed to Django 'request.POST' + fd.append('answers', @answers) + + settings = + type: "POST" + data: fd + processData: false + contentType: false + success: (response) -> + switch response.success + when 'incorrect', 'correct' + @render(response.contents) + @updateProgress response + else + alert(response.success) + $.ajaxWithPrefix("#{@url}/problem_check", settings) + check: => Logger.log 'problem_check', @answers $.postWithPrefix "#{@url}/problem_check", @answers, (response) => diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 80a4ef90fc..5b030161db 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -270,6 +270,16 @@ def modx_dispatch(request, dispatch=None, id=None): - id -- the module id. Used to look up the XModule instance ''' # ''' (fix emacs broken parsing) + + print ' THK: module_render.modx_dispatch' + print dispatch + print request.POST.keys() + print request.FILES.keys() + if request.POST.has_key('answers'): + print request.POST['answers'] + for filename in request.FILES.keys(): + uploadedFile = request.FILES.get(filename) + print uploadedFile.read() student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) From c7084e5240d1fe9292b5df9177e6d2429464b548 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 15:35:25 -0400 Subject: [PATCH 04/36] Comments for 'check_fd' --- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index d23c096eec..0ad1bdbba8 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -45,6 +45,11 @@ class @Problem $('head')[0].appendChild(s[0]) $(placeholder).remove() + ### + 'check_fd' uses FormData to allow file submissions through an AJAX call. + NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; + can consolidate all dispatches to use FormData consistently + ### check_fd: => Logger.log 'problem_check', @answers From 6de2fa5e1e5178cacf61a4fe2517788281507aae Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 17:10:35 -0400 Subject: [PATCH 05/36] LMS interface to xqueue in separate file --- common/lib/capa/capa/responsetypes.py | 76 ++++++++----------- lms/djangoapps/courseware/xqueue_interface.py | 51 +++++++++++++ 2 files changed, 83 insertions(+), 44 deletions(-) create mode 100644 lms/djangoapps/courseware/xqueue_interface.py diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 578352eb44..2734e07cd3 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -24,6 +24,7 @@ import abc # specific library imports from calc import evaluator, UndefinedVariable from correctmap import CorrectMap +from courseware import xqueue_interface from util import * from lxml import etree from lxml.html.soupparser import fromstring as fromstring_bs # uses Beautiful Soup!!! FIXME? @@ -798,10 +799,10 @@ class SymbolicResponse(CustomResponse): class CodeResponse(LoncapaResponse): ''' - Grade student code using an external server, called 'xqueue' - In contrast to ExternalResponse, CodeResponse has following behavior: - 1) Goes through a queueing system - 2) Does not do external request for 'get_answers' + Grade student code using an external queueing server, called 'xqueue' + + External requests are only submitted for student submission grading + (i.e. and not for getting reference answers) ''' response_tag = 'coderesponse' @@ -857,9 +858,33 @@ class CodeResponse(LoncapaResponse): self.context.update({'submission': submission}) extra_payload = {'edX_student_response': submission} - r, queuekey = self._send_to_queue(extra_payload) # TODO: Perform checks on the xqueue response + # Prepare xqueue request + #------------------------------------------------------------ - # Non-null CorrectMap['queuekey'] indicates that the problem has been submitted + # Queuekey generation + h = hashlib.md5() + h.update(str(self.system.seed)) + h.update(str(time.time())) + queuekey = h.hexdigest() + + # Generate header + xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue_callback_url, + lms_key=queuekey, + queue_name=self.queue_name) + + # Generate body + # NOTE: Currently specialized to 6.00x's pyxservers, which follows the ExternalResponse interface + contents = {'xml': etree.tostring(self.xml, pretty_print=True), + 'edX_cmd': 'get_score', + 'edX_tests': self.tests, + 'processor': self.code, + 'edX_student_response': submission} + + # Submit request + xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) + + # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap = CorrectMap() cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to queue') @@ -883,7 +908,7 @@ class CodeResponse(LoncapaResponse): self.context['correct'][0] = admap[ad] # Replace 'oldcmap' with new grading results if queuekey matches. - # If queuekey does not match, we keep waiting for the score_msg that will match + # If queuekey does not match, we keep waiting for the score_msg whose key actually matchs if oldcmap.is_right_queuekey(self.answer_id, queuekey): msg = rxml.find('message').text.replace(' ', ' ') oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None) # Queuekey is consumed @@ -892,8 +917,6 @@ class CodeResponse(LoncapaResponse): return oldcmap - # CodeResponse differentiates from ExternalResponse in the behavior of 'get_answers'. CodeResponse.get_answers - # does NOT require a queue submission, and the answer is computed (extracted from problem XML) locally. def get_answers(self): anshtml = '
%s

' % self.answer return {self.answer_id: anshtml} @@ -901,41 +924,6 @@ class CodeResponse(LoncapaResponse): def get_initial_display(self): return {self.answer_id: self.initial_display} - # CodeResponse._send_to_queue implements the same interface as defined for ExternalResponse's 'get_score' - def _send_to_queue(self, extra_payload): - # Prepare payload - xmlstr = etree.tostring(self.xml, pretty_print=True) - header = {'lms_callback_url': self.system.xqueue_callback_url, - 'queue_name': self.queue_name, - } - - # Queuekey generation - h = hashlib.md5() - h.update(str(self.system.seed)) - h.update(str(time.time())) - queuekey = h.hexdigest() - header.update({'lms_key': queuekey}) - - body = {'xml': xmlstr, - 'edX_cmd': 'get_score', - 'edX_tests': self.tests, - 'processor': self.code, - } - body.update(extra_payload) - - payload = {'xqueue_header': json.dumps(header), - 'xqueue_body' : json.dumps(body), - } - - # Contact queue server - try: - r = requests.post(self.url, data=payload) - except Exception as err: - msg = "Error in CodeResponse %s: cannot connect to queue server url=%s" % (err, self.url) - log.error(msg) - raise Exception(msg) - - return r, queuekey #----------------------------------------------------------------------------- diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py new file mode 100644 index 0000000000..6384adb947 --- /dev/null +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -0,0 +1,51 @@ +# +# LMS Interface to external queueing system (xqueue) +# +import json +import requests + +# TODO: Collection of parameters to be hooked into rest of edX system +XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org/xqueue/submit/' + +def upload_files_to_s3(): + print ' THK: xqueue_interface.upload_files_to_s3' + + +def make_xheader(lms_callback_url, lms_key, queue_name): + ''' + Generate header for delivery and reply of queue request. + + Xqueue header is a JSON-serialized dict: + { 'lms_callback_url': url to which xqueue will return the request (string), + 'lms_key': secret key used by LMS to protect its state (string), + 'queue_name': designate a specific queue within xqueue server, e.g. 'MITx-6.00x' (string) + } + ''' + return json.dumps({ 'lms_callback_url': lms_callback_url, + 'lms_key': lms_key, + 'queue_name': queue_name }) + + +def send_to_queue(header, body, xqueue_url=None): + ''' + Submit a request to xqueue. + + header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader' + + body: Serialized data for the receipient behind the queueing service. The operation of + xqueue is agnostic to the contents of 'body' + + ''' + if xqueue_url is None: + xqueue_url = XQUEUE_SUBMIT_URL + + # Contact queue server + payload = {'xqueue_header': header, + 'xqueue_body' : body} + try: + r = requests.post(xqueue_url, data=payload) + except Exception as err: + msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) + raise Exception(msg) + + #print r.text From d2db4134cdb22dcfc7f0616a6f55c0f93ec82371 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 17:28:12 -0400 Subject: [PATCH 06/36] CodeResponse does basic error handling from xqueue submission --- common/lib/capa/capa/responsetypes.py | 15 +++++++++------ lms/djangoapps/courseware/xqueue_interface.py | 6 +++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 2734e07cd3..d8a3940624 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -881,12 +881,15 @@ class CodeResponse(LoncapaResponse): 'edX_student_response': submission} # Submit request - xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + success = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) - # Non-null CorrectMap['queuekey'] indicates that the problem has been queued - cmap = CorrectMap() - cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to queue') + cmap = CorrectMap() + if success: + # Non-null CorrectMap['queuekey'] indicates that the problem has been queued + cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') + else: + cmap.set(self.answer_id, msg='Unable to deliver submission to grader! Please try again later') return cmap @@ -908,7 +911,7 @@ class CodeResponse(LoncapaResponse): self.context['correct'][0] = admap[ad] # Replace 'oldcmap' with new grading results if queuekey matches. - # If queuekey does not match, we keep waiting for the score_msg whose key actually matchs + # If queuekey does not match, we keep waiting for the score_msg whose key actually matches if oldcmap.is_right_queuekey(self.answer_id, queuekey): msg = rxml.find('message').text.replace(' ', ' ') oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None) # Queuekey is consumed diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 6384adb947..3779a3eb94 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -35,6 +35,7 @@ def send_to_queue(header, body, xqueue_url=None): body: Serialized data for the receipient behind the queueing service. The operation of xqueue is agnostic to the contents of 'body' + Returns a 'success' flag indicating successful submission ''' if xqueue_url is None: xqueue_url = XQUEUE_SUBMIT_URL @@ -48,4 +49,7 @@ def send_to_queue(header, body, xqueue_url=None): msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) - #print r.text + # Xqueue responses are JSON-serialized dicts + xreply = json.loads(r.text) + + return xreply['return_code'] == 0 # return_code == 0 from xqueue indicates successful submission From 988136c401d499d582189d906c895c85b7044524 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 18:34:14 -0400 Subject: [PATCH 07/36] LMS must login before submitting to xqueue --- common/lib/capa/capa/responsetypes.py | 10 +++--- lms/djangoapps/courseware/xqueue_interface.py | 36 +++++++++++++++---- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index d8a3940624..50b5df69d9 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -881,15 +881,15 @@ class CodeResponse(LoncapaResponse): 'edX_student_response': submission} # Submit request - success = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + error = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) cmap = CorrectMap() - if success: + if error: + cmap.set(self.answer_id, msg='Unable to deliver your submission to grader! Please try again later') + else: # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') - else: - cmap.set(self.answer_id, msg='Unable to deliver submission to grader! Please try again later') return cmap diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 3779a3eb94..c9042c3f1e 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -5,7 +5,8 @@ import json import requests # TODO: Collection of parameters to be hooked into rest of edX system -XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org/xqueue/submit/' +XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) +XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' def upload_files_to_s3(): print ' THK: xqueue_interface.upload_files_to_s3' @@ -35,21 +36,42 @@ def send_to_queue(header, body, xqueue_url=None): body: Serialized data for the receipient behind the queueing service. The operation of xqueue is agnostic to the contents of 'body' - Returns a 'success' flag indicating successful submission + Returns an 'error' flag indicating error in xqueue transaction ''' if xqueue_url is None: xqueue_url = XQUEUE_SUBMIT_URL - # Contact queue server - payload = {'xqueue_header': header, - 'xqueue_body' : body} + # First, we login with our credentials + #------------------------------------------------------------ + s = requests.session() try: - r = requests.post(xqueue_url, data=payload) + r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH[0], + 'password': XQUEUE_LMS_AUTH[1] }) except Exception as err: msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) # Xqueue responses are JSON-serialized dicts xreply = json.loads(r.text) + return_code = xreply['return_code'] + if return_code: # Nonzero return code from xqueue indicates error + print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + return 1 # Error - return xreply['return_code'] == 0 # return_code == 0 from xqueue indicates successful submission + # Next, we can make a queueing request + #------------------------------------------------------------ + payload = {'xqueue_header': header, + 'xqueue_body' : body} + try: + # Send request + r = s.post(xqueue_url+'/xqueue/submit/', data=payload) + except Exception as err: + msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) + raise Exception(msg) + + xreply = json.loads(r.text) + return_code = xreply['return_code'] + if return_code: + print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + + return return_code From 5c3c3df1b74b7f4ac23f79e0b5d89f1cab114c42 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:24:25 -0400 Subject: [PATCH 08/36] Unbreak check_fd callback --- .../lib/xmodule/xmodule/js/src/capa/display.coffee | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 0ad1bdbba8..d0d00c256f 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -46,9 +46,11 @@ class @Problem $(placeholder).remove() ### - 'check_fd' uses FormData to allow file submissions through an AJAX call. - NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; - can consolidate all dispatches to use FormData consistently + # 'check_fd' uses FormData to allow file submissions, in addition to simple, + # querystring-based answers, in the 'problem_check' dispatch. + # + # NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; + # perhaps preferrable to consolidate all dispatches to use FormData consistently ### check_fd: => Logger.log 'problem_check', @answers @@ -66,20 +68,21 @@ class @Problem # Simple (non-file) answers, # routed to Django 'request.POST' - fd.append('answers', @answers) + fd.append('_answers_querystring', @answers) settings = type: "POST" data: fd processData: false contentType: false - success: (response) -> + success: (response) => switch response.success when 'incorrect', 'correct' @render(response.contents) @updateProgress response else alert(response.success) + $.ajaxWithPrefix("#{@url}/problem_check", settings) check: => From 51fc69da52787742798d4662b701288a1a8abb9b Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:26:37 -0400 Subject: [PATCH 09/36] Spelling in comment --- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index d0d00c256f..2300a56b02 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -50,7 +50,7 @@ class @Problem # querystring-based answers, in the 'problem_check' dispatch. # # NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; - # perhaps preferrable to consolidate all dispatches to use FormData consistently + # perhaps preferable to consolidate all dispatches to use FormData consistently ### check_fd: => Logger.log 'problem_check', @answers From 236c3dc5766da8dc548b418b285c322d84c7b35b Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:34:33 -0400 Subject: [PATCH 10/36] modx_dispatch handles new/old problem_check ajax --- .../xmodule/js/src/capa/display.coffee | 1 + lms/djangoapps/courseware/module_render.py | 24 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 2300a56b02..4318e5850c 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -14,6 +14,7 @@ class @Problem window.update_schematics() @$('section.action input:button').click @refreshAnswers @$('section.action input.check').click @check_fd + #@$('section.action input.check').click @check @$('section.action input.reset').click @reset @$('section.action input.show').click @show @$('section.action input.save').click @save diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 5b030161db..6fd6305989 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -1,5 +1,6 @@ import json import logging +from urlparse import parse_qs from django.conf import settings from django.http import Http404 @@ -14,6 +15,7 @@ from static_replace import replace_urls from xmodule.exceptions import NotFoundError from xmodule.x_module import ModuleSystem from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule +import xqueue_interface log = logging.getLogger("mitx.courseware") @@ -270,16 +272,16 @@ def modx_dispatch(request, dispatch=None, id=None): - id -- the module id. Used to look up the XModule instance ''' # ''' (fix emacs broken parsing) - - print ' THK: module_render.modx_dispatch' - print dispatch - print request.POST.keys() - print request.FILES.keys() - if request.POST.has_key('answers'): - print request.POST['answers'] - for filename in request.FILES.keys(): - uploadedFile = request.FILES.get(filename) - print uploadedFile.read() + + post = request.POST.copy() + + # Catch the use of FormData in xmodule frontend. After this block, the 'post' dict + # is functionally equivalent before- and after- the use of FormData + # TODO: A more elegant solution? + if request.POST.has_key('_answers_querystring'): + post = parse_qs(request.POST.get('_answers_querystring')) + for key in post.keys(): + post[key] = post[key][0] # parse_qs returns { key: list } student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) @@ -292,7 +294,7 @@ def modx_dispatch(request, dispatch=None, id=None): # Let the module handle the AJAX try: - ajax_return = instance.handle_ajax(dispatch, request.POST) + ajax_return = instance.handle_ajax(dispatch, post) except NotFoundError: log.exception("Module indicating to user that request doesn't exist") raise Http404 From ea36eef6e0ec3ce27873886b2538fb4477129178 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:38:33 -0400 Subject: [PATCH 11/36] modx_dispatch uses copied variable rather than source --- lms/djangoapps/courseware/module_render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 6fd6305989..29e3b527e0 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -278,8 +278,8 @@ def modx_dispatch(request, dispatch=None, id=None): # Catch the use of FormData in xmodule frontend. After this block, the 'post' dict # is functionally equivalent before- and after- the use of FormData # TODO: A more elegant solution? - if request.POST.has_key('_answers_querystring'): - post = parse_qs(request.POST.get('_answers_querystring')) + if post.has_key('_answers_querystring'): + post = parse_qs(post.get('_answers_querystring')) for key in post.keys(): post[key] = post[key][0] # parse_qs returns { key: list } From 35a461d917ea4b056b79140ed08d2eea5db609f7 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 20:22:33 -0400 Subject: [PATCH 12/36] Empty file inputs insert empty string as their submission in ajax; LMS reacts accordingly --- .../xmodule/js/src/capa/display.coffee | 7 +++++-- lms/djangoapps/courseware/module_render.py | 21 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 4318e5850c..c9d26a5ec2 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -65,11 +65,14 @@ class @Problem # For each file input, allow a single file submission, # routed to Django 'request.FILES' @$('input:file').each (index, element) -> - fd.append(element.id, element.files[0]) + if element.files[0] instanceof File + fd.append(element.id, element.files[0]) + else + fd.append(element.id, '') # Even if no file selected, need to include input id # Simple (non-file) answers, # routed to Django 'request.POST' - fd.append('_answers_querystring', @answers) + fd.append('__answers_querystring', @answers) settings = type: "POST" diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 29e3b527e0..18ff2d8305 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -275,13 +275,22 @@ def modx_dispatch(request, dispatch=None, id=None): post = request.POST.copy() - # Catch the use of FormData in xmodule frontend. After this block, the 'post' dict - # is functionally equivalent before- and after- the use of FormData + # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, + # the 'post' dict is functionally equivalent before- and after- the use of FormData # TODO: A more elegant solution? - if post.has_key('_answers_querystring'): - post = parse_qs(post.get('_answers_querystring')) - for key in post.keys(): - post[key] = post[key][0] # parse_qs returns { key: list } + if post.has_key('__answers_querystring'): + qs = post.pop('__answers_querystring')[0] + qsdict = parse_qs(qs, keep_blank_values=True) + for key in qsdict.keys(): + qsdict[key] = qsdict[key][0] # parse_qs returns { key: list } + post.update(qsdict) + + # Check for submitted files + if request.FILES: + print 'Got files!' + + print post.keys() + print request.FILES.keys() student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) From 688f8914e3c7c935f74b900fc7a5f1cfaf0e312f Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 20:32:03 -0400 Subject: [PATCH 13/36] Update filesubmission template to show state --- common/lib/capa/capa/inputtypes.py | 4 ++-- common/lib/capa/capa/templates/filesubmission.html | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index 32ee479414..c3cddf48ed 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -13,6 +13,7 @@ Module containing the problem elements which render into input objects - checkboxgroup - imageinput (for clickable image) - optioninput (for option list) +- filesubmission (upload a file) These are matched by *.html files templates/*.html which are mako templates with the actual html. @@ -306,8 +307,7 @@ def filesubmission(element, value, status, render_template, msg=''): Upload a single file (e.g. for programming assignments) ''' eid = element.get('id') - - context = {'id': eid, } + context = { 'id': eid, 'state': status, 'msg': msg, } html = render_template("filesubmission.html", context) return etree.XML(html) diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html index f9073799d4..08f45c916a 100644 --- a/common/lib/capa/capa/templates/filesubmission.html +++ b/common/lib/capa/capa/templates/filesubmission.html @@ -1,3 +1,16 @@

+ % if state == 'unsubmitted': + + % elif state == 'correct': + + % elif state == 'incorrect': + + % elif state == 'incomplete': + + % endif + (${state}) +
+ ${msg|n} +
From 3b3482b3941f71d1b0839e98a87d25beb10b6f7c Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 20:43:09 -0400 Subject: [PATCH 14/36] Skeleton for S3 upload by LMS --- lms/djangoapps/courseware/module_render.py | 10 +++++----- lms/djangoapps/courseware/xqueue_interface.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 18ff2d8305..b0bb760753 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -285,12 +285,12 @@ def modx_dispatch(request, dispatch=None, id=None): qsdict[key] = qsdict[key][0] # parse_qs returns { key: list } post.update(qsdict) - # Check for submitted files + # Check for submitted files, send it to S3 immediately. LMS/xqueue manipulates only the + # pointer, which is saved as the student "submission" if request.FILES: - print 'Got files!' - - print post.keys() - print request.FILES.keys() + for inputfile_id in request.FILES.keys(): + s3_identifier = xqueue_interface.upload_files_to_s3(request.FILES[inputfile_id]) + post.update({inputfile_id: s3_identifier}) student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index c9042c3f1e..f21c4ad9bb 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -8,8 +8,9 @@ import requests XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' -def upload_files_to_s3(): +def upload_files_to_s3(submission_file): print ' THK: xqueue_interface.upload_files_to_s3' + return '' def make_xheader(lms_callback_url, lms_key, queue_name): From 843f8ae9c32a01170c635743530623065a96f64a Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 21:51:36 -0400 Subject: [PATCH 15/36] Basic uploads to S3 --- lms/djangoapps/courseware/xqueue_interface.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index f21c4ad9bb..8aca6df8ee 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -4,13 +4,39 @@ import json import requests +from boto.s3.connection import S3Connection +from boto.s3.key import Key + # TODO: Collection of parameters to be hooked into rest of edX system XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' +AWS_ACCESS_KEY = 'AKIAIYY272VA3C5R4DSQ' +AWS_SECRET_KEY = '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' + +AWS_BUCKET_NAME = 'XQUEUE' + def upload_files_to_s3(submission_file): - print ' THK: xqueue_interface.upload_files_to_s3' - return '' + ''' + Upload student file submissions to S3. + + Returns the S3 key for accessing the file + ''' + print type(submission_file) + print dir(submission_file) + print submission_file + + conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) + bucket_name = AWS_ACCESS_KEY + AWS_BUCKET_NAME + bucket = conn.create_bucket(bucket_name.lower()) # Bucket names must be lowercase... + + k = Key(bucket) + k.key = submission_file.name + k.set_contents_from_string(submission_file.read) + + s3_identifier = k.generate_url(60) + print s3_identifier + return s3_identifier def make_xheader(lms_callback_url, lms_key, queue_name): From e4609f3abebdd2d74165e0565c32e3822897e3db Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 09:30:03 -0400 Subject: [PATCH 16/36] Remove unnecessary hardcode url in CodeResponse --- common/lib/capa/capa/responsetypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 50b5df69d9..f4ba9231f5 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -811,7 +811,7 @@ class CodeResponse(LoncapaResponse): def setup_response(self): xml = self.xml - self.url = xml.get('url', "http://xqueue.edx.org/xqueue/submit/") # FIXME -- hardcoded url + self.url = xml.get('url') self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename) answer = xml.find('answer') From 3d8ee671b39ba66721e3e580fca6711cc504f3f3 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 09:34:47 -0400 Subject: [PATCH 17/36] Adjust comments --- common/lib/capa/capa/responsetypes.py | 2 +- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 11 +++++------ lms/djangoapps/courseware/module_render.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index f4ba9231f5..bd5a588c27 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -873,7 +873,7 @@ class CodeResponse(LoncapaResponse): queue_name=self.queue_name) # Generate body - # NOTE: Currently specialized to 6.00x's pyxservers, which follows the ExternalResponse interface + # NOTE: Currently specialized to 6.00x's pyxserver, which follows the ExternalResponse interface contents = {'xml': etree.tostring(self.xml, pretty_print=True), 'edX_cmd': 'get_score', 'edX_tests': self.tests, diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index c9d26a5ec2..a06f512a6d 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -47,11 +47,11 @@ class @Problem $(placeholder).remove() ### - # 'check_fd' uses FormData to allow file submissions, in addition to simple, - # querystring-based answers, in the 'problem_check' dispatch. + # 'check_fd' uses FormData to allow file submissions in the 'problem_check' dispatch, + # in addition to simple querystring-based answers # # NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; - # perhaps preferable to consolidate all dispatches to use FormData consistently + # maybe preferable to consolidate all dispatches to use FormData ### check_fd: => Logger.log 'problem_check', @answers @@ -63,15 +63,14 @@ class @Problem fd = new FormData() # For each file input, allow a single file submission, - # routed to Django 'request.FILES' + # which is routed to Django 'request.FILES' @$('input:file').each (index, element) -> if element.files[0] instanceof File fd.append(element.id, element.files[0]) else fd.append(element.id, '') # Even if no file selected, need to include input id - # Simple (non-file) answers, - # routed to Django 'request.POST' + # Simple (non-file) answers, which is routed to Django 'request.POST' fd.append('__answers_querystring', @answers) settings = diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index b0bb760753..0eaaa60196 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -276,7 +276,7 @@ def modx_dispatch(request, dispatch=None, id=None): post = request.POST.copy() # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, - # the 'post' dict is functionally equivalent before- and after- the use of FormData + # the 'post' dict is functionally equivalent before and after the use of FormData # TODO: A more elegant solution? if post.has_key('__answers_querystring'): qs = post.pop('__answers_querystring')[0] From fbed664d4f9ded7aa630cba815308fcaa3f365cc Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 09:39:18 -0400 Subject: [PATCH 18/36] Auth saved as dict rather than tuple --- lms/djangoapps/courseware/xqueue_interface.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 8aca6df8ee..2bb7bed297 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -8,7 +8,8 @@ from boto.s3.connection import S3Connection from boto.s3.key import Key # TODO: Collection of parameters to be hooked into rest of edX system -XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) +XQUEUE_LMS_AUTH = { 'username': 'LMS', + 'password': 'PaloAltoCA' } XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' AWS_ACCESS_KEY = 'AKIAIYY272VA3C5R4DSQ' @@ -72,8 +73,8 @@ def send_to_queue(header, body, xqueue_url=None): #------------------------------------------------------------ s = requests.session() try: - r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH[0], - 'password': XQUEUE_LMS_AUTH[1] }) + r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH['username'], + 'password': XQUEUE_LMS_AUTH['password'] }) except Exception as err: msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) From f023dc6e2689d98f1db7da2bbb252d0805632557 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 10:10:06 -0400 Subject: [PATCH 19/36] xqueue_interface generates hashkeys --- common/lib/capa/capa/responsetypes.py | 12 ++---------- lms/djangoapps/courseware/xqueue_interface.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index bd5a588c27..dc0fd518ee 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -8,7 +8,6 @@ Used by capa_problem.py ''' # standard library imports -import hashlib import inspect import json import logging @@ -17,7 +16,6 @@ import numpy import random import re import requests -import time import traceback import abc @@ -856,18 +854,12 @@ class CodeResponse(LoncapaResponse): raise Exception(err) self.context.update({'submission': submission}) - extra_payload = {'edX_student_response': submission} # Prepare xqueue request #------------------------------------------------------------ - # Queuekey generation - h = hashlib.md5() - h.update(str(self.system.seed)) - h.update(str(time.time())) - queuekey = h.hexdigest() - # Generate header + queuekey = xqueue_interface.make_hashkey(self.system.seed) xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue_callback_url, lms_key=queuekey, queue_name=self.queue_name) @@ -886,7 +878,7 @@ class CodeResponse(LoncapaResponse): cmap = CorrectMap() if error: - cmap.set(self.answer_id, msg='Unable to deliver your submission to grader! Please try again later') + cmap.set(self.answer_id, queuekey=None, msg='Unable to deliver your submission to grader! Please try again later') else: # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 2bb7bed297..8ae8b6b441 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -1,8 +1,10 @@ # # LMS Interface to external queueing system (xqueue) # +import hashlib import json import requests +import time from boto.s3.connection import S3Connection from boto.s3.key import Key @@ -39,6 +41,15 @@ def upload_files_to_s3(submission_file): print s3_identifier return s3_identifier +def make_hashkey(seed=None): + ''' + Generate a string key by hashing + ''' + h = hashlib.md5() + if seed is not None: + h.update(str(seed)) + h.update(str(time.time())) + return h.hexdigest() def make_xheader(lms_callback_url, lms_key, queue_name): ''' From 1be3057d3c79ffd8312ab29cbceab0bbe8f56e78 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 11:49:20 -0400 Subject: [PATCH 20/36] xqueue interface in LMS not responsible for S3 upload --- lms/djangoapps/courseware/xqueue_interface.py | 31 +------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 8ae8b6b441..fa495d60a0 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -6,41 +6,11 @@ import json import requests import time -from boto.s3.connection import S3Connection -from boto.s3.key import Key - # TODO: Collection of parameters to be hooked into rest of edX system XQUEUE_LMS_AUTH = { 'username': 'LMS', 'password': 'PaloAltoCA' } XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' -AWS_ACCESS_KEY = 'AKIAIYY272VA3C5R4DSQ' -AWS_SECRET_KEY = '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' - -AWS_BUCKET_NAME = 'XQUEUE' - -def upload_files_to_s3(submission_file): - ''' - Upload student file submissions to S3. - - Returns the S3 key for accessing the file - ''' - print type(submission_file) - print dir(submission_file) - print submission_file - - conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) - bucket_name = AWS_ACCESS_KEY + AWS_BUCKET_NAME - bucket = conn.create_bucket(bucket_name.lower()) # Bucket names must be lowercase... - - k = Key(bucket) - k.key = submission_file.name - k.set_contents_from_string(submission_file.read) - - s3_identifier = k.generate_url(60) - print s3_identifier - return s3_identifier - def make_hashkey(seed=None): ''' Generate a string key by hashing @@ -51,6 +21,7 @@ def make_hashkey(seed=None): h.update(str(time.time())) return h.hexdigest() + def make_xheader(lms_callback_url, lms_key, queue_name): ''' Generate header for delivery and reply of queue request. From 7a9489a9b8966b707fe93314fe3233754f9d9ec9 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 13:53:31 -0400 Subject: [PATCH 21/36] modx_dispatch combines file and non-file submission into single answer dict --- lms/djangoapps/courseware/module_render.py | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 0eaaa60196..744b84455c 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -273,24 +273,23 @@ def modx_dispatch(request, dispatch=None, id=None): ''' # ''' (fix emacs broken parsing) - post = request.POST.copy() + # Check for submitted files + post = dict() + if request.FILES: + for inputfile_id in request.FILES.keys(): + post[inputfile_id] = request.FILES[inputfile_id] # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, # the 'post' dict is functionally equivalent before and after the use of FormData # TODO: A more elegant solution? - if post.has_key('__answers_querystring'): - qs = post.pop('__answers_querystring')[0] - qsdict = parse_qs(qs, keep_blank_values=True) - for key in qsdict.keys(): - qsdict[key] = qsdict[key][0] # parse_qs returns { key: list } - post.update(qsdict) - - # Check for submitted files, send it to S3 immediately. LMS/xqueue manipulates only the - # pointer, which is saved as the student "submission" - if request.FILES: - for inputfile_id in request.FILES.keys(): - s3_identifier = xqueue_interface.upload_files_to_s3(request.FILES[inputfile_id]) - post.update({inputfile_id: s3_identifier}) + for key in request.POST.keys(): + if key == '__answers_querystring': + qs = request.POST.get(key) + qsdict = parse_qs(qs, keep_blank_values=True) + for qskey in qsdict.keys(): + post[qskey] = qsdict[qskey][0] # parse_qs returns {key: list} + else: + post[key] = request.POST.get(key) student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) From eeaa1e04400eddbcbf881089f67e9d40ea973322 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 13:54:33 -0400 Subject: [PATCH 22/36] Capa util function to turn File objects in dict to filename (string) --- common/lib/capa/capa/util.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/common/lib/capa/capa/util.py b/common/lib/capa/capa/util.py index 63a5f43c03..1de69cd032 100644 --- a/common/lib/capa/capa/util.py +++ b/common/lib/capa/capa/util.py @@ -30,3 +30,14 @@ def contextualize_text(text, context): # private for key in sorted(context, lambda x, y: cmp(len(y), len(x))): text = text.replace('$' + key, str(context[key])) return text + + +def convert_files_to_filenames(answers): + ''' + Check for File objects in the dict of submitted answers, + convert File objects to their filename (string) + ''' + new_answers = dict() + for answer_id in answers.keys(): + new_answers[answer_id] = str(answers[answer_id]) + return new_answers From afcd6fb1d70817b983145b198b4fdcc90171e92b Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:02:49 -0400 Subject: [PATCH 23/36] Safe logging of File-included answers --- common/lib/xmodule/xmodule/capa_module.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 2fae8b94e2..2bf1dd0487 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -16,6 +16,7 @@ from xmodule.exceptions import NotFoundError from progress import Progress from capa.capa_problem import LoncapaProblem from capa.responsetypes import StudentInputError +from capa.util import convert_files_to_filenames log = logging.getLogger("mitx.courseware") @@ -413,10 +414,9 @@ class CapaModule(XModule): event_info = dict() event_info['state'] = self.lcp.get_state() event_info['problem_id'] = self.location.url() - + answers = self.make_dict_of_responses(get) - - event_info['answers'] = answers + event_info['answers'] = convert_files_to_filenames(answers) # Too late. Cannot submit if self.closed(): @@ -424,8 +424,7 @@ class CapaModule(XModule): self.system.track_function('save_problem_check_fail', event_info) raise NotFoundError('Problem is closed') - # Problem submitted. Student should reset before checking - # again. + # Problem submitted. Student should reset before checking again if self.lcp.done and self.rerandomize == "always": event_info['failure'] = 'unreset' self.system.track_function('save_problem_check_fail', event_info) From cbb377d38357352a37e8ceb4ca08b8215b3122d3 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:10:22 -0400 Subject: [PATCH 24/36] Answer dict values remain unicode, rather than str --- common/lib/capa/capa/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/util.py b/common/lib/capa/capa/util.py index 1de69cd032..1dc113cd20 100644 --- a/common/lib/capa/capa/util.py +++ b/common/lib/capa/capa/util.py @@ -39,5 +39,5 @@ def convert_files_to_filenames(answers): ''' new_answers = dict() for answer_id in answers.keys(): - new_answers[answer_id] = str(answers[answer_id]) + new_answers[answer_id] = unicode(answers[answer_id]) return new_answers From bfd441255afce626f57e309aedba84a8b0d0d583 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:16:20 -0400 Subject: [PATCH 25/36] Every path except for 'get_score' gets filename instead of file object --- common/lib/capa/capa/capa_problem.py | 5 +++-- common/lib/capa/capa/responsetypes.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 55b9096021..bdfbbab73d 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -31,7 +31,7 @@ import calc from correctmap import CorrectMap import eia import inputtypes -from util import contextualize_text +from util import contextualize_text, convert_files_to_filenames # to be replaced with auto-registering import responsetypes @@ -228,7 +228,8 @@ class LoncapaProblem(object): Calls the Response for each question in this problem, to do the actual grading. ''' - self.student_answers = answers + + self.student_answers = convert_files_to_filenames(answers) oldcmap = self.correct_map # old CorrectMap newcmap = CorrectMap() # start new with empty CorrectMap # log.debug('Responders: %s' % self.responders) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index dc0fd518ee..2874de7395 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -161,7 +161,7 @@ class LoncapaResponse(object): Returns the new CorrectMap, with (correctness,msg,hint,hintmode) for each answer_id. ''' new_cmap = self.get_score(student_answers) - self.get_hints(student_answers, new_cmap, old_cmap) + self.get_hints(convert_files_to_filenames(student_answers), new_cmap, old_cmap) # log.debug('new_cmap = %s' % new_cmap) return new_cmap From 473bb817e8d89a32021142aa04e3d7b7a58a2b96 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:34:48 -0400 Subject: [PATCH 26/36] File objects passed to responsetype only if responsetype explicitly allows filesubmissions --- common/lib/capa/capa/capa_problem.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index bdfbbab73d..d5f45831ee 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -230,11 +230,16 @@ class LoncapaProblem(object): ''' self.student_answers = convert_files_to_filenames(answers) + oldcmap = self.correct_map # old CorrectMap newcmap = CorrectMap() # start new with empty CorrectMap # log.debug('Responders: %s' % self.responders) - for responder in self.responders.values(): - results = responder.evaluate_answers(answers, oldcmap) # call the responsetype instance to do the actual grading + for responder in self.responders.values(): # Call each responsetype instance to do actual grading + if 'filesubmission' in responder.allowed_inputfields: # File objects are passed only if responsetype + # explicitly allows for file submissions + results = responder.evaluate_answers(answers, oldcmap) + else: + results = responder.evaluate_answers(convert_files_to_filenames(answers), oldcmap) newcmap.update(results) self.correct_map = newcmap # log.debug('%s: in grade_answers, answers=%s, cmap=%s' % (self,answers,newcmap)) From 604596843cb5df250ba7fbba3867ae8c7b54344d Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:39:30 -0400 Subject: [PATCH 27/36] Time to think about file interface with xqueue --- common/lib/capa/capa/responsetypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 2874de7395..6279a2de28 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -853,7 +853,7 @@ class CodeResponse(LoncapaResponse): log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' % (err, self.answer_id, student_answers)) raise Exception(err) - self.context.update({'submission': submission}) + self.context.update({'submission': unicode(submission)}) # Submission could be a file # Prepare xqueue request #------------------------------------------------------------ @@ -870,7 +870,7 @@ class CodeResponse(LoncapaResponse): 'edX_cmd': 'get_score', 'edX_tests': self.tests, 'processor': self.code, - 'edX_student_response': submission} + 'edX_student_response': unicode(submission)} # Submit request error = xqueue_interface.send_to_queue(header=xheader, From af22761778e6eabcd839de99c32ffd407ab5b604 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 15:27:39 -0400 Subject: [PATCH 28/36] modx_dispatch doesn't need xqueue_interface --- lms/djangoapps/courseware/module_render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 744b84455c..7883f1e076 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -15,7 +15,6 @@ from static_replace import replace_urls from xmodule.exceptions import NotFoundError from xmodule.x_module import ModuleSystem from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule -import xqueue_interface log = logging.getLogger("mitx.courseware") From c25eded4fcfcdbdaa05e74ff8a5bfab555db920b Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 15:32:02 -0400 Subject: [PATCH 29/36] Move xqueue_interface to lib/capa --- common/lib/capa/capa/responsetypes.py | 2 +- .../courseware => common/lib/capa/capa}/xqueue_interface.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {lms/djangoapps/courseware => common/lib/capa/capa}/xqueue_interface.py (100%) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 6279a2de28..034753d5fd 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -22,10 +22,10 @@ import abc # specific library imports from calc import evaluator, UndefinedVariable from correctmap import CorrectMap -from courseware import xqueue_interface from util import * from lxml import etree from lxml.html.soupparser import fromstring as fromstring_bs # uses Beautiful Soup!!! FIXME? +import xqueue_interface log = logging.getLogger('mitx.' + __name__) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py similarity index 100% rename from lms/djangoapps/courseware/xqueue_interface.py rename to common/lib/capa/capa/xqueue_interface.py From 9d52c43286b09b0448ef625124f0996ae4fb8534 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 15:49:30 -0400 Subject: [PATCH 30/36] Add 'value' to filesubmission template --- common/lib/capa/capa/inputtypes.py | 2 +- common/lib/capa/capa/templates/filesubmission.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index c3cddf48ed..583d29b82e 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -307,7 +307,7 @@ def filesubmission(element, value, status, render_template, msg=''): Upload a single file (e.g. for programming assignments) ''' eid = element.get('id') - context = { 'id': eid, 'state': status, 'msg': msg, } + context = { 'id': eid, 'state': status, 'msg': msg, 'value': value, } html = render_template("filesubmission.html", context) return etree.XML(html) diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html index 08f45c916a..ff9fc992fd 100644 --- a/common/lib/capa/capa/templates/filesubmission.html +++ b/common/lib/capa/capa/templates/filesubmission.html @@ -1,5 +1,5 @@
-
+
% if state == 'unsubmitted': % elif state == 'correct': From 8eff9f7caa29f71fa05d0fa74e1987c7d8deea56 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 17:11:55 -0400 Subject: [PATCH 31/36] Upload to xqueue --- common/lib/capa/capa/responsetypes.py | 16 +++++++++++----- common/lib/capa/capa/xqueue_interface.py | 12 +++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 034753d5fd..d995e8b902 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -848,12 +848,13 @@ class CodeResponse(LoncapaResponse): def get_score(self, student_answers): try: - submission = student_answers[self.answer_id] + submission = student_answers[self.answer_id] # Note that submission can be a file except Exception as err: - log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' % (err, self.answer_id, student_answers)) + log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' % + (err, self.answer_id, convert_files_to_filenames(student_answers))) raise Exception(err) - self.context.update({'submission': unicode(submission)}) # Submission could be a file + self.context.update({'submission': unicode(submission)}) # Prepare xqueue request #------------------------------------------------------------ @@ -873,8 +874,13 @@ class CodeResponse(LoncapaResponse): 'edX_student_response': unicode(submission)} # Submit request - error = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + if hasattr(submission, 'read'): # Test for whether submission is a file + error = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents), + file_to_upload=submission) + else: + error = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) cmap = CorrectMap() if error: diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py index fa495d60a0..e479be3ed6 100644 --- a/common/lib/capa/capa/xqueue_interface.py +++ b/common/lib/capa/capa/xqueue_interface.py @@ -37,7 +37,7 @@ def make_xheader(lms_callback_url, lms_key, queue_name): 'queue_name': queue_name }) -def send_to_queue(header, body, xqueue_url=None): +def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): ''' Submit a request to xqueue. @@ -46,6 +46,8 @@ def send_to_queue(header, body, xqueue_url=None): body: Serialized data for the receipient behind the queueing service. The operation of xqueue is agnostic to the contents of 'body' + file_to_upload: File object to be uploaded to xqueue along with queue request + Returns an 'error' flag indicating error in xqueue transaction ''' if xqueue_url is None: @@ -72,9 +74,13 @@ def send_to_queue(header, body, xqueue_url=None): #------------------------------------------------------------ payload = {'xqueue_header': header, 'xqueue_body' : body} + + files = None + if file_to_upload is not None: + files = { file_to_upload.name: file_to_upload } + try: - # Send request - r = s.post(xqueue_url+'/xqueue/submit/', data=payload) + r = s.post(xqueue_url+'/xqueue/submit/', data=payload, files=files) except Exception as err: msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) From 8aef6a0e754cbf49079c35764d4ffc61a3583152 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 17:38:34 -0400 Subject: [PATCH 32/36] Ajax doesn't complain about FormData if there's no file to be uploaded --- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index a06f512a6d..12017105a1 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -56,8 +56,13 @@ class @Problem check_fd: => Logger.log 'problem_check', @answers + # If there are no file inputs in the problem, we can fall back on @check + if $('input:file').length == 0 + @check() + return + if not window.FormData - alert "Sorry, your browser does not support file uploads. If you can, please use Chrome or Safari which have been verified to support this feature." + alert "Sorry, your browser does not support file uploads. Your submit request could not be fulfilled. If you can, please use Chrome or Safari which have been verified to support file uploads." return fd = new FormData() From 881cafb88d6829557d542a9504652d771bad0d59 Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 09:14:09 -0400 Subject: [PATCH 33/36] Moved default parameter assignment to function def, parse_xreply util function --- common/lib/capa/capa/xqueue_interface.py | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py index e479be3ed6..38adeb6d48 100644 --- a/common/lib/capa/capa/xqueue_interface.py +++ b/common/lib/capa/capa/xqueue_interface.py @@ -9,7 +9,7 @@ import time # TODO: Collection of parameters to be hooked into rest of edX system XQUEUE_LMS_AUTH = { 'username': 'LMS', 'password': 'PaloAltoCA' } -XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' +XQUEUE_URL = 'http://xqueue.edx.org' def make_hashkey(seed=None): ''' @@ -37,7 +37,7 @@ def make_xheader(lms_callback_url, lms_key, queue_name): 'queue_name': queue_name }) -def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): +def send_to_queue(header, body, file_to_upload=None, xqueue_url=XQUEUE_URL): ''' Submit a request to xqueue. @@ -50,8 +50,6 @@ def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): Returns an 'error' flag indicating error in xqueue transaction ''' - if xqueue_url is None: - xqueue_url = XQUEUE_SUBMIT_URL # First, we login with our credentials #------------------------------------------------------------ @@ -64,10 +62,9 @@ def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): raise Exception(msg) # Xqueue responses are JSON-serialized dicts - xreply = json.loads(r.text) - return_code = xreply['return_code'] + (return_code, msg) = parse_xreply(r.text) if return_code: # Nonzero return code from xqueue indicates error - print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + print ' Error in queue_interface.send_to_queue: %s' % msg return 1 # Error # Next, we can make a queueing request @@ -85,9 +82,20 @@ def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) - xreply = json.loads(r.text) - return_code = xreply['return_code'] + (return_code, msg) = parse_xreply(r.text) if return_code: - print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + print ' Error in queue_interface.send_to_queue: %s' % msg return return_code + +def parse_xreply(xreply): + ''' + Parse the reply from xqueue. Messages are JSON-serialized dict: + { 'return_code': 0 (success), 1 (fail), + 'content': Message from xqueue (string) + } + ''' + xreply = json.loads(xreply) + return_code = xreply['return_code'] + content = xreply['content'] + return (return_code, content) From c950d437e51fcca9548e653e50ce3160211bab8e Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 11:58:00 -0400 Subject: [PATCH 34/36] Add comments on CodeResponse.get_score --- common/lib/capa/capa/responsetypes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index d995e8b902..a1719ddcce 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -867,11 +867,13 @@ class CodeResponse(LoncapaResponse): # Generate body # NOTE: Currently specialized to 6.00x's pyxserver, which follows the ExternalResponse interface + # We should define a common interface for external code graders to CodeResponse contents = {'xml': etree.tostring(self.xml, pretty_print=True), 'edX_cmd': 'get_score', 'edX_tests': self.tests, 'processor': self.code, - 'edX_student_response': unicode(submission)} + 'edX_student_response': unicode(submission), # unicode on File object returns its filename + } # Submit request if hasattr(submission, 'read'): # Test for whether submission is a file From 6c875206bc05a63eac4c6e93e942d8295b9e7c98 Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 12:44:19 -0400 Subject: [PATCH 35/36] Simplify file submission front end --- .../xmodule/js/src/capa/display.coffee | 16 +++++++--------- lms/djangoapps/courseware/module_render.py | 19 +++---------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 12017105a1..0e760e98ff 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -67,16 +67,14 @@ class @Problem fd = new FormData() - # For each file input, allow a single file submission, - # which is routed to Django 'request.FILES' - @$('input:file').each (index, element) -> - if element.files[0] instanceof File - fd.append(element.id, element.files[0]) + @$("[id^=input_#{@element_id.replace(/problem_/, '')}_]").each (index, element) -> + if element.type is 'file' + if element.files[0] instanceof File + fd.append(element.id, element.files[0]) + else + fd.append(element.id, '') else - fd.append(element.id, '') # Even if no file selected, need to include input id - - # Simple (non-file) answers, which is routed to Django 'request.POST' - fd.append('__answers_querystring', @answers) + fd.append(element.id, element.value) settings = type: "POST" diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 7883f1e076..9ce4008597 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -1,6 +1,5 @@ import json import logging -from urlparse import parse_qs from django.conf import settings from django.http import Http404 @@ -273,22 +272,10 @@ def modx_dispatch(request, dispatch=None, id=None): # ''' (fix emacs broken parsing) # Check for submitted files - post = dict() + p = request.POST.copy() if request.FILES: for inputfile_id in request.FILES.keys(): - post[inputfile_id] = request.FILES[inputfile_id] - - # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, - # the 'post' dict is functionally equivalent before and after the use of FormData - # TODO: A more elegant solution? - for key in request.POST.keys(): - if key == '__answers_querystring': - qs = request.POST.get(key) - qsdict = parse_qs(qs, keep_blank_values=True) - for qskey in qsdict.keys(): - post[qskey] = qsdict[qskey][0] # parse_qs returns {key: list} - else: - post[key] = request.POST.get(key) + p[inputfile_id] = request.FILES[inputfile_id] student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) @@ -301,7 +288,7 @@ def modx_dispatch(request, dispatch=None, id=None): # Let the module handle the AJAX try: - ajax_return = instance.handle_ajax(dispatch, post) + ajax_return = instance.handle_ajax(dispatch, p) except NotFoundError: log.exception("Module indicating to user that request doesn't exist") raise Http404 From 110637c0231ed7e15b01f03f53bc5145347f89c9 Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 15:06:19 -0400 Subject: [PATCH 36/36] XqueueInterface is a singleton instantiated object --- common/lib/capa/capa/responsetypes.py | 16 ++-- common/lib/capa/capa/xqueue_interface.py | 117 +++++++++++++---------- 2 files changed, 74 insertions(+), 59 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index a1719ddcce..12f619a881 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -29,6 +29,8 @@ import xqueue_interface log = logging.getLogger('mitx.' + __name__) +qinterface = xqueue_interface.XqueueInterface() + #----------------------------------------------------------------------------- # Exceptions @@ -809,7 +811,6 @@ class CodeResponse(LoncapaResponse): def setup_response(self): xml = self.xml - self.url = xml.get('url') self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename) answer = xml.find('answer') @@ -877,16 +878,17 @@ class CodeResponse(LoncapaResponse): # Submit request if hasattr(submission, 'read'): # Test for whether submission is a file - error = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents), - file_to_upload=submission) + (error, msg) = qinterface.send_to_queue(header=xheader, + body=json.dumps(contents), + file_to_upload=submission) else: - error = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + (error, msg) = qinterface.send_to_queue(header=xheader, + body=json.dumps(contents)) cmap = CorrectMap() if error: - cmap.set(self.answer_id, queuekey=None, msg='Unable to deliver your submission to grader! Please try again later') + cmap.set(self.answer_id, queuekey=None, + msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg) else: # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py index 38adeb6d48..deb068adf6 100644 --- a/common/lib/capa/capa/xqueue_interface.py +++ b/common/lib/capa/capa/xqueue_interface.py @@ -3,6 +3,7 @@ # import hashlib import json +import logging import requests import time @@ -11,6 +12,8 @@ XQUEUE_LMS_AUTH = { 'username': 'LMS', 'password': 'PaloAltoCA' } XQUEUE_URL = 'http://xqueue.edx.org' +log = logging.getLogger('mitx.' + __name__) + def make_hashkey(seed=None): ''' Generate a string key by hashing @@ -37,61 +40,10 @@ def make_xheader(lms_callback_url, lms_key, queue_name): 'queue_name': queue_name }) -def send_to_queue(header, body, file_to_upload=None, xqueue_url=XQUEUE_URL): - ''' - Submit a request to xqueue. - - header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader' - - body: Serialized data for the receipient behind the queueing service. The operation of - xqueue is agnostic to the contents of 'body' - - file_to_upload: File object to be uploaded to xqueue along with queue request - - Returns an 'error' flag indicating error in xqueue transaction - ''' - - # First, we login with our credentials - #------------------------------------------------------------ - s = requests.session() - try: - r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH['username'], - 'password': XQUEUE_LMS_AUTH['password'] }) - except Exception as err: - msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) - raise Exception(msg) - - # Xqueue responses are JSON-serialized dicts - (return_code, msg) = parse_xreply(r.text) - if return_code: # Nonzero return code from xqueue indicates error - print ' Error in queue_interface.send_to_queue: %s' % msg - return 1 # Error - - # Next, we can make a queueing request - #------------------------------------------------------------ - payload = {'xqueue_header': header, - 'xqueue_body' : body} - - files = None - if file_to_upload is not None: - files = { file_to_upload.name: file_to_upload } - - try: - r = s.post(xqueue_url+'/xqueue/submit/', data=payload, files=files) - except Exception as err: - msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) - raise Exception(msg) - - (return_code, msg) = parse_xreply(r.text) - if return_code: - print ' Error in queue_interface.send_to_queue: %s' % msg - - return return_code - def parse_xreply(xreply): ''' Parse the reply from xqueue. Messages are JSON-serialized dict: - { 'return_code': 0 (success), 1 (fail), + { 'return_code': 0 (success), 1 (fail) 'content': Message from xqueue (string) } ''' @@ -99,3 +51,64 @@ def parse_xreply(xreply): return_code = xreply['return_code'] content = xreply['content'] return (return_code, content) + + +class XqueueInterface: + ''' + Interface to the external grading system + ''' + + def __init__(self, url=XQUEUE_URL, auth=XQUEUE_LMS_AUTH): + self.url = url + self.auth = auth + self.s = requests.session() + self._login() + + def send_to_queue(self, header, body, file_to_upload=None): + ''' + Submit a request to xqueue. + + header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader' + + body: Serialized data for the receipient behind the queueing service. The operation of + xqueue is agnostic to the contents of 'body' + + file_to_upload: File object to be uploaded to xqueue along with queue request + + Returns (error_code, msg) where error_code != 0 indicates an error + ''' + # Attempt to send to queue + (error, msg) = self._send_to_queue(header, body, file_to_upload) + + if error and (msg == 'login_required'): # Log in, then try again + self._login() + (error, msg) = self._send_to_queue(header, body, file_to_upload) + + return (error, msg) + + def _login(self): + try: + r = self.s.post(self.url+'/xqueue/login/', data={ 'username': self.auth['username'], + 'password': self.auth['password'] }) + except requests.exceptions.ConnectionError, err: + log.error(err) + return (1, 'cannot connect to server') + + return parse_xreply(r.text) + + def _send_to_queue(self, header, body, file_to_upload=None): + + payload = {'xqueue_header': header, + 'xqueue_body' : body} + + files = None + if file_to_upload is not None: + files = { file_to_upload.name: file_to_upload } + + try: + r = self.s.post(self.url+'/xqueue/submit/', data=payload, files=files) + except requests.exceptions.ConnectionError, err: + log.error(err) + return (1, 'cannot connect to server') + + return parse_xreply(r.text)