From fac76593159f5b2345df71b7a46e1e417256d51a Mon Sep 17 00:00:00 2001 From: kimth Date: Wed, 1 Aug 2012 10:23:30 -0400 Subject: [PATCH 01/66] Use xqueue cname --- common/lib/capa/capa/responsetypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 75ed92ab3e..812ffbc72c 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -810,7 +810,7 @@ class CodeResponse(LoncapaResponse): def setup_response(self): xml = self.xml - self.url = xml.get('url', "http://107.20.215.194/xqueue/submit/") # FIXME -- hardcoded url + self.url = xml.get('url', "http://xqueue.edx.org/xqueue/submit/") # FIXME -- hardcoded url self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename) answer = xml.find('answer') From 8a983b3ced91f69c4d619949203c3c2786ea47e9 Mon Sep 17 00:00:00 2001 From: kimth Date: Wed, 1 Aug 2012 10:39:09 -0400 Subject: [PATCH 02/66] Queuekey does not need to be integer --- common/lib/capa/capa/responsetypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 812ffbc72c..bb62cdfa1c 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -888,7 +888,7 @@ class CodeResponse(LoncapaResponse): msg = rxml.find('message').text.replace(' ', ' ') oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None) # Queuekey is consumed else: - log.debug('CodeResponse: queuekey %d does not match for answer_id=%s.' % (queuekey, self.answer_id)) + log.debug('CodeResponse: queuekey %s does not match for answer_id=%s.' % (queuekey, self.answer_id)) return oldcmap @@ -913,7 +913,7 @@ class CodeResponse(LoncapaResponse): h = hashlib.md5() h.update(str(self.system.seed)) h.update(str(time.time())) - queuekey = int(h.hexdigest(), 16) + queuekey = h.hexdigest() header.update({'lms_key': queuekey}) body = {'xml': xmlstr, From 541f5ecdd28a1fdf9fc3e264886153f0e10c007d Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 14:32:40 -0400 Subject: [PATCH 03/66] Filesubmission frontend --- common/lib/capa/capa/capa_problem.py | 2 +- common/lib/capa/capa/inputtypes.py | 13 +++++++ common/lib/capa/capa/responsetypes.py | 2 +- .../capa/capa/templates/filesubmission.html | 3 ++ .../xmodule/js/src/capa/display.coffee | 34 ++++++++++++++++++- lms/djangoapps/courseware/module_render.py | 10 ++++++ 6 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 common/lib/capa/capa/templates/filesubmission.html diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index ed99c71635..55b9096021 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -39,7 +39,7 @@ import responsetypes # dict of tagname, Response Class -- this should come from auto-registering response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__]) -entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup'] +entry_types = ['textline', 'schematic', 'textbox', 'imageinput', 'optioninput', 'choicegroup', 'radiogroup', 'checkboxgroup', 'filesubmission'] solution_types = ['solution'] # extra things displayed after "show answers" is pressed response_properties = ["responseparam", "answer"] # these get captured as student responses diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index 8b3867be5b..32ee479414 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -299,6 +299,19 @@ def textline_dynamath(element, value, status, render_template, msg=''): return etree.XML(html) +#----------------------------------------------------------------------------- +@register_render_function +def filesubmission(element, value, status, render_template, msg=''): + ''' + Upload a single file (e.g. for programming assignments) + ''' + eid = element.get('id') + + context = {'id': eid, } + html = render_template("filesubmission.html", context) + return etree.XML(html) + + #----------------------------------------------------------------------------- ## TODO: Make a wrapper for @register_render_function diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index bb62cdfa1c..578352eb44 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -805,7 +805,7 @@ class CodeResponse(LoncapaResponse): ''' response_tag = 'coderesponse' - allowed_inputfields = ['textline', 'textbox'] + allowed_inputfields = ['textbox', 'filesubmission'] max_inputfields = 1 def setup_response(self): diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html new file mode 100644 index 0000000000..f9073799d4 --- /dev/null +++ b/common/lib/capa/capa/templates/filesubmission.html @@ -0,0 +1,3 @@ +
+
+
diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 4ee8257e36..d23c096eec 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -13,7 +13,7 @@ class @Problem MathJax.Hub.Queue ["Typeset", MathJax.Hub] window.update_schematics() @$('section.action input:button').click @refreshAnswers - @$('section.action input.check').click @check + @$('section.action input.check').click @check_fd @$('section.action input.reset').click @reset @$('section.action input.show').click @show @$('section.action input.save').click @save @@ -45,6 +45,38 @@ class @Problem $('head')[0].appendChild(s[0]) $(placeholder).remove() + check_fd: => + Logger.log 'problem_check', @answers + + if not window.FormData + alert "Sorry, your browser does not support file uploads. If you can, please use Chrome or Safari which have been verified to support this feature." + return + + fd = new FormData() + + # For each file input, allow a single file submission, + # routed to Django 'request.FILES' + @$('input:file').each (index, element) -> + fd.append(element.id, element.files[0]) + + # Simple (non-file) answers, + # routed to Django 'request.POST' + fd.append('answers', @answers) + + settings = + type: "POST" + data: fd + processData: false + contentType: false + success: (response) -> + switch response.success + when 'incorrect', 'correct' + @render(response.contents) + @updateProgress response + else + alert(response.success) + $.ajaxWithPrefix("#{@url}/problem_check", settings) + check: => Logger.log 'problem_check', @answers $.postWithPrefix "#{@url}/problem_check", @answers, (response) => diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 80a4ef90fc..5b030161db 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -270,6 +270,16 @@ def modx_dispatch(request, dispatch=None, id=None): - id -- the module id. Used to look up the XModule instance ''' # ''' (fix emacs broken parsing) + + print ' THK: module_render.modx_dispatch' + print dispatch + print request.POST.keys() + print request.FILES.keys() + if request.POST.has_key('answers'): + print request.POST['answers'] + for filename in request.FILES.keys(): + uploadedFile = request.FILES.get(filename) + print uploadedFile.read() student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) From c7084e5240d1fe9292b5df9177e6d2429464b548 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 15:35:25 -0400 Subject: [PATCH 04/66] Comments for 'check_fd' --- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index d23c096eec..0ad1bdbba8 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -45,6 +45,11 @@ class @Problem $('head')[0].appendChild(s[0]) $(placeholder).remove() + ### + 'check_fd' uses FormData to allow file submissions through an AJAX call. + NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; + can consolidate all dispatches to use FormData consistently + ### check_fd: => Logger.log 'problem_check', @answers From 6de2fa5e1e5178cacf61a4fe2517788281507aae Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 17:10:35 -0400 Subject: [PATCH 05/66] LMS interface to xqueue in separate file --- common/lib/capa/capa/responsetypes.py | 76 ++++++++----------- lms/djangoapps/courseware/xqueue_interface.py | 51 +++++++++++++ 2 files changed, 83 insertions(+), 44 deletions(-) create mode 100644 lms/djangoapps/courseware/xqueue_interface.py diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 578352eb44..2734e07cd3 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -24,6 +24,7 @@ import abc # specific library imports from calc import evaluator, UndefinedVariable from correctmap import CorrectMap +from courseware import xqueue_interface from util import * from lxml import etree from lxml.html.soupparser import fromstring as fromstring_bs # uses Beautiful Soup!!! FIXME? @@ -798,10 +799,10 @@ class SymbolicResponse(CustomResponse): class CodeResponse(LoncapaResponse): ''' - Grade student code using an external server, called 'xqueue' - In contrast to ExternalResponse, CodeResponse has following behavior: - 1) Goes through a queueing system - 2) Does not do external request for 'get_answers' + Grade student code using an external queueing server, called 'xqueue' + + External requests are only submitted for student submission grading + (i.e. and not for getting reference answers) ''' response_tag = 'coderesponse' @@ -857,9 +858,33 @@ class CodeResponse(LoncapaResponse): self.context.update({'submission': submission}) extra_payload = {'edX_student_response': submission} - r, queuekey = self._send_to_queue(extra_payload) # TODO: Perform checks on the xqueue response + # Prepare xqueue request + #------------------------------------------------------------ - # Non-null CorrectMap['queuekey'] indicates that the problem has been submitted + # Queuekey generation + h = hashlib.md5() + h.update(str(self.system.seed)) + h.update(str(time.time())) + queuekey = h.hexdigest() + + # Generate header + xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue_callback_url, + lms_key=queuekey, + queue_name=self.queue_name) + + # Generate body + # NOTE: Currently specialized to 6.00x's pyxservers, which follows the ExternalResponse interface + contents = {'xml': etree.tostring(self.xml, pretty_print=True), + 'edX_cmd': 'get_score', + 'edX_tests': self.tests, + 'processor': self.code, + 'edX_student_response': submission} + + # Submit request + xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) + + # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap = CorrectMap() cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to queue') @@ -883,7 +908,7 @@ class CodeResponse(LoncapaResponse): self.context['correct'][0] = admap[ad] # Replace 'oldcmap' with new grading results if queuekey matches. - # If queuekey does not match, we keep waiting for the score_msg that will match + # If queuekey does not match, we keep waiting for the score_msg whose key actually matchs if oldcmap.is_right_queuekey(self.answer_id, queuekey): msg = rxml.find('message').text.replace(' ', ' ') oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None) # Queuekey is consumed @@ -892,8 +917,6 @@ class CodeResponse(LoncapaResponse): return oldcmap - # CodeResponse differentiates from ExternalResponse in the behavior of 'get_answers'. CodeResponse.get_answers - # does NOT require a queue submission, and the answer is computed (extracted from problem XML) locally. def get_answers(self): anshtml = '
%s

' % self.answer return {self.answer_id: anshtml} @@ -901,41 +924,6 @@ class CodeResponse(LoncapaResponse): def get_initial_display(self): return {self.answer_id: self.initial_display} - # CodeResponse._send_to_queue implements the same interface as defined for ExternalResponse's 'get_score' - def _send_to_queue(self, extra_payload): - # Prepare payload - xmlstr = etree.tostring(self.xml, pretty_print=True) - header = {'lms_callback_url': self.system.xqueue_callback_url, - 'queue_name': self.queue_name, - } - - # Queuekey generation - h = hashlib.md5() - h.update(str(self.system.seed)) - h.update(str(time.time())) - queuekey = h.hexdigest() - header.update({'lms_key': queuekey}) - - body = {'xml': xmlstr, - 'edX_cmd': 'get_score', - 'edX_tests': self.tests, - 'processor': self.code, - } - body.update(extra_payload) - - payload = {'xqueue_header': json.dumps(header), - 'xqueue_body' : json.dumps(body), - } - - # Contact queue server - try: - r = requests.post(self.url, data=payload) - except Exception as err: - msg = "Error in CodeResponse %s: cannot connect to queue server url=%s" % (err, self.url) - log.error(msg) - raise Exception(msg) - - return r, queuekey #----------------------------------------------------------------------------- diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py new file mode 100644 index 0000000000..6384adb947 --- /dev/null +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -0,0 +1,51 @@ +# +# LMS Interface to external queueing system (xqueue) +# +import json +import requests + +# TODO: Collection of parameters to be hooked into rest of edX system +XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org/xqueue/submit/' + +def upload_files_to_s3(): + print ' THK: xqueue_interface.upload_files_to_s3' + + +def make_xheader(lms_callback_url, lms_key, queue_name): + ''' + Generate header for delivery and reply of queue request. + + Xqueue header is a JSON-serialized dict: + { 'lms_callback_url': url to which xqueue will return the request (string), + 'lms_key': secret key used by LMS to protect its state (string), + 'queue_name': designate a specific queue within xqueue server, e.g. 'MITx-6.00x' (string) + } + ''' + return json.dumps({ 'lms_callback_url': lms_callback_url, + 'lms_key': lms_key, + 'queue_name': queue_name }) + + +def send_to_queue(header, body, xqueue_url=None): + ''' + Submit a request to xqueue. + + header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader' + + body: Serialized data for the receipient behind the queueing service. The operation of + xqueue is agnostic to the contents of 'body' + + ''' + if xqueue_url is None: + xqueue_url = XQUEUE_SUBMIT_URL + + # Contact queue server + payload = {'xqueue_header': header, + 'xqueue_body' : body} + try: + r = requests.post(xqueue_url, data=payload) + except Exception as err: + msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) + raise Exception(msg) + + #print r.text From d2db4134cdb22dcfc7f0616a6f55c0f93ec82371 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 17:28:12 -0400 Subject: [PATCH 06/66] CodeResponse does basic error handling from xqueue submission --- common/lib/capa/capa/responsetypes.py | 15 +++++++++------ lms/djangoapps/courseware/xqueue_interface.py | 6 +++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 2734e07cd3..d8a3940624 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -881,12 +881,15 @@ class CodeResponse(LoncapaResponse): 'edX_student_response': submission} # Submit request - xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + success = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) - # Non-null CorrectMap['queuekey'] indicates that the problem has been queued - cmap = CorrectMap() - cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to queue') + cmap = CorrectMap() + if success: + # Non-null CorrectMap['queuekey'] indicates that the problem has been queued + cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') + else: + cmap.set(self.answer_id, msg='Unable to deliver submission to grader! Please try again later') return cmap @@ -908,7 +911,7 @@ class CodeResponse(LoncapaResponse): self.context['correct'][0] = admap[ad] # Replace 'oldcmap' with new grading results if queuekey matches. - # If queuekey does not match, we keep waiting for the score_msg whose key actually matchs + # If queuekey does not match, we keep waiting for the score_msg whose key actually matches if oldcmap.is_right_queuekey(self.answer_id, queuekey): msg = rxml.find('message').text.replace(' ', ' ') oldcmap.set(self.answer_id, correctness=self.context['correct'][0], msg=msg, queuekey=None) # Queuekey is consumed diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 6384adb947..3779a3eb94 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -35,6 +35,7 @@ def send_to_queue(header, body, xqueue_url=None): body: Serialized data for the receipient behind the queueing service. The operation of xqueue is agnostic to the contents of 'body' + Returns a 'success' flag indicating successful submission ''' if xqueue_url is None: xqueue_url = XQUEUE_SUBMIT_URL @@ -48,4 +49,7 @@ def send_to_queue(header, body, xqueue_url=None): msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) - #print r.text + # Xqueue responses are JSON-serialized dicts + xreply = json.loads(r.text) + + return xreply['return_code'] == 0 # return_code == 0 from xqueue indicates successful submission From 988136c401d499d582189d906c895c85b7044524 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 18:34:14 -0400 Subject: [PATCH 07/66] LMS must login before submitting to xqueue --- common/lib/capa/capa/responsetypes.py | 10 +++--- lms/djangoapps/courseware/xqueue_interface.py | 36 +++++++++++++++---- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index d8a3940624..50b5df69d9 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -881,15 +881,15 @@ class CodeResponse(LoncapaResponse): 'edX_student_response': submission} # Submit request - success = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + error = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) cmap = CorrectMap() - if success: + if error: + cmap.set(self.answer_id, msg='Unable to deliver your submission to grader! Please try again later') + else: # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') - else: - cmap.set(self.answer_id, msg='Unable to deliver submission to grader! Please try again later') return cmap diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 3779a3eb94..c9042c3f1e 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -5,7 +5,8 @@ import json import requests # TODO: Collection of parameters to be hooked into rest of edX system -XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org/xqueue/submit/' +XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) +XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' def upload_files_to_s3(): print ' THK: xqueue_interface.upload_files_to_s3' @@ -35,21 +36,42 @@ def send_to_queue(header, body, xqueue_url=None): body: Serialized data for the receipient behind the queueing service. The operation of xqueue is agnostic to the contents of 'body' - Returns a 'success' flag indicating successful submission + Returns an 'error' flag indicating error in xqueue transaction ''' if xqueue_url is None: xqueue_url = XQUEUE_SUBMIT_URL - # Contact queue server - payload = {'xqueue_header': header, - 'xqueue_body' : body} + # First, we login with our credentials + #------------------------------------------------------------ + s = requests.session() try: - r = requests.post(xqueue_url, data=payload) + r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH[0], + 'password': XQUEUE_LMS_AUTH[1] }) except Exception as err: msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) # Xqueue responses are JSON-serialized dicts xreply = json.loads(r.text) + return_code = xreply['return_code'] + if return_code: # Nonzero return code from xqueue indicates error + print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + return 1 # Error - return xreply['return_code'] == 0 # return_code == 0 from xqueue indicates successful submission + # Next, we can make a queueing request + #------------------------------------------------------------ + payload = {'xqueue_header': header, + 'xqueue_body' : body} + try: + # Send request + r = s.post(xqueue_url+'/xqueue/submit/', data=payload) + except Exception as err: + msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) + raise Exception(msg) + + xreply = json.loads(r.text) + return_code = xreply['return_code'] + if return_code: + print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + + return return_code From 5c3c3df1b74b7f4ac23f79e0b5d89f1cab114c42 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:24:25 -0400 Subject: [PATCH 08/66] Unbreak check_fd callback --- .../lib/xmodule/xmodule/js/src/capa/display.coffee | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 0ad1bdbba8..d0d00c256f 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -46,9 +46,11 @@ class @Problem $(placeholder).remove() ### - 'check_fd' uses FormData to allow file submissions through an AJAX call. - NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; - can consolidate all dispatches to use FormData consistently + # 'check_fd' uses FormData to allow file submissions, in addition to simple, + # querystring-based answers, in the 'problem_check' dispatch. + # + # NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; + # perhaps preferrable to consolidate all dispatches to use FormData consistently ### check_fd: => Logger.log 'problem_check', @answers @@ -66,20 +68,21 @@ class @Problem # Simple (non-file) answers, # routed to Django 'request.POST' - fd.append('answers', @answers) + fd.append('_answers_querystring', @answers) settings = type: "POST" data: fd processData: false contentType: false - success: (response) -> + success: (response) => switch response.success when 'incorrect', 'correct' @render(response.contents) @updateProgress response else alert(response.success) + $.ajaxWithPrefix("#{@url}/problem_check", settings) check: => From 51fc69da52787742798d4662b701288a1a8abb9b Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:26:37 -0400 Subject: [PATCH 09/66] Spelling in comment --- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index d0d00c256f..2300a56b02 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -50,7 +50,7 @@ class @Problem # querystring-based answers, in the 'problem_check' dispatch. # # NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; - # perhaps preferrable to consolidate all dispatches to use FormData consistently + # perhaps preferable to consolidate all dispatches to use FormData consistently ### check_fd: => Logger.log 'problem_check', @answers From 236c3dc5766da8dc548b418b285c322d84c7b35b Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:34:33 -0400 Subject: [PATCH 10/66] modx_dispatch handles new/old problem_check ajax --- .../xmodule/js/src/capa/display.coffee | 1 + lms/djangoapps/courseware/module_render.py | 24 ++++++++++--------- 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 2300a56b02..4318e5850c 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -14,6 +14,7 @@ class @Problem window.update_schematics() @$('section.action input:button').click @refreshAnswers @$('section.action input.check').click @check_fd + #@$('section.action input.check').click @check @$('section.action input.reset').click @reset @$('section.action input.show').click @show @$('section.action input.save').click @save diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 5b030161db..6fd6305989 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -1,5 +1,6 @@ import json import logging +from urlparse import parse_qs from django.conf import settings from django.http import Http404 @@ -14,6 +15,7 @@ from static_replace import replace_urls from xmodule.exceptions import NotFoundError from xmodule.x_module import ModuleSystem from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule +import xqueue_interface log = logging.getLogger("mitx.courseware") @@ -270,16 +272,16 @@ def modx_dispatch(request, dispatch=None, id=None): - id -- the module id. Used to look up the XModule instance ''' # ''' (fix emacs broken parsing) - - print ' THK: module_render.modx_dispatch' - print dispatch - print request.POST.keys() - print request.FILES.keys() - if request.POST.has_key('answers'): - print request.POST['answers'] - for filename in request.FILES.keys(): - uploadedFile = request.FILES.get(filename) - print uploadedFile.read() + + post = request.POST.copy() + + # Catch the use of FormData in xmodule frontend. After this block, the 'post' dict + # is functionally equivalent before- and after- the use of FormData + # TODO: A more elegant solution? + if request.POST.has_key('_answers_querystring'): + post = parse_qs(request.POST.get('_answers_querystring')) + for key in post.keys(): + post[key] = post[key][0] # parse_qs returns { key: list } student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) @@ -292,7 +294,7 @@ def modx_dispatch(request, dispatch=None, id=None): # Let the module handle the AJAX try: - ajax_return = instance.handle_ajax(dispatch, request.POST) + ajax_return = instance.handle_ajax(dispatch, post) except NotFoundError: log.exception("Module indicating to user that request doesn't exist") raise Http404 From ea36eef6e0ec3ce27873886b2538fb4477129178 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 19:38:33 -0400 Subject: [PATCH 11/66] modx_dispatch uses copied variable rather than source --- lms/djangoapps/courseware/module_render.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 6fd6305989..29e3b527e0 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -278,8 +278,8 @@ def modx_dispatch(request, dispatch=None, id=None): # Catch the use of FormData in xmodule frontend. After this block, the 'post' dict # is functionally equivalent before- and after- the use of FormData # TODO: A more elegant solution? - if request.POST.has_key('_answers_querystring'): - post = parse_qs(request.POST.get('_answers_querystring')) + if post.has_key('_answers_querystring'): + post = parse_qs(post.get('_answers_querystring')) for key in post.keys(): post[key] = post[key][0] # parse_qs returns { key: list } From 35a461d917ea4b056b79140ed08d2eea5db609f7 Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 20:22:33 -0400 Subject: [PATCH 12/66] Empty file inputs insert empty string as their submission in ajax; LMS reacts accordingly --- .../xmodule/js/src/capa/display.coffee | 7 +++++-- lms/djangoapps/courseware/module_render.py | 21 +++++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 4318e5850c..c9d26a5ec2 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -65,11 +65,14 @@ class @Problem # For each file input, allow a single file submission, # routed to Django 'request.FILES' @$('input:file').each (index, element) -> - fd.append(element.id, element.files[0]) + if element.files[0] instanceof File + fd.append(element.id, element.files[0]) + else + fd.append(element.id, '') # Even if no file selected, need to include input id # Simple (non-file) answers, # routed to Django 'request.POST' - fd.append('_answers_querystring', @answers) + fd.append('__answers_querystring', @answers) settings = type: "POST" diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 29e3b527e0..18ff2d8305 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -275,13 +275,22 @@ def modx_dispatch(request, dispatch=None, id=None): post = request.POST.copy() - # Catch the use of FormData in xmodule frontend. After this block, the 'post' dict - # is functionally equivalent before- and after- the use of FormData + # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, + # the 'post' dict is functionally equivalent before- and after- the use of FormData # TODO: A more elegant solution? - if post.has_key('_answers_querystring'): - post = parse_qs(post.get('_answers_querystring')) - for key in post.keys(): - post[key] = post[key][0] # parse_qs returns { key: list } + if post.has_key('__answers_querystring'): + qs = post.pop('__answers_querystring')[0] + qsdict = parse_qs(qs, keep_blank_values=True) + for key in qsdict.keys(): + qsdict[key] = qsdict[key][0] # parse_qs returns { key: list } + post.update(qsdict) + + # Check for submitted files + if request.FILES: + print 'Got files!' + + print post.keys() + print request.FILES.keys() student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) From 688f8914e3c7c935f74b900fc7a5f1cfaf0e312f Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 20:32:03 -0400 Subject: [PATCH 13/66] Update filesubmission template to show state --- common/lib/capa/capa/inputtypes.py | 4 ++-- common/lib/capa/capa/templates/filesubmission.html | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index 32ee479414..c3cddf48ed 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -13,6 +13,7 @@ Module containing the problem elements which render into input objects - checkboxgroup - imageinput (for clickable image) - optioninput (for option list) +- filesubmission (upload a file) These are matched by *.html files templates/*.html which are mako templates with the actual html. @@ -306,8 +307,7 @@ def filesubmission(element, value, status, render_template, msg=''): Upload a single file (e.g. for programming assignments) ''' eid = element.get('id') - - context = {'id': eid, } + context = { 'id': eid, 'state': status, 'msg': msg, } html = render_template("filesubmission.html", context) return etree.XML(html) diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html index f9073799d4..08f45c916a 100644 --- a/common/lib/capa/capa/templates/filesubmission.html +++ b/common/lib/capa/capa/templates/filesubmission.html @@ -1,3 +1,16 @@

+ % if state == 'unsubmitted': + + % elif state == 'correct': + + % elif state == 'incorrect': + + % elif state == 'incomplete': + + % endif + (${state}) +
+ ${msg|n} +
From 3b3482b3941f71d1b0839e98a87d25beb10b6f7c Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 20:43:09 -0400 Subject: [PATCH 14/66] Skeleton for S3 upload by LMS --- lms/djangoapps/courseware/module_render.py | 10 +++++----- lms/djangoapps/courseware/xqueue_interface.py | 3 ++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 18ff2d8305..b0bb760753 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -285,12 +285,12 @@ def modx_dispatch(request, dispatch=None, id=None): qsdict[key] = qsdict[key][0] # parse_qs returns { key: list } post.update(qsdict) - # Check for submitted files + # Check for submitted files, send it to S3 immediately. LMS/xqueue manipulates only the + # pointer, which is saved as the student "submission" if request.FILES: - print 'Got files!' - - print post.keys() - print request.FILES.keys() + for inputfile_id in request.FILES.keys(): + s3_identifier = xqueue_interface.upload_files_to_s3(request.FILES[inputfile_id]) + post.update({inputfile_id: s3_identifier}) student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index c9042c3f1e..f21c4ad9bb 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -8,8 +8,9 @@ import requests XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' -def upload_files_to_s3(): +def upload_files_to_s3(submission_file): print ' THK: xqueue_interface.upload_files_to_s3' + return '' def make_xheader(lms_callback_url, lms_key, queue_name): From 843f8ae9c32a01170c635743530623065a96f64a Mon Sep 17 00:00:00 2001 From: kimth Date: Thu, 2 Aug 2012 21:51:36 -0400 Subject: [PATCH 15/66] Basic uploads to S3 --- lms/djangoapps/courseware/xqueue_interface.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index f21c4ad9bb..8aca6df8ee 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -4,13 +4,39 @@ import json import requests +from boto.s3.connection import S3Connection +from boto.s3.key import Key + # TODO: Collection of parameters to be hooked into rest of edX system XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' +AWS_ACCESS_KEY = 'AKIAIYY272VA3C5R4DSQ' +AWS_SECRET_KEY = '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' + +AWS_BUCKET_NAME = 'XQUEUE' + def upload_files_to_s3(submission_file): - print ' THK: xqueue_interface.upload_files_to_s3' - return '' + ''' + Upload student file submissions to S3. + + Returns the S3 key for accessing the file + ''' + print type(submission_file) + print dir(submission_file) + print submission_file + + conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) + bucket_name = AWS_ACCESS_KEY + AWS_BUCKET_NAME + bucket = conn.create_bucket(bucket_name.lower()) # Bucket names must be lowercase... + + k = Key(bucket) + k.key = submission_file.name + k.set_contents_from_string(submission_file.read) + + s3_identifier = k.generate_url(60) + print s3_identifier + return s3_identifier def make_xheader(lms_callback_url, lms_key, queue_name): From e4609f3abebdd2d74165e0565c32e3822897e3db Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 09:30:03 -0400 Subject: [PATCH 16/66] Remove unnecessary hardcode url in CodeResponse --- common/lib/capa/capa/responsetypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 50b5df69d9..f4ba9231f5 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -811,7 +811,7 @@ class CodeResponse(LoncapaResponse): def setup_response(self): xml = self.xml - self.url = xml.get('url', "http://xqueue.edx.org/xqueue/submit/") # FIXME -- hardcoded url + self.url = xml.get('url') self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename) answer = xml.find('answer') From 3d8ee671b39ba66721e3e580fca6711cc504f3f3 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 09:34:47 -0400 Subject: [PATCH 17/66] Adjust comments --- common/lib/capa/capa/responsetypes.py | 2 +- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 11 +++++------ lms/djangoapps/courseware/module_render.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index f4ba9231f5..bd5a588c27 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -873,7 +873,7 @@ class CodeResponse(LoncapaResponse): queue_name=self.queue_name) # Generate body - # NOTE: Currently specialized to 6.00x's pyxservers, which follows the ExternalResponse interface + # NOTE: Currently specialized to 6.00x's pyxserver, which follows the ExternalResponse interface contents = {'xml': etree.tostring(self.xml, pretty_print=True), 'edX_cmd': 'get_score', 'edX_tests': self.tests, diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index c9d26a5ec2..a06f512a6d 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -47,11 +47,11 @@ class @Problem $(placeholder).remove() ### - # 'check_fd' uses FormData to allow file submissions, in addition to simple, - # querystring-based answers, in the 'problem_check' dispatch. + # 'check_fd' uses FormData to allow file submissions in the 'problem_check' dispatch, + # in addition to simple querystring-based answers # # NOTE: The dispatch 'problem_check' is being singled out for the use of FormData; - # perhaps preferable to consolidate all dispatches to use FormData consistently + # maybe preferable to consolidate all dispatches to use FormData ### check_fd: => Logger.log 'problem_check', @answers @@ -63,15 +63,14 @@ class @Problem fd = new FormData() # For each file input, allow a single file submission, - # routed to Django 'request.FILES' + # which is routed to Django 'request.FILES' @$('input:file').each (index, element) -> if element.files[0] instanceof File fd.append(element.id, element.files[0]) else fd.append(element.id, '') # Even if no file selected, need to include input id - # Simple (non-file) answers, - # routed to Django 'request.POST' + # Simple (non-file) answers, which is routed to Django 'request.POST' fd.append('__answers_querystring', @answers) settings = diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index b0bb760753..0eaaa60196 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -276,7 +276,7 @@ def modx_dispatch(request, dispatch=None, id=None): post = request.POST.copy() # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, - # the 'post' dict is functionally equivalent before- and after- the use of FormData + # the 'post' dict is functionally equivalent before and after the use of FormData # TODO: A more elegant solution? if post.has_key('__answers_querystring'): qs = post.pop('__answers_querystring')[0] From fbed664d4f9ded7aa630cba815308fcaa3f365cc Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 09:39:18 -0400 Subject: [PATCH 18/66] Auth saved as dict rather than tuple --- lms/djangoapps/courseware/xqueue_interface.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 8aca6df8ee..2bb7bed297 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -8,7 +8,8 @@ from boto.s3.connection import S3Connection from boto.s3.key import Key # TODO: Collection of parameters to be hooked into rest of edX system -XQUEUE_LMS_AUTH = ('LMS','PaloAltoCA') # (username, password) +XQUEUE_LMS_AUTH = { 'username': 'LMS', + 'password': 'PaloAltoCA' } XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' AWS_ACCESS_KEY = 'AKIAIYY272VA3C5R4DSQ' @@ -72,8 +73,8 @@ def send_to_queue(header, body, xqueue_url=None): #------------------------------------------------------------ s = requests.session() try: - r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH[0], - 'password': XQUEUE_LMS_AUTH[1] }) + r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH['username'], + 'password': XQUEUE_LMS_AUTH['password'] }) except Exception as err: msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) From f023dc6e2689d98f1db7da2bbb252d0805632557 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 10:10:06 -0400 Subject: [PATCH 19/66] xqueue_interface generates hashkeys --- common/lib/capa/capa/responsetypes.py | 12 ++---------- lms/djangoapps/courseware/xqueue_interface.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index bd5a588c27..dc0fd518ee 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -8,7 +8,6 @@ Used by capa_problem.py ''' # standard library imports -import hashlib import inspect import json import logging @@ -17,7 +16,6 @@ import numpy import random import re import requests -import time import traceback import abc @@ -856,18 +854,12 @@ class CodeResponse(LoncapaResponse): raise Exception(err) self.context.update({'submission': submission}) - extra_payload = {'edX_student_response': submission} # Prepare xqueue request #------------------------------------------------------------ - # Queuekey generation - h = hashlib.md5() - h.update(str(self.system.seed)) - h.update(str(time.time())) - queuekey = h.hexdigest() - # Generate header + queuekey = xqueue_interface.make_hashkey(self.system.seed) xheader = xqueue_interface.make_xheader(lms_callback_url=self.system.xqueue_callback_url, lms_key=queuekey, queue_name=self.queue_name) @@ -886,7 +878,7 @@ class CodeResponse(LoncapaResponse): cmap = CorrectMap() if error: - cmap.set(self.answer_id, msg='Unable to deliver your submission to grader! Please try again later') + cmap.set(self.answer_id, queuekey=None, msg='Unable to deliver your submission to grader! Please try again later') else: # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 2bb7bed297..8ae8b6b441 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -1,8 +1,10 @@ # # LMS Interface to external queueing system (xqueue) # +import hashlib import json import requests +import time from boto.s3.connection import S3Connection from boto.s3.key import Key @@ -39,6 +41,15 @@ def upload_files_to_s3(submission_file): print s3_identifier return s3_identifier +def make_hashkey(seed=None): + ''' + Generate a string key by hashing + ''' + h = hashlib.md5() + if seed is not None: + h.update(str(seed)) + h.update(str(time.time())) + return h.hexdigest() def make_xheader(lms_callback_url, lms_key, queue_name): ''' From 1be3057d3c79ffd8312ab29cbceab0bbe8f56e78 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 11:49:20 -0400 Subject: [PATCH 20/66] xqueue interface in LMS not responsible for S3 upload --- lms/djangoapps/courseware/xqueue_interface.py | 31 +------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/lms/djangoapps/courseware/xqueue_interface.py index 8ae8b6b441..fa495d60a0 100644 --- a/lms/djangoapps/courseware/xqueue_interface.py +++ b/lms/djangoapps/courseware/xqueue_interface.py @@ -6,41 +6,11 @@ import json import requests import time -from boto.s3.connection import S3Connection -from boto.s3.key import Key - # TODO: Collection of parameters to be hooked into rest of edX system XQUEUE_LMS_AUTH = { 'username': 'LMS', 'password': 'PaloAltoCA' } XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' -AWS_ACCESS_KEY = 'AKIAIYY272VA3C5R4DSQ' -AWS_SECRET_KEY = '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' - -AWS_BUCKET_NAME = 'XQUEUE' - -def upload_files_to_s3(submission_file): - ''' - Upload student file submissions to S3. - - Returns the S3 key for accessing the file - ''' - print type(submission_file) - print dir(submission_file) - print submission_file - - conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) - bucket_name = AWS_ACCESS_KEY + AWS_BUCKET_NAME - bucket = conn.create_bucket(bucket_name.lower()) # Bucket names must be lowercase... - - k = Key(bucket) - k.key = submission_file.name - k.set_contents_from_string(submission_file.read) - - s3_identifier = k.generate_url(60) - print s3_identifier - return s3_identifier - def make_hashkey(seed=None): ''' Generate a string key by hashing @@ -51,6 +21,7 @@ def make_hashkey(seed=None): h.update(str(time.time())) return h.hexdigest() + def make_xheader(lms_callback_url, lms_key, queue_name): ''' Generate header for delivery and reply of queue request. From 7a9489a9b8966b707fe93314fe3233754f9d9ec9 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 13:53:31 -0400 Subject: [PATCH 21/66] modx_dispatch combines file and non-file submission into single answer dict --- lms/djangoapps/courseware/module_render.py | 27 +++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 0eaaa60196..744b84455c 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -273,24 +273,23 @@ def modx_dispatch(request, dispatch=None, id=None): ''' # ''' (fix emacs broken parsing) - post = request.POST.copy() + # Check for submitted files + post = dict() + if request.FILES: + for inputfile_id in request.FILES.keys(): + post[inputfile_id] = request.FILES[inputfile_id] # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, # the 'post' dict is functionally equivalent before and after the use of FormData # TODO: A more elegant solution? - if post.has_key('__answers_querystring'): - qs = post.pop('__answers_querystring')[0] - qsdict = parse_qs(qs, keep_blank_values=True) - for key in qsdict.keys(): - qsdict[key] = qsdict[key][0] # parse_qs returns { key: list } - post.update(qsdict) - - # Check for submitted files, send it to S3 immediately. LMS/xqueue manipulates only the - # pointer, which is saved as the student "submission" - if request.FILES: - for inputfile_id in request.FILES.keys(): - s3_identifier = xqueue_interface.upload_files_to_s3(request.FILES[inputfile_id]) - post.update({inputfile_id: s3_identifier}) + for key in request.POST.keys(): + if key == '__answers_querystring': + qs = request.POST.get(key) + qsdict = parse_qs(qs, keep_blank_values=True) + for qskey in qsdict.keys(): + post[qskey] = qsdict[qskey][0] # parse_qs returns {key: list} + else: + post[key] = request.POST.get(key) student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) From eeaa1e04400eddbcbf881089f67e9d40ea973322 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 13:54:33 -0400 Subject: [PATCH 22/66] Capa util function to turn File objects in dict to filename (string) --- common/lib/capa/capa/util.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/common/lib/capa/capa/util.py b/common/lib/capa/capa/util.py index 63a5f43c03..1de69cd032 100644 --- a/common/lib/capa/capa/util.py +++ b/common/lib/capa/capa/util.py @@ -30,3 +30,14 @@ def contextualize_text(text, context): # private for key in sorted(context, lambda x, y: cmp(len(y), len(x))): text = text.replace('$' + key, str(context[key])) return text + + +def convert_files_to_filenames(answers): + ''' + Check for File objects in the dict of submitted answers, + convert File objects to their filename (string) + ''' + new_answers = dict() + for answer_id in answers.keys(): + new_answers[answer_id] = str(answers[answer_id]) + return new_answers From afcd6fb1d70817b983145b198b4fdcc90171e92b Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:02:49 -0400 Subject: [PATCH 23/66] Safe logging of File-included answers --- common/lib/xmodule/xmodule/capa_module.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/common/lib/xmodule/xmodule/capa_module.py b/common/lib/xmodule/xmodule/capa_module.py index 2fae8b94e2..2bf1dd0487 100644 --- a/common/lib/xmodule/xmodule/capa_module.py +++ b/common/lib/xmodule/xmodule/capa_module.py @@ -16,6 +16,7 @@ from xmodule.exceptions import NotFoundError from progress import Progress from capa.capa_problem import LoncapaProblem from capa.responsetypes import StudentInputError +from capa.util import convert_files_to_filenames log = logging.getLogger("mitx.courseware") @@ -413,10 +414,9 @@ class CapaModule(XModule): event_info = dict() event_info['state'] = self.lcp.get_state() event_info['problem_id'] = self.location.url() - + answers = self.make_dict_of_responses(get) - - event_info['answers'] = answers + event_info['answers'] = convert_files_to_filenames(answers) # Too late. Cannot submit if self.closed(): @@ -424,8 +424,7 @@ class CapaModule(XModule): self.system.track_function('save_problem_check_fail', event_info) raise NotFoundError('Problem is closed') - # Problem submitted. Student should reset before checking - # again. + # Problem submitted. Student should reset before checking again if self.lcp.done and self.rerandomize == "always": event_info['failure'] = 'unreset' self.system.track_function('save_problem_check_fail', event_info) From cbb377d38357352a37e8ceb4ca08b8215b3122d3 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:10:22 -0400 Subject: [PATCH 24/66] Answer dict values remain unicode, rather than str --- common/lib/capa/capa/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/lib/capa/capa/util.py b/common/lib/capa/capa/util.py index 1de69cd032..1dc113cd20 100644 --- a/common/lib/capa/capa/util.py +++ b/common/lib/capa/capa/util.py @@ -39,5 +39,5 @@ def convert_files_to_filenames(answers): ''' new_answers = dict() for answer_id in answers.keys(): - new_answers[answer_id] = str(answers[answer_id]) + new_answers[answer_id] = unicode(answers[answer_id]) return new_answers From bfd441255afce626f57e309aedba84a8b0d0d583 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:16:20 -0400 Subject: [PATCH 25/66] Every path except for 'get_score' gets filename instead of file object --- common/lib/capa/capa/capa_problem.py | 5 +++-- common/lib/capa/capa/responsetypes.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index 55b9096021..bdfbbab73d 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -31,7 +31,7 @@ import calc from correctmap import CorrectMap import eia import inputtypes -from util import contextualize_text +from util import contextualize_text, convert_files_to_filenames # to be replaced with auto-registering import responsetypes @@ -228,7 +228,8 @@ class LoncapaProblem(object): Calls the Response for each question in this problem, to do the actual grading. ''' - self.student_answers = answers + + self.student_answers = convert_files_to_filenames(answers) oldcmap = self.correct_map # old CorrectMap newcmap = CorrectMap() # start new with empty CorrectMap # log.debug('Responders: %s' % self.responders) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index dc0fd518ee..2874de7395 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -161,7 +161,7 @@ class LoncapaResponse(object): Returns the new CorrectMap, with (correctness,msg,hint,hintmode) for each answer_id. ''' new_cmap = self.get_score(student_answers) - self.get_hints(student_answers, new_cmap, old_cmap) + self.get_hints(convert_files_to_filenames(student_answers), new_cmap, old_cmap) # log.debug('new_cmap = %s' % new_cmap) return new_cmap From 473bb817e8d89a32021142aa04e3d7b7a58a2b96 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:34:48 -0400 Subject: [PATCH 26/66] File objects passed to responsetype only if responsetype explicitly allows filesubmissions --- common/lib/capa/capa/capa_problem.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/capa_problem.py b/common/lib/capa/capa/capa_problem.py index bdfbbab73d..d5f45831ee 100644 --- a/common/lib/capa/capa/capa_problem.py +++ b/common/lib/capa/capa/capa_problem.py @@ -230,11 +230,16 @@ class LoncapaProblem(object): ''' self.student_answers = convert_files_to_filenames(answers) + oldcmap = self.correct_map # old CorrectMap newcmap = CorrectMap() # start new with empty CorrectMap # log.debug('Responders: %s' % self.responders) - for responder in self.responders.values(): - results = responder.evaluate_answers(answers, oldcmap) # call the responsetype instance to do the actual grading + for responder in self.responders.values(): # Call each responsetype instance to do actual grading + if 'filesubmission' in responder.allowed_inputfields: # File objects are passed only if responsetype + # explicitly allows for file submissions + results = responder.evaluate_answers(answers, oldcmap) + else: + results = responder.evaluate_answers(convert_files_to_filenames(answers), oldcmap) newcmap.update(results) self.correct_map = newcmap # log.debug('%s: in grade_answers, answers=%s, cmap=%s' % (self,answers,newcmap)) From 604596843cb5df250ba7fbba3867ae8c7b54344d Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 14:39:30 -0400 Subject: [PATCH 27/66] Time to think about file interface with xqueue --- common/lib/capa/capa/responsetypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 2874de7395..6279a2de28 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -853,7 +853,7 @@ class CodeResponse(LoncapaResponse): log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' % (err, self.answer_id, student_answers)) raise Exception(err) - self.context.update({'submission': submission}) + self.context.update({'submission': unicode(submission)}) # Submission could be a file # Prepare xqueue request #------------------------------------------------------------ @@ -870,7 +870,7 @@ class CodeResponse(LoncapaResponse): 'edX_cmd': 'get_score', 'edX_tests': self.tests, 'processor': self.code, - 'edX_student_response': submission} + 'edX_student_response': unicode(submission)} # Submit request error = xqueue_interface.send_to_queue(header=xheader, From af22761778e6eabcd839de99c32ffd407ab5b604 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 15:27:39 -0400 Subject: [PATCH 28/66] modx_dispatch doesn't need xqueue_interface --- lms/djangoapps/courseware/module_render.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 744b84455c..7883f1e076 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -15,7 +15,6 @@ from static_replace import replace_urls from xmodule.exceptions import NotFoundError from xmodule.x_module import ModuleSystem from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule -import xqueue_interface log = logging.getLogger("mitx.courseware") From c25eded4fcfcdbdaa05e74ff8a5bfab555db920b Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 15:32:02 -0400 Subject: [PATCH 29/66] Move xqueue_interface to lib/capa --- common/lib/capa/capa/responsetypes.py | 2 +- .../courseware => common/lib/capa/capa}/xqueue_interface.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {lms/djangoapps/courseware => common/lib/capa/capa}/xqueue_interface.py (100%) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 6279a2de28..034753d5fd 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -22,10 +22,10 @@ import abc # specific library imports from calc import evaluator, UndefinedVariable from correctmap import CorrectMap -from courseware import xqueue_interface from util import * from lxml import etree from lxml.html.soupparser import fromstring as fromstring_bs # uses Beautiful Soup!!! FIXME? +import xqueue_interface log = logging.getLogger('mitx.' + __name__) diff --git a/lms/djangoapps/courseware/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py similarity index 100% rename from lms/djangoapps/courseware/xqueue_interface.py rename to common/lib/capa/capa/xqueue_interface.py From 9d52c43286b09b0448ef625124f0996ae4fb8534 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 15:49:30 -0400 Subject: [PATCH 30/66] Add 'value' to filesubmission template --- common/lib/capa/capa/inputtypes.py | 2 +- common/lib/capa/capa/templates/filesubmission.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/lib/capa/capa/inputtypes.py b/common/lib/capa/capa/inputtypes.py index c3cddf48ed..583d29b82e 100644 --- a/common/lib/capa/capa/inputtypes.py +++ b/common/lib/capa/capa/inputtypes.py @@ -307,7 +307,7 @@ def filesubmission(element, value, status, render_template, msg=''): Upload a single file (e.g. for programming assignments) ''' eid = element.get('id') - context = { 'id': eid, 'state': status, 'msg': msg, } + context = { 'id': eid, 'state': status, 'msg': msg, 'value': value, } html = render_template("filesubmission.html", context) return etree.XML(html) diff --git a/common/lib/capa/capa/templates/filesubmission.html b/common/lib/capa/capa/templates/filesubmission.html index 08f45c916a..ff9fc992fd 100644 --- a/common/lib/capa/capa/templates/filesubmission.html +++ b/common/lib/capa/capa/templates/filesubmission.html @@ -1,5 +1,5 @@
-
+
% if state == 'unsubmitted': % elif state == 'correct': From 8eff9f7caa29f71fa05d0fa74e1987c7d8deea56 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 17:11:55 -0400 Subject: [PATCH 31/66] Upload to xqueue --- common/lib/capa/capa/responsetypes.py | 16 +++++++++++----- common/lib/capa/capa/xqueue_interface.py | 12 +++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index 034753d5fd..d995e8b902 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -848,12 +848,13 @@ class CodeResponse(LoncapaResponse): def get_score(self, student_answers): try: - submission = student_answers[self.answer_id] + submission = student_answers[self.answer_id] # Note that submission can be a file except Exception as err: - log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' % (err, self.answer_id, student_answers)) + log.error('Error in CodeResponse %s: cannot get student answer for %s; student_answers=%s' % + (err, self.answer_id, convert_files_to_filenames(student_answers))) raise Exception(err) - self.context.update({'submission': unicode(submission)}) # Submission could be a file + self.context.update({'submission': unicode(submission)}) # Prepare xqueue request #------------------------------------------------------------ @@ -873,8 +874,13 @@ class CodeResponse(LoncapaResponse): 'edX_student_response': unicode(submission)} # Submit request - error = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + if hasattr(submission, 'read'): # Test for whether submission is a file + error = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents), + file_to_upload=submission) + else: + error = xqueue_interface.send_to_queue(header=xheader, + body=json.dumps(contents)) cmap = CorrectMap() if error: diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py index fa495d60a0..e479be3ed6 100644 --- a/common/lib/capa/capa/xqueue_interface.py +++ b/common/lib/capa/capa/xqueue_interface.py @@ -37,7 +37,7 @@ def make_xheader(lms_callback_url, lms_key, queue_name): 'queue_name': queue_name }) -def send_to_queue(header, body, xqueue_url=None): +def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): ''' Submit a request to xqueue. @@ -46,6 +46,8 @@ def send_to_queue(header, body, xqueue_url=None): body: Serialized data for the receipient behind the queueing service. The operation of xqueue is agnostic to the contents of 'body' + file_to_upload: File object to be uploaded to xqueue along with queue request + Returns an 'error' flag indicating error in xqueue transaction ''' if xqueue_url is None: @@ -72,9 +74,13 @@ def send_to_queue(header, body, xqueue_url=None): #------------------------------------------------------------ payload = {'xqueue_header': header, 'xqueue_body' : body} + + files = None + if file_to_upload is not None: + files = { file_to_upload.name: file_to_upload } + try: - # Send request - r = s.post(xqueue_url+'/xqueue/submit/', data=payload) + r = s.post(xqueue_url+'/xqueue/submit/', data=payload, files=files) except Exception as err: msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) From 8aef6a0e754cbf49079c35764d4ffc61a3583152 Mon Sep 17 00:00:00 2001 From: kimth Date: Fri, 3 Aug 2012 17:38:34 -0400 Subject: [PATCH 32/66] Ajax doesn't complain about FormData if there's no file to be uploaded --- common/lib/xmodule/xmodule/js/src/capa/display.coffee | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index a06f512a6d..12017105a1 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -56,8 +56,13 @@ class @Problem check_fd: => Logger.log 'problem_check', @answers + # If there are no file inputs in the problem, we can fall back on @check + if $('input:file').length == 0 + @check() + return + if not window.FormData - alert "Sorry, your browser does not support file uploads. If you can, please use Chrome or Safari which have been verified to support this feature." + alert "Sorry, your browser does not support file uploads. Your submit request could not be fulfilled. If you can, please use Chrome or Safari which have been verified to support file uploads." return fd = new FormData() From 84ed806f0dd4da076f5396816b1242a9a5e68451 Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 20:11:58 -0400 Subject: [PATCH 33/66] start on lms migration path: view for loaded modules, and reload method --- common/lib/xmodule/xmodule/modulestore/xml.py | 35 ++++++++++++------- common/lib/xmodule/xmodule/x_module.py | 2 ++ lms/envs/dev.py | 5 +++ lms/urls.py | 6 ++++ 4 files changed, 36 insertions(+), 12 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 0567e4e7a7..46fcf19469 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -146,19 +146,30 @@ class XMLModuleStore(ModuleStoreBase): os.path.exists(self.data_dir / d / "course.xml")] for course_dir in course_dirs: - try: - # Special-case code here, since we don't have a location for the - # course before it loads. - # So, make a tracker to track load-time errors, then put in the right - # place after the course loads and we have its location - errorlog = make_error_tracker() - course_descriptor = self.load_course(course_dir, errorlog.tracker) - self.courses[course_dir] = course_descriptor - self._location_errors[course_descriptor.location] = errorlog - except: - msg = "Failed to load course '%s'" % course_dir - log.exception(msg) + self.try_load_course(course_dir) + def try_load_course(self,course_dir): + ''' + Load a course, keeping track of errors as we go along. + ''' + try: + # Special-case code here, since we don't have a location for the + # course before it loads. + # So, make a tracker to track load-time errors, then put in the right + # place after the course loads and we have its location + errorlog = make_error_tracker() + course_descriptor = self.load_course(course_dir, errorlog.tracker) + self.courses[course_dir] = course_descriptor + self._location_errors[course_descriptor.location] = errorlog + except: + msg = "Failed to load course '%s'" % course_dir + log.exception(msg) + + def __unicode__(self): + ''' + String representation - for debugging + ''' + return 'data_dir=%s, %d courses, %d modules' % (self.data_dir,len(self.courses),len(self.modules)) def load_course(self, course_dir, tracker): """ diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index f6a43f2612..60670767f7 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -204,6 +204,8 @@ class XModule(HTMLSnippet): ''' return self.metadata.get('display_name', self.url_name.replace('_', ' ')) + def __unicode__(self): + return '' % (self.name, self.category, self.id) def get_children(self): ''' diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 813471fb54..a4655bf763 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -58,6 +58,11 @@ CACHE_TIMEOUT = 0 # Dummy secret key for dev SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' +################################ LMS Migration ################################# +MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True + +LMS_MIGRATION_ALLOWED_IPS = ['any'] + ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True diff --git a/lms/urls.py b/lms/urls.py index 78198b2dfb..c74e92ea6e 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -169,6 +169,12 @@ if settings.MITX_FEATURES.get('AUTH_USE_OPENID'): url(r'^openid/logo.gif$', 'django_openid_auth.views.logo', name='openid-logo'), ) +if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + urlpatterns += ( + url(r'^migrate/modules$', 'lms_migration.migrate.manage_modulestores'), + url(r'^migrate/reload/(?P[^/]+)$', 'lms_migration.migrate.manage_modulestores'), + ) + urlpatterns = patterns(*urlpatterns) if settings.DEBUG: From f1ba26b007a181fb2962d493ddd7f1a825bf47ed Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 20:36:17 -0400 Subject: [PATCH 34/66] require login and enrollment in course to be able to view its courseware --- lms/djangoapps/courseware/views.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 18b710e108..831e8ced29 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -150,6 +150,7 @@ def render_accordion(request, course, chapter, section): return render_to_string('accordion.html', context) +@login_required @ensure_csrf_cookie @cache_control(no_cache=True, no_store=True, must_revalidate=True) def index(request, course_id, chapter=None, section=None, @@ -172,6 +173,10 @@ def index(request, course_id, chapter=None, section=None, - HTTPresponse ''' course = check_course(course_id) + registered = registered_for_course(course, request.user) + if not registered: + log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url())) + return redirect('/') try: context = { @@ -266,14 +271,18 @@ def course_info(request, course_id): return render_to_response('info.html', {'course': course}) +def registered_for_course(course, user): + '''Return CourseEnrollment if user is registered for course, else False''' + if user is None: + return False + if user.is_authenticated(): + return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists() + else: + return False + @ensure_csrf_cookie @cache_if_anonymous def course_about(request, course_id): - def registered_for_course(course, user): - if user.is_authenticated(): - return CourseEnrollment.objects.filter(user=user, course_id=course.id).exists() - else: - return False course = check_course(course_id, course_must_be_open=False) registered = registered_for_course(course, request.user) return render_to_response('portal/course_about.html', {'course': course, 'registered': registered}) From 30922fb4491d752a46f269150446618b7abdb444 Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 21:39:23 -0400 Subject: [PATCH 35/66] add ACCESS_REQUIRE_STAFF_FOR_COURSE feature for enrollment check --- common/djangoapps/student/views.py | 9 +++++++++ lms/djangoapps/courseware/courses.py | 20 ++++++++++++++++++++ lms/envs/dev.py | 3 ++- lms/templates/portal/course_about.html | 3 +++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index 35ce225011..ace1f8f576 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -37,6 +37,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment from datetime import date from collections import namedtuple +from courseware.courses import course_staff_group_name, has_staff_access_to_course log = logging.getLogger("mitx.student") Article = namedtuple('Article', 'title url author image deck publication publish_date') @@ -184,6 +185,14 @@ def change_enrollment(request): .format(user.username, enrollment.course_id)) return {'success': False, 'error': 'The course requested does not exist.'} + if settings.MITX_FEATURES.get('ACCESS_REQUIRE_STAFF_FOR_COURSE'): + # require that user be in the staff_* group (or be an overall admin) to be able to enroll + # eg staff_6.002x or staff_6.00x + if not has_staff_access_to_course(user,course): + staff_group = course_staff_group_name(course) + log.debug('user %s denied enrollment to %s ; not in %s' % (user,course.location.url(),staff_group)) + return {'success': False, 'error' : '%s membership required to access course.' % staff_group} + enrollment, created = CourseEnrollment.objects.get_or_create(user=user, course_id=course.id) return {'success': True} diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 19eef3ee80..e11fb566f4 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -114,3 +114,23 @@ def get_course_info_section(course, section_key): return "! Info section missing !" raise KeyError("Invalid about key " + str(section_key)) + +def course_staff_group_name(course): + return 'staff_%s' % course.metadata['course'] + +def has_staff_access_to_course(user,course): + ''' + Returns True if the given user has staff access to the course. + This means that user is in the staff_* group, or is an overall admin. + ''' + if user.is_staff: + return True + user_groups = [x[1] for x in user.groups.values_list()] # note this is the Auth group, not UserTestGroup + log.debug('user is in groups %s' % user_groups) + staff_group = course_staff_group_name(course) + if staff_group in user_groups: + return True + return False + + + diff --git a/lms/envs/dev.py b/lms/envs/dev.py index a4655bf763..3e681e6b0d 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -60,12 +60,13 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ################################ LMS Migration ################################# MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True LMS_MIGRATION_ALLOWED_IPS = ['any'] ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True -MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True +MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True # require that user be in the staff_* group to be able to enroll INSTALLED_APPS += ('external_auth',) INSTALLED_APPS += ('django_openid_auth',) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index e6359d0542..afecc2f795 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -19,6 +19,8 @@ $(document).delegate('#class_enroll_form', 'ajax:success', function(data, json, xhr) { if(json.success) { location.href="${reverse('dashboard')}"; + }else{ + document.getElementById('register_message').innerHTML = "

" + json.error + "

"; } }); })(this) @@ -63,6 +65,7 @@ You are registered for this course (${course.number}). %else: Register for ${course.number} +
%endif %else: Register for ${course.number} From 10f53d62e3a312b58509d399ea97e2bb7128e4d2 Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 22:38:33 -0400 Subject: [PATCH 36/66] migration views - see modulestore contents, and force reload of course --- lms/djangoapps/lms_migration/__init__.py | 0 lms/djangoapps/lms_migration/migrate.py | 104 +++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 lms/djangoapps/lms_migration/__init__.py create mode 100644 lms/djangoapps/lms_migration/migrate.py diff --git a/lms/djangoapps/lms_migration/__init__.py b/lms/djangoapps/lms_migration/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py new file mode 100644 index 0000000000..4da346c657 --- /dev/null +++ b/lms/djangoapps/lms_migration/migrate.py @@ -0,0 +1,104 @@ +# +# migration tools for content team to go from stable-edx4edx to LMS+CMS +# + +import logging +from pprint import pprint +import xmodule.modulestore.django as xmodule_django +from xmodule.modulestore.django import modulestore + +from django.http import HttpResponse +from django.conf import settings + +log = logging.getLogger("mitx.lms_migrate") +LOCAL_DEBUG = True +ALLOWED_IPS = settings.LMS_MIGRATION_ALLOWED_IPS + +def escape(s): + """escape HTML special characters in string""" + return str(s).replace('<','<').replace('>','>') + +def manage_modulestores(request,reload_dir=None): + ''' + Manage the static in-memory modulestores. + + If reload_dir is not None, then instruct the xml loader to reload that course directory. + ''' + html = "" + + def_ms = modulestore() + courses = def_ms.get_courses() + + #---------------------------------------- + # check on IP address of requester + + ip = request.META.get('HTTP_X_REAL_IP','') # nginx reverse proxy + if not ip: + ip = request.META.get('REMOTE_ADDR','None') + + if LOCAL_DEBUG: + html += '

IP address: %s ' % ip + + if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): + html += 'Permission denied' + html += "" + return HttpResponse(html) + + #---------------------------------------- + # reload course if specified + + if reload_dir is not None: + if reload_dir not in def_ms.courses: + html += "

Error: '%s' is not a valid course directory

" % reload_dir + else: + html += "

Reloaded course directory '%s'

" % reload_dir + def_ms.try_load_course(reload_dir) + + #---------------------------------------- + + html += '

Courses loaded in the modulestore

' + html += '
    ' + for cdir, course in def_ms.courses.items(): + html += '
  1. %s (%s)
    2. ' % (settings.MITX_ROOT_URL, + escape(cdir), + escape(cdir), + course.location.url()) + html += '
' + + #---------------------------------------- + + dumpfields = ['definition','location','metadata'] + + for cdir, course in def_ms.courses.items(): + html += '
' + html += '

Course: %s (%s)

' % (course.metadata['display_name'],cdir) + + for field in dumpfields: + data = getattr(course,field) + html += '

%s

' % field + if type(data)==dict: + html += '
    ' + for k,v in data.items(): + html += '
  • %s:%s
    • ' % (escape(k),escape(v)) + html += '
' + else: + html += '
  • %s
' % escape(data) + + + #---------------------------------------- + + html += '
' + html += "courses: 
%s
" % escape(courses) + + ms = xmodule_django._MODULESTORES + html += "modules: 
%s
" % escape(ms) + html += "default modulestore: 
%s
" % escape(unicode(def_ms)) + + #---------------------------------------- + + log.debug('_MODULESTORES=%s' % ms) + log.debug('courses=%s' % courses) + log.debug('def_ms=%s' % unicode(def_ms)) + + html += "" + return HttpResponse(html) From 9de6e28180021c4f0310a981c808635d33f4309b Mon Sep 17 00:00:00 2001 From: ichuang Date: Fri, 3 Aug 2012 22:52:48 -0400 Subject: [PATCH 37/66] limit course reload to localhost or user.is_staff --- lms/djangoapps/lms_migration/migrate.py | 11 ++++++++--- lms/envs/dev.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py index 4da346c657..285b26b483 100644 --- a/lms/djangoapps/lms_migration/migrate.py +++ b/lms/djangoapps/lms_migration/migrate.py @@ -38,11 +38,16 @@ def manage_modulestores(request,reload_dir=None): if LOCAL_DEBUG: html += '

IP address: %s ' % ip + log.debug('request from ip=%s' % ip) if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): - html += 'Permission denied' - html += "" - return HttpResponse(html) + if request.user and request.user.is_staff: + log.debug('request allowed because user=%s is staff' % request.user) + else: + html += 'Permission denied' + html += "" + log.debug('request denied, ALLOWED_IPS=%s' % ALLOWED_IPS) + return HttpResponse(html) #---------------------------------------- # reload course if specified diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 3e681e6b0d..d61c2e8b39 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -62,7 +62,7 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True -LMS_MIGRATION_ALLOWED_IPS = ['any'] +LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True From c20ebd13c47f6053516fe25b5b15719e6abea5e2 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 08:22:06 -0400 Subject: [PATCH 38/66] in course_about.html make "You are registered..." a link to course --- lms/templates/portal/course_about.html | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index afecc2f795..bdea0a47d1 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -62,7 +62,20 @@
%if user.is_authenticated(): %if registered: + <% + if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']: + course_target = reverse('info', args=[course.id]) + else: + course_target = reverse('about_course', args=[course.id]) + show_link = settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION') + %> + %if show_link: + + %endif You are registered for this course (${course.number}). + %if show_link: + + %endif %else: Register for ${course.number}
From 3f83904c128bf3bd8dcf01bb23039c96cf5c1d61 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 10:19:54 -0400 Subject: [PATCH 39/66] if AUTH_REQUIRE_STAFF_FOR_COURSE then course list = those accessible --- common/djangoapps/student/views.py | 14 +++++--------- lms/djangoapps/courseware/courses.py | 23 ++++++++++++++++++++++- lms/djangoapps/courseware/views.py | 18 +++++------------- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/common/djangoapps/student/views.py b/common/djangoapps/student/views.py index ace1f8f576..87490786c1 100644 --- a/common/djangoapps/student/views.py +++ b/common/djangoapps/student/views.py @@ -8,7 +8,6 @@ import uuid import feedparser import urllib import itertools -from collections import defaultdict from django.conf import settings from django.contrib.auth import logout, authenticate, login @@ -37,7 +36,7 @@ from xmodule.modulestore.exceptions import ItemNotFoundError from models import Registration, UserProfile, PendingNameChange, PendingEmailChange, CourseEnrollment from datetime import date from collections import namedtuple -from courseware.courses import course_staff_group_name, has_staff_access_to_course +from courseware.courses import course_staff_group_name, has_staff_access_to_course, get_courses_by_university log = logging.getLogger("mitx.student") Article = namedtuple('Article', 'title url author image deck publication publish_date') @@ -65,9 +64,9 @@ def index(request): from external_auth.views import edXauth_ssl_login return edXauth_ssl_login(request) - return main_index() + return main_index(user=request.user) -def main_index(extra_context = {}): +def main_index(extra_context = {}, user=None): ''' Render the edX main page. @@ -89,11 +88,8 @@ def main_index(extra_context = {}): entry.image = soup.img['src'] if soup.img else None entry.summary = soup.getText() - universities = defaultdict(list) - courses = sorted(modulestore().get_courses(), key=lambda course: course.number) - for course in courses: - universities[course.org].append(course) - + # The course selection work is done in courseware.courses. + universities = get_courses_by_university(None) context = {'universities': universities, 'entries': entries} context.update(extra_context) return render_to_response('index.html', context) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index e11fb566f4..c050084fff 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -1,3 +1,4 @@ +from collections import defaultdict from fs.errors import ResourceNotFoundError from functools import wraps import logging @@ -123,6 +124,8 @@ def has_staff_access_to_course(user,course): Returns True if the given user has staff access to the course. This means that user is in the staff_* group, or is an overall admin. ''' + if user is None or (not user.is_authenticated()) or course is None: + return False if user.is_staff: return True user_groups = [x[1] for x in user.groups.values_list()] # note this is the Auth group, not UserTestGroup @@ -132,5 +135,23 @@ def has_staff_access_to_course(user,course): return True return False - +def get_courses_by_university(user): + ''' + Returns dict of lists of courses available, keyed by course.org (ie university). + Courses are sorted by course.number. + + if ACCESS_REQUIRE_STAFF_FOR_COURSE then list only includes those accessible to user. + ''' + # TODO: Clean up how 'error' is done. + # filter out any courses that errored. + courses = [c for c in modulestore().get_courses() + if isinstance(c, CourseDescriptor)] + courses = sorted(courses, key=lambda course: course.number) + universities = defaultdict(list) + for course in courses: + if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + if not has_staff_access_to_course(user,course): + continue + universities[course.org].append(course) + return universities diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 831e8ced29..41b2101b44 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -1,4 +1,3 @@ -from collections import defaultdict import json import logging import urllib @@ -28,7 +27,7 @@ from xmodule.course_module import CourseDescriptor from util.cache import cache, cache_if_anonymous from student.models import UserTestGroup, CourseEnrollment from courseware import grades -from courseware.courses import check_course +from courseware.courses import check_course, get_courses_by_university log = logging.getLogger("mitx.courseware") @@ -58,19 +57,12 @@ def user_groups(user): @ensure_csrf_cookie @cache_if_anonymous def courses(request): - # TODO: Clean up how 'error' is done. - - # filter out any courses that errored. - courses = [c for c in modulestore().get_courses() - if isinstance(c, CourseDescriptor)] - courses = sorted(courses, key=lambda course: course.number) - universities = defaultdict(list) - for course in courses: - universities[course.org].append(course) - + ''' + Render "find courses" page. The course selection work is done in courseware.courses. + ''' + universities = get_courses_by_university(request.user) return render_to_response("courses.html", {'universities': universities}) - @cache_control(no_cache=True, no_store=True, must_revalidate=True) def gradebook(request, course_id): if 'course_admin' not in user_groups(request.user): From fb7b48e10af14f643ce038459f1d3cf56197414e Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 10:28:05 -0400 Subject: [PATCH 40/66] minor - have migrate also show user when debugging --- lms/djangoapps/lms_migration/migrate.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/lms_migration/migrate.py b/lms/djangoapps/lms_migration/migrate.py index 285b26b483..2bf893507b 100644 --- a/lms/djangoapps/lms_migration/migrate.py +++ b/lms/djangoapps/lms_migration/migrate.py @@ -38,7 +38,8 @@ def manage_modulestores(request,reload_dir=None): if LOCAL_DEBUG: html += '

IP address: %s ' % ip - log.debug('request from ip=%s' % ip) + html += '

User: %s ' % request.user + log.debug('request from ip=%s, user=%s' % (ip,request.user)) if not (ip in ALLOWED_IPS or 'any' in ALLOWED_IPS): if request.user and request.user.is_staff: From d50af5765e009ff9c142f24c9f87b061007d20b0 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 11:03:54 -0400 Subject: [PATCH 41/66] make university profile pages also use get_courses_by_university --- common/lib/xmodule/xmodule/xml_module.py | 1 + lms/djangoapps/courseware/courses.py | 7 ++++++- lms/djangoapps/courseware/views.py | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index b0a289d149..d46304b067 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -41,6 +41,7 @@ class XmlDescriptor(XModuleDescriptor): # to definition_from_xml, and from the xml returned by definition_to_xml metadata_attributes = ('format', 'graceperiod', 'showanswer', 'rerandomize', 'start', 'due', 'graded', 'display_name', 'url_name', 'hide_from_toc', + 'ispublic', # if True, then course is listed for all users; see # VS[compat] Remove once unused. 'name', 'slug') diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index c050084fff..78025c2fae 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -135,6 +135,11 @@ def has_staff_access_to_course(user,course): return True return False +def has_access_to_course(user,course): + if course.metadata.get('ispublic'): + return True + return has_staff_access_to_course(user,course) + def get_courses_by_university(user): ''' Returns dict of lists of courses available, keyed by course.org (ie university). @@ -150,7 +155,7 @@ def get_courses_by_university(user): universities = defaultdict(list) for course in courses: if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): - if not has_staff_access_to_course(user,course): + if not has_access_to_course(user,course): continue universities[course.org].append(course) return universities diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 41b2101b44..5db3bcf91a 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -289,7 +289,7 @@ def university_profile(request, org_id): raise Http404("University Profile not found for {0}".format(org_id)) # Only grab courses for this org... - courses = [c for c in all_courses if c.org == org_id] + courses = get_courses_by_university(request.user)[org_id] context = dict(courses=courses, org_id=org_id) template_file = "university_profile/{0}.html".format(org_id).lower() From b8ae026c2937b0a3bbe66278033fe2fc9fc326e4 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 11:16:47 -0400 Subject: [PATCH 42/66] fail gracefully if course.xml missing metadata in course_staff_group_name --- lms/djangoapps/courseware/courses.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 78025c2fae..133a593ac8 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -117,7 +117,10 @@ def get_course_info_section(course, section_key): raise KeyError("Invalid about key " + str(section_key)) def course_staff_group_name(course): - return 'staff_%s' % course.metadata['course'] + coursename = course.metadata.get('course','') + if not coursename: # Fall 2012: not all course.xml have metadata correct yet + coursename = course.metadata.get('data_dir','UnknownCourseName') + return 'staff_%s' % coursename def has_staff_access_to_course(user,course): ''' From 85af1d88cf46c2279caa4974ba364ebf5bbfb020 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 15:30:36 -0400 Subject: [PATCH 43/66] utility scripts to create new users, and staff_* groups for courses --- utility-scripts/create_groups.py | 32 ++++++++ utility-scripts/create_user.py | 137 +++++++++++++++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 utility-scripts/create_groups.py create mode 100644 utility-scripts/create_user.py diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py new file mode 100644 index 0000000000..841242fd54 --- /dev/null +++ b/utility-scripts/create_groups.py @@ -0,0 +1,32 @@ +#!/usr/bin/python +# +# File: create_groups.py +# Date: 04-Aug-12 +# Author: I. Chuang +# +# Create all staff_* groups for classes in data directory. + +import os, sys, string, re + +sys.path.append(os.path.abspath('.')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' +from lms.envs.dev import * + +from django.conf import settings +from django.contrib.auth.models import User, Group +from path import path + +data_dir = settings.DATA_DIR +print "data_dir = %s" % data_dir + +for course_dir in os.listdir(data_dir): + # print course_dir + if not os.path.isdir(path(data_dir) / course_dir): + continue + gname = 'staff_%s' % course_dir + if Group.objects.filter(name=gname): + print "group exists for %s" % gname + continue + g = Group(name=gname) + g.save() + print "created group %s" % gname diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py new file mode 100644 index 0000000000..19bdb6e743 --- /dev/null +++ b/utility-scripts/create_user.py @@ -0,0 +1,137 @@ +#!/usr/bin/python +# +# File: create_user.py +# Date: 04-Aug-12 +# Author: I. Chuang +# +# Create user. Prompt for groups and ExternalAuthMap + +import os, sys, string, re +import datetime +from getpass import getpass +import json +import readline + +sys.path.append(os.path.abspath('.')) +os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' +from lms.envs.dev import * + +from student.models import UserProfile, Registration +from external_auth.models import ExternalAuthMap +from django.contrib.auth.models import User, Group +from random import choice + +class MyCompleter(object): # Custom completer + + def __init__(self, options): + self.options = sorted(options) + + def complete(self, text, state): + if state == 0: # on first trigger, build possible matches + if text: # cache matches (entries that start with entered text) + self.matches = [s for s in self.options + if s and s.startswith(text)] + else: # no text entered, all matches possible + self.matches = self.options[:] + + # return match indexed by state + try: + return self.matches[state] + except IndexError: + return None + +def GenPasswd(length=8, chars=string.letters + string.digits): + return ''.join([choice(chars) for i in range(length)]) + +#----------------------------------------------------------------------------- +# main + +while True: + uname = raw_input('username: ') + if User.objects.filter(username=uname): + print "username %s already taken" % uname + else: + break + +while True: + email = raw_input('email: ') + if User.objects.filter(email=email): + print "email %s already taken" % email + else: + break + +name = raw_input('Full name: ') + +make_eamap = False +if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y': + if not email.endswith('@MIT.EDU'): + print "Failed - email must be @MIT.EDU" + sys.exit(-1) + mit_domain = 'ssl:MIT' + if ExternalAuthMap.objects.filter(external_id = email, external_domain = mit_domain): + print "Failed - email %s already exists as external_id" % email + sys.exit(-1) + make_eamap = True + password = GenPasswd(12) +else: + while True: + password = getpass() + password2 = getpass() + if password == password2: + break + print "Oops, passwords do not match, please retry" + +user = User(username=uname, email=email, is_active=True) +user.set_password(password) +try: + user.save() +except IntegrityError: + print "Oops, failed to create user %s, IntegrityError" % user + raise + +r = Registration() +r.register(user) + +up = UserProfile(user=user) +up.name = name +up.save() + +if make_eamap: + credentials = "/C=US/ST=Massachusetts/O=Massachusetts Institute of Technology/OU=Client CA v1/CN=%s/emailAddress=%s" % (name,email) + eamap = ExternalAuthMap(external_id = email, + external_email = email, + external_domain = mit_domain, + external_name = name, + internal_password = password, + external_credentials = json.dumps(credentials), + ) + eamap.user = user + eamap.dtsignup = datetime.datetime.now() + eamap.save() + +print "User %s created successfully!" % user + +if not raw_input('Add user %s to any groups? [n] ' % user).lower()=='y': + sys.exit(0) + +print "Here are the groups available:" + +groups = [str(g.name) for g in Group.objects.all()] +print groups + +completer = MyCompleter(groups) +readline.set_completer(completer.complete) +readline.parse_and_bind('tab: complete') + +while True: + gname = raw_input("Add group (tab to autocomplete, empty line to end): ") + if not gname: + break + if not gname in groups: + print "Unknown group %s" % gname + continue + g = Group.objects.get(name=gname) + user.groups.add(g) + print "Added %s to group %s" % (user,g) + +print "Done!" From 7fe75030cc64050e8997d011bab39174aa9c24ad Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 17:52:14 -0400 Subject: [PATCH 44/66] fix staff edit link in module content display (goes to github) --- common/djangoapps/xmodule_modifiers.py | 35 +++++++++++++++------- common/lib/xmodule/xmodule/html_module.py | 10 ++++++- common/lib/xmodule/xmodule/xml_module.py | 11 +++++-- lms/djangoapps/courseware/courses.py | 8 ++++- lms/djangoapps/courseware/module_render.py | 5 +++- lms/templates/staff_problem_info.html | 8 ++--- 6 files changed, 57 insertions(+), 20 deletions(-) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index 4d412000ec..082c5f5122 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -1,9 +1,15 @@ +import re import json +import logging + from django.conf import settings from functools import wraps from static_replace import replace_urls from mitxmako.shortcuts import render_to_string +from xmodule.seq_module import SequenceModule +from xmodule.vertical_module import VerticalModule +log = logging.getLogger("mitx.xmodule_modifiers") def wrap_xmodule(get_html, module, template): """ @@ -69,27 +75,33 @@ def add_histogram(get_html, module): the output of the old get_html function with additional information for admin users only, including a histogram of student answers and the definition of the xmodule + + Does nothing if module is a SequenceModule """ @wraps(get_html) def _get_html(): + + if type(module) in [SequenceModule, VerticalModule]: # TODO: make this more general, eg use an XModule attribute instead + return get_html() + module_id = module.id histogram = grade_histogram(module_id) render_histogram = len(histogram) > 0 - # TODO: fixme - no filename in module.xml in general (this code block - # for edx4edx) the following if block is for summer 2012 edX course - # development; it will change when the CMS comes online - if settings.MITX_FEATURES.get('DISPLAY_EDIT_LINK') and settings.DEBUG and module_xml.get('filename') is not None: - coursename = multicourse_settings.get_coursename_from_request(request) - github_url = multicourse_settings.get_course_github_url(coursename) - fn = module_xml.get('filename') - if module_xml.tag=='problem': fn = 'problems/' + fn # grrr - edit_link = (github_url + '/tree/master/' + fn) if github_url is not None else None - if module_xml.tag=='problem': edit_link += '.xml' # grrr + # TODO (ichuang): Remove after fall 2012 LMS migration done + if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): + filename = module.definition.get('filename','') + log.debug('filename = %s' % filename) + data_dir = module.system.filestore.root_path.rsplit('/')[-1] + edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filename) + log.debug('edit_link = %s' % edit_link) + log.debug('module = %s' % dir(module)) + log.debug('module type = %s' % type(module)) + log.debug('location = %s' % str(module.location)) else: edit_link = False - staff_context = {'definition': json.dumps(module.definition, indent=4), + staff_context = {'definition': module.definition.get('data'), 'metadata': json.dumps(module.metadata, indent=4), 'element_id': module.location.html_id(), 'edit_link': edit_link, @@ -99,3 +111,4 @@ def add_histogram(get_html, module): return render_to_string("staff_problem_info.html", staff_context) return _get_html + diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 7a09004e33..8af1a9c5ba 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -94,7 +94,15 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): msg = "Couldn't parse html in {0}.".format(filepath) log.warning(msg) system.error_tracker("Warning: " + msg) - return {'data' : html} + + definition = {'data' : html} + + # TODO (ichuang): remove this after migration + # for Fall 2012 LMS migration: keep filename + definition['filename'] = filepath + + return definition + except (ResourceNotFoundError) as err: msg = 'Unable to load file contents at path {0}: {1} '.format( filepath, err) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index d46304b067..dee87921d9 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -9,7 +9,7 @@ from fs.errors import ResourceNotFoundError import os import sys -log = logging.getLogger(__name__) +log = logging.getLogger('mitx.' + __name__) _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') @@ -110,6 +110,7 @@ class XmlDescriptor(XModuleDescriptor): filename = xml_object.get('filename') if filename is None: definition_xml = copy.deepcopy(xml_object) + filepath = '' else: filepath = cls._format_filepath(xml_object.tag, filename) @@ -137,7 +138,13 @@ class XmlDescriptor(XModuleDescriptor): raise Exception, msg, sys.exc_info()[2] cls.clean_metadata_from_xml(definition_xml) - return cls.definition_from_xml(definition_xml, system) + definition = cls.definition_from_xml(definition_xml, system) + + # TODO (ichuang): remove this after migration + # for Fall 2012 LMS migration: keep filename + definition['filename'] = filepath + + return definition @classmethod diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 133a593ac8..e568f97f56 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -117,7 +117,13 @@ def get_course_info_section(course, section_key): raise KeyError("Invalid about key " + str(section_key)) def course_staff_group_name(course): - coursename = course.metadata.get('course','') + ''' + course should be either a CourseDescriptor instance, or a string (the .course entry of a Location) + ''' + if type(course)==str: + coursename = course + else: + coursename = course.metadata.get('course','') if not coursename: # Fall 2012: not all course.xml have metadata correct yet coursename = course.metadata.get('data_dir','UnknownCourseName') return 'staff_%s' % coursename diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 9260e15c61..cdb9dc40f3 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -15,6 +15,8 @@ from xmodule.exceptions import NotFoundError from xmodule.x_module import ModuleSystem from xmodule_modifiers import replace_static_urls, add_histogram, wrap_xmodule +from courseware.courses import has_staff_access_to_course + log = logging.getLogger("mitx.courseware") @@ -188,7 +190,8 @@ def get_module(user, request, location, student_module_cache, position=None): module.metadata['data_dir'] ) - if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and user.is_staff: + if (settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and + (user.is_staff or has_staff_access_to_course(user, module.location.course))): module.get_html = add_histogram(module.get_html, module) # If StudentModule for this instance wasn't already in the database, diff --git a/lms/templates/staff_problem_info.html b/lms/templates/staff_problem_info.html index f9fa999ae9..c9b92c51db 100644 --- a/lms/templates/staff_problem_info.html +++ b/lms/templates/staff_problem_info.html @@ -1,11 +1,11 @@ ${module_content} -
-definition = ${definition | h} -metadata = ${metadata | h} -
%if edit_link: % endif +
+definition = 
${definition | h}
+metadata = ${metadata | h} +
%if render_histogram:
%endif From 23669f5aa1fc23b049f53032fa073f171e2aec89 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 17:56:32 -0400 Subject: [PATCH 45/66] add some error handling to utility scripts --- utility-scripts/create_groups.py | 7 ++++++- utility-scripts/create_user.py | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py index 841242fd54..063d2ae392 100644 --- a/utility-scripts/create_groups.py +++ b/utility-scripts/create_groups.py @@ -10,7 +10,12 @@ import os, sys, string, re sys.path.append(os.path.abspath('.')) os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' -from lms.envs.dev import * + +try: + from lms.envs.dev import * +except Exception as err: + print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory." + sys.exit(-1) from django.conf import settings from django.contrib.auth.models import User, Group diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py index 19bdb6e743..c9708f537d 100644 --- a/utility-scripts/create_user.py +++ b/utility-scripts/create_user.py @@ -14,7 +14,12 @@ import readline sys.path.append(os.path.abspath('.')) os.environ['DJANGO_SETTINGS_MODULE'] = 'lms.envs.dev' -from lms.envs.dev import * + +try: + from lms.envs.dev import * +except Exception as err: + print "Run this script from the top-level mitx directory (mitx_all/mitx), not a subdirectory." + sys.exit(-1) from student.models import UserProfile, Registration from external_auth.models import ExternalAuthMap From ebe6bf4888611364f8f36a2b38632bf4f29121a1 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 21:10:50 -0400 Subject: [PATCH 46/66] remove some unnecessary debugging lines in xmodule_modifiers --- common/djangoapps/xmodule_modifiers.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index 082c5f5122..221ad31116 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -91,13 +91,8 @@ def add_histogram(get_html, module): # TODO (ichuang): Remove after fall 2012 LMS migration done if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): filename = module.definition.get('filename','') - log.debug('filename = %s' % filename) data_dir = module.system.filestore.root_path.rsplit('/')[-1] edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filename) - log.debug('edit_link = %s' % edit_link) - log.debug('module = %s' % dir(module)) - log.debug('module type = %s' % type(module)) - log.debug('location = %s' % str(module.location)) else: edit_link = False From 9db88b0b52c23513b757b30bc47ef828e99bde8a Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 21:13:43 -0400 Subject: [PATCH 47/66] fix comment in dev.py --- lms/envs/dev.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lms/envs/dev.py b/lms/envs/dev.py index d61c2e8b39..50062e0513 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -60,13 +60,13 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ################################ LMS Migration ################################# MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True -MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] ################################ OpenID Auth ################################# MITX_FEATURES['AUTH_USE_OPENID'] = True -MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True # require that user be in the staff_* group to be able to enroll +MITX_FEATURES['BYPASS_ACTIVATION_EMAIL_FOR_EXTAUTH'] = True INSTALLED_APPS += ('external_auth',) INSTALLED_APPS += ('django_openid_auth',) From 1ff49aa3f9da856c2ca213f108f11bc6eb3dd3fe Mon Sep 17 00:00:00 2001 From: ichuang Date: Sat, 4 Aug 2012 21:15:51 -0400 Subject: [PATCH 48/66] remove unnecessary comments from util-scripts/* --- utility-scripts/create_groups.py | 2 -- utility-scripts/create_user.py | 2 -- 2 files changed, 4 deletions(-) diff --git a/utility-scripts/create_groups.py b/utility-scripts/create_groups.py index 063d2ae392..33c563127f 100644 --- a/utility-scripts/create_groups.py +++ b/utility-scripts/create_groups.py @@ -1,8 +1,6 @@ #!/usr/bin/python # # File: create_groups.py -# Date: 04-Aug-12 -# Author: I. Chuang # # Create all staff_* groups for classes in data directory. diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py index c9708f537d..e5cb5aed2c 100644 --- a/utility-scripts/create_user.py +++ b/utility-scripts/create_user.py @@ -1,8 +1,6 @@ #!/usr/bin/python # # File: create_user.py -# Date: 04-Aug-12 -# Author: I. Chuang # # Create user. Prompt for groups and ExternalAuthMap From 3c23235885d3d78a2530bde95c4e7aa893456ef7 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 11:39:56 -0400 Subject: [PATCH 49/66] fix for some broken github edit links - avoids symlinks --- common/djangoapps/xmodule_modifiers.py | 9 ++++++--- common/lib/xmodule/xmodule/html_module.py | 4 ++-- common/lib/xmodule/xmodule/xml_module.py | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/common/djangoapps/xmodule_modifiers.py b/common/djangoapps/xmodule_modifiers.py index 221ad31116..843d2eaa38 100644 --- a/common/djangoapps/xmodule_modifiers.py +++ b/common/djangoapps/xmodule_modifiers.py @@ -90,9 +90,12 @@ def add_histogram(get_html, module): # TODO (ichuang): Remove after fall 2012 LMS migration done if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): - filename = module.definition.get('filename','') - data_dir = module.system.filestore.root_path.rsplit('/')[-1] - edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filename) + [filepath, filename] = module.definition.get('filename','') + osfs = module.system.filestore + if osfs.exists(filename): + filepath = filename # if original, unmangled filename exists then use it (github doesn't like symlinks) + data_dir = osfs.root_path.rsplit('/')[-1] + edit_link = "https://github.com/MITx/%s/tree/master/%s" % (data_dir,filepath) else: edit_link = False diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 8af1a9c5ba..260b84278b 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -98,8 +98,8 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): definition = {'data' : html} # TODO (ichuang): remove this after migration - # for Fall 2012 LMS migration: keep filename - definition['filename'] = filepath + # for Fall 2012 LMS migration: keep filename (and unmangled filename) + definition['filename'] = [ filepath, filename ] return definition diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index dee87921d9..fbb17fd236 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -141,8 +141,8 @@ class XmlDescriptor(XModuleDescriptor): definition = cls.definition_from_xml(definition_xml, system) # TODO (ichuang): remove this after migration - # for Fall 2012 LMS migration: keep filename - definition['filename'] = filepath + # for Fall 2012 LMS migration: keep filename (and unmangled filename) + definition['filename'] = [ filepath, filename ] return definition From 3ee224e3994fb66709dcab2a460d77329d3d4a6f Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 12:39:13 -0400 Subject: [PATCH 50/66] improve create_user script slightly, to auto-grab fullname for MIT users --- utility-scripts/create_user.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/utility-scripts/create_user.py b/utility-scripts/create_user.py index e5cb5aed2c..3ce9ce0ecf 100644 --- a/utility-scripts/create_user.py +++ b/utility-scripts/create_user.py @@ -56,17 +56,9 @@ while True: else: break -while True: - email = raw_input('email: ') - if User.objects.filter(email=email): - print "email %s already taken" % email - else: - break - -name = raw_input('Full name: ') - make_eamap = False if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y': + email = '%s@MIT.EDU' % uname if not email.endswith('@MIT.EDU'): print "Failed - email must be @MIT.EDU" sys.exit(-1) @@ -76,6 +68,13 @@ if raw_input('Create MIT ExternalAuth? [n] ').lower()=='y': sys.exit(-1) make_eamap = True password = GenPasswd(12) + + # get name from kerberos + kname = os.popen("finger %s | grep 'name:'" % email).read().strip().split('name: ')[1].strip() + name = raw_input('Full name: [%s] ' % kname).strip() + if name=='': + name = kname + print "name = %s" % name else: while True: password = getpass() @@ -84,6 +83,16 @@ else: break print "Oops, passwords do not match, please retry" + while True: + email = raw_input('email: ') + if User.objects.filter(email=email): + print "email %s already taken" % email + else: + break + + name = raw_input('Full name: ') + + user = User(username=uname, email=email, is_active=True) user.set_password(password) try: From c42960c172604472ae8a8a98349b2cdd986717ba Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 16:32:58 -0400 Subject: [PATCH 51/66] add feature ENABLE_SQL_TRACKING_LOGS and url view /event_logs --- common/djangoapps/track/models.py | 20 +++++++++++++++++++- common/djangoapps/track/views.py | 29 +++++++++++++++++++++++++++-- lms/envs/dev.py | 1 + lms/urls.py | 5 +++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/common/djangoapps/track/models.py b/common/djangoapps/track/models.py index 71a8362390..401fa2832f 100644 --- a/common/djangoapps/track/models.py +++ b/common/djangoapps/track/models.py @@ -1,3 +1,21 @@ from django.db import models -# Create your models here. +from django.db import models + +class TrackingLog(models.Model): + dtcreated = models.DateTimeField('creation date',auto_now_add=True) + username = models.CharField(max_length=32,blank=True) + ip = models.CharField(max_length=32,blank=True) + event_source = models.CharField(max_length=32) + event_type = models.CharField(max_length=32,blank=True) + event = models.TextField(blank=True) + agent = models.CharField(max_length=256,blank=True) + page = models.CharField(max_length=32,blank=True,null=True) + time = models.DateTimeField('event time') + + def __unicode__(self): + s = "[%s] %s@%s: %s | %s | %s | %s" % (self.time, self.username, self.ip, self.event_source, + self.event_type, self.page, self.event) + return s + + diff --git a/common/djangoapps/track/views.py b/common/djangoapps/track/views.py index a60d8bef28..1123b8d30a 100644 --- a/common/djangoapps/track/views.py +++ b/common/djangoapps/track/views.py @@ -2,19 +2,32 @@ import json import logging import os import datetime +import dateutil.parser -# Create your views here. +from django.contrib.auth.decorators import login_required from django.http import HttpResponse from django.http import Http404 +from django.shortcuts import redirect from django.conf import settings +from mitxmako.shortcuts import render_to_response + +from django_future.csrf import ensure_csrf_cookie +from track.models import TrackingLog log = logging.getLogger("tracking") +LOGFIELDS = ['username','ip','event_source','event_type','event','agent','page','time'] def log_event(event): event_str = json.dumps(event) log.info(event_str[:settings.TRACK_MAX_EVENT]) - + if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): + event['time'] = dateutil.parser.parse(event['time']) + tldat = TrackingLog(**dict([(x,event[x]) for x in LOGFIELDS])) + try: + tldat.save() + except Exception as err: + log.debug(err) def user_track(request): try: # TODO: Do the same for many of the optional META parameters @@ -70,4 +83,16 @@ def server_track(request, event_type, event, page=None): "page": page, "time": datetime.datetime.utcnow().isoformat(), } + + if event_type=="/event_logs" and request.user.is_staff: # don't log + return log_event(event) + +@login_required +@ensure_csrf_cookie +def view_tracking_log(request): + if not request.user.is_staff: + return redirect('/') + record_instances = TrackingLog.objects.all().order_by('-time')[0:100] + return render_to_response('tracking_log.html',{'records':record_instances}) + diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 50062e0513..204fcec04b 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -14,6 +14,7 @@ DEBUG = True TEMPLATE_DEBUG = True MITX_FEATURES['DISABLE_START_DATES'] = True +MITX_FEATURES['ENABLE_SQL_TRACKING_LOGS'] = True WIKI_ENABLED = True diff --git a/lms/urls.py b/lms/urls.py index c74e92ea6e..9dc317039e 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -175,6 +175,11 @@ if settings.MITX_FEATURES.get('ENABLE_LMS_MIGRATION'): url(r'^migrate/reload/(?P[^/]+)$', 'lms_migration.migrate.manage_modulestores'), ) +if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): + urlpatterns += ( + url(r'^event_logs$', 'track.views.view_tracking_log'), + ) + urlpatterns = patterns(*urlpatterns) if settings.DEBUG: From 190f1f8f892e352ad88a13f0f16aa1331906e014 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 16:35:32 -0400 Subject: [PATCH 52/66] tracking_log template --- lms/templates/tracking_log.html | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 lms/templates/tracking_log.html diff --git a/lms/templates/tracking_log.html b/lms/templates/tracking_log.html new file mode 100644 index 0000000000..66d375c2f3 --- /dev/null +++ b/lms/templates/tracking_log.html @@ -0,0 +1,14 @@ + +

Tracking Log

+ +% for rec in records: + + + + + + + +% endfor +
datetimeusernameipaddrsourcetype
${rec.time}${rec.username}${rec.ip}${rec.event_source}${rec.event_type}
+ \ No newline at end of file From 0347eb498c40efb45637045c3ec8e6e50deb1e99 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 20:24:00 -0400 Subject: [PATCH 53/66] add MITX_FEATURES flags to enable textbook and discussion, and modify course_navigation correspondingly --- lms/envs/common.py | 11 +++ lms/envs/dev_ike.py | 107 +-------------------------- lms/templates/course_navigation.html | 6 ++ 3 files changed, 21 insertions(+), 103 deletions(-) diff --git a/lms/envs/common.py b/lms/envs/common.py index d89e6760a7..83a4bd4181 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -48,6 +48,17 @@ MITX_FEATURES = { ## DO NOT SET TO True IN THIS FILE ## Doing so will cause all courses to be released on production 'DISABLE_START_DATES': False, # When True, all courses will be active, regardless of start date + + 'ENABLE_TEXTBOOK' : True, + 'ENABLE_DISCUSSION' : True, + + 'ENABLE_SQL_TRACKING_LOGS': False, + 'ENABLE_LMS_MIGRATION': False, + + # extrernal access methods + 'ACCESS_REQUIRE_STAFF_FOR_COURSE': False, + 'AUTH_USE_OPENID': False, + 'AUTH_USE_MIT_CERTIFICATES' : False, } # Used for A/B testing diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py index fb7d980550..b6cd67dfd8 100644 --- a/lms/envs/dev_ike.py +++ b/lms/envs/dev_ike.py @@ -9,108 +9,9 @@ sessions. Assumes structure: """ from .common import * from .logsettings import get_logger_config +from .dev import * -DEBUG = True -TEMPLATE_DEBUG = True +WIKI_ENABLED = False +MITX_FEATURES['ENABLE_TEXTBOOK'] = False +MITX_FEATURES['ENABLE_DISCUSSION'] = False -MITX_FEATURES['DISABLE_START_DATES'] = True - -WIKI_ENABLED = True - -LOGGING = get_logger_config(ENV_ROOT / "log", - logging_env="dev", - tracking_filename="tracking.log", - debug=True) - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': ENV_ROOT / "db" / "mitx.db", - } -} - -CACHES = { - # This is the cache used for most things. Askbot will not work without a - # functioning cache -- it relies on caching to load its settings in places. - # In staging/prod envs, the sessions also live here. - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'mitx_loc_mem_cache', - 'KEY_FUNCTION': 'util.memcache.safe_key', - }, - - # The general cache is what you get if you use our util.cache. It's used for - # things like caching the course.xml file for different A/B test groups. - # We set it to be a DummyCache to force reloading of course.xml in dev. - # In staging environments, we would grab VERSION from data uploaded by the - # push process. - 'general': { - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - 'KEY_PREFIX': 'general', - 'VERSION': 4, - 'KEY_FUNCTION': 'util.memcache.safe_key', - } -} - -# Dummy secret key for dev -SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' - -################################ OpenID Auth ################################# -MITX_FEATURES['AUTH_USE_OPENID'] = True - -INSTALLED_APPS += ('external_auth',) -INSTALLED_APPS += ('django_openid_auth',) -#INSTALLED_APPS += ('ssl_auth',) - -#MIDDLEWARE_CLASSES += ( -# #'ssl_auth.ssl_auth.NginxProxyHeaderMiddleware', # ssl authentication behind nginx proxy -# ) - -#AUTHENTICATION_BACKENDS = ( -# 'django_openid_auth.auth.OpenIDBackend', -# 'django.contrib.auth.backends.ModelBackend', -# ) - -OPENID_CREATE_USERS = False -OPENID_UPDATE_DETAILS_FROM_SREG = True -OPENID_SSO_SERVER_URL = 'https://www.google.com/accounts/o8/id' -OPENID_USE_AS_ADMIN_LOGIN = False -#import external_auth.views as edXauth -#OPENID_RENDER_FAILURE = edXauth.edXauth_openid - -################################ DEBUG TOOLBAR ################################# -INSTALLED_APPS += ('debug_toolbar',) -MIDDLEWARE_CLASSES += ('debug_toolbar.middleware.DebugToolbarMiddleware',) -INTERNAL_IPS = ('127.0.0.1',) - -DEBUG_TOOLBAR_PANELS = ( - 'debug_toolbar.panels.version.VersionDebugPanel', - 'debug_toolbar.panels.timer.TimerDebugPanel', - 'debug_toolbar.panels.settings_vars.SettingsVarsDebugPanel', - 'debug_toolbar.panels.headers.HeaderDebugPanel', - 'debug_toolbar.panels.request_vars.RequestVarsDebugPanel', - 'debug_toolbar.panels.sql.SQLDebugPanel', - 'debug_toolbar.panels.signals.SignalDebugPanel', - 'debug_toolbar.panels.logger.LoggingPanel', - -# Enabling the profiler has a weird bug as of django-debug-toolbar==0.9.4 and -# Django=1.3.1/1.4 where requests to views get duplicated (your method gets -# hit twice). So you can uncomment when you need to diagnose performance -# problems, but you shouldn't leave it on. -# 'debug_toolbar.panels.profiling.ProfilingDebugPanel', -) - -############################ FILE UPLOADS (ASKBOT) ############################# -DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage' -MEDIA_ROOT = ENV_ROOT / "uploads" -MEDIA_URL = "/static/uploads/" -STATICFILES_DIRS.append(("uploads", MEDIA_ROOT)) -FILE_UPLOAD_TEMP_DIR = ENV_ROOT / "uploads" -FILE_UPLOAD_HANDLERS = ( - 'django.core.files.uploadhandler.MemoryFileUploadHandler', - 'django.core.files.uploadhandler.TemporaryFileUploadHandler', -) - -########################### PIPELINE ################################# - -PIPELINE_SASS_ARGUMENTS = '-r {proj_dir}/static/sass/bourbon/lib/bourbon.rb'.format(proj_dir=PROJECT_ROOT) diff --git a/lms/templates/course_navigation.html b/lms/templates/course_navigation.html index 8bda22148d..84b0c04ca0 100644 --- a/lms/templates/course_navigation.html +++ b/lms/templates/course_navigation.html @@ -14,10 +14,16 @@ def url_class(url):
  • Courseware
  • Course Info
    • % if user.is_authenticated(): +% if settings.MITX_FEATURES.get('ENABLE_TEXTBOOK'):
  • Textbook
    • +% endif +% if settings.MITX_FEATURES.get('ENABLE_DISCUSSION'):
  • Discussion
    • +% endif % endif +% if settings.WIKI_ENABLED:
  • Wiki
    • +% endif % if user.is_authenticated():
  • Profile
    • % endif From 553f7046b470c1716619f6d359ca16a55d76b709 Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 21:12:56 -0400 Subject: [PATCH 54/66] suggested username for ssl auth is conjoined name with no spaces --- common/djangoapps/external_auth/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/djangoapps/external_auth/views.py b/common/djangoapps/external_auth/views.py index d00a0a7182..0425f3e158 100644 --- a/common/djangoapps/external_auth/views.py +++ b/common/djangoapps/external_auth/views.py @@ -150,7 +150,7 @@ def edXauth_signup(request, eamap=None): context = {'has_extauth_info': True, 'show_signup_immediately' : True, 'extauth_email': eamap.external_email, - 'extauth_username' : eamap.external_name.split(' ')[0], + 'extauth_username' : eamap.external_name.replace(' ',''), # default - conjoin name, no spaces 'extauth_name': eamap.external_name, } From 76074442869a533ab90ea767db33018640226b8a Mon Sep 17 00:00:00 2001 From: ichuang Date: Sun, 5 Aug 2012 23:26:31 -0400 Subject: [PATCH 55/66] fix bug: course staff group based on dir_name, not course number --- lms/djangoapps/courseware/courses.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index e568f97f56..31ae3e7fda 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -123,9 +123,9 @@ def course_staff_group_name(course): if type(course)==str: coursename = course else: - coursename = course.metadata.get('course','') - if not coursename: # Fall 2012: not all course.xml have metadata correct yet coursename = course.metadata.get('data_dir','UnknownCourseName') + if not coursename: # Fall 2012: not all course.xml have metadata correct yet + coursename = course.metadata.get('course','') return 'staff_%s' % coursename def has_staff_access_to_course(user,course): @@ -138,8 +138,8 @@ def has_staff_access_to_course(user,course): if user.is_staff: return True user_groups = [x[1] for x in user.groups.values_list()] # note this is the Auth group, not UserTestGroup - log.debug('user is in groups %s' % user_groups) staff_group = course_staff_group_name(course) + log.debug('course %s user %s groups %s' % (staff_group, user, user_groups)) if staff_group in user_groups: return True return False From 881cafb88d6829557d542a9504652d771bad0d59 Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 09:14:09 -0400 Subject: [PATCH 56/66] Moved default parameter assignment to function def, parse_xreply util function --- common/lib/capa/capa/xqueue_interface.py | 28 +++++++++++++++--------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py index e479be3ed6..38adeb6d48 100644 --- a/common/lib/capa/capa/xqueue_interface.py +++ b/common/lib/capa/capa/xqueue_interface.py @@ -9,7 +9,7 @@ import time # TODO: Collection of parameters to be hooked into rest of edX system XQUEUE_LMS_AUTH = { 'username': 'LMS', 'password': 'PaloAltoCA' } -XQUEUE_SUBMIT_URL = 'http://xqueue.edx.org' +XQUEUE_URL = 'http://xqueue.edx.org' def make_hashkey(seed=None): ''' @@ -37,7 +37,7 @@ def make_xheader(lms_callback_url, lms_key, queue_name): 'queue_name': queue_name }) -def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): +def send_to_queue(header, body, file_to_upload=None, xqueue_url=XQUEUE_URL): ''' Submit a request to xqueue. @@ -50,8 +50,6 @@ def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): Returns an 'error' flag indicating error in xqueue transaction ''' - if xqueue_url is None: - xqueue_url = XQUEUE_SUBMIT_URL # First, we login with our credentials #------------------------------------------------------------ @@ -64,10 +62,9 @@ def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): raise Exception(msg) # Xqueue responses are JSON-serialized dicts - xreply = json.loads(r.text) - return_code = xreply['return_code'] + (return_code, msg) = parse_xreply(r.text) if return_code: # Nonzero return code from xqueue indicates error - print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + print ' Error in queue_interface.send_to_queue: %s' % msg return 1 # Error # Next, we can make a queueing request @@ -85,9 +82,20 @@ def send_to_queue(header, body, file_to_upload=None, xqueue_url=None): msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) raise Exception(msg) - xreply = json.loads(r.text) - return_code = xreply['return_code'] + (return_code, msg) = parse_xreply(r.text) if return_code: - print ' Error in queue_interface.send_to_queue: %s' % xreply['content'] + print ' Error in queue_interface.send_to_queue: %s' % msg return return_code + +def parse_xreply(xreply): + ''' + Parse the reply from xqueue. Messages are JSON-serialized dict: + { 'return_code': 0 (success), 1 (fail), + 'content': Message from xqueue (string) + } + ''' + xreply = json.loads(xreply) + return_code = xreply['return_code'] + content = xreply['content'] + return (return_code, content) From c950d437e51fcca9548e653e50ce3160211bab8e Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 11:58:00 -0400 Subject: [PATCH 57/66] Add comments on CodeResponse.get_score --- common/lib/capa/capa/responsetypes.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index d995e8b902..a1719ddcce 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -867,11 +867,13 @@ class CodeResponse(LoncapaResponse): # Generate body # NOTE: Currently specialized to 6.00x's pyxserver, which follows the ExternalResponse interface + # We should define a common interface for external code graders to CodeResponse contents = {'xml': etree.tostring(self.xml, pretty_print=True), 'edX_cmd': 'get_score', 'edX_tests': self.tests, 'processor': self.code, - 'edX_student_response': unicode(submission)} + 'edX_student_response': unicode(submission), # unicode on File object returns its filename + } # Submit request if hasattr(submission, 'read'): # Test for whether submission is a file From 6c875206bc05a63eac4c6e93e942d8295b9e7c98 Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 12:44:19 -0400 Subject: [PATCH 58/66] Simplify file submission front end --- .../xmodule/js/src/capa/display.coffee | 16 +++++++--------- lms/djangoapps/courseware/module_render.py | 19 +++---------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/common/lib/xmodule/xmodule/js/src/capa/display.coffee b/common/lib/xmodule/xmodule/js/src/capa/display.coffee index 12017105a1..0e760e98ff 100644 --- a/common/lib/xmodule/xmodule/js/src/capa/display.coffee +++ b/common/lib/xmodule/xmodule/js/src/capa/display.coffee @@ -67,16 +67,14 @@ class @Problem fd = new FormData() - # For each file input, allow a single file submission, - # which is routed to Django 'request.FILES' - @$('input:file').each (index, element) -> - if element.files[0] instanceof File - fd.append(element.id, element.files[0]) + @$("[id^=input_#{@element_id.replace(/problem_/, '')}_]").each (index, element) -> + if element.type is 'file' + if element.files[0] instanceof File + fd.append(element.id, element.files[0]) + else + fd.append(element.id, '') else - fd.append(element.id, '') # Even if no file selected, need to include input id - - # Simple (non-file) answers, which is routed to Django 'request.POST' - fd.append('__answers_querystring', @answers) + fd.append(element.id, element.value) settings = type: "POST" diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 7883f1e076..9ce4008597 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -1,6 +1,5 @@ import json import logging -from urlparse import parse_qs from django.conf import settings from django.http import Http404 @@ -273,22 +272,10 @@ def modx_dispatch(request, dispatch=None, id=None): # ''' (fix emacs broken parsing) # Check for submitted files - post = dict() + p = request.POST.copy() if request.FILES: for inputfile_id in request.FILES.keys(): - post[inputfile_id] = request.FILES[inputfile_id] - - # Catch the use of FormData in xmodule frontend for 'problem_check'. After this block, - # the 'post' dict is functionally equivalent before and after the use of FormData - # TODO: A more elegant solution? - for key in request.POST.keys(): - if key == '__answers_querystring': - qs = request.POST.get(key) - qsdict = parse_qs(qs, keep_blank_values=True) - for qskey in qsdict.keys(): - post[qskey] = qsdict[qskey][0] # parse_qs returns {key: list} - else: - post[key] = request.POST.get(key) + p[inputfile_id] = request.FILES[inputfile_id] student_module_cache = StudentModuleCache(request.user, modulestore().get_item(id)) instance, instance_module, shared_module, module_type = get_module(request.user, request, id, student_module_cache) @@ -301,7 +288,7 @@ def modx_dispatch(request, dispatch=None, id=None): # Let the module handle the AJAX try: - ajax_return = instance.handle_ajax(dispatch, post) + ajax_return = instance.handle_ajax(dispatch, p) except NotFoundError: log.exception("Module indicating to user that request doesn't exist") raise Http404 From a46a37d1c0445bbdec154077b0a976119e818e77 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 13:58:42 -0400 Subject: [PATCH 59/66] log.debug -> log.exception; revert log change in xml_module --- common/djangoapps/track/views.py | 2 +- common/lib/xmodule/xmodule/xml_module.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common/djangoapps/track/views.py b/common/djangoapps/track/views.py index 1123b8d30a..31878bee26 100644 --- a/common/djangoapps/track/views.py +++ b/common/djangoapps/track/views.py @@ -27,7 +27,7 @@ def log_event(event): try: tldat.save() except Exception as err: - log.debug(err) + log.exception(err) def user_track(request): try: # TODO: Do the same for many of the optional META parameters diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index fbb17fd236..7a12ed869d 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -9,7 +9,7 @@ from fs.errors import ResourceNotFoundError import os import sys -log = logging.getLogger('mitx.' + __name__) +log = logging.getLogger(__name__) _AttrMapBase = namedtuple('_AttrMap', 'metadata_key to_metadata from_metadata') From 3484f5382cb593386f3871848060de7e29fab6ee Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:10:00 -0400 Subject: [PATCH 60/66] isinstance instead of type --- lms/djangoapps/courseware/courses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 31ae3e7fda..8193988d67 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -120,7 +120,7 @@ def course_staff_group_name(course): ''' course should be either a CourseDescriptor instance, or a string (the .course entry of a Location) ''' - if type(course)==str: + if isinstance(course,str): coursename = course else: coursename = course.metadata.get('data_dir','UnknownCourseName') From 871ed954be3638326c8cd472bd5f573973ef5790 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:16:11 -0400 Subject: [PATCH 61/66] ACCESS_REQUIRE_STAFF_FOR_COURSE default False in lms.envs.dev --- lms/envs/dev.py | 2 +- lms/envs/dev_ike.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lms/envs/dev.py b/lms/envs/dev.py index 204fcec04b..bc5b621b32 100644 --- a/lms/envs/dev.py +++ b/lms/envs/dev.py @@ -61,7 +61,7 @@ SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd' ################################ LMS Migration ################################# MITX_FEATURES['ENABLE_LMS_MIGRATION'] = True -MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = False # require that user be in the staff_* group to be able to enroll LMS_MIGRATION_ALLOWED_IPS = ['127.0.0.1'] diff --git a/lms/envs/dev_ike.py b/lms/envs/dev_ike.py index b6cd67dfd8..2256decb46 100644 --- a/lms/envs/dev_ike.py +++ b/lms/envs/dev_ike.py @@ -14,4 +14,5 @@ from .dev import * WIKI_ENABLED = False MITX_FEATURES['ENABLE_TEXTBOOK'] = False MITX_FEATURES['ENABLE_DISCUSSION'] = False +MITX_FEATURES['ACCESS_REQUIRE_STAFF_FOR_COURSE'] = True # require that user be in the staff_* group to be able to enroll From 6f894c816cef3f7c6ee7541ffd1ffd31a77a8cce Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:19:45 -0400 Subject: [PATCH 62/66] use jquery for error msg in course_about --- lms/templates/portal/course_about.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index bdea0a47d1..c2c1e3b747 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -20,7 +20,7 @@ if(json.success) { location.href="${reverse('dashboard')}"; }else{ - document.getElementById('register_message').innerHTML = "

    " + json.error + "

    "; + $('#register_message).html("

    " + json.error + "

    ") } }); })(this) From b1ddff838c15f85d3616582b94e2e8a8725ad466 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:23:55 -0400 Subject: [PATCH 63/66] add comment about course start date logic --- lms/templates/portal/course_about.html | 1 + 1 file changed, 1 insertion(+) diff --git a/lms/templates/portal/course_about.html b/lms/templates/portal/course_about.html index c2c1e3b747..a3bf8dd755 100644 --- a/lms/templates/portal/course_about.html +++ b/lms/templates/portal/course_about.html @@ -63,6 +63,7 @@ %if user.is_authenticated(): %if registered: <% + ## TODO: move this logic into a view if course.has_started() or settings.MITX_FEATURES['DISABLE_START_DATES']: course_target = reverse('info', args=[course.id]) else: From 8a1747770a1bae1d3274692aee856288425d1067 Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:30:53 -0400 Subject: [PATCH 64/66] redirect to course_about page if hit internal course page unregistered for --- lms/djangoapps/courseware/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lms/djangoapps/courseware/views.py b/lms/djangoapps/courseware/views.py index 5db3bcf91a..f014e3fcb5 100644 --- a/lms/djangoapps/courseware/views.py +++ b/lms/djangoapps/courseware/views.py @@ -168,7 +168,7 @@ def index(request, course_id, chapter=None, section=None, registered = registered_for_course(course, request.user) if not registered: log.debug('User %s tried to view course %s but is not enrolled' % (request.user,course.location.url())) - return redirect('/') + return redirect(reverse('about_course', args=[course.id])) try: context = { From 9805ed89620f6567c5237ae2ab8b7688cf04903c Mon Sep 17 00:00:00 2001 From: ichuang Date: Mon, 6 Aug 2012 14:37:17 -0400 Subject: [PATCH 65/66] cleanup syntax, split long if into two lines --- common/djangoapps/track/views.py | 2 +- lms/djangoapps/courseware/module_render.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/djangoapps/track/views.py b/common/djangoapps/track/views.py index 31878bee26..b5f9c54665 100644 --- a/common/djangoapps/track/views.py +++ b/common/djangoapps/track/views.py @@ -23,7 +23,7 @@ def log_event(event): log.info(event_str[:settings.TRACK_MAX_EVENT]) if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'): event['time'] = dateutil.parser.parse(event['time']) - tldat = TrackingLog(**dict([(x,event[x]) for x in LOGFIELDS])) + tldat = TrackingLog(**dict( (x,event[x]) for x in LOGFIELDS )) try: tldat.save() except Exception as err: diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index cdb9dc40f3..b6ba381a26 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -190,9 +190,9 @@ def get_module(user, request, location, student_module_cache, position=None): module.metadata['data_dir'] ) - if (settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF') and - (user.is_staff or has_staff_access_to_course(user, module.location.course))): - module.get_html = add_histogram(module.get_html, module) + if settings.MITX_FEATURES.get('DISPLAY_HISTOGRAMS_TO_STAFF'): + if has_staff_access_to_course(user, module.location.course): + module.get_html = add_histogram(module.get_html, module) # If StudentModule for this instance wasn't already in the database, # and this isn't a guest user, create it. From 110637c0231ed7e15b01f03f53bc5145347f89c9 Mon Sep 17 00:00:00 2001 From: kimth Date: Mon, 6 Aug 2012 15:06:19 -0400 Subject: [PATCH 66/66] XqueueInterface is a singleton instantiated object --- common/lib/capa/capa/responsetypes.py | 16 ++-- common/lib/capa/capa/xqueue_interface.py | 117 +++++++++++++---------- 2 files changed, 74 insertions(+), 59 deletions(-) diff --git a/common/lib/capa/capa/responsetypes.py b/common/lib/capa/capa/responsetypes.py index a1719ddcce..12f619a881 100644 --- a/common/lib/capa/capa/responsetypes.py +++ b/common/lib/capa/capa/responsetypes.py @@ -29,6 +29,8 @@ import xqueue_interface log = logging.getLogger('mitx.' + __name__) +qinterface = xqueue_interface.XqueueInterface() + #----------------------------------------------------------------------------- # Exceptions @@ -809,7 +811,6 @@ class CodeResponse(LoncapaResponse): def setup_response(self): xml = self.xml - self.url = xml.get('url') self.queue_name = xml.get('queuename', self.system.xqueue_default_queuename) answer = xml.find('answer') @@ -877,16 +878,17 @@ class CodeResponse(LoncapaResponse): # Submit request if hasattr(submission, 'read'): # Test for whether submission is a file - error = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents), - file_to_upload=submission) + (error, msg) = qinterface.send_to_queue(header=xheader, + body=json.dumps(contents), + file_to_upload=submission) else: - error = xqueue_interface.send_to_queue(header=xheader, - body=json.dumps(contents)) + (error, msg) = qinterface.send_to_queue(header=xheader, + body=json.dumps(contents)) cmap = CorrectMap() if error: - cmap.set(self.answer_id, queuekey=None, msg='Unable to deliver your submission to grader! Please try again later') + cmap.set(self.answer_id, queuekey=None, + msg='Unable to deliver your submission to grader. (Reason: %s.) Please try again later.' % msg) else: # Non-null CorrectMap['queuekey'] indicates that the problem has been queued cmap.set(self.answer_id, queuekey=queuekey, msg='Submitted to grader') diff --git a/common/lib/capa/capa/xqueue_interface.py b/common/lib/capa/capa/xqueue_interface.py index 38adeb6d48..deb068adf6 100644 --- a/common/lib/capa/capa/xqueue_interface.py +++ b/common/lib/capa/capa/xqueue_interface.py @@ -3,6 +3,7 @@ # import hashlib import json +import logging import requests import time @@ -11,6 +12,8 @@ XQUEUE_LMS_AUTH = { 'username': 'LMS', 'password': 'PaloAltoCA' } XQUEUE_URL = 'http://xqueue.edx.org' +log = logging.getLogger('mitx.' + __name__) + def make_hashkey(seed=None): ''' Generate a string key by hashing @@ -37,61 +40,10 @@ def make_xheader(lms_callback_url, lms_key, queue_name): 'queue_name': queue_name }) -def send_to_queue(header, body, file_to_upload=None, xqueue_url=XQUEUE_URL): - ''' - Submit a request to xqueue. - - header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader' - - body: Serialized data for the receipient behind the queueing service. The operation of - xqueue is agnostic to the contents of 'body' - - file_to_upload: File object to be uploaded to xqueue along with queue request - - Returns an 'error' flag indicating error in xqueue transaction - ''' - - # First, we login with our credentials - #------------------------------------------------------------ - s = requests.session() - try: - r = s.post(xqueue_url+'/xqueue/login/', data={ 'username': XQUEUE_LMS_AUTH['username'], - 'password': XQUEUE_LMS_AUTH['password'] }) - except Exception as err: - msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) - raise Exception(msg) - - # Xqueue responses are JSON-serialized dicts - (return_code, msg) = parse_xreply(r.text) - if return_code: # Nonzero return code from xqueue indicates error - print ' Error in queue_interface.send_to_queue: %s' % msg - return 1 # Error - - # Next, we can make a queueing request - #------------------------------------------------------------ - payload = {'xqueue_header': header, - 'xqueue_body' : body} - - files = None - if file_to_upload is not None: - files = { file_to_upload.name: file_to_upload } - - try: - r = s.post(xqueue_url+'/xqueue/submit/', data=payload, files=files) - except Exception as err: - msg = 'Error in xqueue_interface.send_to_queue %s: Cannot connect to server url=%s' % (err, xqueue_url) - raise Exception(msg) - - (return_code, msg) = parse_xreply(r.text) - if return_code: - print ' Error in queue_interface.send_to_queue: %s' % msg - - return return_code - def parse_xreply(xreply): ''' Parse the reply from xqueue. Messages are JSON-serialized dict: - { 'return_code': 0 (success), 1 (fail), + { 'return_code': 0 (success), 1 (fail) 'content': Message from xqueue (string) } ''' @@ -99,3 +51,64 @@ def parse_xreply(xreply): return_code = xreply['return_code'] content = xreply['content'] return (return_code, content) + + +class XqueueInterface: + ''' + Interface to the external grading system + ''' + + def __init__(self, url=XQUEUE_URL, auth=XQUEUE_LMS_AUTH): + self.url = url + self.auth = auth + self.s = requests.session() + self._login() + + def send_to_queue(self, header, body, file_to_upload=None): + ''' + Submit a request to xqueue. + + header: JSON-serialized dict in the format described in 'xqueue_interface.make_xheader' + + body: Serialized data for the receipient behind the queueing service. The operation of + xqueue is agnostic to the contents of 'body' + + file_to_upload: File object to be uploaded to xqueue along with queue request + + Returns (error_code, msg) where error_code != 0 indicates an error + ''' + # Attempt to send to queue + (error, msg) = self._send_to_queue(header, body, file_to_upload) + + if error and (msg == 'login_required'): # Log in, then try again + self._login() + (error, msg) = self._send_to_queue(header, body, file_to_upload) + + return (error, msg) + + def _login(self): + try: + r = self.s.post(self.url+'/xqueue/login/', data={ 'username': self.auth['username'], + 'password': self.auth['password'] }) + except requests.exceptions.ConnectionError, err: + log.error(err) + return (1, 'cannot connect to server') + + return parse_xreply(r.text) + + def _send_to_queue(self, header, body, file_to_upload=None): + + payload = {'xqueue_header': header, + 'xqueue_body' : body} + + files = None + if file_to_upload is not None: + files = { file_to_upload.name: file_to_upload } + + try: + r = self.s.post(self.url+'/xqueue/submit/', data=payload, files=files) + except requests.exceptions.ConnectionError, err: + log.error(err) + return (1, 'cannot connect to server') + + return parse_xreply(r.text)