Merge pull request #40 from edx/feature/brian/regrade-celery-2
Feature/brian/regrade celery 2
This commit is contained in:
brianhw
@@ -1,13 +1,11 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pytz
|
||||
import datetime
|
||||
import dateutil.parser
|
||||
|
||||
from django.contrib.auth.decorators import login_required
|
||||
from django.http import HttpResponse
|
||||
from django.http import Http404
|
||||
from django.shortcuts import redirect
|
||||
from django.conf import settings
|
||||
from mitxmako.shortcuts import render_to_response
|
||||
@@ -22,6 +20,7 @@ LOGFIELDS = ['username', 'ip', 'event_source', 'event_type', 'event', 'agent', '
|
||||
|
||||
|
||||
def log_event(event):
|
||||
"""Write tracking event to log file, and optionally to TrackingLog model."""
|
||||
event_str = json.dumps(event)
|
||||
log.info(event_str[:settings.TRACK_MAX_EVENT])
|
||||
if settings.MITX_FEATURES.get('ENABLE_SQL_TRACKING_LOGS'):
|
||||
@@ -34,6 +33,11 @@ def log_event(event):
|
||||
|
||||
|
||||
def user_track(request):
|
||||
"""
|
||||
Log when GET call to "event" URL is made by a user.
|
||||
|
||||
GET call should provide "event_type", "event", and "page" arguments.
|
||||
"""
|
||||
try: # TODO: Do the same for many of the optional META parameters
|
||||
username = request.user.username
|
||||
except:
|
||||
@@ -50,7 +54,6 @@ def user_track(request):
|
||||
except:
|
||||
agent = ''
|
||||
|
||||
# TODO: Move a bunch of this into log_event
|
||||
event = {
|
||||
"username": username,
|
||||
"session": scookie,
|
||||
@@ -68,6 +71,7 @@ def user_track(request):
|
||||
|
||||
|
||||
def server_track(request, event_type, event, page=None):
|
||||
"""Log events related to server requests."""
|
||||
try:
|
||||
username = request.user.username
|
||||
except:
|
||||
@@ -95,9 +99,52 @@ def server_track(request, event_type, event, page=None):
|
||||
log_event(event)
|
||||
|
||||
|
||||
def task_track(request_info, task_info, event_type, event, page=None):
|
||||
"""
|
||||
Logs tracking information for events occuring within celery tasks.
|
||||
|
||||
The `event_type` is a string naming the particular event being logged,
|
||||
while `event` is a dict containing whatever additional contextual information
|
||||
is desired.
|
||||
|
||||
The `request_info` is a dict containing information about the original
|
||||
task request. Relevant keys are `username`, `ip`, `agent`, and `host`.
|
||||
While the dict is required, the values in it are not, so that {} can be
|
||||
passed in.
|
||||
|
||||
In addition, a `task_info` dict provides more information about the current
|
||||
task, to be stored with the `event` dict. This may also be an empty dict.
|
||||
|
||||
The `page` parameter is optional, and allows the name of the page to
|
||||
be provided.
|
||||
"""
|
||||
|
||||
# supplement event information with additional information
|
||||
# about the task in which it is running.
|
||||
full_event = dict(event, **task_info)
|
||||
|
||||
# All fields must be specified, in case the tracking information is
|
||||
# also saved to the TrackingLog model. Get values from the task-level
|
||||
# information, or just add placeholder values.
|
||||
event = {
|
||||
"username": request_info.get('username', 'unknown'),
|
||||
"ip": request_info.get('ip', 'unknown'),
|
||||
"event_source": "task",
|
||||
"event_type": event_type,
|
||||
"event": full_event,
|
||||
"agent": request_info.get('agent', 'unknown'),
|
||||
"page": page,
|
||||
"time": datetime.datetime.utcnow().isoformat(),
|
||||
"host": request_info.get('host', 'unknown')
|
||||
}
|
||||
|
||||
log_event(event)
|
||||
|
||||
|
||||
@login_required
|
||||
@ensure_csrf_cookie
|
||||
def view_tracking_log(request, args=''):
|
||||
"""View to output contents of TrackingLog model. For staff use only."""
|
||||
if not request.user.is_staff:
|
||||
return redirect('/')
|
||||
nlen = 100
|
||||
|
||||
@@ -15,25 +15,22 @@ This is used by capa_module.
|
||||
|
||||
from datetime import datetime
|
||||
import logging
|
||||
import math
|
||||
import numpy
|
||||
import os.path
|
||||
import re
|
||||
import sys
|
||||
|
||||
from lxml import etree
|
||||
from xml.sax.saxutils import unescape
|
||||
from copy import deepcopy
|
||||
|
||||
from .correctmap import CorrectMap
|
||||
import inputtypes
|
||||
import customrender
|
||||
from .util import contextualize_text, convert_files_to_filenames
|
||||
import xqueue_interface
|
||||
from capa.correctmap import CorrectMap
|
||||
import capa.inputtypes as inputtypes
|
||||
import capa.customrender as customrender
|
||||
from capa.util import contextualize_text, convert_files_to_filenames
|
||||
import capa.xqueue_interface as xqueue_interface
|
||||
|
||||
# to be replaced with auto-registering
|
||||
import responsetypes
|
||||
import safe_exec
|
||||
import capa.responsetypes as responsetypes
|
||||
from capa.safe_exec import safe_exec
|
||||
|
||||
# dict of tagname, Response Class -- this should come from auto-registering
|
||||
response_tag_dict = dict([(x.response_tag, x) for x in responsetypes.__all__])
|
||||
@@ -46,8 +43,8 @@ response_properties = ["codeparam", "responseparam", "answer", "openendedparam"]
|
||||
|
||||
# special problem tags which should be turned into innocuous HTML
|
||||
html_transforms = {'problem': {'tag': 'div'},
|
||||
"text": {'tag': 'span'},
|
||||
"math": {'tag': 'span'},
|
||||
'text': {'tag': 'span'},
|
||||
'math': {'tag': 'span'},
|
||||
}
|
||||
|
||||
# These should be removed from HTML output, including all subelements
|
||||
@@ -134,7 +131,6 @@ class LoncapaProblem(object):
|
||||
|
||||
self.extracted_tree = self._extract_html(self.tree)
|
||||
|
||||
|
||||
def do_reset(self):
|
||||
'''
|
||||
Reset internal state to unfinished, with no answers
|
||||
@@ -175,7 +171,7 @@ class LoncapaProblem(object):
|
||||
Return the maximum score for this problem.
|
||||
'''
|
||||
maxscore = 0
|
||||
for response, responder in self.responders.iteritems():
|
||||
for responder in self.responders.values():
|
||||
maxscore += responder.get_max_score()
|
||||
return maxscore
|
||||
|
||||
@@ -220,7 +216,7 @@ class LoncapaProblem(object):
|
||||
def ungraded_response(self, xqueue_msg, queuekey):
|
||||
'''
|
||||
Handle any responses from the xqueue that do not contain grades
|
||||
Will try to pass the queue message to all inputtypes that can handle ungraded responses
|
||||
Will try to pass the queue message to all inputtypes that can handle ungraded responses
|
||||
|
||||
Does not return any value
|
||||
'''
|
||||
@@ -230,7 +226,6 @@ class LoncapaProblem(object):
|
||||
if hasattr(the_input, 'ungraded_response'):
|
||||
the_input.ungraded_response(xqueue_msg, queuekey)
|
||||
|
||||
|
||||
def is_queued(self):
|
||||
'''
|
||||
Returns True if any part of the problem has been submitted to an external queue
|
||||
@@ -238,7 +233,6 @@ class LoncapaProblem(object):
|
||||
'''
|
||||
return any(self.correct_map.is_queued(answer_id) for answer_id in self.correct_map)
|
||||
|
||||
|
||||
def get_recentmost_queuetime(self):
|
||||
'''
|
||||
Returns a DateTime object that represents the timestamp of the most recent
|
||||
@@ -256,11 +250,11 @@ class LoncapaProblem(object):
|
||||
|
||||
return max(queuetimes)
|
||||
|
||||
|
||||
def grade_answers(self, answers):
|
||||
'''
|
||||
Grade student responses. Called by capa_module.check_problem.
|
||||
answers is a dict of all the entries from request.POST, but with the first part
|
||||
|
||||
`answers` is a dict of all the entries from request.POST, but with the first part
|
||||
of each key removed (the string before the first "_").
|
||||
|
||||
Thus, for example, input_ID123 -> ID123, and input_fromjs_ID123 -> fromjs_ID123
|
||||
@@ -270,24 +264,72 @@ class LoncapaProblem(object):
|
||||
|
||||
# if answers include File objects, convert them to filenames.
|
||||
self.student_answers = convert_files_to_filenames(answers)
|
||||
return self._grade_answers(answers)
|
||||
|
||||
def supports_rescoring(self):
|
||||
"""
|
||||
Checks that the current problem definition permits rescoring.
|
||||
|
||||
More precisely, it checks that there are no response types in
|
||||
the current problem that are not fully supported (yet) for rescoring.
|
||||
|
||||
This includes responsetypes for which the student's answer
|
||||
is not properly stored in state, i.e. file submissions. At present,
|
||||
we have no way to know if an existing response was actually a real
|
||||
answer or merely the filename of a file submitted as an answer.
|
||||
|
||||
It turns out that because rescoring is a background task, limiting
|
||||
it to responsetypes that don't support file submissions also means
|
||||
that the responsetypes are synchronous. This is convenient as it
|
||||
permits rescoring to be complete when the rescoring call returns.
|
||||
"""
|
||||
return all('filesubmission' not in responder.allowed_inputfields for responder in self.responders.values())
|
||||
|
||||
def rescore_existing_answers(self):
|
||||
"""
|
||||
Rescore student responses. Called by capa_module.rescore_problem.
|
||||
"""
|
||||
return self._grade_answers(None)
|
||||
|
||||
def _grade_answers(self, student_answers):
|
||||
"""
|
||||
Internal grading call used for checking new 'student_answers' and also
|
||||
rescoring existing student_answers.
|
||||
|
||||
For new student_answers being graded, `student_answers` is a dict of all the
|
||||
entries from request.POST, but with the first part of each key removed
|
||||
(the string before the first "_"). Thus, for example,
|
||||
input_ID123 -> ID123, and input_fromjs_ID123 -> fromjs_ID123.
|
||||
|
||||
For rescoring, `student_answers` is None.
|
||||
|
||||
Calls the Response for each question in this problem, to do the actual grading.
|
||||
"""
|
||||
# old CorrectMap
|
||||
oldcmap = self.correct_map
|
||||
|
||||
# start new with empty CorrectMap
|
||||
newcmap = CorrectMap()
|
||||
# log.debug('Responders: %s' % self.responders)
|
||||
|
||||
# Call each responsetype instance to do actual grading
|
||||
for responder in self.responders.values():
|
||||
# File objects are passed only if responsetype explicitly allows for file
|
||||
# submissions
|
||||
if 'filesubmission' in responder.allowed_inputfields:
|
||||
results = responder.evaluate_answers(answers, oldcmap)
|
||||
# File objects are passed only if responsetype explicitly allows
|
||||
# for file submissions. But we have no way of knowing if
|
||||
# student_answers contains a proper answer or the filename of
|
||||
# an earlier submission, so for now skip these entirely.
|
||||
# TODO: figure out where to get file submissions when rescoring.
|
||||
if 'filesubmission' in responder.allowed_inputfields and student_answers is None:
|
||||
raise Exception("Cannot rescore problems with possible file submissions")
|
||||
|
||||
# use 'student_answers' only if it is provided, and if it might contain a file
|
||||
# submission that would not exist in the persisted "student_answers".
|
||||
if 'filesubmission' in responder.allowed_inputfields and student_answers is not None:
|
||||
results = responder.evaluate_answers(student_answers, oldcmap)
|
||||
else:
|
||||
results = responder.evaluate_answers(convert_files_to_filenames(answers), oldcmap)
|
||||
results = responder.evaluate_answers(self.student_answers, oldcmap)
|
||||
newcmap.update(results)
|
||||
|
||||
self.correct_map = newcmap
|
||||
# log.debug('%s: in grade_answers, answers=%s, cmap=%s' % (self,answers,newcmap))
|
||||
return newcmap
|
||||
|
||||
def get_question_answers(self):
|
||||
@@ -331,7 +373,6 @@ class LoncapaProblem(object):
|
||||
html = contextualize_text(etree.tostring(self._extract_html(self.tree)), self.context)
|
||||
return html
|
||||
|
||||
|
||||
def handle_input_ajax(self, get):
|
||||
'''
|
||||
InputTypes can support specialized AJAX calls. Find the correct input and pass along the correct data
|
||||
@@ -348,8 +389,6 @@ class LoncapaProblem(object):
|
||||
log.warning("Could not find matching input for id: %s" % input_id)
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
# ======= Private Methods Below ========
|
||||
|
||||
def _process_includes(self):
|
||||
@@ -359,16 +398,16 @@ class LoncapaProblem(object):
|
||||
'''
|
||||
includes = self.tree.findall('.//include')
|
||||
for inc in includes:
|
||||
file = inc.get('file')
|
||||
if file is not None:
|
||||
filename = inc.get('file')
|
||||
if filename is not None:
|
||||
try:
|
||||
# open using ModuleSystem OSFS filestore
|
||||
ifp = self.system.filestore.open(file)
|
||||
ifp = self.system.filestore.open(filename)
|
||||
except Exception as err:
|
||||
log.warning('Error %s in problem xml include: %s' % (
|
||||
err, etree.tostring(inc, pretty_print=True)))
|
||||
log.warning('Cannot find file %s in %s' % (
|
||||
file, self.system.filestore))
|
||||
filename, self.system.filestore))
|
||||
# if debugging, don't fail - just log error
|
||||
# TODO (vshnayder): need real error handling, display to users
|
||||
if not self.system.get('DEBUG'):
|
||||
@@ -381,7 +420,7 @@ class LoncapaProblem(object):
|
||||
except Exception as err:
|
||||
log.warning('Error %s in problem xml include: %s' % (
|
||||
err, etree.tostring(inc, pretty_print=True)))
|
||||
log.warning('Cannot parse XML in %s' % (file))
|
||||
log.warning('Cannot parse XML in %s' % (filename))
|
||||
# if debugging, don't fail - just log error
|
||||
# TODO (vshnayder): same as above
|
||||
if not self.system.get('DEBUG'):
|
||||
@@ -389,11 +428,11 @@ class LoncapaProblem(object):
|
||||
else:
|
||||
continue
|
||||
|
||||
# insert new XML into tree in place of inlcude
|
||||
# insert new XML into tree in place of include
|
||||
parent = inc.getparent()
|
||||
parent.insert(parent.index(inc), incxml)
|
||||
parent.remove(inc)
|
||||
log.debug('Included %s into %s' % (file, self.problem_id))
|
||||
log.debug('Included %s into %s' % (filename, self.problem_id))
|
||||
|
||||
def _extract_system_path(self, script):
|
||||
"""
|
||||
@@ -463,7 +502,7 @@ class LoncapaProblem(object):
|
||||
|
||||
if all_code:
|
||||
try:
|
||||
safe_exec.safe_exec(
|
||||
safe_exec(
|
||||
all_code,
|
||||
context,
|
||||
random_seed=self.seed,
|
||||
@@ -519,18 +558,18 @@ class LoncapaProblem(object):
|
||||
value = ""
|
||||
if self.student_answers and problemid in self.student_answers:
|
||||
value = self.student_answers[problemid]
|
||||
|
||||
|
||||
if input_id not in self.input_state:
|
||||
self.input_state[input_id] = {}
|
||||
|
||||
|
||||
# do the rendering
|
||||
state = {'value': value,
|
||||
'status': status,
|
||||
'id': input_id,
|
||||
'input_state': self.input_state[input_id],
|
||||
'feedback': {'message': msg,
|
||||
'hint': hint,
|
||||
'hintmode': hintmode, }}
|
||||
'status': status,
|
||||
'id': input_id,
|
||||
'input_state': self.input_state[input_id],
|
||||
'feedback': {'message': msg,
|
||||
'hint': hint,
|
||||
'hintmode': hintmode, }}
|
||||
|
||||
input_type_cls = inputtypes.registry.get_class_for_tag(problemtree.tag)
|
||||
# save the input type so that we can make ajax calls on it if we need to
|
||||
@@ -554,7 +593,7 @@ class LoncapaProblem(object):
|
||||
for item in problemtree:
|
||||
item_xhtml = self._extract_html(item)
|
||||
if item_xhtml is not None:
|
||||
tree.append(item_xhtml)
|
||||
tree.append(item_xhtml)
|
||||
|
||||
if tree.tag in html_transforms:
|
||||
tree.tag = html_transforms[problemtree.tag]['tag']
|
||||
|
||||
@@ -4,7 +4,6 @@ Tests of responsetypes
|
||||
|
||||
from datetime import datetime
|
||||
import json
|
||||
from nose.plugins.skip import SkipTest
|
||||
import os
|
||||
import random
|
||||
import unittest
|
||||
@@ -56,9 +55,18 @@ class ResponseTest(unittest.TestCase):
|
||||
self.assertEqual(result, 'incorrect',
|
||||
msg="%s should be marked incorrect" % str(input_str))
|
||||
|
||||
def _get_random_number_code(self):
|
||||
"""Returns code to be used to generate a random result."""
|
||||
return "str(random.randint(0, 1e9))"
|
||||
|
||||
def _get_random_number_result(self, seed_value):
|
||||
"""Returns a result that should be generated using the random_number_code."""
|
||||
rand = random.Random(seed_value)
|
||||
return str(rand.randint(0, 1e9))
|
||||
|
||||
|
||||
class MultiChoiceResponseTest(ResponseTest):
|
||||
from response_xml_factory import MultipleChoiceResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import MultipleChoiceResponseXMLFactory
|
||||
xml_factory_class = MultipleChoiceResponseXMLFactory
|
||||
|
||||
def test_multiple_choice_grade(self):
|
||||
@@ -80,7 +88,7 @@ class MultiChoiceResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class TrueFalseResponseTest(ResponseTest):
|
||||
from response_xml_factory import TrueFalseResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import TrueFalseResponseXMLFactory
|
||||
xml_factory_class = TrueFalseResponseXMLFactory
|
||||
|
||||
def test_true_false_grade(self):
|
||||
@@ -120,7 +128,7 @@ class TrueFalseResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class ImageResponseTest(ResponseTest):
|
||||
from response_xml_factory import ImageResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import ImageResponseXMLFactory
|
||||
xml_factory_class = ImageResponseXMLFactory
|
||||
|
||||
def test_rectangle_grade(self):
|
||||
@@ -184,7 +192,7 @@ class ImageResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class SymbolicResponseTest(ResponseTest):
|
||||
from response_xml_factory import SymbolicResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import SymbolicResponseXMLFactory
|
||||
xml_factory_class = SymbolicResponseXMLFactory
|
||||
|
||||
def test_grade_single_input(self):
|
||||
@@ -224,8 +232,8 @@ class SymbolicResponseTest(ResponseTest):
|
||||
|
||||
def test_complex_number_grade(self):
|
||||
problem = self.build_problem(math_display=True,
|
||||
expect="[[cos(theta),i*sin(theta)],[i*sin(theta),cos(theta)]]",
|
||||
options=["matrix", "imaginary"])
|
||||
expect="[[cos(theta),i*sin(theta)],[i*sin(theta),cos(theta)]]",
|
||||
options=["matrix", "imaginary"])
|
||||
|
||||
# For LaTeX-style inputs, symmath_check() will try to contact
|
||||
# a server to convert the input to MathML.
|
||||
@@ -312,16 +320,16 @@ class SymbolicResponseTest(ResponseTest):
|
||||
# Should not allow multiple inputs, since we specify
|
||||
# only one "expect" value
|
||||
with self.assertRaises(Exception):
|
||||
problem = self.build_problem(math_display=True,
|
||||
expect="2*x+3*y",
|
||||
num_inputs=3)
|
||||
self.build_problem(math_display=True,
|
||||
expect="2*x+3*y",
|
||||
num_inputs=3)
|
||||
|
||||
def _assert_symbolic_grade(self, problem,
|
||||
student_input,
|
||||
dynamath_input,
|
||||
expected_correctness):
|
||||
student_input,
|
||||
dynamath_input,
|
||||
expected_correctness):
|
||||
input_dict = {'1_2_1': str(student_input),
|
||||
'1_2_1_dynamath': str(dynamath_input)}
|
||||
'1_2_1_dynamath': str(dynamath_input)}
|
||||
|
||||
correct_map = problem.grade_answers(input_dict)
|
||||
|
||||
@@ -330,7 +338,7 @@ class SymbolicResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class OptionResponseTest(ResponseTest):
|
||||
from response_xml_factory import OptionResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import OptionResponseXMLFactory
|
||||
xml_factory_class = OptionResponseXMLFactory
|
||||
|
||||
def test_grade(self):
|
||||
@@ -350,7 +358,7 @@ class FormulaResponseTest(ResponseTest):
|
||||
"""
|
||||
Test the FormulaResponse class
|
||||
"""
|
||||
from response_xml_factory import FormulaResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import FormulaResponseXMLFactory
|
||||
xml_factory_class = FormulaResponseXMLFactory
|
||||
|
||||
def test_grade(self):
|
||||
@@ -570,7 +578,7 @@ class FormulaResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class StringResponseTest(ResponseTest):
|
||||
from response_xml_factory import StringResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import StringResponseXMLFactory
|
||||
xml_factory_class = StringResponseXMLFactory
|
||||
|
||||
def test_case_sensitive(self):
|
||||
@@ -647,19 +655,18 @@ class StringResponseTest(ResponseTest):
|
||||
hintfn="gimme_a_random_hint",
|
||||
script=textwrap.dedent("""
|
||||
def gimme_a_random_hint(answer_ids, student_answers, new_cmap, old_cmap):
|
||||
answer = str(random.randint(0, 1e9))
|
||||
answer = {code}
|
||||
new_cmap.set_hint_and_mode(answer_ids[0], answer, "always")
|
||||
|
||||
""")
|
||||
""".format(code=self._get_random_number_code()))
|
||||
)
|
||||
correct_map = problem.grade_answers({'1_2_1': '2'})
|
||||
hint = correct_map.get_hint('1_2_1')
|
||||
r = random.Random(problem.seed)
|
||||
self.assertEqual(hint, str(r.randint(0, 1e9)))
|
||||
self.assertEqual(hint, self._get_random_number_result(problem.seed))
|
||||
|
||||
|
||||
class CodeResponseTest(ResponseTest):
|
||||
from response_xml_factory import CodeResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import CodeResponseXMLFactory
|
||||
xml_factory_class = CodeResponseXMLFactory
|
||||
|
||||
def setUp(self):
|
||||
@@ -673,6 +680,7 @@ class CodeResponseTest(ResponseTest):
|
||||
|
||||
@staticmethod
|
||||
def make_queuestate(key, time):
|
||||
"""Create queuestate dict"""
|
||||
timestr = datetime.strftime(time, dateformat)
|
||||
return {'key': key, 'time': timestr}
|
||||
|
||||
@@ -710,7 +718,7 @@ class CodeResponseTest(ResponseTest):
|
||||
old_cmap = CorrectMap()
|
||||
for i, answer_id in enumerate(answer_ids):
|
||||
queuekey = 1000 + i
|
||||
queuestate = CodeResponseTest.make_queuestate(1000 + i, datetime.now())
|
||||
queuestate = CodeResponseTest.make_queuestate(queuekey, datetime.now())
|
||||
old_cmap.update(CorrectMap(answer_id=answer_ids[i], queuestate=queuestate))
|
||||
|
||||
# Message format common to external graders
|
||||
@@ -771,7 +779,7 @@ class CodeResponseTest(ResponseTest):
|
||||
for i, answer_id in enumerate(answer_ids):
|
||||
queuekey = 1000 + i
|
||||
latest_timestamp = datetime.now()
|
||||
queuestate = CodeResponseTest.make_queuestate(1000 + i, latest_timestamp)
|
||||
queuestate = CodeResponseTest.make_queuestate(queuekey, latest_timestamp)
|
||||
cmap.update(CorrectMap(answer_id=answer_id, queuestate=queuestate))
|
||||
self.problem.correct_map.update(cmap)
|
||||
|
||||
@@ -796,7 +804,7 @@ class CodeResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class ChoiceResponseTest(ResponseTest):
|
||||
from response_xml_factory import ChoiceResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import ChoiceResponseXMLFactory
|
||||
xml_factory_class = ChoiceResponseXMLFactory
|
||||
|
||||
def test_radio_group_grade(self):
|
||||
@@ -828,7 +836,7 @@ class ChoiceResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class JavascriptResponseTest(ResponseTest):
|
||||
from response_xml_factory import JavascriptResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import JavascriptResponseXMLFactory
|
||||
xml_factory_class = JavascriptResponseXMLFactory
|
||||
|
||||
def test_grade(self):
|
||||
@@ -858,7 +866,7 @@ class JavascriptResponseTest(ResponseTest):
|
||||
system.can_execute_unsafe_code = lambda: False
|
||||
|
||||
with self.assertRaises(LoncapaProblemError):
|
||||
problem = self.build_problem(
|
||||
self.build_problem(
|
||||
system=system,
|
||||
generator_src="test_problem_generator.js",
|
||||
grader_src="test_problem_grader.js",
|
||||
@@ -869,7 +877,7 @@ class JavascriptResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class NumericalResponseTest(ResponseTest):
|
||||
from response_xml_factory import NumericalResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import NumericalResponseXMLFactory
|
||||
xml_factory_class = NumericalResponseXMLFactory
|
||||
|
||||
def test_grade_exact(self):
|
||||
@@ -961,7 +969,7 @@ class NumericalResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class CustomResponseTest(ResponseTest):
|
||||
from response_xml_factory import CustomResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import CustomResponseXMLFactory
|
||||
xml_factory_class = CustomResponseXMLFactory
|
||||
|
||||
def test_inline_code(self):
|
||||
@@ -1000,15 +1008,14 @@ class CustomResponseTest(ResponseTest):
|
||||
|
||||
def test_inline_randomization(self):
|
||||
# Make sure the seed from the problem gets fed into the script execution.
|
||||
inline_script = """messages[0] = str(random.randint(0, 1e9))"""
|
||||
inline_script = "messages[0] = {code}".format(code=self._get_random_number_code())
|
||||
problem = self.build_problem(answer=inline_script)
|
||||
|
||||
input_dict = {'1_2_1': '0'}
|
||||
correctmap = problem.grade_answers(input_dict)
|
||||
|
||||
input_msg = correctmap.get_msg('1_2_1')
|
||||
r = random.Random(problem.seed)
|
||||
self.assertEqual(input_msg, str(r.randint(0, 1e9)))
|
||||
self.assertEqual(input_msg, self._get_random_number_result(problem.seed))
|
||||
|
||||
def test_function_code_single_input(self):
|
||||
# For function code, we pass in these arguments:
|
||||
@@ -1241,25 +1248,23 @@ class CustomResponseTest(ResponseTest):
|
||||
def test_setup_randomization(self):
|
||||
# Ensure that the problem setup script gets the random seed from the problem.
|
||||
script = textwrap.dedent("""
|
||||
num = random.randint(0, 1e9)
|
||||
""")
|
||||
num = {code}
|
||||
""".format(code=self._get_random_number_code()))
|
||||
problem = self.build_problem(script=script)
|
||||
r = random.Random(problem.seed)
|
||||
self.assertEqual(r.randint(0, 1e9), problem.context['num'])
|
||||
self.assertEqual(problem.context['num'], self._get_random_number_result(problem.seed))
|
||||
|
||||
def test_check_function_randomization(self):
|
||||
# The check function should get random-seeded from the problem.
|
||||
script = textwrap.dedent("""
|
||||
def check_func(expect, answer_given):
|
||||
return {'ok': True, 'msg': str(random.randint(0, 1e9))}
|
||||
""")
|
||||
return {{'ok': True, 'msg': {code} }}
|
||||
""".format(code=self._get_random_number_code()))
|
||||
|
||||
problem = self.build_problem(script=script, cfn="check_func", expect="42")
|
||||
input_dict = {'1_2_1': '42'}
|
||||
correct_map = problem.grade_answers(input_dict)
|
||||
msg = correct_map.get_msg('1_2_1')
|
||||
r = random.Random(problem.seed)
|
||||
self.assertEqual(msg, str(r.randint(0, 1e9)))
|
||||
self.assertEqual(msg, self._get_random_number_result(problem.seed))
|
||||
|
||||
def test_module_imports_inline(self):
|
||||
'''
|
||||
@@ -1320,7 +1325,7 @@ class CustomResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class SchematicResponseTest(ResponseTest):
|
||||
from response_xml_factory import SchematicResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import SchematicResponseXMLFactory
|
||||
xml_factory_class = SchematicResponseXMLFactory
|
||||
|
||||
def test_grade(self):
|
||||
@@ -1349,11 +1354,10 @@ class SchematicResponseTest(ResponseTest):
|
||||
|
||||
def test_check_function_randomization(self):
|
||||
# The check function should get a random seed from the problem.
|
||||
script = "correct = ['correct' if (submission[0]['num'] == random.randint(0, 1e9)) else 'incorrect']"
|
||||
script = "correct = ['correct' if (submission[0]['num'] == {code}) else 'incorrect']".format(code=self._get_random_number_code())
|
||||
problem = self.build_problem(answer=script)
|
||||
|
||||
r = random.Random(problem.seed)
|
||||
submission_dict = {'num': r.randint(0, 1e9)}
|
||||
submission_dict = {'num': self._get_random_number_result(problem.seed)}
|
||||
input_dict = {'1_2_1': json.dumps(submission_dict)}
|
||||
correct_map = problem.grade_answers(input_dict)
|
||||
|
||||
@@ -1372,7 +1376,7 @@ class SchematicResponseTest(ResponseTest):
|
||||
|
||||
|
||||
class AnnotationResponseTest(ResponseTest):
|
||||
from response_xml_factory import AnnotationResponseXMLFactory
|
||||
from capa.tests.response_xml_factory import AnnotationResponseXMLFactory
|
||||
xml_factory_class = AnnotationResponseXMLFactory
|
||||
|
||||
def test_grade(self):
|
||||
@@ -1393,7 +1397,7 @@ class AnnotationResponseTest(ResponseTest):
|
||||
{'correctness': incorrect, 'points': 0, 'answers': {answer_id: 'null'}},
|
||||
]
|
||||
|
||||
for (index, test) in enumerate(tests):
|
||||
for test in tests:
|
||||
expected_correctness = test['correctness']
|
||||
expected_points = test['points']
|
||||
answers = test['answers']
|
||||
|
||||
@@ -424,7 +424,7 @@ class CapaModule(CapaFields, XModule):
|
||||
|
||||
# If we cannot construct the problem HTML,
|
||||
# then generate an error message instead.
|
||||
except Exception, err:
|
||||
except Exception as err:
|
||||
html = self.handle_problem_html_error(err)
|
||||
|
||||
# The convention is to pass the name of the check button
|
||||
@@ -655,7 +655,7 @@ class CapaModule(CapaFields, XModule):
|
||||
@staticmethod
|
||||
def make_dict_of_responses(get):
|
||||
'''Make dictionary of student responses (aka "answers")
|
||||
get is POST dictionary (Djano QueryDict).
|
||||
get is POST dictionary (Django QueryDict).
|
||||
|
||||
The *get* dict has keys of the form 'x_y', which are mapped
|
||||
to key 'y' in the returned dict. For example,
|
||||
@@ -739,13 +739,13 @@ class CapaModule(CapaFields, XModule):
|
||||
# Too late. Cannot submit
|
||||
if self.closed():
|
||||
event_info['failure'] = 'closed'
|
||||
self.system.track_function('save_problem_check_fail', event_info)
|
||||
self.system.track_function('problem_check_fail', event_info)
|
||||
raise NotFoundError('Problem is closed')
|
||||
|
||||
# Problem submitted. Student should reset before checking again
|
||||
if self.done and self.rerandomize == "always":
|
||||
event_info['failure'] = 'unreset'
|
||||
self.system.track_function('save_problem_check_fail', event_info)
|
||||
self.system.track_function('problem_check_fail', event_info)
|
||||
raise NotFoundError('Problem must be reset before it can be checked again')
|
||||
|
||||
# Problem queued. Students must wait a specified waittime before they are allowed to submit
|
||||
@@ -759,6 +759,8 @@ class CapaModule(CapaFields, XModule):
|
||||
|
||||
try:
|
||||
correct_map = self.lcp.grade_answers(answers)
|
||||
self.attempts = self.attempts + 1
|
||||
self.lcp.done = True
|
||||
self.set_state_from_lcp()
|
||||
|
||||
except (StudentInputError, ResponseError, LoncapaProblemError) as inst:
|
||||
@@ -778,17 +780,13 @@ class CapaModule(CapaFields, XModule):
|
||||
|
||||
return {'success': msg}
|
||||
|
||||
except Exception, err:
|
||||
except Exception as err:
|
||||
if self.system.DEBUG:
|
||||
msg = "Error checking problem: " + str(err)
|
||||
msg += '\nTraceback:\n' + traceback.format_exc()
|
||||
return {'success': msg}
|
||||
raise
|
||||
|
||||
self.attempts = self.attempts + 1
|
||||
self.lcp.done = True
|
||||
|
||||
self.set_state_from_lcp()
|
||||
self.publish_grade()
|
||||
|
||||
# success = correct if ALL questions in this problem are correct
|
||||
@@ -802,7 +800,7 @@ class CapaModule(CapaFields, XModule):
|
||||
event_info['correct_map'] = correct_map.get_dict()
|
||||
event_info['success'] = success
|
||||
event_info['attempts'] = self.attempts
|
||||
self.system.track_function('save_problem_check', event_info)
|
||||
self.system.track_function('problem_check', event_info)
|
||||
|
||||
if hasattr(self.system, 'psychometrics_handler'): # update PsychometricsData using callback
|
||||
self.system.psychometrics_handler(self.get_state_for_lcp())
|
||||
@@ -814,12 +812,92 @@ class CapaModule(CapaFields, XModule):
|
||||
'contents': html,
|
||||
}
|
||||
|
||||
def rescore_problem(self):
|
||||
"""
|
||||
Checks whether the existing answers to a problem are correct.
|
||||
|
||||
This is called when the correct answer to a problem has been changed,
|
||||
and the grade should be re-evaluated.
|
||||
|
||||
Returns a dict with one key:
|
||||
{'success' : 'correct' | 'incorrect' | AJAX alert msg string }
|
||||
|
||||
Raises NotFoundError if called on a problem that has not yet been
|
||||
answered, or NotImplementedError if it's a problem that cannot be rescored.
|
||||
|
||||
Returns the error messages for exceptions occurring while performing
|
||||
the rescoring, rather than throwing them.
|
||||
"""
|
||||
event_info = {'state': self.lcp.get_state(), 'problem_id': self.location.url()}
|
||||
|
||||
if not self.lcp.supports_rescoring():
|
||||
event_info['failure'] = 'unsupported'
|
||||
self.system.track_function('problem_rescore_fail', event_info)
|
||||
raise NotImplementedError("Problem's definition does not support rescoring")
|
||||
|
||||
if not self.done:
|
||||
event_info['failure'] = 'unanswered'
|
||||
self.system.track_function('problem_rescore_fail', event_info)
|
||||
raise NotFoundError('Problem must be answered before it can be graded again')
|
||||
|
||||
# get old score, for comparison:
|
||||
orig_score = self.lcp.get_score()
|
||||
event_info['orig_score'] = orig_score['score']
|
||||
event_info['orig_total'] = orig_score['total']
|
||||
|
||||
try:
|
||||
correct_map = self.lcp.rescore_existing_answers()
|
||||
|
||||
except (StudentInputError, ResponseError, LoncapaProblemError) as inst:
|
||||
log.warning("Input error in capa_module:problem_rescore", exc_info=True)
|
||||
event_info['failure'] = 'input_error'
|
||||
self.system.track_function('problem_rescore_fail', event_info)
|
||||
return {'success': u"Error: {0}".format(inst.message)}
|
||||
|
||||
except Exception as err:
|
||||
event_info['failure'] = 'unexpected'
|
||||
self.system.track_function('problem_rescore_fail', event_info)
|
||||
if self.system.DEBUG:
|
||||
msg = u"Error checking problem: {0}".format(err.message)
|
||||
msg += u'\nTraceback:\n' + traceback.format_exc()
|
||||
return {'success': msg}
|
||||
raise
|
||||
|
||||
# rescoring should have no effect on attempts, so don't
|
||||
# need to increment here, or mark done. Just save.
|
||||
self.set_state_from_lcp()
|
||||
|
||||
self.publish_grade()
|
||||
|
||||
new_score = self.lcp.get_score()
|
||||
event_info['new_score'] = new_score['score']
|
||||
event_info['new_total'] = new_score['total']
|
||||
|
||||
# success = correct if ALL questions in this problem are correct
|
||||
success = 'correct'
|
||||
for answer_id in correct_map:
|
||||
if not correct_map.is_correct(answer_id):
|
||||
success = 'incorrect'
|
||||
|
||||
# NOTE: We are logging both full grading and queued-grading submissions. In the latter,
|
||||
# 'success' will always be incorrect
|
||||
event_info['correct_map'] = correct_map.get_dict()
|
||||
event_info['success'] = success
|
||||
event_info['attempts'] = self.attempts
|
||||
self.system.track_function('problem_rescore', event_info)
|
||||
|
||||
# psychometrics should be called on rescoring requests in the same way as check-problem
|
||||
if hasattr(self.system, 'psychometrics_handler'): # update PsychometricsData using callback
|
||||
self.system.psychometrics_handler(self.get_state_for_lcp())
|
||||
|
||||
return {'success': success}
|
||||
|
||||
def save_problem(self, get):
|
||||
'''
|
||||
"""
|
||||
Save the passed in answers.
|
||||
Returns a dict { 'success' : bool, ['error' : error-msg]},
|
||||
with the error key only present if success is False.
|
||||
'''
|
||||
Returns a dict { 'success' : bool, 'msg' : message }
|
||||
The message is informative on success, and an error message on failure.
|
||||
"""
|
||||
event_info = dict()
|
||||
event_info['state'] = self.lcp.get_state()
|
||||
event_info['problem_id'] = self.location.url()
|
||||
|
||||
@@ -19,6 +19,7 @@ from django.http import QueryDict
|
||||
|
||||
from . import test_system
|
||||
from pytz import UTC
|
||||
from capa.correctmap import CorrectMap
|
||||
|
||||
|
||||
class CapaFactory(object):
|
||||
@@ -597,6 +598,85 @@ class CapaModuleTest(unittest.TestCase):
|
||||
# Expect that the problem was NOT reset
|
||||
self.assertTrue('success' in result and not result['success'])
|
||||
|
||||
def test_rescore_problem_correct(self):
|
||||
|
||||
module = CapaFactory.create(attempts=1, done=True)
|
||||
|
||||
# Simulate that all answers are marked correct, no matter
|
||||
# what the input is, by patching LoncapaResponse.evaluate_answers()
|
||||
with patch('capa.responsetypes.LoncapaResponse.evaluate_answers') as mock_evaluate_answers:
|
||||
mock_evaluate_answers.return_value = CorrectMap(CapaFactory.answer_key(), 'correct')
|
||||
result = module.rescore_problem()
|
||||
|
||||
# Expect that the problem is marked correct
|
||||
self.assertEqual(result['success'], 'correct')
|
||||
|
||||
# Expect that we get no HTML
|
||||
self.assertFalse('contents' in result)
|
||||
|
||||
# Expect that the number of attempts is not incremented
|
||||
self.assertEqual(module.attempts, 1)
|
||||
|
||||
def test_rescore_problem_incorrect(self):
|
||||
# make sure it also works when attempts have been reset,
|
||||
# so add this to the test:
|
||||
module = CapaFactory.create(attempts=0, done=True)
|
||||
|
||||
# Simulate that all answers are marked incorrect, no matter
|
||||
# what the input is, by patching LoncapaResponse.evaluate_answers()
|
||||
with patch('capa.responsetypes.LoncapaResponse.evaluate_answers') as mock_evaluate_answers:
|
||||
mock_evaluate_answers.return_value = CorrectMap(CapaFactory.answer_key(), 'incorrect')
|
||||
result = module.rescore_problem()
|
||||
|
||||
# Expect that the problem is marked incorrect
|
||||
self.assertEqual(result['success'], 'incorrect')
|
||||
|
||||
# Expect that the number of attempts is not incremented
|
||||
self.assertEqual(module.attempts, 0)
|
||||
|
||||
def test_rescore_problem_not_done(self):
|
||||
# Simulate that the problem is NOT done
|
||||
module = CapaFactory.create(done=False)
|
||||
|
||||
# Try to rescore the problem, and get exception
|
||||
with self.assertRaises(xmodule.exceptions.NotFoundError):
|
||||
module.rescore_problem()
|
||||
|
||||
def test_rescore_problem_not_supported(self):
|
||||
module = CapaFactory.create(done=True)
|
||||
|
||||
# Try to rescore the problem, and get exception
|
||||
with patch('capa.capa_problem.LoncapaProblem.supports_rescoring') as mock_supports_rescoring:
|
||||
mock_supports_rescoring.return_value = False
|
||||
with self.assertRaises(NotImplementedError):
|
||||
module.rescore_problem()
|
||||
|
||||
def _rescore_problem_error_helper(self, exception_class):
|
||||
"""Helper to allow testing all errors that rescoring might return."""
|
||||
# Create the module
|
||||
module = CapaFactory.create(attempts=1, done=True)
|
||||
|
||||
# Simulate answering a problem that raises the exception
|
||||
with patch('capa.capa_problem.LoncapaProblem.rescore_existing_answers') as mock_rescore:
|
||||
mock_rescore.side_effect = exception_class(u'test error \u03a9')
|
||||
result = module.rescore_problem()
|
||||
|
||||
# Expect an AJAX alert message in 'success'
|
||||
expected_msg = u'Error: test error \u03a9'
|
||||
self.assertEqual(result['success'], expected_msg)
|
||||
|
||||
# Expect that the number of attempts is NOT incremented
|
||||
self.assertEqual(module.attempts, 1)
|
||||
|
||||
def test_rescore_problem_student_input_error(self):
|
||||
self._rescore_problem_error_helper(StudentInputError)
|
||||
|
||||
def test_rescore_problem_problem_error(self):
|
||||
self._rescore_problem_error_helper(LoncapaProblemError)
|
||||
|
||||
def test_rescore_problem_response_error(self):
|
||||
self._rescore_problem_error_helper(ResponseError)
|
||||
|
||||
def test_save_problem(self):
|
||||
module = CapaFactory.create(done=False)
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ from . import test_system
|
||||
|
||||
class DummySystem(ImportSystem):
|
||||
|
||||
@patch('xmodule.modulestore.xml.OSFS', lambda dir: MemoryFS())
|
||||
@patch('xmodule.modulestore.xml.OSFS', lambda directory: MemoryFS())
|
||||
def __init__(self, load_error_modules):
|
||||
|
||||
xmlstore = XMLModuleStore("data_dir", course_dirs=[], load_error_modules=load_error_modules)
|
||||
@@ -41,7 +41,8 @@ class DummySystem(ImportSystem):
|
||||
)
|
||||
|
||||
def render_template(self, template, context):
|
||||
raise Exception("Shouldn't be called")
|
||||
raise Exception("Shouldn't be called")
|
||||
|
||||
|
||||
class ConditionalFactory(object):
|
||||
"""
|
||||
@@ -93,7 +94,7 @@ class ConditionalFactory(object):
|
||||
# return dict:
|
||||
return {'cond_module': cond_module,
|
||||
'source_module': source_module,
|
||||
'child_module': child_module }
|
||||
'child_module': child_module}
|
||||
|
||||
|
||||
class ConditionalModuleBasicTest(unittest.TestCase):
|
||||
@@ -109,12 +110,11 @@ class ConditionalModuleBasicTest(unittest.TestCase):
|
||||
'''verify that get_icon_class works independent of condition satisfaction'''
|
||||
modules = ConditionalFactory.create(self.test_system)
|
||||
for attempted in ["false", "true"]:
|
||||
for icon_class in [ 'other', 'problem', 'video']:
|
||||
for icon_class in ['other', 'problem', 'video']:
|
||||
modules['source_module'].is_attempted = attempted
|
||||
modules['child_module'].get_icon_class = lambda: icon_class
|
||||
self.assertEqual(modules['cond_module'].get_icon_class(), icon_class)
|
||||
|
||||
|
||||
def test_get_html(self):
|
||||
modules = ConditionalFactory.create(self.test_system)
|
||||
# because test_system returns the repr of the context dict passed to render_template,
|
||||
@@ -224,4 +224,3 @@ class ConditionalModuleXmlTest(unittest.TestCase):
|
||||
print "post-attempt ajax: ", ajax
|
||||
html = ajax['html']
|
||||
self.assertTrue(any(['This is a secret' in item for item in html]))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user