fix: Report Custom Python Errors to Instructors (#28199)
Partner Support commonly raises the issue of instructors' custom Python problems not generating any response report on the instructor dashboard. Such errors are due to the operating restrictions placed on codejail. Sometimes not all answers can be processed by the server, which kills off some to accommodate. Instead of spiking the whole report, this change logs not only the error in our system, but also allows the mostly complete response to reach the instructor. This PR will decrease friction not only for Partner support and instructors, but T&L, who have periodically implemented workarounds to the problem. The PR merely implements exception handling for generating reports which logged exceptions and added them to the report, continuing the work done in TNL-8218 which did the same for grading.
This commit is contained in:
connorhaugh
@@ -683,50 +683,64 @@ class ProblemBlock(
|
||||
|
||||
if 'student_answers' not in user_state.state:
|
||||
continue
|
||||
try:
|
||||
lcp = LoncapaProblem(
|
||||
problem_text=self.data,
|
||||
id=self.location.html_id(),
|
||||
capa_system=capa_system,
|
||||
# We choose to run without a fully initialized CapaModule
|
||||
capa_module=None,
|
||||
state={
|
||||
'done': user_state.state.get('done'),
|
||||
'correct_map': user_state.state.get('correct_map'),
|
||||
'student_answers': user_state.state.get('student_answers'),
|
||||
'has_saved_answers': user_state.state.get('has_saved_answers'),
|
||||
'input_state': user_state.state.get('input_state'),
|
||||
'seed': user_state.state.get('seed'),
|
||||
},
|
||||
seed=user_state.state.get('seed'),
|
||||
# extract_tree=False allows us to work without a fully initialized CapaModule
|
||||
# We'll still be able to find particular data in the XML when we need it
|
||||
extract_tree=False,
|
||||
)
|
||||
|
||||
lcp = LoncapaProblem(
|
||||
problem_text=self.data,
|
||||
id=self.location.html_id(),
|
||||
capa_system=capa_system,
|
||||
# We choose to run without a fully initialized CapaModule
|
||||
capa_module=None,
|
||||
state={
|
||||
'done': user_state.state.get('done'),
|
||||
'correct_map': user_state.state.get('correct_map'),
|
||||
'student_answers': user_state.state.get('student_answers'),
|
||||
'has_saved_answers': user_state.state.get('has_saved_answers'),
|
||||
'input_state': user_state.state.get('input_state'),
|
||||
'seed': user_state.state.get('seed'),
|
||||
},
|
||||
seed=user_state.state.get('seed'),
|
||||
# extract_tree=False allows us to work without a fully initialized CapaModule
|
||||
# We'll still be able to find particular data in the XML when we need it
|
||||
extract_tree=False,
|
||||
)
|
||||
for answer_id, orig_answers in lcp.student_answers.items():
|
||||
# Some types of problems have data in lcp.student_answers that isn't in lcp.problem_data.
|
||||
# E.g. formulae do this to store the MathML version of the answer.
|
||||
# We exclude these rows from the report because we only need the text-only answer.
|
||||
if answer_id.endswith('_dynamath'):
|
||||
continue
|
||||
|
||||
for answer_id, orig_answers in lcp.student_answers.items():
|
||||
# Some types of problems have data in lcp.student_answers that isn't in lcp.problem_data.
|
||||
# E.g. formulae do this to store the MathML version of the answer.
|
||||
# We exclude these rows from the report because we only need the text-only answer.
|
||||
if answer_id.endswith('_dynamath'):
|
||||
continue
|
||||
if limit_responses and count >= limit_responses:
|
||||
# End the iterator here
|
||||
return
|
||||
|
||||
if limit_responses and count >= limit_responses:
|
||||
# End the iterator here
|
||||
return
|
||||
question_text = lcp.find_question_label(answer_id)
|
||||
answer_text = lcp.find_answer_text(answer_id, current_answer=orig_answers)
|
||||
correct_answer_text = lcp.find_correct_answer_text(answer_id)
|
||||
|
||||
question_text = lcp.find_question_label(answer_id)
|
||||
answer_text = lcp.find_answer_text(answer_id, current_answer=orig_answers)
|
||||
correct_answer_text = lcp.find_correct_answer_text(answer_id)
|
||||
|
||||
count += 1
|
||||
count += 1
|
||||
report = {
|
||||
_("Answer ID"): answer_id,
|
||||
_("Question"): question_text,
|
||||
_("Answer"): answer_text,
|
||||
}
|
||||
if correct_answer_text is not None:
|
||||
report[_("Correct Answer")] = correct_answer_text
|
||||
yield (user_state.username, report)
|
||||
except LoncapaProblemError:
|
||||
# Capture a backtrace for errors from failed loncapa problems
|
||||
log.exception(
|
||||
"An error occurred generating a problem report on course %s, problem %s, and student %s",
|
||||
self.course_id, self.scope_ids.usage_id,
|
||||
self.scope_ids.user_id
|
||||
)
|
||||
# Also input error in report
|
||||
report = {
|
||||
_("Answer ID"): answer_id,
|
||||
_("Question"): question_text,
|
||||
_("Answer"): answer_text,
|
||||
_("Answer ID"): "Python Error",
|
||||
_("Question"): "Generating a report on the problem failed.",
|
||||
_("Answer"): "Python Error: No Answer Retrieved",
|
||||
}
|
||||
if correct_answer_text is not None:
|
||||
report[_("Correct Answer")] = correct_answer_text
|
||||
yield (user_state.username, report)
|
||||
|
||||
@property
|
||||
|
||||
@@ -3236,3 +3236,16 @@ class ProblemBlockReportGenerationTest(unittest.TestCase):
|
||||
iterator = iter([self._user_state(suffix='_dynamath')])
|
||||
report_data = list(descriptor.generate_report_data(iterator))
|
||||
assert 0 == len(report_data)
|
||||
|
||||
def test_generate_report_data_report_loncapa_error(self):
|
||||
#Test to make sure reports continue despite loncappa errors, and write them into the report.
|
||||
descriptor = self._get_descriptor()
|
||||
with patch('xmodule.capa_module.LoncapaProblem') as mock_LoncapaProblem:
|
||||
mock_LoncapaProblem.side_effect = LoncapaProblemError
|
||||
report_data = list(descriptor.generate_report_data(
|
||||
self._mock_user_state_generator(
|
||||
user_count=1,
|
||||
response_count=5,
|
||||
)
|
||||
))
|
||||
assert 'Python Error: No Answer Retrieved' in list(report_data[0][1].values())
|
||||
|
||||
Reference in New Issue
Block a user