Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add endpoint and background task for downloading of submission files

Browse Source
This commit is contained in:
0x29a
2020-07-19 23:51:06 +02:00
parent 66e2c7a5ea
commit 921875b70b
13 changed files with 395 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
lms/djangoapps/instructor/tests/test_api.py
View File

@@ -149,6 +149,7 @@ INSTRUCTOR_POST_ENDPOINTS = set([
'calculate_grades_csv',
'change_due_date',
'export_ora2_data',
'export_ora2_submission_files',
'get_grading_config',
'get_problem_responses',
'get_proctored_exam_results',
@@ -428,6 +429,7 @@ class TestInstructorAPIDenyLevels(SharedModuleStoreTestCase, LoginEnrollmentTest
('get_proctored_exam_results', {}),
('get_problem_responses', {}),
('export_ora2_data', {}),
('export_ora2_submission_files', {}),
('rescore_problem',
{'problem_to_reset': self.problem_urlname, 'unique_student_identifier': self.user.email}),
('override_problem_score',
@@ -2875,6 +2877,32 @@ class TestInstructorAPILevelsDataDump(SharedModuleStoreTestCase, LoginEnrollment
self.assertContains(response, already_running_status, status_code=400)
def test_get_ora2_submission_files_success(self):
url = reverse('export_ora2_submission_files', kwargs={'course_id': text_type(self.course.id)})
with patch(
'lms.djangoapps.instructor_task.api.submit_export_ora2_submission_files'
) as mock_submit_ora2_task:
mock_submit_ora2_task.return_value = True
response = self.client.post(url, {})
success_status = 'Attachments archive is being created.'
self.assertContains(response, success_status)
def test_get_ora2_submission_files_already_running(self):
url = reverse('export_ora2_submission_files', kwargs={'course_id': text_type(self.course.id)})
task_type = 'export_ora2_submission_files'
already_running_status = generate_already_running_error_message(task_type)
with patch(
'lms.djangoapps.instructor_task.api.submit_export_ora2_submission_files'
) as mock_submit_ora2_task:
mock_submit_ora2_task.side_effect = AlreadyRunningError(already_running_status)
response = self.client.post(url, {})
self.assertContains(response, already_running_status, status_code=400)
def test_get_student_progress_url(self):
""" Test that progress_url is in the successful response. """
url = reverse('get_student_progress_url', kwargs={'course_id': text_type(self.course.id)})

22
lms/djangoapps/instructor/views/api.py
View File

@@ -2049,6 +2049,28 @@ def export_ora2_data(request, course_id):
return JsonResponse({"status": success_status})
@transaction.non_atomic_requests
@require_POST
@ensure_csrf_cookie
@cache_control(no_cache=True, no_store=True, must_revalidate=True)
@require_course_permission(permissions.CAN_RESEARCH)
@common_exceptions_400
def export_ora2_submission_files(request, course_id):
"""
Pushes a Celery task which will download and compress all submission
files (texts, attachments) into a zip archive.
"""
course_key = CourseKey.from_string(course_id)
task_api.submit_export_ora2_submission_files(request, course_key)
return JsonResponse({
"status": _(
"Attachments archive is being created."
)
})
@transaction.non_atomic_requests
@require_POST
@ensure_csrf_cookie

3
lms/djangoapps/instructor/views/api_urls.py
View File

@@ -54,6 +54,9 @@ urlpatterns = [
url(r'^get_course_survey_results$', api.get_course_survey_results, name='get_course_survey_results'),
url(r'^export_ora2_data', api.export_ora2_data, name='export_ora2_data'),
url(r'^export_ora2_submission_files', api.export_ora2_submission_files,
name='export_ora2_submission_files'),
# spoc gradebook
url(r'^gradebook$', gradebook_api.spoc_gradebook, name='spoc_gradebook'),

3
lms/djangoapps/instructor/views/instructor_dashboard.py
View File

@@ -628,6 +628,9 @@ def _section_data_download(course, access):
'get_course_survey_results', kwargs={'course_id': six.text_type(course_key)}
),
'export_ora2_data_url': reverse('export_ora2_data', kwargs={'course_id': six.text_type(course_key)}),
'export_ora2_submission_files_url': reverse(
'export_ora2_submission_files', kwargs={'course_id': six.text_type(course_key)}
),
}
if not access.get('data_researcher'):
section_data['is_hidden'] = True

14
lms/djangoapps/instructor_task/api.py
View File

@@ -35,6 +35,7 @@ from lms.djangoapps.instructor_task.tasks import (
course_survey_report_csv,
delete_problem_state,
export_ora2_data,
export_ora2_submission_files,
generate_certificates,
override_problem_score,
proctored_exam_results_csv,
@@ -450,6 +451,19 @@ def submit_export_ora2_data(request, course_key):
return submit_task(request, task_type, task_class, course_key, task_input, task_key)
def submit_export_ora2_submission_files(request, course_key):
"""
Submits a task to download and compress all submissions
files (texts, attachments) for given course.
"""
task_type = 'export_ora2_submission_files'
task_class = export_ora2_submission_files
task_input = {}
task_key = ''
return submit_task(request, task_type, task_class, course_key, task_input, task_key)
def generate_certificates_for_students(request, course_key, student_set=None, specific_student_id=None):
"""
Submits a task to generate certificates for given students enrolled in the course.

9
lms/djangoapps/instructor_task/models.py
View File

@@ -280,9 +280,14 @@ class DjangoStorageReportStore(ReportStore):
"""
path = self.path_to(course_id, filename)
# See https://github.com/boto/boto/issues/2868
# Boto doesn't play nice with unicod in python3
# Boto doesn't play nice with unicode in python3
if not six.PY2:
buff = ContentFile(buff.read().encode('utf-8'))
buff_contents = buff.read()
if not isinstance(buff_contents, bytes):
buff_contents = buff_contents.encode('utf-8')
buff = ContentFile(buff_contents)
self.storage.save(path, buff)

12
lms/djangoapps/instructor_task/tasks.py
View File

@@ -40,6 +40,7 @@ from lms.djangoapps.instructor_task.tasks_helper.misc import (
cohort_students_and_upload,
upload_course_survey_report,
upload_ora2_data,
upload_ora2_submission_files,
upload_proctored_exam_results_report
)
from lms.djangoapps.instructor_task.tasks_helper.module_state import (
@@ -292,3 +293,14 @@ def export_ora2_data(entry_id, xmodule_instance_args):
action_name = ugettext_noop('generated')
task_fn = partial(upload_ora2_data, xmodule_instance_args)
return run_main_task(entry_id, task_fn, action_name)
@task(base=BaseInstructorTask)
def export_ora2_submission_files(entry_id, xmodule_instance_args):
"""
Download all submission files, generate csv downloads list,
put all this into zip archive and push it to S3.
"""
action_name = ugettext_noop('compressed')
task_fn = partial(upload_ora2_submission_files, xmodule_instance_args)
return run_main_task(entry_id, task_fn, action_name)

119
lms/djangoapps/instructor_task/tasks_helper/misc.py
View File

@@ -7,16 +7,21 @@ running state of a course.
import logging
from collections import OrderedDict
from contextlib import contextmanager
from datetime import datetime
from io import StringIO
from tempfile import TemporaryFile
from time import time
from zipfile import ZipFile
import csv
import os
import unicodecsv
import six
from django.contrib.auth.models import User
from django.core.exceptions import ValidationError
from django.core.files.storage import DefaultStorage
from openassessment.data import OraAggregateData
from openassessment.data import OraAggregateData, OraDownloadData
from pytz import UTC
from lms.djangoapps.instructor_analytics.basic import get_proctored_exam_results
@@ -27,7 +32,12 @@ from survey.models import SurveyAnswer
from util.file import UniversalNewlineIterator
from .runner import TaskProgress
from .utils import UPDATE_STATUS_FAILED, UPDATE_STATUS_SUCCEEDED, upload_csv_to_report_store
from .utils import (
UPDATE_STATUS_FAILED,
UPDATE_STATUS_SUCCEEDED,
upload_csv_to_report_store,
upload_zip_to_report_store,
)
# define different loggers for use within tasks and on client side
TASK_LOG = logging.getLogger('edx.celery.task')
@@ -340,3 +350,108 @@ def upload_ora2_data(
TASK_LOG.info(u'%s, Task type: %s, Upload complete.', task_info_string, action_name)
return UPDATE_STATUS_SUCCEEDED
def _task_step(task_progress, task_info_string, action_name):
"""
Returns a context manager, that logs error and updates TaskProgress
filures counter in case inner block throws an exception.
"""
@contextmanager
def _step_context_manager(step_description, exception_text, step_error_description):
curr_step = {'step': step_description}
TASK_LOG.info(
'%s, Task type: %s, Current step: %s',
task_info_string,
action_name,
curr_step,
)
task_progress.update_task_state(extra_meta=curr_step)
try:
yield
# Update progress to failed regardless of error type
except Exception: # pylint: disable=broad-except
TASK_LOG.exception(exception_text)
task_progress.failed = 1
task_progress.update_task_state(extra_meta={'step': step_error_description})
return _step_context_manager
def upload_ora2_submission_files(
_xmodule_instance_args, _entry_id, course_id, _task_input, action_name
):
"""
Creates zip archive with submission files in three steps:
1. Collect all files information using ORA download helper.
2. Download all submission attachments, put them in temporary zip
file along with submission texts and csv downloads list.
3. Upload zip file into reports storage.
"""
start_time = time()
start_date = datetime.now(UTC)
num_attempted = 1
num_total = 1
fmt = 'Task: {task_id}, InstructorTask ID: {entry_id}, Course: {course_id}, Input: {task_input}'
task_info_string = fmt.format(
task_id=_xmodule_instance_args.get('task_id') if _xmodule_instance_args is not None else None,
entry_id=_entry_id,
course_id=course_id,
task_input=_task_input
)
TASK_LOG.info(u'%s, Task type: %s, Starting task execution', task_info_string, action_name)
task_progress = TaskProgress(action_name, num_total, start_time)
task_progress.attempted = num_attempted
step_manager = _task_step(task_progress, task_info_string, action_name)
submission_files_data = None
with step_manager(
'Collecting attachments data',
'Failed to get ORA submissions attachments data.',
'Error while collecting data',
):
submission_files_data = OraDownloadData.collect_ora2_submission_files(course_id)
if submission_files_data is None:
return UPDATE_STATUS_FAILED
with TemporaryFile('rb+') as zip_file:
compressed = None
with step_manager(
'Downloading and compressing attachments files',
'Failed to download and compress submissions attachments.',
'Error while downloading and compressing submissions attachments',
):
compressed = OraDownloadData.create_zip_with_attachments(zip_file, course_id, submission_files_data)
if compressed is None:
return UPDATE_STATUS_FAILED
zip_filename = None
with step_manager(
'Uploading zip file to storage',
'Failed to upload zip file to storage.',
'Error while uploading zip file to storage',
):
zip_filename = upload_zip_to_report_store(zip_file, 'submission_files', course_id, start_date),
if not zip_filename:
return UPDATE_STATUS_FAILED
task_progress.succeeded = 1
curr_step = {'step': 'Finalizing attachments extracting'}
task_progress.update_task_state(extra_meta=curr_step)
TASK_LOG.info(u'%s, Task type: %s, Upload complete.', task_info_string, action_name)
return UPDATE_STATUS_SUCCEEDED

17
lms/djangoapps/instructor_task/tasks_helper/utils.py
View File

@@ -49,6 +49,23 @@ def upload_csv_to_report_store(rows, csv_name, course_id, timestamp, config_name
return report_name, report_path
def upload_zip_to_report_store(file, zip_name, course_id, timestamp, config_name='GRADES_DOWNLOAD'):
"""
Upload given file buffer as a zip file using ReportStore.
"""
report_store = ReportStore.from_config(config_name)
report_name = u"{course_prefix}_{zip_name}_{timestamp_str}.zip".format(
course_prefix=course_filename_prefix_generator(course_id),
zip_name=zip_name,
timestamp_str=timestamp.strftime("%Y-%m-%d-%H%M")
)
report_store.store(course_id, report_name, file)
tracker_emit(zip_name)
return report_name
def tracker_emit(report_name):
"""
Emits a 'report.requested' event for the given report.

19
lms/djangoapps/instructor_task/tests/test_api.py
View File

@@ -27,6 +27,7 @@ from lms.djangoapps.instructor_task.api import (
submit_delete_entrance_exam_state_for_student,
submit_delete_problem_state_for_all_students,
submit_export_ora2_data,
submit_export_ora2_submission_files,
submit_override_score,
submit_rescore_entrance_exam_for_student,
submit_rescore_problem_for_all_students,
@@ -36,7 +37,7 @@ from lms.djangoapps.instructor_task.api import (
)
from lms.djangoapps.instructor_task.api_helper import AlreadyRunningError, QueueConnectionError
from lms.djangoapps.instructor_task.models import PROGRESS, InstructorTask
from lms.djangoapps.instructor_task.tasks import export_ora2_data
from lms.djangoapps.instructor_task.tasks import export_ora2_data, export_ora2_submission_files
from lms.djangoapps.instructor_task.tests.test_base import (
TEST_COURSE_KEY,
InstructorTaskCourseTestCase,
@@ -282,6 +283,22 @@ class InstructorTaskCourseSubmitTest(TestReportMixin, InstructorTaskCourseTestCa
mock_submit_task.assert_called_once_with(
request, 'export_ora2_data', export_ora2_data, self.course.id, {}, '')
def test_submit_export_ora2_submission_files(self):
request = self.create_task_request(self.instructor)
with patch('lms.djangoapps.instructor_task.api.submit_task') as mock_submit_task:
mock_submit_task.return_value = MagicMock()
submit_export_ora2_submission_files(request, self.course.id)
mock_submit_task.assert_called_once_with(
request,
'export_ora2_submission_files',
export_ora2_submission_files,
self.course.id,
{},
''
)
def test_submit_generate_certs_students(self):
"""
Tests certificates generation task submission api

31
lms/djangoapps/instructor_task/tests/test_tasks.py
View File

@@ -25,6 +25,7 @@ from lms.djangoapps.instructor_task.models import InstructorTask
from lms.djangoapps.instructor_task.tasks import (
delete_problem_state,
export_ora2_data,
export_ora2_submission_files,
generate_certificates,
override_problem_score,
rescore_problem,
@@ -684,3 +685,33 @@ class TestOra2ResponsesInstructorTask(TestInstructorTasks):
assert args[0] == task_entry.id
assert callable(args[1])
assert args[2] == action_name
class TestOra2ExportSubmissionFilesInstructorTask(TestInstructorTasks):
"""Tests instructor task that exports ora2 submission files archive."""
def test_ora2_missing_current_task(self):
self._test_missing_current_task(export_ora2_submission_files)
def test_ora2_with_failure(self):
self._test_run_with_failure(export_ora2_submission_files, 'We expected this to fail')
def test_ora2_with_long_error_msg(self):
self._test_run_with_long_error_msg(export_ora2_submission_files)
def test_ora2_with_short_error_msg(self):
self._test_run_with_short_error_msg(export_ora2_submission_files)
def test_ora2_runs_task(self):
task_entry = self._create_input_entry()
task_xmodule_args = self._get_xmodule_instance_args()
with patch('lms.djangoapps.instructor_task.tasks.run_main_task') as mock_main_task:
export_ora2_submission_files(task_entry.id, task_xmodule_args)
action_name = ugettext_noop('compressed')
assert mock_main_task.call_count == 1
args = mock_main_task.call_args[0]
assert args[0] == task_entry.id
assert callable(args[1])
assert args[2] == action_name

132
lms/djangoapps/instructor_task/tests/test_tasks_helper.py
View File

@@ -12,8 +12,10 @@ Unit tests for LMS instructor-initiated background tasks helper functions.
import os
import shutil
import tempfile
from contextlib import contextmanager
from contextlib import contextmanager, ExitStack
from datetime import datetime, timedelta
from io import BytesIO
from zipfile import ZipFile
import ddt
import unicodecsv
@@ -57,7 +59,8 @@ from lms.djangoapps.instructor_task.tasks_helper.grades import (
from lms.djangoapps.instructor_task.tasks_helper.misc import (
cohort_students_and_upload,
upload_course_survey_report,
upload_ora2_data
upload_ora2_data,
upload_ora2_submission_files
)
from lms.djangoapps.instructor_task.tests.test_base import (
InstructorTaskCourseTestCase,
@@ -2539,25 +2542,126 @@ class TestInstructorOra2Report(SharedModuleStoreTestCase):
self.assertEqual(response, UPDATE_STATUS_FAILED)
def test_report_stores_results(self):
with freeze_time('2001-01-01 00:00:00'):
with ExitStack() as stack:
stack.enter_context(freeze_time('2001-01-01 00:00:00'))
mock_current_task = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
)
mock_collect_data = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraAggregateData.collect_ora2_data')
)
mock_store_rows = stack.enter_context(
patch('lms.djangoapps.instructor_task.models.DjangoStorageReportStore.store_rows')
)
mock_current_task.return_value = self.current_task
test_header = ['field1', 'field2']
test_rows = [['row1_field1', 'row1_field2'], ['row2_field1', 'row2_field2']]
mock_collect_data.return_value = (test_header, test_rows)
return_val = upload_ora2_data(None, None, self.course.id, None, 'generated')
timestamp_str = datetime.now(UTC).strftime('%Y-%m-%d-%H%M')
course_id_string = quote(text_type(self.course.id).replace('/', '_'))
filename = u'{}_ORA_data_{}.csv'.format(course_id_string, timestamp_str)
self.assertEqual(return_val, UPDATE_STATUS_SUCCEEDED)
mock_store_rows.assert_called_once_with(self.course.id, filename, [test_header] + test_rows)
class TestInstructorOra2AttachmentsExport(SharedModuleStoreTestCase):
"""
Tests that ORA2 submission files export works.
"""
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.course = CourseFactory.create()
def setUp(self):
super().setUp()
self.current_task = Mock()
self.current_task.update_state = Mock()
def test_export_fails_if_error_on_collect_step(self):
with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task') as mock_current_task:
mock_current_task.return_value = self.current_task
with patch(
'lms.djangoapps.instructor_task.tasks_helper.misc.OraAggregateData.collect_ora2_data'
'lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files'
) as mock_collect_data:
mock_collect_data.return_value = (test_header, test_rows)
with patch(
'lms.djangoapps.instructor_task.models.DjangoStorageReportStore.store_rows'
) as mock_store_rows:
return_val = upload_ora2_data(None, None, self.course.id, None, 'generated')
mock_collect_data.side_effect = KeyError
timestamp_str = datetime.now(UTC).strftime('%Y-%m-%d-%H%M')
course_id_string = quote(text_type(self.course.id).replace('/', '_'))
filename = u'{}_ORA_data_{}.csv'.format(course_id_string, timestamp_str)
response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
self.assertEqual(response, UPDATE_STATUS_FAILED)
self.assertEqual(return_val, UPDATE_STATUS_SUCCEEDED)
mock_store_rows.assert_called_once_with(self.course.id, filename, [test_header] + test_rows)
def test_export_fails_if_error_on_create_zip_step(self):
with ExitStack() as stack:
mock_current_task = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
)
mock_current_task.return_value = self.current_task
stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files')
)
create_zip_mock = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.create_zip_with_attachments')
)
create_zip_mock.side_effect = KeyError
response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
self.assertEqual(response, UPDATE_STATUS_FAILED)
def test_export_fails_if_error_on_upload_step(self):
with ExitStack() as stack:
mock_current_task = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
)
mock_current_task.return_value = self.current_task
stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files')
)
stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.create_zip_with_attachments')
)
upload_mock = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.upload_zip_to_report_store')
)
upload_mock.side_effect = KeyError
response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
self.assertEqual(response, UPDATE_STATUS_FAILED)
def test_task_stores_zip_with_attachments(self):
with ExitStack() as stack:
mock_current_task = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
)
mock_collect_files = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files')
)
mock_create_zip = stack.enter_context(
patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.create_zip_with_attachments')
)
mock_store = stack.enter_context(
patch('lms.djangoapps.instructor_task.models.DjangoStorageReportStore.store')
)
mock_current_task.return_value = self.current_task
response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
mock_collect_files.assert_called_once()
mock_create_zip.assert_called_once()
mock_store.assert_called_once()
self.assertEqual(response, UPDATE_STATUS_SUCCEEDED)

5
lms/templates/instructor/instructor_dashboard_2/data_download.html
View File

@@ -94,6 +94,11 @@ from openedx.core.djangolib.markup import HTML, Text
<input type="button" name="problem-grade-report" class="async-report-btn" value="${_("Generate Problem Grade Report")}" data-endpoint="${ section_data['problem_grade_report_url'] }"/>
<input type="button" name="export-ora2-data" class="async-report-btn" value="${_("Generate ORA Data Report")}" data-endpoint="${ section_data['export_ora2_data_url'] }"/>
</p>
<p>${_("Click to generate a ZIP file that contains all submission texts and attachments.")}</p>
<p><input type="button" name="export-ora2-data" class="async-report-btn" value="${_("Generate Submission Files Archive")}" data-endpoint="${ section_data['export_ora2_submission_files_url'] }"/></p>
%endif
<div class="request-response msg msg-confirm copy" id="report-request-response"></div>