Add endpoint and background task for downloading of submission files

2020-07-19 23:51:06 +02:00
parent 66e2c7a5ea
commit 921875b70b
13 changed files with 395 additions and 19 deletions
--- a/lms/djangoapps/instructor_task/tests/test_api.py
+++ b/lms/djangoapps/instructor_task/tests/test_api.py
@@ -27,6 +27,7 @@ from lms.djangoapps.instructor_task.api import (
    submit_delete_entrance_exam_state_for_student,
    submit_delete_problem_state_for_all_students,
    submit_export_ora2_data,
+    submit_export_ora2_submission_files,
    submit_override_score,
    submit_rescore_entrance_exam_for_student,
    submit_rescore_problem_for_all_students,
@@ -36,7 +37,7 @@ from lms.djangoapps.instructor_task.api import (
 )
 from lms.djangoapps.instructor_task.api_helper import AlreadyRunningError, QueueConnectionError
 from lms.djangoapps.instructor_task.models import PROGRESS, InstructorTask
-from lms.djangoapps.instructor_task.tasks import export_ora2_data
+from lms.djangoapps.instructor_task.tasks import export_ora2_data, export_ora2_submission_files
 from lms.djangoapps.instructor_task.tests.test_base import (
    TEST_COURSE_KEY,
    InstructorTaskCourseTestCase,
@@ -282,6 +283,22 @@ class InstructorTaskCourseSubmitTest(TestReportMixin, InstructorTaskCourseTestCa
            mock_submit_task.assert_called_once_with(
                request, 'export_ora2_data', export_ora2_data, self.course.id, {}, '')

+    def test_submit_export_ora2_submission_files(self):
+        request = self.create_task_request(self.instructor)
+
+        with patch('lms.djangoapps.instructor_task.api.submit_task') as mock_submit_task:
+            mock_submit_task.return_value = MagicMock()
+            submit_export_ora2_submission_files(request, self.course.id)
+
+            mock_submit_task.assert_called_once_with(
+                request,
+                'export_ora2_submission_files',
+                export_ora2_submission_files,
+                self.course.id,
+                {},
+                ''
+            )
+
    def test_submit_generate_certs_students(self):
        """
        Tests certificates generation task submission api
--- a/lms/djangoapps/instructor_task/tests/test_tasks.py
+++ b/lms/djangoapps/instructor_task/tests/test_tasks.py
@@ -25,6 +25,7 @@ from lms.djangoapps.instructor_task.models import InstructorTask
 from lms.djangoapps.instructor_task.tasks import (
    delete_problem_state,
    export_ora2_data,
+    export_ora2_submission_files,
    generate_certificates,
    override_problem_score,
    rescore_problem,
@@ -684,3 +685,33 @@ class TestOra2ResponsesInstructorTask(TestInstructorTasks):
            assert args[0] == task_entry.id
            assert callable(args[1])
            assert args[2] == action_name
+
+
+class TestOra2ExportSubmissionFilesInstructorTask(TestInstructorTasks):
+    """Tests instructor task that exports ora2 submission files archive."""
+
+    def test_ora2_missing_current_task(self):
+        self._test_missing_current_task(export_ora2_submission_files)
+
+    def test_ora2_with_failure(self):
+        self._test_run_with_failure(export_ora2_submission_files, 'We expected this to fail')
+
+    def test_ora2_with_long_error_msg(self):
+        self._test_run_with_long_error_msg(export_ora2_submission_files)
+
+    def test_ora2_with_short_error_msg(self):
+        self._test_run_with_short_error_msg(export_ora2_submission_files)
+
+    def test_ora2_runs_task(self):
+        task_entry = self._create_input_entry()
+        task_xmodule_args = self._get_xmodule_instance_args()
+
+        with patch('lms.djangoapps.instructor_task.tasks.run_main_task') as mock_main_task:
+            export_ora2_submission_files(task_entry.id, task_xmodule_args)
+            action_name = ugettext_noop('compressed')
+
+            assert mock_main_task.call_count == 1
+            args = mock_main_task.call_args[0]
+            assert args[0] == task_entry.id
+            assert callable(args[1])
+            assert args[2] == action_name
--- a/lms/djangoapps/instructor_task/tests/test_tasks_helper.py
+++ b/lms/djangoapps/instructor_task/tests/test_tasks_helper.py
@@ -12,8 +12,10 @@ Unit tests for LMS instructor-initiated background tasks helper functions.
 import os
 import shutil
 import tempfile
-from contextlib import contextmanager
+from contextlib import contextmanager, ExitStack
 from datetime import datetime, timedelta
+from io import BytesIO
+from zipfile import ZipFile

 import ddt
 import unicodecsv
@@ -57,7 +59,8 @@ from lms.djangoapps.instructor_task.tasks_helper.grades import (
 from lms.djangoapps.instructor_task.tasks_helper.misc import (
    cohort_students_and_upload,
    upload_course_survey_report,
-    upload_ora2_data
+    upload_ora2_data,
+    upload_ora2_submission_files
 )
 from lms.djangoapps.instructor_task.tests.test_base import (
    InstructorTaskCourseTestCase,
@@ -2539,25 +2542,126 @@ class TestInstructorOra2Report(SharedModuleStoreTestCase):
                self.assertEqual(response, UPDATE_STATUS_FAILED)

    def test_report_stores_results(self):
-        with freeze_time('2001-01-01 00:00:00'):
+        with ExitStack() as stack:
+            stack.enter_context(freeze_time('2001-01-01 00:00:00'))
+
+            mock_current_task = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
+            )
+            mock_collect_data = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraAggregateData.collect_ora2_data')
+            )
+            mock_store_rows = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.models.DjangoStorageReportStore.store_rows')
+            )
+
+            mock_current_task.return_value = self.current_task
+
            test_header = ['field1', 'field2']
            test_rows = [['row1_field1', 'row1_field2'], ['row2_field1', 'row2_field2']]

+            mock_collect_data.return_value = (test_header, test_rows)
+
+            return_val = upload_ora2_data(None, None, self.course.id, None, 'generated')
+
+            timestamp_str = datetime.now(UTC).strftime('%Y-%m-%d-%H%M')
+            course_id_string = quote(text_type(self.course.id).replace('/', '_'))
+            filename = u'{}_ORA_data_{}.csv'.format(course_id_string, timestamp_str)
+
+            self.assertEqual(return_val, UPDATE_STATUS_SUCCEEDED)
+            mock_store_rows.assert_called_once_with(self.course.id, filename, [test_header] + test_rows)
+
+
+class TestInstructorOra2AttachmentsExport(SharedModuleStoreTestCase):
+    """
+    Tests that ORA2 submission files export works.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.course = CourseFactory.create()
+
+    def setUp(self):
+        super().setUp()
+
+        self.current_task = Mock()
+        self.current_task.update_state = Mock()
+
+    def test_export_fails_if_error_on_collect_step(self):
        with patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task') as mock_current_task:
            mock_current_task.return_value = self.current_task

            with patch(
-                'lms.djangoapps.instructor_task.tasks_helper.misc.OraAggregateData.collect_ora2_data'
+                'lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files'
            ) as mock_collect_data:
-                mock_collect_data.return_value = (test_header, test_rows)
-                with patch(
-                    'lms.djangoapps.instructor_task.models.DjangoStorageReportStore.store_rows'
-                ) as mock_store_rows:
-                    return_val = upload_ora2_data(None, None, self.course.id, None, 'generated')
+                mock_collect_data.side_effect = KeyError

-                    timestamp_str = datetime.now(UTC).strftime('%Y-%m-%d-%H%M')
-                    course_id_string = quote(text_type(self.course.id).replace('/', '_'))
-                    filename = u'{}_ORA_data_{}.csv'.format(course_id_string, timestamp_str)
+                response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
+                self.assertEqual(response, UPDATE_STATUS_FAILED)

-                    self.assertEqual(return_val, UPDATE_STATUS_SUCCEEDED)
-                    mock_store_rows.assert_called_once_with(self.course.id, filename, [test_header] + test_rows)
+    def test_export_fails_if_error_on_create_zip_step(self):
+        with ExitStack() as stack:
+            mock_current_task = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
+            )
+            mock_current_task.return_value = self.current_task
+
+            stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files')
+            )
+            create_zip_mock = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.create_zip_with_attachments')
+            )
+
+            create_zip_mock.side_effect = KeyError
+
+            response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
+            self.assertEqual(response, UPDATE_STATUS_FAILED)
+
+    def test_export_fails_if_error_on_upload_step(self):
+        with ExitStack() as stack:
+            mock_current_task = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
+            )
+            mock_current_task.return_value = self.current_task
+
+            stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files')
+            )
+            stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.create_zip_with_attachments')
+            )
+            upload_mock = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.upload_zip_to_report_store')
+            )
+
+            upload_mock.side_effect = KeyError
+
+            response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
+            self.assertEqual(response, UPDATE_STATUS_FAILED)
+
+    def test_task_stores_zip_with_attachments(self):
+        with ExitStack() as stack:
+            mock_current_task = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.runner._get_current_task')
+            )
+            mock_collect_files = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.collect_ora2_submission_files')
+            )
+            mock_create_zip = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.tasks_helper.misc.OraDownloadData.create_zip_with_attachments')
+            )
+            mock_store = stack.enter_context(
+                patch('lms.djangoapps.instructor_task.models.DjangoStorageReportStore.store')
+            )
+
+            mock_current_task.return_value = self.current_task
+
+            response = upload_ora2_submission_files(None, None, self.course.id, None, 'compressed')
+
+            mock_collect_files.assert_called_once()
+            mock_create_zip.assert_called_once()
+            mock_store.assert_called_once()
+
+            self.assertEqual(response, UPDATE_STATUS_SUCCEEDED)