Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PLAT-1121 Export courses asynchronously

Browse Source
This commit is contained in:
Jeremy Bowman
2017-03-01 15:42:34 -05:00
parent d40e8c05e8
commit 8d36c3825c
15 changed files with 1383 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cms/djangoapps/contentstore/storage.py
View File

@@ -17,7 +17,7 @@ class ImportExportS3Storage(S3BotoStorage): # pylint: disable=abstract-method
def __init__(self):
bucket = setting('COURSE_IMPORT_EXPORT_BUCKET', settings.AWS_STORAGE_BUCKET_NAME)
super(ImportExportS3Storage, self).__init__(bucket=bucket, querystring_auth=True)
super(ImportExportS3Storage, self).__init__(bucket=bucket, custom_domain=None, querystring_auth=True)
# pylint: disable=invalid-name
course_import_export_storage = get_storage_class(settings.COURSE_IMPORT_EXPORT_STORAGE)()

138
cms/djangoapps/contentstore/tasks.py
View File

@@ -5,11 +5,11 @@ from __future__ import absolute_import
import base64
import json
import logging
import os
import shutil
import tarfile
from datetime import datetime
from tempfile import NamedTemporaryFile, mkdtemp
from celery.task import task
from celery.utils.log import get_task_logger
@@ -20,17 +20,19 @@ from six import iteritems, text_type
from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import SuspiciousOperation
from django.core.files import File
from django.test import RequestFactory
from django.utils.text import get_valid_filename
from django.utils.translation import ugettext as _
from djcelery.common import respect_language
from user_tasks.models import UserTaskArtifact, UserTaskStatus
from user_tasks.tasks import UserTask
import dogstats_wrapper as dog_stats_api
from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer, SearchIndexingError
from contentstore.storage import course_import_export_storage
from contentstore.utils import initialize_permissions
from contentstore.utils import initialize_permissions, reverse_usage_url
from course_action_state.models import CourseRerunState
from models.settings.course_metadata import CourseMetadata
from opaque_keys.edx.keys import CourseKey
@@ -39,9 +41,11 @@ from openedx.core.lib.extract_tar import safetar_extractall
from student.auth import has_course_author_access
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseFields
from xmodule.exceptions import SerializationError
from xmodule.modulestore import COURSE_ROOT, LIBRARY_ROOT
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError
from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml
from xmodule.modulestore.xml_importer import import_course_from_xml, import_library_from_xml
@@ -155,6 +159,136 @@ def push_course_update_task(course_key_string, course_subscription_id, course_di
send_push_course_update(course_key_string, course_subscription_id, course_display_name)
class CourseExportTask(UserTask): # pylint: disable=abstract-method
"""
Base class for course and library export tasks.
"""
@staticmethod
def calculate_total_steps(arguments_dict):
"""
Get the number of in-progress steps in the export process, as shown in the UI.
For reference, these are:
1. Exporting
2. Compressing
"""
return 2
@classmethod
def generate_name(cls, arguments_dict):
"""
Create a name for this particular import task instance.
Arguments:
arguments_dict (dict): The arguments given to the task function
Returns:
text_type: The generated name
"""
key = arguments_dict[u'course_key_string']
return u'Export of {}'.format(key)
@task(base=CourseExportTask, bind=True)
def export_olx(self, user_id, course_key_string, language):
"""
Export a course or library to an OLX .tar.gz archive and prepare it for download.
"""
courselike_key = CourseKey.from_string(course_key_string)
try:
user = User.objects.get(pk=user_id)
except User.DoesNotExist:
with respect_language(language):
self.status.fail(_(u'Unknown User ID: {0}').format(user_id))
return
if not has_course_author_access(user, courselike_key):
with respect_language(language):
self.status.fail(_(u'Permission denied'))
return
if isinstance(courselike_key, LibraryLocator):
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
try:
self.status.set_state(u'Exporting')
tarball = create_export_tarball(courselike_module, courselike_key, {}, self.status)
artifact = UserTaskArtifact(status=self.status, name=u'Output')
artifact.file.save(name=tarball.name, content=File(tarball)) # pylint: disable=no-member
artifact.save()
# catch all exceptions so we can record useful error messages
except Exception as exception: # pylint: disable=broad-except
LOGGER.exception(u'Error exporting course %s', courselike_key)
if self.status.state != UserTaskStatus.FAILED:
self.status.fail({'raw_error_msg': text_type(exception)})
return
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s', course_key)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']}))
raise
except Exception as exc:
LOGGER.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
class CourseImportTask(UserTask): # pylint: disable=abstract-method
"""
Base class for course and library import tasks.

108
cms/djangoapps/contentstore/tests/test_tasks.py Normal file
View File

@@ -0,0 +1,108 @@
"""
Unit tests for course import and export Celery tasks
"""
from __future__ import absolute_import, division, print_function
import copy
import json
import mock
from uuid import uuid4
from django.conf import settings
from django.contrib.auth.models import User
from django.test.utils import override_settings
from user_tasks.models import UserTaskArtifact, UserTaskStatus
from contentstore.tasks import export_olx
from contentstore.tests.test_libraries import LibraryTestCase
from contentstore.tests.utils import CourseTestCase
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
def side_effect_exception(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect for mocking which raises an exception
"""
raise Exception('Boom!')
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ExportCourseTestCase(CourseTestCase):
"""
Tests of the export_olx task applied to courses
"""
def test_success(self):
"""
Verify that a routine course export task succeeds
"""
key = str(self.course.location.course_key)
result = export_olx.delay(self.user.id, key, u'en')
status = UserTaskStatus.objects.get(task_id=result.id)
self.assertEqual(status.state, UserTaskStatus.SUCCEEDED)
artifacts = UserTaskArtifact.objects.filter(status=status)
self.assertEqual(len(artifacts), 1)
output = artifacts[0]
self.assertEqual(output.name, 'Output')
@mock.patch('contentstore.tasks.export_course_to_xml', side_effect=side_effect_exception)
def test_exception(self, mock_export): # pylint: disable=unused-argument
"""
The export task should fail gracefully if an exception is thrown
"""
key = str(self.course.location.course_key)
result = export_olx.delay(self.user.id, key, u'en')
self._assert_failed(result, json.dumps({u'raw_error_msg': u'Boom!'}))
def test_invalid_user_id(self):
"""
Verify that attempts to export a course as an invalid user fail
"""
user_id = User.objects.order_by(u'-id').first().pk + 100
key = str(self.course.location.course_key)
result = export_olx.delay(user_id, key, u'en')
self._assert_failed(result, u'Unknown User ID: {}'.format(user_id))
def test_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to export it
"""
_, nonstaff_user = self.create_non_staff_authed_user_client()
key = str(self.course.location.course_key)
result = export_olx.delay(nonstaff_user.id, key, u'en')
self._assert_failed(result, u'Permission denied')
def _assert_failed(self, task_result, error_message):
"""
Verify that a task failed with the specified error message
"""
status = UserTaskStatus.objects.get(task_id=task_result.id)
self.assertEqual(status.state, UserTaskStatus.FAILED)
artifacts = UserTaskArtifact.objects.filter(status=status)
self.assertEqual(len(artifacts), 1)
error = artifacts[0]
self.assertEqual(error.name, u'Error')
self.assertEqual(error.text, error_message)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ExportLibraryTestCase(LibraryTestCase):
"""
Tests of the export_olx task applied to libraries
"""
def test_success(self):
"""
Verify that a routine library export task succeeds
"""
key = str(self.lib_key)
result = export_olx.delay(self.user.id, key, u'en') # pylint: disable=no-member
status = UserTaskStatus.objects.get(task_id=result.id)
self.assertEqual(status.state, UserTaskStatus.SUCCEEDED)
artifacts = UserTaskArtifact.objects.filter(status=status)
self.assertEqual(len(artifacts), 1)
output = artifacts[0]
self.assertEqual(output.name, 'Output')

214
cms/djangoapps/contentstore/views/import_export.py
View File

@@ -3,13 +3,12 @@ These views handle all actions in Studio related to import and exporting of
courses
"""
import base64
import json
import logging
import os
import re
import shutil
import tarfile
from path import Path as path
from tempfile import mkdtemp
from six import text_type
@@ -17,7 +16,6 @@ from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django.core.files import File
from django.core.files.temp import NamedTemporaryFile
from django.core.servers.basehttp import FileWrapper
from django.db import transaction
from django.http import HttpResponse, HttpResponseNotFound, Http404
@@ -26,28 +24,26 @@ from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_http_methods, require_GET
from edxmako.shortcuts import render_to_response
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import SerializationError
from xmodule.modulestore.django import modulestore
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocator
from user_tasks.conf import settings as user_tasks_settings
from user_tasks.models import UserTaskStatus
from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml
from user_tasks.models import UserTaskArtifact, UserTaskStatus
from student.auth import has_course_author_access
from util.json_request import JsonResponse
from util.views import ensure_valid_course_key
from contentstore.storage import course_import_export_storage
from contentstore.tasks import CourseImportTask, import_olx
from contentstore.tasks import CourseExportTask, CourseImportTask, create_export_tarball, export_olx, import_olx
from contentstore.utils import reverse_course_url, reverse_usage_url, reverse_library_url
from contentstore.utils import reverse_course_url, reverse_library_url
__all__ = [
'import_handler', 'import_status_handler',
'export_handler',
'export_handler', 'export_output_handler', 'export_status_handler',
]
@@ -279,64 +275,6 @@ def import_status_handler(request, course_key_string, filename=None):
return JsonResponse({"ImportStatus": status})
def create_export_tarball(course_module, course_key, context):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting %s', course_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
raise
except Exception as exc:
log.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'unit': None,
'raw_err_msg': str(exc)})
raise
finally:
shutil.rmtree(root_dir / name)
return export_file
def send_tarball(tarball):
"""
Renders a tarball to response, for use when sending a tar.gz file to the user.
@@ -351,7 +289,7 @@ def send_tarball(tarball):
@transaction.non_atomic_requests
@ensure_csrf_cookie
@login_required
@require_http_methods(("GET",))
@require_http_methods(('GET', 'POST'))
@ensure_valid_course_key
def export_handler(request, course_key_string):
"""
@@ -361,15 +299,21 @@ def export_handler(request, course_key_string):
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
POST
Start a Celery task to export the course
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
Note that there are 3 ways to request the tar.gz file. The Studio UI uses
a POST request to start the export asynchronously, with a link appearing
on the page once it's ready. Additionally, for backwards compatibility
reasons the request header can specify application/x-tgz via HTTP_ACCEPT,
or a query parameter can be used (?_accept=application/x-tgz); this will
export the course synchronously and return the resulting file (unless the
request times out for a large course).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
If the tar.gz file has been requested but the export operation fails, the
import page will be returned including a description of the error.
"""
course_key = CourseKey.from_string(course_key_string)
export_url = reverse_course_url('export_handler', course_key)
if not has_course_author_access(request.user, course_key):
raise PermissionDenied()
@@ -389,22 +333,134 @@ def export_handler(request, course_key_string):
'courselike_home_url': reverse_course_url("course_handler", course_key),
'library': False
}
context['export_url'] = export_url + '?_accept=application/x-tgz'
context['status_url'] = reverse_course_url('export_status_handler', course_key)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.GET.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html'))
if 'application/x-tgz' in requested_format:
if request.method == 'POST':
export_olx.delay(request.user.id, course_key_string, request.LANGUAGE_CODE)
return JsonResponse({'ExportStatus': 1})
elif 'application/x-tgz' in requested_format:
try:
tarball = create_export_tarball(courselike_module, course_key, context)
return send_tarball(tarball)
except SerializationError:
return render_to_response('export.html', context)
return send_tarball(tarball)
elif 'text/html' in requested_format:
return render_to_response('export.html', context)
else:
# Only HTML or x-tgz request formats are supported (no JSON).
return HttpResponse(status=406)
@transaction.non_atomic_requests
@require_GET
@ensure_csrf_cookie
@login_required
@ensure_valid_course_key
def export_status_handler(request, course_key_string):
"""
Returns an integer corresponding to the status of a file export. These are:
-X : Export unsuccessful due to some error with X as stage [0-3]
0 : No status info found (export done or task not yet created)
1 : Exporting
2 : Compressing
3 : Export successful
If the export was successful, a URL for the generated .tar.gz file is also
returned.
"""
course_key = CourseKey.from_string(course_key_string)
if not has_course_author_access(request.user, course_key):
raise PermissionDenied()
# The task status record is authoritative once it's been created
task_status = _latest_task_status(request, course_key_string, export_status_handler)
output_url = None
error = None
if task_status is None:
# The task hasn't been initialized yet; did we store info in the session already?
try:
session_status = request.session["export_status"]
status = session_status[course_key_string]
except KeyError:
status = 0
elif task_status.state == UserTaskStatus.SUCCEEDED:
status = 3
artifact = UserTaskArtifact.objects.get(status=task_status, name='Output')
if hasattr(artifact.file.storage, 'bucket'):
filename = os.path.basename(artifact.file.name).encode('utf-8')
disposition = 'attachment; filename="{}"'.format(filename)
output_url = artifact.file.storage.url(artifact.file.name, response_headers={
'response-content-disposition': disposition,
'response-content-encoding': 'application/octet-stream',
'response-content-type': 'application/x-tgz'
})
else:
# local file, serve from the authorization wrapper view
output_url = reverse_course_url('export_output_handler', course_key)
elif task_status.state in (UserTaskStatus.FAILED, UserTaskStatus.CANCELED):
status = max(-(task_status.completed_steps + 1), -2)
errors = UserTaskArtifact.objects.filter(status=task_status, name='Error')
if len(errors):
error = errors[0].text
try:
error = json.loads(error)
except ValueError:
# Wasn't JSON, just use the value as a string
pass
else:
status = min(task_status.completed_steps + 1, 2)
response = {"ExportStatus": status}
if output_url:
response['ExportOutput'] = output_url
elif error:
response['ExportError'] = error
return JsonResponse(response)
@transaction.non_atomic_requests
@require_GET
@ensure_csrf_cookie
@login_required
@ensure_valid_course_key
def export_output_handler(request, course_key_string):
"""
Returns the OLX .tar.gz produced by a file export. Only used in
environments such as devstack where the output is stored in a local
filesystem instead of an external service like S3.
"""
course_key = CourseKey.from_string(course_key_string)
if not has_course_author_access(request.user, course_key):
raise PermissionDenied()
task_status = _latest_task_status(request, course_key_string, export_output_handler)
if task_status and task_status.state == UserTaskStatus.SUCCEEDED:
artifact = None
try:
artifact = UserTaskArtifact.objects.get(status=task_status, name='Output')
tarball = course_import_export_storage.open(artifact.file.name)
return send_tarball(tarball)
except UserTaskArtifact.DoesNotExist:
raise Http404
finally:
if artifact:
artifact.file.close()
else:
raise Http404
def _latest_task_status(request, course_key_string, view_func=None):
"""
Get the most recent export status update for the specified course/library
key.
"""
args = {u'course_key_string': course_key_string}
name = CourseExportTask.generate_name(args)
task_status = UserTaskStatus.objects.filter(name=name)
for status_filter in STATUS_FILTERS:
task_status = status_filter().filter_queryset(request, task_status, view_func)
return task_status.order_by(u'-created').first()

69
cms/djangoapps/contentstore/views/tests/test_import_export.py
View File

@@ -531,6 +531,7 @@ class ExportTestCase(CourseTestCase):
"""
super(ExportTestCase, self).setUp()
self.url = reverse_course_url('export_handler', self.course.id)
self.status_url = reverse_course_url('export_status_handler', self.course.id)
def test_export_html(self):
"""
@@ -547,6 +548,21 @@ class ExportTestCase(CourseTestCase):
resp = self.client.get(self.url, HTTP_ACCEPT='application/json')
self.assertEquals(resp.status_code, 406)
def test_export_async(self):
"""
Get tar.gz file, using asynchronous background task
"""
resp = self.client.post(self.url)
self.assertEquals(resp.status_code, 200)
resp = self.client.get(self.status_url)
result = json.loads(resp.content)
status = result['ExportStatus']
self.assertEquals(status, 3)
self.assertIn('ExportOutput', result)
output_url = result['ExportOutput']
resp = self.client.get(output_url)
self._verify_export_succeeded(resp)
def test_export_targz(self):
"""
Get tar.gz file, using HTTP_ACCEPT.
@@ -588,11 +604,16 @@ class ExportTestCase(CourseTestCase):
def _verify_export_failure(self, expected_text):
""" Export failure helper method. """
resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz')
resp = self.client.post(self.url)
self.assertEquals(resp.status_code, 200)
self.assertIsNone(resp.get('Content-Disposition'))
self.assertContains(resp, 'Unable to create xml for module')
self.assertContains(resp, expected_text)
resp = self.client.get(self.status_url)
self.assertEquals(resp.status_code, 200)
result = json.loads(resp.content)
self.assertNotIn('ExportOutput', result)
self.assertIn('ExportError', result)
error = result['ExportError']
self.assertIn('Unable to create xml for module', error['raw_error_msg'])
self.assertIn(expected_text, error['edit_unit_url'])
def test_library_export(self):
"""
@@ -639,19 +660,53 @@ class ExportTestCase(CourseTestCase):
data=xml_string
)
self.test_export_targz_urlparam()
self.test_export_async()
@ddt.data(
'/export/non.1/existence_1/Run_1', # For mongo
'/export/course-v1:non1+existence1+Run1', # For split
)
def test_export_course_doest_not_exist(self, url):
def test_export_course_does_not_exist(self, url):
"""
Export failure if course is not exist
Export failure if course does not exist
"""
resp = self.client.get_html(url)
self.assertEquals(resp.status_code, 404)
def test_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to export it
"""
client, _ = self.create_non_staff_authed_user_client()
resp = client.get(self.url)
self.assertEqual(resp.status_code, 403)
def test_status_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to see the status of export tasks
"""
client, _ = self.create_non_staff_authed_user_client()
resp = client.get(self.status_url)
self.assertEqual(resp.status_code, 403)
def test_status_missing_record(self):
"""
Attempting to get the status of an export task which isn't currently
represented in the database should yield a useful result
"""
resp = self.client.get(self.status_url)
self.assertEqual(resp.status_code, 200)
result = json.loads(resp.content)
self.assertEqual(result['ExportStatus'], 0)
def test_output_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to see the output of export tasks
"""
client, _ = self.create_non_staff_authed_user_client()
resp = client.get(reverse_course_url('export_output_handler', self.course.id))
self.assertEqual(resp.status_code, 403)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestLibraryImportExport(CourseTestCase):