Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge pull request #14635 from edx/jmbowman/async_course_export_2

Browse Source 
PLAT-1121 Export courses asynchronously
This commit is contained in:
Jeremy Bowman
2017-03-13 11:34:59 -04:00
committed by GitHub
parent d40e8c05e8 8d36c3825c
commit 54d9bbd848
15 changed files with 1383 additions and 338 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cms/djangoapps/contentstore/storage.py
View File

@@ -17,7 +17,7 @@ class ImportExportS3Storage(S3BotoStorage): # pylint: disable=abstract-method
def __init__(self):
bucket = setting('COURSE_IMPORT_EXPORT_BUCKET', settings.AWS_STORAGE_BUCKET_NAME)
super(ImportExportS3Storage, self).__init__(bucket=bucket, querystring_auth=True)
super(ImportExportS3Storage, self).__init__(bucket=bucket, custom_domain=None, querystring_auth=True)
# pylint: disable=invalid-name
course_import_export_storage = get_storage_class(settings.COURSE_IMPORT_EXPORT_STORAGE)()

138
cms/djangoapps/contentstore/tasks.py
View File

@@ -5,11 +5,11 @@ from __future__ import absolute_import
import base64
import json
import logging
import os
import shutil
import tarfile
from datetime import datetime
from tempfile import NamedTemporaryFile, mkdtemp
from celery.task import task
from celery.utils.log import get_task_logger
@@ -20,17 +20,19 @@ from six import iteritems, text_type
from django.conf import settings
from django.contrib.auth.models import User
from django.core.exceptions import SuspiciousOperation
from django.core.files import File
from django.test import RequestFactory
from django.utils.text import get_valid_filename
from django.utils.translation import ugettext as _
from djcelery.common import respect_language
from user_tasks.models import UserTaskArtifact, UserTaskStatus
from user_tasks.tasks import UserTask
import dogstats_wrapper as dog_stats_api
from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer, SearchIndexingError
from contentstore.storage import course_import_export_storage
from contentstore.utils import initialize_permissions
from contentstore.utils import initialize_permissions, reverse_usage_url
from course_action_state.models import CourseRerunState
from models.settings.course_metadata import CourseMetadata
from opaque_keys.edx.keys import CourseKey
@@ -39,9 +41,11 @@ from openedx.core.lib.extract_tar import safetar_extractall
from student.auth import has_course_author_access
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseFields
from xmodule.exceptions import SerializationError
from xmodule.modulestore import COURSE_ROOT, LIBRARY_ROOT
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError
from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml
from xmodule.modulestore.xml_importer import import_course_from_xml, import_library_from_xml
@@ -155,6 +159,136 @@ def push_course_update_task(course_key_string, course_subscription_id, course_di
send_push_course_update(course_key_string, course_subscription_id, course_display_name)
class CourseExportTask(UserTask): # pylint: disable=abstract-method
"""
Base class for course and library export tasks.
"""
@staticmethod
def calculate_total_steps(arguments_dict):
"""
Get the number of in-progress steps in the export process, as shown in the UI.
For reference, these are:
1. Exporting
2. Compressing
"""
return 2
@classmethod
def generate_name(cls, arguments_dict):
"""
Create a name for this particular import task instance.
Arguments:
arguments_dict (dict): The arguments given to the task function
Returns:
text_type: The generated name
"""
key = arguments_dict[u'course_key_string']
return u'Export of {}'.format(key)
@task(base=CourseExportTask, bind=True)
def export_olx(self, user_id, course_key_string, language):
"""
Export a course or library to an OLX .tar.gz archive and prepare it for download.
"""
courselike_key = CourseKey.from_string(course_key_string)
try:
user = User.objects.get(pk=user_id)
except User.DoesNotExist:
with respect_language(language):
self.status.fail(_(u'Unknown User ID: {0}').format(user_id))
return
if not has_course_author_access(user, courselike_key):
with respect_language(language):
self.status.fail(_(u'Permission denied'))
return
if isinstance(courselike_key, LibraryLocator):
courselike_module = modulestore().get_library(courselike_key)
else:
courselike_module = modulestore().get_course(courselike_key)
try:
self.status.set_state(u'Exporting')
tarball = create_export_tarball(courselike_module, courselike_key, {}, self.status)
artifact = UserTaskArtifact(status=self.status, name=u'Output')
artifact.file.save(name=tarball.name, content=File(tarball)) # pylint: disable=no-member
artifact.save()
# catch all exceptions so we can record useful error messages
except Exception as exception: # pylint: disable=broad-except
LOGGER.exception(u'Error exporting course %s', courselike_key)
if self.status.state != UserTaskStatus.FAILED:
self.status.fail({'raw_error_msg': text_type(exception)})
return
def create_export_tarball(course_module, course_key, context, status=None):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
if status:
status.set_state(u'Compressing')
status.increment_completed_steps()
LOGGER.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
LOGGER.exception(u'There was an error exporting %s', course_key)
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'],
'edit_unit_url': context['edit_unit_url']}))
raise
except Exception as exc:
LOGGER.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'edit_unit_url': None,
'raw_err_msg': str(exc)})
if status:
status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']}))
raise
finally:
if os.path.exists(root_dir / name):
shutil.rmtree(root_dir / name)
return export_file
class CourseImportTask(UserTask): # pylint: disable=abstract-method
"""
Base class for course and library import tasks.

108
cms/djangoapps/contentstore/tests/test_tasks.py Normal file
View File

@@ -0,0 +1,108 @@
"""
Unit tests for course import and export Celery tasks
"""
from __future__ import absolute_import, division, print_function
import copy
import json
import mock
from uuid import uuid4
from django.conf import settings
from django.contrib.auth.models import User
from django.test.utils import override_settings
from user_tasks.models import UserTaskArtifact, UserTaskStatus
from contentstore.tasks import export_olx
from contentstore.tests.test_libraries import LibraryTestCase
from contentstore.tests.utils import CourseTestCase
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
def side_effect_exception(*args, **kwargs): # pylint: disable=unused-argument
"""
Side effect for mocking which raises an exception
"""
raise Exception('Boom!')
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ExportCourseTestCase(CourseTestCase):
"""
Tests of the export_olx task applied to courses
"""
def test_success(self):
"""
Verify that a routine course export task succeeds
"""
key = str(self.course.location.course_key)
result = export_olx.delay(self.user.id, key, u'en')
status = UserTaskStatus.objects.get(task_id=result.id)
self.assertEqual(status.state, UserTaskStatus.SUCCEEDED)
artifacts = UserTaskArtifact.objects.filter(status=status)
self.assertEqual(len(artifacts), 1)
output = artifacts[0]
self.assertEqual(output.name, 'Output')
@mock.patch('contentstore.tasks.export_course_to_xml', side_effect=side_effect_exception)
def test_exception(self, mock_export): # pylint: disable=unused-argument
"""
The export task should fail gracefully if an exception is thrown
"""
key = str(self.course.location.course_key)
result = export_olx.delay(self.user.id, key, u'en')
self._assert_failed(result, json.dumps({u'raw_error_msg': u'Boom!'}))
def test_invalid_user_id(self):
"""
Verify that attempts to export a course as an invalid user fail
"""
user_id = User.objects.order_by(u'-id').first().pk + 100
key = str(self.course.location.course_key)
result = export_olx.delay(user_id, key, u'en')
self._assert_failed(result, u'Unknown User ID: {}'.format(user_id))
def test_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to export it
"""
_, nonstaff_user = self.create_non_staff_authed_user_client()
key = str(self.course.location.course_key)
result = export_olx.delay(nonstaff_user.id, key, u'en')
self._assert_failed(result, u'Permission denied')
def _assert_failed(self, task_result, error_message):
"""
Verify that a task failed with the specified error message
"""
status = UserTaskStatus.objects.get(task_id=task_result.id)
self.assertEqual(status.state, UserTaskStatus.FAILED)
artifacts = UserTaskArtifact.objects.filter(status=status)
self.assertEqual(len(artifacts), 1)
error = artifacts[0]
self.assertEqual(error.name, u'Error')
self.assertEqual(error.text, error_message)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ExportLibraryTestCase(LibraryTestCase):
"""
Tests of the export_olx task applied to libraries
"""
def test_success(self):
"""
Verify that a routine library export task succeeds
"""
key = str(self.lib_key)
result = export_olx.delay(self.user.id, key, u'en') # pylint: disable=no-member
status = UserTaskStatus.objects.get(task_id=result.id)
self.assertEqual(status.state, UserTaskStatus.SUCCEEDED)
artifacts = UserTaskArtifact.objects.filter(status=status)
self.assertEqual(len(artifacts), 1)
output = artifacts[0]
self.assertEqual(output.name, 'Output')

214
cms/djangoapps/contentstore/views/import_export.py
View File

@@ -3,13 +3,12 @@ These views handle all actions in Studio related to import and exporting of
courses
"""
import base64
import json
import logging
import os
import re
import shutil
import tarfile
from path import Path as path
from tempfile import mkdtemp
from six import text_type
@@ -17,7 +16,6 @@ from django.conf import settings
from django.contrib.auth.decorators import login_required
from django.core.exceptions import PermissionDenied
from django.core.files import File
from django.core.files.temp import NamedTemporaryFile
from django.core.servers.basehttp import FileWrapper
from django.db import transaction
from django.http import HttpResponse, HttpResponseNotFound, Http404
@@ -26,28 +24,26 @@ from django.views.decorators.csrf import ensure_csrf_cookie
from django.views.decorators.http import require_http_methods, require_GET
from edxmako.shortcuts import render_to_response
from xmodule.contentstore.django import contentstore
from xmodule.exceptions import SerializationError
from xmodule.modulestore.django import modulestore
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.locator import LibraryLocator
from user_tasks.conf import settings as user_tasks_settings
from user_tasks.models import UserTaskStatus
from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml
from user_tasks.models import UserTaskArtifact, UserTaskStatus
from student.auth import has_course_author_access
from util.json_request import JsonResponse
from util.views import ensure_valid_course_key
from contentstore.storage import course_import_export_storage
from contentstore.tasks import CourseImportTask, import_olx
from contentstore.tasks import CourseExportTask, CourseImportTask, create_export_tarball, export_olx, import_olx
from contentstore.utils import reverse_course_url, reverse_usage_url, reverse_library_url
from contentstore.utils import reverse_course_url, reverse_library_url
__all__ = [
'import_handler', 'import_status_handler',
'export_handler',
'export_handler', 'export_output_handler', 'export_status_handler',
]
@@ -279,64 +275,6 @@ def import_status_handler(request, course_key_string, filename=None):
return JsonResponse({"ImportStatus": status})
def create_export_tarball(course_module, course_key, context):
"""
Generates the export tarball, or returns None if there was an error.
Updates the context with any error information if applicable.
"""
name = course_module.url_name
export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz")
root_dir = path(mkdtemp())
try:
if isinstance(course_key, LibraryLocator):
export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name)
else:
export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name)
logging.debug(u'tar file being generated at %s', export_file.name)
with tarfile.open(name=export_file.name, mode='w:gz') as tar_file:
tar_file.add(root_dir / name, arcname=name)
except SerializationError as exc:
log.exception(u'There was an error exporting %s', course_key)
unit = None
failed_item = None
parent = None
try:
failed_item = modulestore().get_item(exc.location)
parent_loc = modulestore().get_parent_location(failed_item.location)
if parent_loc is not None:
parent = modulestore().get_item(parent_loc)
if parent.location.category == 'vertical':
unit = parent
except: # pylint: disable=bare-except
# if we have a nested exception, then we'll show the more generic error message
pass
context.update({
'in_err': True,
'raw_err_msg': str(exc),
'failed_module': failed_item,
'unit': unit,
'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "",
})
raise
except Exception as exc:
log.exception('There was an error exporting %s', course_key)
context.update({
'in_err': True,
'unit': None,
'raw_err_msg': str(exc)})
raise
finally:
shutil.rmtree(root_dir / name)
return export_file
def send_tarball(tarball):
"""
Renders a tarball to response, for use when sending a tar.gz file to the user.
@@ -351,7 +289,7 @@ def send_tarball(tarball):
@transaction.non_atomic_requests
@ensure_csrf_cookie
@login_required
@require_http_methods(("GET",))
@require_http_methods(('GET', 'POST'))
@ensure_valid_course_key
def export_handler(request, course_key_string):
"""
@@ -361,15 +299,21 @@ def export_handler(request, course_key_string):
html: return html page for import page
application/x-tgz: return tar.gz file containing exported course
json: not supported
POST
Start a Celery task to export the course
Note that there are 2 ways to request the tar.gz file. The request header can specify
application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz).
Note that there are 3 ways to request the tar.gz file. The Studio UI uses
a POST request to start the export asynchronously, with a link appearing
on the page once it's ready. Additionally, for backwards compatibility
reasons the request header can specify application/x-tgz via HTTP_ACCEPT,
or a query parameter can be used (?_accept=application/x-tgz); this will
export the course synchronously and return the resulting file (unless the
request times out for a large course).
If the tar.gz file has been requested but the export operation fails, an HTML page will be returned
which describes the error.
If the tar.gz file has been requested but the export operation fails, the
import page will be returned including a description of the error.
"""
course_key = CourseKey.from_string(course_key_string)
export_url = reverse_course_url('export_handler', course_key)
if not has_course_author_access(request.user, course_key):
raise PermissionDenied()
@@ -389,22 +333,134 @@ def export_handler(request, course_key_string):
'courselike_home_url': reverse_course_url("course_handler", course_key),
'library': False
}
context['export_url'] = export_url + '?_accept=application/x-tgz'
context['status_url'] = reverse_course_url('export_status_handler', course_key)
# an _accept URL parameter will be preferred over HTTP_ACCEPT in the header.
requested_format = request.GET.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html'))
if 'application/x-tgz' in requested_format:
if request.method == 'POST':
export_olx.delay(request.user.id, course_key_string, request.LANGUAGE_CODE)
return JsonResponse({'ExportStatus': 1})
elif 'application/x-tgz' in requested_format:
try:
tarball = create_export_tarball(courselike_module, course_key, context)
return send_tarball(tarball)
except SerializationError:
return render_to_response('export.html', context)
return send_tarball(tarball)
elif 'text/html' in requested_format:
return render_to_response('export.html', context)
else:
# Only HTML or x-tgz request formats are supported (no JSON).
return HttpResponse(status=406)
@transaction.non_atomic_requests
@require_GET
@ensure_csrf_cookie
@login_required
@ensure_valid_course_key
def export_status_handler(request, course_key_string):
"""
Returns an integer corresponding to the status of a file export. These are:
-X : Export unsuccessful due to some error with X as stage [0-3]
0 : No status info found (export done or task not yet created)
1 : Exporting
2 : Compressing
3 : Export successful
If the export was successful, a URL for the generated .tar.gz file is also
returned.
"""
course_key = CourseKey.from_string(course_key_string)
if not has_course_author_access(request.user, course_key):
raise PermissionDenied()
# The task status record is authoritative once it's been created
task_status = _latest_task_status(request, course_key_string, export_status_handler)
output_url = None
error = None
if task_status is None:
# The task hasn't been initialized yet; did we store info in the session already?
try:
session_status = request.session["export_status"]
status = session_status[course_key_string]
except KeyError:
status = 0
elif task_status.state == UserTaskStatus.SUCCEEDED:
status = 3
artifact = UserTaskArtifact.objects.get(status=task_status, name='Output')
if hasattr(artifact.file.storage, 'bucket'):
filename = os.path.basename(artifact.file.name).encode('utf-8')
disposition = 'attachment; filename="{}"'.format(filename)
output_url = artifact.file.storage.url(artifact.file.name, response_headers={
'response-content-disposition': disposition,
'response-content-encoding': 'application/octet-stream',
'response-content-type': 'application/x-tgz'
})
else:
# local file, serve from the authorization wrapper view
output_url = reverse_course_url('export_output_handler', course_key)
elif task_status.state in (UserTaskStatus.FAILED, UserTaskStatus.CANCELED):
status = max(-(task_status.completed_steps + 1), -2)
errors = UserTaskArtifact.objects.filter(status=task_status, name='Error')
if len(errors):
error = errors[0].text
try:
error = json.loads(error)
except ValueError:
# Wasn't JSON, just use the value as a string
pass
else:
status = min(task_status.completed_steps + 1, 2)
response = {"ExportStatus": status}
if output_url:
response['ExportOutput'] = output_url
elif error:
response['ExportError'] = error
return JsonResponse(response)
@transaction.non_atomic_requests
@require_GET
@ensure_csrf_cookie
@login_required
@ensure_valid_course_key
def export_output_handler(request, course_key_string):
"""
Returns the OLX .tar.gz produced by a file export. Only used in
environments such as devstack where the output is stored in a local
filesystem instead of an external service like S3.
"""
course_key = CourseKey.from_string(course_key_string)
if not has_course_author_access(request.user, course_key):
raise PermissionDenied()
task_status = _latest_task_status(request, course_key_string, export_output_handler)
if task_status and task_status.state == UserTaskStatus.SUCCEEDED:
artifact = None
try:
artifact = UserTaskArtifact.objects.get(status=task_status, name='Output')
tarball = course_import_export_storage.open(artifact.file.name)
return send_tarball(tarball)
except UserTaskArtifact.DoesNotExist:
raise Http404
finally:
if artifact:
artifact.file.close()
else:
raise Http404
def _latest_task_status(request, course_key_string, view_func=None):
"""
Get the most recent export status update for the specified course/library
key.
"""
args = {u'course_key_string': course_key_string}
name = CourseExportTask.generate_name(args)
task_status = UserTaskStatus.objects.filter(name=name)
for status_filter in STATUS_FILTERS:
task_status = status_filter().filter_queryset(request, task_status, view_func)
return task_status.order_by(u'-created').first()

69
cms/djangoapps/contentstore/views/tests/test_import_export.py
View File

@@ -531,6 +531,7 @@ class ExportTestCase(CourseTestCase):
"""
super(ExportTestCase, self).setUp()
self.url = reverse_course_url('export_handler', self.course.id)
self.status_url = reverse_course_url('export_status_handler', self.course.id)
def test_export_html(self):
"""
@@ -547,6 +548,21 @@ class ExportTestCase(CourseTestCase):
resp = self.client.get(self.url, HTTP_ACCEPT='application/json')
self.assertEquals(resp.status_code, 406)
def test_export_async(self):
"""
Get tar.gz file, using asynchronous background task
"""
resp = self.client.post(self.url)
self.assertEquals(resp.status_code, 200)
resp = self.client.get(self.status_url)
result = json.loads(resp.content)
status = result['ExportStatus']
self.assertEquals(status, 3)
self.assertIn('ExportOutput', result)
output_url = result['ExportOutput']
resp = self.client.get(output_url)
self._verify_export_succeeded(resp)
def test_export_targz(self):
"""
Get tar.gz file, using HTTP_ACCEPT.
@@ -588,11 +604,16 @@ class ExportTestCase(CourseTestCase):
def _verify_export_failure(self, expected_text):
""" Export failure helper method. """
resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz')
resp = self.client.post(self.url)
self.assertEquals(resp.status_code, 200)
self.assertIsNone(resp.get('Content-Disposition'))
self.assertContains(resp, 'Unable to create xml for module')
self.assertContains(resp, expected_text)
resp = self.client.get(self.status_url)
self.assertEquals(resp.status_code, 200)
result = json.loads(resp.content)
self.assertNotIn('ExportOutput', result)
self.assertIn('ExportError', result)
error = result['ExportError']
self.assertIn('Unable to create xml for module', error['raw_error_msg'])
self.assertIn(expected_text, error['edit_unit_url'])
def test_library_export(self):
"""
@@ -639,19 +660,53 @@ class ExportTestCase(CourseTestCase):
data=xml_string
)
self.test_export_targz_urlparam()
self.test_export_async()
@ddt.data(
'/export/non.1/existence_1/Run_1', # For mongo
'/export/course-v1:non1+existence1+Run1', # For split
)
def test_export_course_doest_not_exist(self, url):
def test_export_course_does_not_exist(self, url):
"""
Export failure if course is not exist
Export failure if course does not exist
"""
resp = self.client.get_html(url)
self.assertEquals(resp.status_code, 404)
def test_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to export it
"""
client, _ = self.create_non_staff_authed_user_client()
resp = client.get(self.url)
self.assertEqual(resp.status_code, 403)
def test_status_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to see the status of export tasks
"""
client, _ = self.create_non_staff_authed_user_client()
resp = client.get(self.status_url)
self.assertEqual(resp.status_code, 403)
def test_status_missing_record(self):
"""
Attempting to get the status of an export task which isn't currently
represented in the database should yield a useful result
"""
resp = self.client.get(self.status_url)
self.assertEqual(resp.status_code, 200)
result = json.loads(resp.content)
self.assertEqual(result['ExportStatus'], 0)
def test_output_non_course_author(self):
"""
Verify that users who aren't authors of the course are unable to see the output of export tasks
"""
client, _ = self.create_non_staff_authed_user_client()
resp = client.get(reverse_course_url('export_output_handler', self.course.id))
self.assertEqual(resp.status_code, 403)
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class TestLibraryImportExport(CourseTestCase):

103
cms/static/js/factories/export.js
View File

@@ -1,66 +1,57 @@
define(['gettext', 'common/js/components/views/feedback_prompt'], function(gettext, PromptView) {
define([
'domReady', 'js/views/export', 'jquery', 'gettext'
], function(domReady, Export, $, gettext) {
'use strict';
return function(hasUnit, editUnitUrl, courselikeHomeUrl, library, errMsg) {
var dialog;
if (hasUnit) {
dialog = new PromptView({
title: gettext('There has been an error while exporting.'),
message: gettext('There has been a failure to export to XML at least one component. It is recommended that you go to the edit page and repair the error before attempting another export. Please check that all components on the page are valid and do not display any error messages.'),
intent: 'error',
actions: {
primary: {
text: gettext('Correct failed component'),
click: function(view) {
view.hide();
document.location = editUnitUrl;
}
},
secondary: {
text: gettext('Return to Export'),
click: function(view) {
view.hide();
return function(courselikeHomeUrl, library, statusUrl) {
var $submitBtn = $('.action-export'),
unloading = false,
previousExport = Export.storedExport(courselikeHomeUrl);
var onComplete = function() {
$submitBtn.show();
};
var startExport = function(e) {
e.preventDefault();
$submitBtn.hide();
Export.reset(library);
Export.start(statusUrl).then(onComplete);
$.ajax({
type: 'POST',
url: window.location.pathname,
data: {},
success: function(result, textStatus, xhr) {
if (xhr.status === 200) {
setTimeout(function() { Export.pollStatus(result); }, 1000);
} else {
// It could be that the user is simply refreshing the page
// so we need to be sure this is an actual error from the server
if (!unloading) {
$(window).off('beforeunload.import');
Export.reset(library);
onComplete();
Export.showError(gettext('Your export has failed.'));
}
}
}
});
} else {
var msg = '<p>';
var action;
if (library) {
msg += gettext('Your library could not be exported to XML. There is not enough information to identify the failed component. Inspect your library to identify any problematic components and try again.');
action = gettext('Take me to the main library page');
} else {
msg += gettext('Your course could not be exported to XML. There is not enough information to identify the failed component. Inspect your course to identify any problematic components and try again.');
action = gettext('Take me to the main course page');
};
$(window).on('beforeunload', function() { unloading = true; });
// Display the status of last file upload on page load
if (previousExport) {
if (previousExport.completed !== true) {
$submitBtn.hide();
}
msg += '</p><p>' + gettext('The raw error message is:') + '</p>' + errMsg;
dialog = new PromptView({
title: gettext('There has been an error with your export.'),
message: msg,
intent: 'error',
actions: {
primary: {
text: action,
click: function(view) {
view.hide();
document.location = courselikeHomeUrl;
}
},
secondary: {
text: gettext('Cancel'),
click: function(view) {
view.hide();
}
}
}
});
Export.resume(library).then(onComplete);
}
// The CSS animation for the dialog relies on the 'js' class
// being on the body. This happens after this JavaScript is executed,
// causing a 'bouncing' of the dialog after it is initially shown.
// As a workaround, add this class first.
$('body').addClass('js');
dialog.show();
domReady(function() {
// export form setup
$submitBtn.bind('click', startExport);
});
};
});

12
cms/static/js/factories/import.js
View File

@@ -29,7 +29,7 @@ define([
var onComplete = function() {
bar.hide();
chooseBtn
.find('.copy').html(gettext('Choose new file')).end()
.find('.copy').text(gettext('Choose new file')).end()
.show();
};
@@ -38,7 +38,9 @@ define([
// Display the status of last file upload on page load
if (previousImport) {
$('.file-name-block')
.find('.file-name').html(previousImport.file.name).end()
.find('.file-name')
.text(previousImport.file.name)
.end()
.show();
if (previousImport.completed !== true) {
@@ -123,7 +125,7 @@ define([
setTimeout(function() { Import.pollStatus(); }, 3000);
} else {
bar.show();
fill.width(percentVal).html(percentVal);
fill.width(percentVal).text(percentVal);
}
},
sequentialUploads: true,
@@ -136,7 +138,7 @@ define([
if (filepath.substr(filepath.length - 6, 6) === 'tar.gz') {
$('.error-block').hide();
$('.file-name').html($(this).val().replace('C:\\fakepath\\', ''));
$('.file-name').text($(this).val().replace('C:\\fakepath\\', ''));
$('.file-name-block').show();
chooseBtn.hide();
submitBtn.show();
@@ -145,7 +147,7 @@ define([
var msg = gettext('File format not supported. Please upload a file with a {file_extension} extension.')
.replace('{file_extension}', '<code>tar.gz</code>');
$('.error-block').html(msg).show();
$('.error-block').text(msg).show();
}
};

400
cms/static/js/views/export.js Normal file
View File

@@ -0,0 +1,400 @@
/**
* Course export-related js.
*/
define([
'jquery', 'underscore', 'gettext', 'moment', 'common/js/components/views/feedback_prompt',
'edx-ui-toolkit/js/utils/html-utils', 'jquery.cookie'
], function($, _, gettext, moment, PromptView, HtmlUtils) {
'use strict';
/** ******** Private properties ****************************************/
var COOKIE_NAME = 'lastexport';
var STAGE = {
PREPARING: 0,
EXPORTING: 1,
COMPRESSING: 2,
SUCCESS: 3
};
var STATE = {
READY: 1,
IN_PROGRESS: 2,
SUCCESS: 3,
ERROR: 4
};
var courselikeHomeUrl;
var current = {stage: 0, state: STATE.READY, downloadUrl: null};
var deferred = null;
var isLibrary = false;
var statusUrl = null;
var successUnixDate = null;
var timeout = {id: null, delay: 1000};
var $dom = {
downloadLink: $('#download-exported-button'),
stages: $('ol.status-progress').children(),
successStage: $('.item-progresspoint-success'),
wrapper: $('div.wrapper-status')
};
/** ******** Private functions *****************************************/
/**
* Makes Export feedback status list visible
*
*/
var displayFeedbackList = function() {
$dom.wrapper.removeClass('is-hidden');
};
/**
* Updates the Export feedback status list
*
* @param {string} [currStageMsg=''] The message to show on the
* current stage (for now only in case of error)
*/
var updateFeedbackList = function(currStageMsg) {
var $checkmark, $curr, $prev, $next;
var date, stageMsg, time;
$checkmark = $dom.successStage.find('.icon');
stageMsg = currStageMsg || '';
function completeStage(stage) {
$(stage)
.removeClass('is-not-started is-started')
.addClass('is-complete');
}
function errorStage(stage) {
if (!$(stage).hasClass('has-error')) {
stageMsg = HtmlUtils.joinHtml(
HtmlUtils.HTML('<p class="copy error">'),
stageMsg,
HtmlUtils.HTML('</p>')
);
$(stage)
.removeClass('is-started')
.addClass('has-error')
.find('p.copy')
.hide()
.after(HtmlUtils.ensureHtml(stageMsg).toString());
}
}
function resetStage(stage) {
$(stage)
.removeClass('is-complete is-started has-error')
.addClass('is-not-started')
.find('p.error')
.remove()
.end()
.find('p.copy')
.show();
}
switch (current.state) {
case STATE.READY:
_.map($dom.stages, resetStage);
break;
case STATE.IN_PROGRESS:
$prev = $dom.stages.slice(0, current.stage);
$curr = $dom.stages.eq(current.stage);
_.map($prev, completeStage);
$curr.removeClass('is-not-started').addClass('is-started');
break;
case STATE.SUCCESS:
date = moment(successUnixDate).utc().format('MM/DD/YYYY');
time = moment(successUnixDate).utc().format('HH:mm');
_.map($dom.stages, completeStage);
$dom.successStage
.find('.item-progresspoint-success-date')
.text('(' + date + ' at ' + time + ' UTC)');
break;
case STATE.ERROR:
// Make all stages up to, and including, the error stage 'complete'.
$prev = $dom.stages.slice(0, current.stage + 1);
$curr = $dom.stages.eq(current.stage);
$next = $dom.stages.slice(current.stage + 1);
_.map($prev, completeStage);
_.map($next, resetStage);
errorStage($curr);
break;
default:
// Invalid state, don't change anything
return;
}
if (current.state === STATE.SUCCESS) {
$checkmark.removeClass('fa-square-o').addClass('fa-check-square-o');
$dom.downloadLink.attr('href', current.downloadUrl);
} else {
$checkmark.removeClass('fa-check-square-o').addClass('fa-square-o');
$dom.downloadLink.attr('href', '#');
}
};
/**
* Sets the Export in the "error" status.
*
* Immediately stops any further polling from the server.
* Displays the error message at the list element that corresponds
* to the stage where the error occurred.
*
* @param {string} msg Error message to display.
* @param {int} [stage=current.stage] Stage of export process at which error occurred.
*/
var error = function(msg, stage) {
current.stage = Math.abs(stage || current.stage); // Could be negative
current.state = STATE.ERROR;
clearTimeout(timeout.id);
updateFeedbackList(msg);
deferred.resolve();
};
/**
* Stores in a cookie the current export data
*
* @param {boolean} [completed=false] If the export has been completed or not
*/
var storeExport = function(completed) {
$.cookie(COOKIE_NAME, JSON.stringify({
statusUrl: statusUrl,
date: moment().valueOf(),
completed: completed || false
}), {path: window.location.pathname});
};
/** ******** Public functions ******************************************/
var CourseExport = {
/**
* Fetches the previous stored export
*
* @param {string} contentHomeUrl the full URL to the course or library being exported
* @return {JSON} the data of the previous export
*/
storedExport: function(contentHomeUrl) {
var storedData = JSON.parse($.cookie(COOKIE_NAME));
if (storedData) {
successUnixDate = storedData.date;
}
if (contentHomeUrl) {
courselikeHomeUrl = contentHomeUrl;
}
return storedData;
},
/**
* Sets the Export on the "success" status
*
* If it wasn't already, marks the stored export as "completed",
* and updates its date timestamp
*/
success: function() {
current.state = STATE.SUCCESS;
if (this.storedExport().completed !== true) {
storeExport(true);
}
updateFeedbackList();
deferred.resolve();
},
/**
* Entry point for server feedback
*
* Checks for export status updates every `timeout` milliseconds,
* and updates the page accordingly.
*
* @param {int} [stage=0] Starting stage.
*/
pollStatus: function(data) {
var editUnitUrl = null,
msg = data;
if (current.state !== STATE.IN_PROGRESS) {
return;
}
current.stage = data.ExportStatus || STAGE.PREPARING;
if (current.stage === STAGE.SUCCESS) {
current.downloadUrl = data.ExportOutput;
this.success();
} else if (current.stage < STAGE.PREPARING) { // Failed
if (data.ExportError) {
msg = data.ExportError;
}
if (msg.raw_error_msg) {
editUnitUrl = msg.edit_unit_url;
msg = msg.raw_error_msg;
}
error(msg);
this.showError(editUnitUrl, msg);
} else { // In progress
updateFeedbackList();
$.getJSON(statusUrl, function(result) {
timeout.id = setTimeout(function() {
this.pollStatus(result);
}.bind(this), timeout.delay);
}.bind(this));
}
},
/**
* Resets the Export internally and visually
*
*/
reset: function(library) {
current.stage = STAGE.PREPARING;
current.state = STATE.READY;
current.downloadUrl = null;
isLibrary = library;
clearTimeout(timeout.id);
updateFeedbackList();
},
/**
* Show last export status from server and start sending requests
* to the server for status updates
*
* @return {jQuery promise}
*/
resume: function(library) {
deferred = $.Deferred();
isLibrary = library;
statusUrl = this.storedExport().statusUrl;
$.getJSON(statusUrl, function(data) {
current.stage = data.ExportStatus;
current.downloadUrl = data.ExportOutput;
displayFeedbackList();
current.state = STATE.IN_PROGRESS;
this.pollStatus(data);
}.bind(this));
return deferred.promise();
},
/**
* Show a dialog giving further information about the details of an export error.
*
* @param {string} editUnitUrl URL of the unit in which the error occurred, if known
* @param {string} errMsg Detailed error message
*/
showError: function(editUnitUrl, errMsg) {
var action,
dialog,
msg = '';
if (editUnitUrl) {
dialog = new PromptView({
title: gettext('There has been an error while exporting.'),
message: gettext('There has been a failure to export to XML at least one component. ' +
'It is recommended that you go to the edit page and repair the error before attempting ' +
'another export. Please check that all components on the page are valid and do not display ' +
'any error messages.'),
intent: 'error',
actions: {
primary: {
text: gettext('Correct failed component'),
click: function(view) {
view.hide();
document.location = editUnitUrl;
}
},
secondary: {
text: gettext('Return to Export'),
click: function(view) {
view.hide();
}
}
}
});
} else {
if (isLibrary) {
msg += gettext('Your library could not be exported to XML. There is not enough information to ' +
'identify the failed component. Inspect your library to identify any problematic components ' +
'and try again.');
action = gettext('Take me to the main library page');
} else {
msg += gettext('Your course could not be exported to XML. There is not enough information to ' +
'identify the failed component. Inspect your course to identify any problematic components ' +
'and try again.');
action = gettext('Take me to the main course page');
}
msg += ' ' + gettext('The raw error message is:') + ' ' + errMsg;
dialog = new PromptView({
title: gettext('There has been an error with your export.'),
message: msg,
intent: 'error',
actions: {
primary: {
text: action,
click: function(view) {
view.hide();
document.location = courselikeHomeUrl;
}
},
secondary: {
text: gettext('Cancel'),
click: function(view) {
view.hide();
}
}
}
});
}
// The CSS animation for the dialog relies on the 'js' class
// being on the body. This happens after this JavaScript is executed,
// causing a 'bouncing' of the dialog after it is initially shown.
// As a workaround, add this class first.
$('body').addClass('js');
dialog.show();
},
/**
* Starts the exporting process.
* Makes status list visible and starts showing export progress.
*
* @param {string} url The full URL to use to query the server
* about the export status
* @return {jQuery promise}
*/
start: function(url) {
current.state = STATE.IN_PROGRESS;
deferred = $.Deferred();
statusUrl = url;
storeExport();
displayFeedbackList();
updateFeedbackList();
return deferred.promise();
}
};
return CourseExport;
});

17
cms/static/js/views/import.js
View File

@@ -2,8 +2,8 @@
* Course import-related js.
*/
define(
['jquery', 'underscore', 'gettext', 'moment', 'jquery.cookie'],
function($, _, gettext, moment) {
['jquery', 'underscore', 'gettext', 'moment', 'edx-ui-toolkit/js/utils/html-utils', 'jquery.cookie'],
function($, _, gettext, moment, HtmlUtils) {
'use strict';
/** ******** Private properties ****************************************/
@@ -127,10 +127,10 @@ define(
*/
var updateFeedbackList = function(currStageMsg) {
var $checkmark, $curr, $prev, $next;
var date, successUnix, time;
var date, stageMsg, successUnix, time;
$checkmark = $dom.successStage.find('.icon');
currStageMsg = currStageMsg || '';
stageMsg = currStageMsg || '';
function completeStage(stage) {
$(stage)
@@ -140,12 +140,17 @@ define(
function errorStage(stage) {
if (!$(stage).hasClass('has-error')) {
stageMsg = HtmlUtils.joinHtml(
HtmlUtils.HTML('<p class="copy error">'),
stageMsg,
HtmlUtils.HTML('</p>')
);
$(stage)
.removeClass('is-started')
.addClass('has-error')
.find('p.copy')
.hide()
.after("<p class='copy error'>" + currStageMsg + '</p>');
.after(HtmlUtils.ensureHtml(stageMsg).toString());
}
}
@@ -181,7 +186,7 @@ define(
$dom.successStage
.find('.item-progresspoint-success-date')
.html('(' + date + ' at ' + time + ' UTC)');
.text('(' + date + ' at ' + time + ' UTC)');
break;

253
cms/static/sass/views/_export.scss
View File

@@ -86,98 +86,183 @@
}
}
// OLD
.description {
@extend %t-copy-sub1;
float: left;
width: 62%;
margin-right: 3%;
// ====================
h2 {
@extend %t-title5;
@extend %t-strong;
// UI: upload progress
.wrapper-status {
@include transition(opacity $tmg-f2 ease-in-out 0);
opacity: 1.0;
// STATE: hidden
&.is-hidden {
opacity: 0.0;
display: none;
}
> .title {
@extend %t-title4;
margin-bottom: $baseline;
border-bottom: 1px solid $gray-l3;
padding-bottom: ($baseline/2);
}
// elem - progress list
.list-progress {
width: flex-grid(9, 9);
strong {
@extend %t-strong;
}
.status-visual {
position: relative;
float: left;
width: flex-grid(1,9);
p + p {
margin-top: $baseline;
}
ul {
margin: 20px 0;
list-style: disc inside;
li {
margin: 0 0 5px 0;
}
}
}
.export-form-wrapper {
.export-form {
float: left;
width: 35%;
padding: 25px 30px 35px;
@include box-sizing(border-box);
border: 1px solid $mediumGrey;
border-radius: 3px;
background: $lightGrey;
text-align: center;
h2 {
@extend %t-title4;
@extend %t-light;
margin-bottom: ($baseline*1.5);
}
.error-block {
@extend %t-copy-sub1;
display: none;
margin-bottom: ($baseline*0.75);
}
.error-block {
color: $error-red;
}
.button-export {
@include green-button();
@extend %t-action1;
padding: 10px 50px 11px;
}
.message-status {
@extend %t-copy-sub2;
margin-top: ($baseline/2);
}
.progress-bar {
display: none;
width: 350px;
height: 30px;
margin: 30px auto 10px;
border: 1px solid $blue;
&.loaded {
border-color: #66b93d;
.progress-fill {
background: #66b93d;
}
.icon {
@include transition(opacity $tmg-f1 ease-in-out 0);
@extend %t-icon4;
position: absolute;
top: ($baseline/2);
left: $baseline;
}
}
.progress-fill {
width: 0%;
height: 30px;
background: $blue;
color: $white;
line-height: 48px;
.status-detail {
float: left;
width: flex-grid(8,9);
margin-left: ($baseline*3);
.title {
@extend %t-title5;
@extend %t-strong;
}
.copy {
@extend %t-copy-base;
color: $gray-l2;
}
}
.item-progresspoint {
@include clearfix();
@include transition(opacity $tmg-f1 ease-in-out 0);
margin-bottom: $baseline;
border-bottom: 1px solid $gray-l4;
padding-bottom: $baseline;
&:last-child {
margin-bottom: 0;
border-bottom: none;
padding-bottom: 0;
}
// CASE: has actions
&.has-actions {
.list-actions {
display: none;
.action-primary {
@extend %btn-primary-blue;
}
}
}
// TYPE: success
&.item-progresspoint-success {
.item-progresspoint-success-date {
@include margin-left($baseline/4);
display: none;
}
&.is-complete {
.item-progresspoint-success-date {
display: inline;
}
}
}
// STATE: not started
&.is-not-started {
opacity: 0.5;
.fa-warning {
visibility: hidden;
opacity: 0.0;
}
.fa-cog {
visibility: visible;
opacity: 1.0;
}
.fa-check {
opacity: 0.3;
}
}
// STATE: started
&.is-started {
.fa-warning {
visibility: hidden;
opacity: 0.0;
}
.fa-cog {
@include animation(fa-spin 2s infinite linear);
visibility: visible;
opacity: 1.0;
}
}
// STATE: completed
&.is-complete {
.fa-cog {
visibility: visible;
opacity: 1.0;
}
.fa-warning {
visibility: hidden;
opacity: 0.0;
}
.icon {
color: $green;
}
.status-detail .title {
color: $green;
}
.list-actions {
display: block;
}
}
// STATE: error
&.has-error {
.fa-cog {
visibility: hidden;
opacity: 0.0;
}
.fa-warning {
visibility: visible;
opacity: 1.0;
}
.icon {
color: $red;
}
.status-detail .title, .status-detail .copy {
color: $red;
}
}
}
}
}

98
cms/templates/export.html
View File

@@ -27,17 +27,13 @@ else:
<%block name="bodyclass">is-signedin course tools view-export</%block>
<%block name="requirejs">
% if in_err:
var hasUnit = ${bool(unit) | n, dump_js_escaped_json},
editUnitUrl = "${edit_unit_url | n, js_escaped_string}",
courselikeHomeUrl = "${courselike_home_url | n, js_escaped_string}",
is_library = ${library | n, dump_js_escaped_json}
errMsg = "${raw_err_msg | n, js_escaped_string}";
var courselikeHomeUrl = "${courselike_home_url | n, js_escaped_string}",
is_library = ${library | n, dump_js_escaped_json},
statusUrl = "${status_url | n, js_escaped_string}";
require(["js/factories/export"], function(ExportFactory) {
ExportFactory(hasUnit, editUnitUrl, courselikeHomeUrl, is_library, errMsg);
ExportFactory(courselikeHomeUrl, is_library, statusUrl);
});
%endif
</%block>
<%block name="content">
@@ -93,7 +89,7 @@ else:
<ul class="list-actions">
<li class="item-action">
<a class="action action-export action-primary" href="${export_url}">
<a class="action action-export action-primary" href="#">
<span class="icon fa fa-arrow-circle-o-down" aria-hidden="true"></span>
<span class="copy">
%if library:
@@ -105,6 +101,90 @@ else:
</li>
</ul>
</div>
<div class="wrapper wrapper-status is-hidden">
<h3 class="title">
%if library:
${_("Library Export Status")}
%else:
${_("Course Export Status")}
%endif
</h3>
<ol class="status-progress list-progress">
<li class="item-progresspoint item-progresspoint-prepare is-complete">
<span class="deco status-visual">
<span class="icon fa fa-cog" aria-hidden="true"></span>
<span class="icon fa fa-warning" aria-hidden="true"></span>
</span>
<div class="status-detail">
<h3 class="title">${_("Preparing")}</h3>
<div class="progress-bar">
<div class="progress-fill"></div>
</div>
<p class="copy">${_("Preparing to start the export")}</p>
</div>
</li>
<li class="item-progresspoint item-progresspoint-export is-started">
<span class="deco status-visual">
<span class="icon fa fa-cog" aria-hidden="true"></span>
<span class="icon fa fa-warning" aria-hidden="true"></span>
</span>
<div class="status-detail">
<h3 class="title">${_("Exporting")}</h3>
<p class="copy">${_("Creating the export data files (You can now leave this page safely, but avoid making drastic changes to content until this export is complete)")}</p>
</div>
</li>
<li class="item-progresspoint item-progresspoint-compress is-not-started">
<span class="deco status-visual">
<span class="icon fa fa-cog" aria-hidden="true"></span>
<span class="icon fa fa-warning" aria-hidden="true"></span>
</span>
<div class="status-detail">
<h3 class="title">${_("Compressing")}</h3>
<p class="copy">${_("Compressing the exported data and preparing it for download")}</p>
</div>
</li>
<li class="item-progresspoint item-progresspoint-success has-actions is-not-started">
<span class="deco status-visual">
<span class="icon fa fa-square-o" aria-hidden="true"></span>
</span>
<div class="status-detail">
<h3 class="title">
${_("Success")}
<span class="item-progresspoint-success-date"></span>
</h3>
<p class="copy">
%if library:
${_("Your exported library can now be downloaded")}
%else:
${_("Your exported course can now be downloaded")}
%endif
</p>
<ul class="list-actions">
<li class="item-action">
<a href="#" id="download-exported-button" class="action action-primary">
%if library:
${_("Download Exported Library")}
%else:
${_("Download Exported Course")}
%endif
</a>
</li>
</ul>
</div>
</li>
</ol>
</div>
%if not library:
<div class="export-contents">
<div class="export-includes">

6
cms/templates/import.html
View File

@@ -1,3 +1,4 @@
<%page expression_filter="h"/>
<%inherit file="base.html" />
<%def name="online_help_token()">
<%
@@ -13,6 +14,7 @@ else:
from openedx.core.djangolib.js_utils import (
dump_js_escaped_json, js_escaped_string
)
from openedx.core.djangolib.markup import HTML, Text
%>
<%block name="title">
%if library:
@@ -44,11 +46,11 @@ else:
<div class="introduction">
## Translators: ".tar.gz" is a file extension, and files with that extension are called "gzipped tar files": these terms should not be translated
%if library:
<p>${_("Be sure you want to import a library before continuing. The contents of the imported library will replace the contents of the existing library. {em_start}You cannot undo a library import{em_end}. Before you proceed, we recommend that you export the current library, so that you have a backup copy of it.").format(em_start='<strong>', em_end="</strong>")}</p>
<p>${Text(_("Be sure you want to import a library before continuing. The contents of the imported library will replace the contents of the existing library. {em_start}You cannot undo a library import{em_end}. Before you proceed, we recommend that you export the current library, so that you have a backup copy of it.")).format(em_start=HTML('<strong>'), em_end=HTML("</strong>"))}</p>
<p>${_("The library that you import must be in a .tar.gz file (that is, a .tar file compressed with GNU Zip). This .tar.gz file must contain a library.xml file. It may also contain other files.")}</p>
<p>${_("The import process has five stages. During the first two stages, you must stay on this page. You can leave this page after the Unpacking stage has completed. We recommend, however, that you don't make important changes to your library until the import operation has completed.")}</p>
%else:
<p>${_("Be sure you want to import a course before continuing. The contents of the imported course will replace the contents of the existing course. {em_start}You cannot undo a course import{em_end}. Before you proceed, we recommend that you export the current course, so that you have a backup copy of it.").format(em_start='<strong>', em_end="</strong>")}</p>
<p>${Text(_("Be sure you want to import a course before continuing. The contents of the imported course will replace the contents of the existing course. {em_start}You cannot undo a course import{em_end}. Before you proceed, we recommend that you export the current course, so that you have a backup copy of it.")).format(em_start=HTML('<strong>'), em_end=HTML("</strong>"))}</p>
<p>${_("The course that you import must be in a .tar.gz file (that is, a .tar file compressed with GNU Zip). This .tar.gz file must contain a course.xml file. It may also contain other files.")}</p>
<p>${_("The import process has five stages. During the first two stages, you must stay on this page. You can leave this page after the Unpacking stage has completed. We recommend, however, that you don't make important changes to your course until the import operation has completed.")}</p>
%endif

2
cms/urls.py
View File

@@ -96,6 +96,8 @@ urlpatterns += patterns(
url(r'^import/{}$'.format(COURSELIKE_KEY_PATTERN), 'import_handler'),
url(r'^import_status/{}/(?P<filename>.+)$'.format(COURSELIKE_KEY_PATTERN), 'import_status_handler'),
url(r'^export/{}$'.format(COURSELIKE_KEY_PATTERN), 'export_handler'),
url(r'^export_output/{}$'.format(COURSELIKE_KEY_PATTERN), 'export_output_handler'),
url(r'^export_status/{}$'.format(COURSELIKE_KEY_PATTERN), 'export_status_handler'),
url(r'^xblock/outline/{}$'.format(settings.USAGE_KEY_PATTERN), 'xblock_outline_handler'),
url(r'^xblock/container/{}$'.format(settings.USAGE_KEY_PATTERN), 'xblock_container_handler'),
url(r'^xblock/{}/(?P<view_name>[^/]+)$'.format(settings.USAGE_KEY_PATTERN), 'xblock_view_handler'),

223
common/test/acceptance/pages/studio/import_export.py
View File

@@ -29,19 +29,116 @@ class TemplateCheckMixin(object):
return self.q(css='h1.page-header')[0].text.split('\n')[-1]
class ExportMixin(object):
class ImportExportMixin(object):
"""
Mixin for functionality common to both the import and export pages
"""
def is_task_list_showing(self):
"""
The task list shows a series of steps being performed during import or
export. It is normally hidden until the process begins.
Tell us whether it's currently visible.
"""
return self.q(css='.wrapper-status').visible
def is_timestamp_visible(self):
"""
Checks if the UTC timestamp of the last successful import/export is visible
"""
return self.q(css='.item-progresspoint-success-date').visible
@property
def parsed_timestamp(self):
"""
Return python datetime object from the parsed timestamp tuple (date, time)
"""
timestamp = "{0} {1}".format(*self.timestamp)
formatted_timestamp = time.strptime(timestamp, "%m/%d/%Y %H:%M")
return datetime.fromtimestamp(time.mktime(formatted_timestamp))
@property
def timestamp(self):
"""
The timestamp is displayed on the page as "(MM/DD/YYYY at HH:mm)"
It parses the timestamp and returns a (date, time) tuple
"""
string = self.q(css='.item-progresspoint-success-date').text[0]
return re.match(r'\(([^ ]+).+?(\d{2}:\d{2})', string).groups()
def wait_for_tasks(self, completed=False, fail_on=None):
"""
Wait for all of the items in the task list to be set to the correct state.
"""
if fail_on:
# Makes no sense to include this if the tasks haven't run.
completed = True
state, desc_template = self._task_properties(completed)
for desc, css_class in self.task_classes.items():
desc_text = desc_template.format(desc)
# pylint: disable=cell-var-from-loop
EmptyPromise(lambda: self.q(css='.{}.{}'.format(css_class, state)).present, desc_text, timeout=30)
if fail_on == desc:
EmptyPromise(
lambda: self.q(css='.{}.is-complete.has-error'.format(css_class)).present,
"{} checkpoint marked as failed".format(desc),
timeout=30
)
# The rest should never run.
state, desc_template = self._task_properties(False)
def wait_for_timestamp_visible(self):
"""
Wait for the timestamp of the last successful import/export to be visible.
"""
EmptyPromise(self.is_timestamp_visible, 'Timestamp Visible', timeout=30).fulfill()
@staticmethod
def _task_properties(completed):
"""
Outputs the CSS class and promise description for task states based on completion.
"""
if completed:
return 'is-complete', "'{}' is marked complete"
else:
return 'is-not-started', "'{}' is in not-yet-started status"
class ExportMixin(ImportExportMixin):
"""
Export page Mixin.
"""
url_path = "export"
task_classes = {
'Preparing': 'item-progresspoint-prepare',
'Exporting': 'item-progresspoint-export',
'Compressing': 'item-progresspoint-compress',
'Success': 'item-progresspoint-success'
}
def is_browser_on_page(self):
"""
Verify this is the export page
"""
return self.q(css='body.view-export').present
def is_click_handler_registered(self):
"""
Check if the click handler for the export button has been registered yet
"""
script = """
var $ = require('jquery'),
buttonEvents = $._data($('a.action-primary')[0], 'events');
return buttonEvents && buttonEvents.hasOwnProperty('click');"""
stripped_script = ''.join([line.strip() for line in script.split('\n')])
return self.browser.execute_script(stripped_script)
def _get_tarball(self, url):
"""
Download tarball at `url`
@@ -61,14 +158,13 @@ class ExportMixin(object):
"""
Downloads the course or library in tarball form.
"""
tarball_url = self.q(css='a.action-export').attrs('href')[0]
tarball_url = self.q(css='#download-exported-button')[0].get_attribute('href')
good_status, headers = self._get_tarball(tarball_url)
return good_status, headers['content-type'] == 'application/x-tgz'
def click_export(self):
"""
Click the export button. Should only be used if expected to fail, as
otherwise a browser dialog for saving the file will be presented.
Click the export button.
"""
self.q(css='a.action-export').click()
@@ -78,6 +174,13 @@ class ExportMixin(object):
"""
return self.q(css='.prompt.error').visible
def is_export_finished(self):
"""
Checks if the 'Download Exported Course/Library' button is showing.
"""
button = self.q(css='#download-exported-button')[0]
return button.is_displayed() and button.get_attribute('href')
def click_modal_button(self):
"""
Click the button on the modal dialog that appears when there's a problem.
@@ -90,6 +193,18 @@ class ExportMixin(object):
"""
EmptyPromise(self.is_error_modal_showing, 'Error Modal Displayed', timeout=30).fulfill()
def wait_for_export(self):
"""
Wait for the export process to finish.
"""
EmptyPromise(self.is_export_finished, 'Export Finished', timeout=30).fulfill()
def wait_for_export_click_handler(self):
"""
Wait for the export button click handler to be registered
"""
EmptyPromise(self.is_click_handler_registered, 'Export Button Click Handler Registered', timeout=30).fulfill()
class LibraryLoader(object):
"""
@@ -117,31 +232,20 @@ class ExportLibraryPage(ExportMixin, TemplateCheckMixin, LibraryLoader, LibraryP
"""
class ImportMixin(object):
class ImportMixin(ImportExportMixin):
"""
Import page mixin
"""
url_path = "import"
@property
def timestamp(self):
"""
The timestamp is displayed on the page as "(MM/DD/YYYY at HH:mm)"
It parses the timestamp and returns a (date, time) tuple
"""
string = self.q(css='.item-progresspoint-success-date').text[0]
return re.match(r'\(([^ ]+).+?(\d{2}:\d{2})', string).groups()
@property
def parsed_timestamp(self):
"""
Return python datetime object from the parsed timestamp tuple (date, time)
"""
timestamp = "{0} {1}".format(*self.timestamp)
formatted_timestamp = time.strptime(timestamp, "%m/%d/%Y %H:%M")
return datetime.fromtimestamp(time.mktime(formatted_timestamp))
task_classes = {
'Uploading': 'item-progresspoint-upload',
'Unpacking': 'item-progresspoint-unpack',
'Verifying': 'item-progresspoint-verify',
'Updating': 'item-progresspoint-import',
'Success': 'item-progresspoint-success'
}
def is_browser_on_page(self):
"""
@@ -149,6 +253,17 @@ class ImportMixin(object):
"""
return self.q(css='.choose-file-button').present
def is_click_handler_registered(self):
"""
Check if the click handler for the file selector button has been registered yet
"""
script = """
var $ = require('jquery'),
buttonEvents = $._data($('a.choose-file-button')[0], 'events');
return buttonEvents && buttonEvents.hasOwnProperty('click');"""
stripped_script = ''.join([line.strip() for line in script.split('\n')])
return self.browser.execute_script(stripped_script)
@staticmethod
def file_path(filename):
"""
@@ -196,46 +311,6 @@ class ImportMixin(object):
"""
return self.q(css='#view-updated-button').visible
@staticmethod
def _task_properties(completed):
"""
Outputs the CSS class and promise description for task states based on completion.
"""
if completed:
return 'is-complete', "'{}' is marked complete"
else:
return 'is-not-started', "'{}' is in not-yet-started status"
def wait_for_tasks(self, completed=False, fail_on=None):
"""
Wait for all of the items in the task list to be set to the correct state.
"""
classes = {
'Uploading': 'item-progresspoint-upload',
'Unpacking': 'item-progresspoint-unpack',
'Verifying': 'item-progresspoint-verify',
'Updating': 'item-progresspoint-import',
'Success': 'item-progresspoint-success'
}
if fail_on:
# Makes no sense to include this if the tasks haven't run.
completed = True
state, desc_template = self._task_properties(completed)
for desc, css_class in classes.items():
desc_text = desc_template.format(desc)
# pylint: disable=cell-var-from-loop
EmptyPromise(lambda: self.q(css='.{}.{}'.format(css_class, state)).present, desc_text, timeout=30)
if fail_on == desc:
EmptyPromise(
lambda: self.q(css='.{}.is-complete.has-error'.format(css_class)).present,
"{} checkpoint marked as failed".format(desc),
timeout=30
)
# The rest should never run.
state, desc_template = self._task_properties(False)
def wait_for_upload(self):
"""
Wait for the upload to be confirmed.
@@ -250,26 +325,12 @@ class ImportMixin(object):
"""
return self.q(css='#fileupload .error-block').visible
def is_task_list_showing(self):
def wait_for_choose_file_click_handler(self):
"""
The task list shows a series of steps being performed during import. It is normally
hidden until the upload begins.
Tell us whether it's currently visible.
Wait for the choose file button click handler to be registered
"""
return self.q(css='.wrapper-status').visible
def is_timestamp_visible(self):
"""
Checks if the UTC timestamp of the last successful import is visible
"""
return self.q(css='.item-progresspoint-success-date').visible
def wait_for_timestamp_visible(self):
"""
Wait for the timestamp of the last successful import to be visible.
"""
EmptyPromise(self.is_timestamp_visible, 'Timestamp Visible', timeout=30).fulfill()
EmptyPromise(self.is_click_handler_registered, 'Choose File Button Click Handler Registered',
timeout=30).fulfill()
def wait_for_filename_error(self):
"""

76
common/test/acceptance/tests/studio/test_import_export.py
View File

@@ -4,8 +4,6 @@ Acceptance tests for the Import and Export pages
from nose.plugins.attrib import attr
from datetime import datetime
from flaky import flaky
from abc import abstractmethod
from common.test.acceptance.tests.studio.base_studio_test import StudioLibraryTest, StudioCourseTest
@@ -32,10 +30,70 @@ class ExportTestMixin(object):
The download will succeed
And the file will be of the right MIME type.
"""
self.export_page.wait_for_export_click_handler()
self.export_page.click_export()
self.export_page.wait_for_export()
good_status, is_tarball_mimetype = self.export_page.download_tarball()
self.assertTrue(good_status)
self.assertTrue(is_tarball_mimetype)
def test_export_timestamp(self):
"""
Scenario: I perform a course / library export
On export success, the page displays a UTC timestamp previously not visible
And if I refresh the page, the timestamp is still displayed
"""
self.assertFalse(self.export_page.is_timestamp_visible())
# Get the time when the export has started.
# export_page timestamp is in (MM/DD/YYYY at HH:mm) so replacing (second, microsecond) to
# keep the comparison consistent
export_start_time = datetime.utcnow().replace(microsecond=0, second=0)
self.export_page.wait_for_export_click_handler()
self.export_page.click_export()
self.export_page.wait_for_export()
# Get the time when the export has finished.
# export_page timestamp is in (MM/DD/YYYY at HH:mm) so replacing (second, microsecond) to
# keep the comparison consistent
export_finish_time = datetime.utcnow().replace(microsecond=0, second=0)
export_timestamp = self.export_page.parsed_timestamp
self.export_page.wait_for_timestamp_visible()
# Verify that 'export_timestamp' is between start and finish upload time
self.assertLessEqual(
export_start_time,
export_timestamp,
"Course export timestamp should be export_start_time <= export_timestamp <= export_end_time"
)
self.assertGreaterEqual(
export_finish_time,
export_timestamp,
"Course export timestamp should be export_start_time <= export_timestamp <= export_end_time"
)
self.export_page.visit()
self.export_page.wait_for_tasks(completed=True)
self.export_page.wait_for_timestamp_visible()
def test_task_list(self):
"""
Scenario: I should see feedback checkpoints when exporting a course or library
Given that I am on an export page
No task checkpoint list should be showing
When I export the course or library
Each task in the checklist should be marked confirmed
And the task list should be visible
"""
# The task list shouldn't be visible to start.
self.assertFalse(self.export_page.is_task_list_showing(), "Task list shown too early.")
self.export_page.wait_for_tasks()
self.export_page.wait_for_export_click_handler()
self.export_page.click_export()
self.export_page.wait_for_tasks(completed=True)
self.assertTrue(self.export_page.is_task_list_showing(), "Task list did not display.")
@attr(shard=7)
class TestCourseExport(ExportTestMixin, StudioCourseTest):
@@ -101,7 +159,6 @@ class ImportTestMixin(object):
"""
return []
@flaky # TODO fix this, see PLAT-1186
def test_upload(self):
"""
Scenario: I want to upload a course or library for import.
@@ -110,6 +167,7 @@ class ImportTestMixin(object):
I can select the file and upload it
And the page will give me confirmation that it uploaded successfully
"""
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.import_page.wait_for_upload()
@@ -125,6 +183,7 @@ class ImportTestMixin(object):
# import_page timestamp is in (MM/DD/YYYY at HH:mm) so replacing (second, microsecond) to
# keep the comparison consistent
upload_start_time = datetime.utcnow().replace(microsecond=0, second=0)
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.import_page.wait_for_upload()
@@ -158,6 +217,7 @@ class ImportTestMixin(object):
Given that I upload a library or course
A button will appear that contains the URL to the library or course's main page
"""
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.assertEqual(self.import_page.finished_target_url(), self.landing_page.url)
@@ -167,6 +227,7 @@ class ImportTestMixin(object):
Given that I select a file that is an .mp4 for upload
An error message will appear
"""
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball('funny_cat_video.mp4')
self.import_page.wait_for_filename_error()
@@ -182,6 +243,7 @@ class ImportTestMixin(object):
# The task list shouldn't be visible to start.
self.assertFalse(self.import_page.is_task_list_showing(), "Task list shown too early.")
self.import_page.wait_for_tasks()
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.import_page.wait_for_tasks(completed=True)
self.assertTrue(self.import_page.is_task_list_showing(), "Task list did not display.")
@@ -195,6 +257,7 @@ class ImportTestMixin(object):
And the 'Updating' task should be marked failed
And the remaining tasks should not be marked as started
"""
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.bad_tarball_name)
self.import_page.wait_for_tasks(fail_on='Updating')
@@ -212,7 +275,6 @@ class TestEntranceExamCourseImport(ImportTestMixin, StudioCourseTest):
def page_args(self):
return [self.browser, self.course_info['org'], self.course_info['number'], self.course_info['run']]
@flaky # TODO fix this, see PLAT-1186
def test_course_updated_with_entrance_exam(self):
"""
Given that I visit an empty course before import
@@ -229,6 +291,7 @@ class TestEntranceExamCourseImport(ImportTestMixin, StudioCourseTest):
self.assertRaises(IndexError, self.landing_page.section, "Section")
self.assertRaises(IndexError, self.landing_page.section, "Entrance Exam")
self.import_page.visit()
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.import_page.wait_for_upload()
self.landing_page.visit()
@@ -259,7 +322,6 @@ class TestCourseImport(ImportTestMixin, StudioCourseTest):
def page_args(self):
return [self.browser, self.course_info['org'], self.course_info['number'], self.course_info['run']]
@flaky # TODO fix this, see PLAT-1186
def test_course_updated(self):
"""
Given that I visit an empty course before import
@@ -273,6 +335,7 @@ class TestCourseImport(ImportTestMixin, StudioCourseTest):
# Should not exist yet.
self.assertRaises(IndexError, self.landing_page.section, "Section")
self.import_page.visit()
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.import_page.wait_for_upload()
self.landing_page.visit()
@@ -299,6 +362,7 @@ class TestCourseImport(ImportTestMixin, StudioCourseTest):
Then timestamp is not visible
"""
self.import_page.visit()
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.import_page.wait_for_upload()
self.assertTrue(self.import_page.is_timestamp_visible())
@@ -330,7 +394,6 @@ class TestLibraryImport(ImportTestMixin, StudioLibraryTest):
def page_args(self):
return [self.browser, self.library_key]
@flaky # TODO fix this, see PLAT-1186
def test_library_updated(self):
"""
Given that I visit an empty library
@@ -345,6 +408,7 @@ class TestLibraryImport(ImportTestMixin, StudioLibraryTest):
# No items should be in the library to start.
self.assertEqual(len(self.landing_page.xblocks), 0)
self.import_page.visit()
self.import_page.wait_for_choose_file_click_handler()
self.import_page.upload_tarball(self.tarball_name)
self.import_page.wait_for_upload()
self.landing_page.visit()