diff --git a/cms/djangoapps/contentstore/storage.py b/cms/djangoapps/contentstore/storage.py index 2779b25809..72dc99d062 100644 --- a/cms/djangoapps/contentstore/storage.py +++ b/cms/djangoapps/contentstore/storage.py @@ -17,7 +17,7 @@ class ImportExportS3Storage(S3BotoStorage): # pylint: disable=abstract-method def __init__(self): bucket = setting('COURSE_IMPORT_EXPORT_BUCKET', settings.AWS_STORAGE_BUCKET_NAME) - super(ImportExportS3Storage, self).__init__(bucket=bucket, querystring_auth=True) + super(ImportExportS3Storage, self).__init__(bucket=bucket, custom_domain=None, querystring_auth=True) # pylint: disable=invalid-name course_import_export_storage = get_storage_class(settings.COURSE_IMPORT_EXPORT_STORAGE)() diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index 2604cea5c4..bbc260d968 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -5,11 +5,11 @@ from __future__ import absolute_import import base64 import json -import logging import os import shutil import tarfile from datetime import datetime +from tempfile import NamedTemporaryFile, mkdtemp from celery.task import task from celery.utils.log import get_task_logger @@ -20,17 +20,19 @@ from six import iteritems, text_type from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import SuspiciousOperation +from django.core.files import File from django.test import RequestFactory from django.utils.text import get_valid_filename from django.utils.translation import ugettext as _ from djcelery.common import respect_language +from user_tasks.models import UserTaskArtifact, UserTaskStatus from user_tasks.tasks import UserTask import dogstats_wrapper as dog_stats_api from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer, SearchIndexingError from contentstore.storage import course_import_export_storage -from contentstore.utils import initialize_permissions +from contentstore.utils import initialize_permissions, reverse_usage_url from course_action_state.models import CourseRerunState from models.settings.course_metadata import CourseMetadata from opaque_keys.edx.keys import CourseKey @@ -39,9 +41,11 @@ from openedx.core.lib.extract_tar import safetar_extractall from student.auth import has_course_author_access from xmodule.contentstore.django import contentstore from xmodule.course_module import CourseFields +from xmodule.exceptions import SerializationError from xmodule.modulestore import COURSE_ROOT, LIBRARY_ROOT from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError +from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml from xmodule.modulestore.xml_importer import import_course_from_xml, import_library_from_xml @@ -155,6 +159,136 @@ def push_course_update_task(course_key_string, course_subscription_id, course_di send_push_course_update(course_key_string, course_subscription_id, course_display_name) +class CourseExportTask(UserTask): # pylint: disable=abstract-method + """ + Base class for course and library export tasks. + """ + + @staticmethod + def calculate_total_steps(arguments_dict): + """ + Get the number of in-progress steps in the export process, as shown in the UI. + + For reference, these are: + + 1. Exporting + 2. Compressing + """ + return 2 + + @classmethod + def generate_name(cls, arguments_dict): + """ + Create a name for this particular import task instance. + + Arguments: + arguments_dict (dict): The arguments given to the task function + + Returns: + text_type: The generated name + """ + key = arguments_dict[u'course_key_string'] + return u'Export of {}'.format(key) + + +@task(base=CourseExportTask, bind=True) +def export_olx(self, user_id, course_key_string, language): + """ + Export a course or library to an OLX .tar.gz archive and prepare it for download. + """ + courselike_key = CourseKey.from_string(course_key_string) + + try: + user = User.objects.get(pk=user_id) + except User.DoesNotExist: + with respect_language(language): + self.status.fail(_(u'Unknown User ID: {0}').format(user_id)) + return + if not has_course_author_access(user, courselike_key): + with respect_language(language): + self.status.fail(_(u'Permission denied')) + return + + if isinstance(courselike_key, LibraryLocator): + courselike_module = modulestore().get_library(courselike_key) + else: + courselike_module = modulestore().get_course(courselike_key) + + try: + self.status.set_state(u'Exporting') + tarball = create_export_tarball(courselike_module, courselike_key, {}, self.status) + artifact = UserTaskArtifact(status=self.status, name=u'Output') + artifact.file.save(name=tarball.name, content=File(tarball)) # pylint: disable=no-member + artifact.save() + # catch all exceptions so we can record useful error messages + except Exception as exception: # pylint: disable=broad-except + LOGGER.exception(u'Error exporting course %s', courselike_key) + if self.status.state != UserTaskStatus.FAILED: + self.status.fail({'raw_error_msg': text_type(exception)}) + return + + +def create_export_tarball(course_module, course_key, context, status=None): + """ + Generates the export tarball, or returns None if there was an error. + + Updates the context with any error information if applicable. + """ + name = course_module.url_name + export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") + root_dir = path(mkdtemp()) + + try: + if isinstance(course_key, LibraryLocator): + export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) + else: + export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) + + if status: + status.set_state(u'Compressing') + status.increment_completed_steps() + LOGGER.debug(u'tar file being generated at %s', export_file.name) + with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: + tar_file.add(root_dir / name, arcname=name) + + except SerializationError as exc: + LOGGER.exception(u'There was an error exporting %s', course_key) + parent = None + try: + failed_item = modulestore().get_item(exc.location) + parent_loc = modulestore().get_parent_location(failed_item.location) + + if parent_loc is not None: + parent = modulestore().get_item(parent_loc) + except: # pylint: disable=bare-except + # if we have a nested exception, then we'll show the more generic error message + pass + + context.update({ + 'in_err': True, + 'raw_err_msg': str(exc), + 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", + }) + if status: + status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'], + 'edit_unit_url': context['edit_unit_url']})) + raise + except Exception as exc: + LOGGER.exception('There was an error exporting %s', course_key) + context.update({ + 'in_err': True, + 'edit_unit_url': None, + 'raw_err_msg': str(exc)}) + if status: + status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']})) + raise + finally: + if os.path.exists(root_dir / name): + shutil.rmtree(root_dir / name) + + return export_file + + class CourseImportTask(UserTask): # pylint: disable=abstract-method """ Base class for course and library import tasks. diff --git a/cms/djangoapps/contentstore/tests/test_tasks.py b/cms/djangoapps/contentstore/tests/test_tasks.py new file mode 100644 index 0000000000..86eab6efb5 --- /dev/null +++ b/cms/djangoapps/contentstore/tests/test_tasks.py @@ -0,0 +1,108 @@ +""" +Unit tests for course import and export Celery tasks +""" +from __future__ import absolute_import, division, print_function + +import copy +import json +import mock +from uuid import uuid4 + +from django.conf import settings +from django.contrib.auth.models import User +from django.test.utils import override_settings + +from user_tasks.models import UserTaskArtifact, UserTaskStatus + +from contentstore.tasks import export_olx +from contentstore.tests.test_libraries import LibraryTestCase +from contentstore.tests.utils import CourseTestCase + +TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) +TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex + + +def side_effect_exception(*args, **kwargs): # pylint: disable=unused-argument + """ + Side effect for mocking which raises an exception + """ + raise Exception('Boom!') + + +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +class ExportCourseTestCase(CourseTestCase): + """ + Tests of the export_olx task applied to courses + """ + + def test_success(self): + """ + Verify that a routine course export task succeeds + """ + key = str(self.course.location.course_key) + result = export_olx.delay(self.user.id, key, u'en') + status = UserTaskStatus.objects.get(task_id=result.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + artifacts = UserTaskArtifact.objects.filter(status=status) + self.assertEqual(len(artifacts), 1) + output = artifacts[0] + self.assertEqual(output.name, 'Output') + + @mock.patch('contentstore.tasks.export_course_to_xml', side_effect=side_effect_exception) + def test_exception(self, mock_export): # pylint: disable=unused-argument + """ + The export task should fail gracefully if an exception is thrown + """ + key = str(self.course.location.course_key) + result = export_olx.delay(self.user.id, key, u'en') + self._assert_failed(result, json.dumps({u'raw_error_msg': u'Boom!'})) + + def test_invalid_user_id(self): + """ + Verify that attempts to export a course as an invalid user fail + """ + user_id = User.objects.order_by(u'-id').first().pk + 100 + key = str(self.course.location.course_key) + result = export_olx.delay(user_id, key, u'en') + self._assert_failed(result, u'Unknown User ID: {}'.format(user_id)) + + def test_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to export it + """ + _, nonstaff_user = self.create_non_staff_authed_user_client() + key = str(self.course.location.course_key) + result = export_olx.delay(nonstaff_user.id, key, u'en') + self._assert_failed(result, u'Permission denied') + + def _assert_failed(self, task_result, error_message): + """ + Verify that a task failed with the specified error message + """ + status = UserTaskStatus.objects.get(task_id=task_result.id) + self.assertEqual(status.state, UserTaskStatus.FAILED) + artifacts = UserTaskArtifact.objects.filter(status=status) + self.assertEqual(len(artifacts), 1) + error = artifacts[0] + self.assertEqual(error.name, u'Error') + self.assertEqual(error.text, error_message) + + +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +class ExportLibraryTestCase(LibraryTestCase): + """ + Tests of the export_olx task applied to libraries + """ + + def test_success(self): + """ + Verify that a routine library export task succeeds + """ + key = str(self.lib_key) + result = export_olx.delay(self.user.id, key, u'en') # pylint: disable=no-member + status = UserTaskStatus.objects.get(task_id=result.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + artifacts = UserTaskArtifact.objects.filter(status=status) + self.assertEqual(len(artifacts), 1) + output = artifacts[0] + self.assertEqual(output.name, 'Output') diff --git a/cms/djangoapps/contentstore/views/import_export.py b/cms/djangoapps/contentstore/views/import_export.py index a424bbc734..9f7e212c3e 100644 --- a/cms/djangoapps/contentstore/views/import_export.py +++ b/cms/djangoapps/contentstore/views/import_export.py @@ -3,13 +3,12 @@ These views handle all actions in Studio related to import and exporting of courses """ import base64 +import json import logging import os import re import shutil -import tarfile from path import Path as path -from tempfile import mkdtemp from six import text_type @@ -17,7 +16,6 @@ from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied from django.core.files import File -from django.core.files.temp import NamedTemporaryFile from django.core.servers.basehttp import FileWrapper from django.db import transaction from django.http import HttpResponse, HttpResponseNotFound, Http404 @@ -26,28 +24,26 @@ from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.http import require_http_methods, require_GET from edxmako.shortcuts import render_to_response -from xmodule.contentstore.django import contentstore from xmodule.exceptions import SerializationError from xmodule.modulestore.django import modulestore from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import LibraryLocator from user_tasks.conf import settings as user_tasks_settings -from user_tasks.models import UserTaskStatus -from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml +from user_tasks.models import UserTaskArtifact, UserTaskStatus from student.auth import has_course_author_access from util.json_request import JsonResponse from util.views import ensure_valid_course_key from contentstore.storage import course_import_export_storage -from contentstore.tasks import CourseImportTask, import_olx +from contentstore.tasks import CourseExportTask, CourseImportTask, create_export_tarball, export_olx, import_olx -from contentstore.utils import reverse_course_url, reverse_usage_url, reverse_library_url +from contentstore.utils import reverse_course_url, reverse_library_url __all__ = [ 'import_handler', 'import_status_handler', - 'export_handler', + 'export_handler', 'export_output_handler', 'export_status_handler', ] @@ -279,64 +275,6 @@ def import_status_handler(request, course_key_string, filename=None): return JsonResponse({"ImportStatus": status}) -def create_export_tarball(course_module, course_key, context): - """ - Generates the export tarball, or returns None if there was an error. - - Updates the context with any error information if applicable. - """ - name = course_module.url_name - export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") - root_dir = path(mkdtemp()) - - try: - if isinstance(course_key, LibraryLocator): - export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) - else: - export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) - - logging.debug(u'tar file being generated at %s', export_file.name) - with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: - tar_file.add(root_dir / name, arcname=name) - - except SerializationError as exc: - log.exception(u'There was an error exporting %s', course_key) - unit = None - failed_item = None - parent = None - try: - failed_item = modulestore().get_item(exc.location) - parent_loc = modulestore().get_parent_location(failed_item.location) - - if parent_loc is not None: - parent = modulestore().get_item(parent_loc) - if parent.location.category == 'vertical': - unit = parent - except: # pylint: disable=bare-except - # if we have a nested exception, then we'll show the more generic error message - pass - - context.update({ - 'in_err': True, - 'raw_err_msg': str(exc), - 'failed_module': failed_item, - 'unit': unit, - 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", - }) - raise - except Exception as exc: - log.exception('There was an error exporting %s', course_key) - context.update({ - 'in_err': True, - 'unit': None, - 'raw_err_msg': str(exc)}) - raise - finally: - shutil.rmtree(root_dir / name) - - return export_file - - def send_tarball(tarball): """ Renders a tarball to response, for use when sending a tar.gz file to the user. @@ -351,7 +289,7 @@ def send_tarball(tarball): @transaction.non_atomic_requests @ensure_csrf_cookie @login_required -@require_http_methods(("GET",)) +@require_http_methods(('GET', 'POST')) @ensure_valid_course_key def export_handler(request, course_key_string): """ @@ -361,15 +299,21 @@ def export_handler(request, course_key_string): html: return html page for import page application/x-tgz: return tar.gz file containing exported course json: not supported + POST + Start a Celery task to export the course - Note that there are 2 ways to request the tar.gz file. The request header can specify - application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz). + Note that there are 3 ways to request the tar.gz file. The Studio UI uses + a POST request to start the export asynchronously, with a link appearing + on the page once it's ready. Additionally, for backwards compatibility + reasons the request header can specify application/x-tgz via HTTP_ACCEPT, + or a query parameter can be used (?_accept=application/x-tgz); this will + export the course synchronously and return the resulting file (unless the + request times out for a large course). - If the tar.gz file has been requested but the export operation fails, an HTML page will be returned - which describes the error. + If the tar.gz file has been requested but the export operation fails, the + import page will be returned including a description of the error. """ course_key = CourseKey.from_string(course_key_string) - export_url = reverse_course_url('export_handler', course_key) if not has_course_author_access(request.user, course_key): raise PermissionDenied() @@ -389,22 +333,134 @@ def export_handler(request, course_key_string): 'courselike_home_url': reverse_course_url("course_handler", course_key), 'library': False } - - context['export_url'] = export_url + '?_accept=application/x-tgz' + context['status_url'] = reverse_course_url('export_status_handler', course_key) # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header. requested_format = request.GET.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html')) - if 'application/x-tgz' in requested_format: + if request.method == 'POST': + export_olx.delay(request.user.id, course_key_string, request.LANGUAGE_CODE) + return JsonResponse({'ExportStatus': 1}) + elif 'application/x-tgz' in requested_format: try: tarball = create_export_tarball(courselike_module, course_key, context) + return send_tarball(tarball) except SerializationError: return render_to_response('export.html', context) - return send_tarball(tarball) - elif 'text/html' in requested_format: return render_to_response('export.html', context) - else: # Only HTML or x-tgz request formats are supported (no JSON). return HttpResponse(status=406) + + +@transaction.non_atomic_requests +@require_GET +@ensure_csrf_cookie +@login_required +@ensure_valid_course_key +def export_status_handler(request, course_key_string): + """ + Returns an integer corresponding to the status of a file export. These are: + + -X : Export unsuccessful due to some error with X as stage [0-3] + 0 : No status info found (export done or task not yet created) + 1 : Exporting + 2 : Compressing + 3 : Export successful + + If the export was successful, a URL for the generated .tar.gz file is also + returned. + """ + course_key = CourseKey.from_string(course_key_string) + if not has_course_author_access(request.user, course_key): + raise PermissionDenied() + + # The task status record is authoritative once it's been created + task_status = _latest_task_status(request, course_key_string, export_status_handler) + output_url = None + error = None + if task_status is None: + # The task hasn't been initialized yet; did we store info in the session already? + try: + session_status = request.session["export_status"] + status = session_status[course_key_string] + except KeyError: + status = 0 + elif task_status.state == UserTaskStatus.SUCCEEDED: + status = 3 + artifact = UserTaskArtifact.objects.get(status=task_status, name='Output') + if hasattr(artifact.file.storage, 'bucket'): + filename = os.path.basename(artifact.file.name).encode('utf-8') + disposition = 'attachment; filename="{}"'.format(filename) + output_url = artifact.file.storage.url(artifact.file.name, response_headers={ + 'response-content-disposition': disposition, + 'response-content-encoding': 'application/octet-stream', + 'response-content-type': 'application/x-tgz' + }) + else: + # local file, serve from the authorization wrapper view + output_url = reverse_course_url('export_output_handler', course_key) + elif task_status.state in (UserTaskStatus.FAILED, UserTaskStatus.CANCELED): + status = max(-(task_status.completed_steps + 1), -2) + errors = UserTaskArtifact.objects.filter(status=task_status, name='Error') + if len(errors): + error = errors[0].text + try: + error = json.loads(error) + except ValueError: + # Wasn't JSON, just use the value as a string + pass + else: + status = min(task_status.completed_steps + 1, 2) + + response = {"ExportStatus": status} + if output_url: + response['ExportOutput'] = output_url + elif error: + response['ExportError'] = error + return JsonResponse(response) + + +@transaction.non_atomic_requests +@require_GET +@ensure_csrf_cookie +@login_required +@ensure_valid_course_key +def export_output_handler(request, course_key_string): + """ + Returns the OLX .tar.gz produced by a file export. Only used in + environments such as devstack where the output is stored in a local + filesystem instead of an external service like S3. + """ + course_key = CourseKey.from_string(course_key_string) + if not has_course_author_access(request.user, course_key): + raise PermissionDenied() + + task_status = _latest_task_status(request, course_key_string, export_output_handler) + if task_status and task_status.state == UserTaskStatus.SUCCEEDED: + artifact = None + try: + artifact = UserTaskArtifact.objects.get(status=task_status, name='Output') + tarball = course_import_export_storage.open(artifact.file.name) + return send_tarball(tarball) + except UserTaskArtifact.DoesNotExist: + raise Http404 + finally: + if artifact: + artifact.file.close() + else: + raise Http404 + + +def _latest_task_status(request, course_key_string, view_func=None): + """ + Get the most recent export status update for the specified course/library + key. + """ + args = {u'course_key_string': course_key_string} + name = CourseExportTask.generate_name(args) + task_status = UserTaskStatus.objects.filter(name=name) + for status_filter in STATUS_FILTERS: + task_status = status_filter().filter_queryset(request, task_status, view_func) + return task_status.order_by(u'-created').first() diff --git a/cms/djangoapps/contentstore/views/tests/test_import_export.py b/cms/djangoapps/contentstore/views/tests/test_import_export.py index aa65668810..d320bac5b0 100644 --- a/cms/djangoapps/contentstore/views/tests/test_import_export.py +++ b/cms/djangoapps/contentstore/views/tests/test_import_export.py @@ -531,6 +531,7 @@ class ExportTestCase(CourseTestCase): """ super(ExportTestCase, self).setUp() self.url = reverse_course_url('export_handler', self.course.id) + self.status_url = reverse_course_url('export_status_handler', self.course.id) def test_export_html(self): """ @@ -547,6 +548,21 @@ class ExportTestCase(CourseTestCase): resp = self.client.get(self.url, HTTP_ACCEPT='application/json') self.assertEquals(resp.status_code, 406) + def test_export_async(self): + """ + Get tar.gz file, using asynchronous background task + """ + resp = self.client.post(self.url) + self.assertEquals(resp.status_code, 200) + resp = self.client.get(self.status_url) + result = json.loads(resp.content) + status = result['ExportStatus'] + self.assertEquals(status, 3) + self.assertIn('ExportOutput', result) + output_url = result['ExportOutput'] + resp = self.client.get(output_url) + self._verify_export_succeeded(resp) + def test_export_targz(self): """ Get tar.gz file, using HTTP_ACCEPT. @@ -588,11 +604,16 @@ class ExportTestCase(CourseTestCase): def _verify_export_failure(self, expected_text): """ Export failure helper method. """ - resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz') + resp = self.client.post(self.url) self.assertEquals(resp.status_code, 200) - self.assertIsNone(resp.get('Content-Disposition')) - self.assertContains(resp, 'Unable to create xml for module') - self.assertContains(resp, expected_text) + resp = self.client.get(self.status_url) + self.assertEquals(resp.status_code, 200) + result = json.loads(resp.content) + self.assertNotIn('ExportOutput', result) + self.assertIn('ExportError', result) + error = result['ExportError'] + self.assertIn('Unable to create xml for module', error['raw_error_msg']) + self.assertIn(expected_text, error['edit_unit_url']) def test_library_export(self): """ @@ -639,19 +660,53 @@ class ExportTestCase(CourseTestCase): data=xml_string ) - self.test_export_targz_urlparam() + self.test_export_async() @ddt.data( '/export/non.1/existence_1/Run_1', # For mongo '/export/course-v1:non1+existence1+Run1', # For split ) - def test_export_course_doest_not_exist(self, url): + def test_export_course_does_not_exist(self, url): """ - Export failure if course is not exist + Export failure if course does not exist """ resp = self.client.get_html(url) self.assertEquals(resp.status_code, 404) + def test_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to export it + """ + client, _ = self.create_non_staff_authed_user_client() + resp = client.get(self.url) + self.assertEqual(resp.status_code, 403) + + def test_status_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to see the status of export tasks + """ + client, _ = self.create_non_staff_authed_user_client() + resp = client.get(self.status_url) + self.assertEqual(resp.status_code, 403) + + def test_status_missing_record(self): + """ + Attempting to get the status of an export task which isn't currently + represented in the database should yield a useful result + """ + resp = self.client.get(self.status_url) + self.assertEqual(resp.status_code, 200) + result = json.loads(resp.content) + self.assertEqual(result['ExportStatus'], 0) + + def test_output_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to see the output of export tasks + """ + client, _ = self.create_non_staff_authed_user_client() + resp = client.get(reverse_course_url('export_output_handler', self.course.id)) + self.assertEqual(resp.status_code, 403) + @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) class TestLibraryImportExport(CourseTestCase): diff --git a/cms/static/js/factories/export.js b/cms/static/js/factories/export.js index 355c84d928..2da33eab8b 100644 --- a/cms/static/js/factories/export.js +++ b/cms/static/js/factories/export.js @@ -1,66 +1,57 @@ -define(['gettext', 'common/js/components/views/feedback_prompt'], function(gettext, PromptView) { +define([ + 'domReady', 'js/views/export', 'jquery', 'gettext' +], function(domReady, Export, $, gettext) { 'use strict'; - return function(hasUnit, editUnitUrl, courselikeHomeUrl, library, errMsg) { - var dialog; - if (hasUnit) { - dialog = new PromptView({ - title: gettext('There has been an error while exporting.'), - message: gettext('There has been a failure to export to XML at least one component. It is recommended that you go to the edit page and repair the error before attempting another export. Please check that all components on the page are valid and do not display any error messages.'), - intent: 'error', - actions: { - primary: { - text: gettext('Correct failed component'), - click: function(view) { - view.hide(); - document.location = editUnitUrl; - } - }, - secondary: { - text: gettext('Return to Export'), - click: function(view) { - view.hide(); + return function(courselikeHomeUrl, library, statusUrl) { + var $submitBtn = $('.action-export'), + unloading = false, + previousExport = Export.storedExport(courselikeHomeUrl); + + var onComplete = function() { + $submitBtn.show(); + }; + + var startExport = function(e) { + e.preventDefault(); + $submitBtn.hide(); + Export.reset(library); + Export.start(statusUrl).then(onComplete); + $.ajax({ + type: 'POST', + url: window.location.pathname, + data: {}, + success: function(result, textStatus, xhr) { + if (xhr.status === 200) { + setTimeout(function() { Export.pollStatus(result); }, 1000); + } else { + // It could be that the user is simply refreshing the page + // so we need to be sure this is an actual error from the server + if (!unloading) { + $(window).off('beforeunload.import'); + + Export.reset(library); + onComplete(); + + Export.showError(gettext('Your export has failed.')); } } } }); - } else { - var msg = '

'; - var action; - if (library) { - msg += gettext('Your library could not be exported to XML. There is not enough information to identify the failed component. Inspect your library to identify any problematic components and try again.'); - action = gettext('Take me to the main library page'); - } else { - msg += gettext('Your course could not be exported to XML. There is not enough information to identify the failed component. Inspect your course to identify any problematic components and try again.'); - action = gettext('Take me to the main course page'); + }; + + $(window).on('beforeunload', function() { unloading = true; }); + + // Display the status of last file upload on page load + if (previousExport) { + if (previousExport.completed !== true) { + $submitBtn.hide(); } - msg += '

' + gettext('The raw error message is:') + '

' + errMsg; - dialog = new PromptView({ - title: gettext('There has been an error with your export.'), - message: msg, - intent: 'error', - actions: { - primary: { - text: action, - click: function(view) { - view.hide(); - document.location = courselikeHomeUrl; - } - }, - secondary: { - text: gettext('Cancel'), - click: function(view) { - view.hide(); - } - } - } - }); + Export.resume(library).then(onComplete); } - // The CSS animation for the dialog relies on the 'js' class - // being on the body. This happens after this JavaScript is executed, - // causing a 'bouncing' of the dialog after it is initially shown. - // As a workaround, add this class first. - $('body').addClass('js'); - dialog.show(); + domReady(function() { + // export form setup + $submitBtn.bind('click', startExport); + }); }; }); diff --git a/cms/static/js/factories/import.js b/cms/static/js/factories/import.js index fba6b83410..3c5f0483d6 100644 --- a/cms/static/js/factories/import.js +++ b/cms/static/js/factories/import.js @@ -29,7 +29,7 @@ define([ var onComplete = function() { bar.hide(); chooseBtn - .find('.copy').html(gettext('Choose new file')).end() + .find('.copy').text(gettext('Choose new file')).end() .show(); }; @@ -38,7 +38,9 @@ define([ // Display the status of last file upload on page load if (previousImport) { $('.file-name-block') - .find('.file-name').html(previousImport.file.name).end() + .find('.file-name') + .text(previousImport.file.name) + .end() .show(); if (previousImport.completed !== true) { @@ -123,7 +125,7 @@ define([ setTimeout(function() { Import.pollStatus(); }, 3000); } else { bar.show(); - fill.width(percentVal).html(percentVal); + fill.width(percentVal).text(percentVal); } }, sequentialUploads: true, @@ -136,7 +138,7 @@ define([ if (filepath.substr(filepath.length - 6, 6) === 'tar.gz') { $('.error-block').hide(); - $('.file-name').html($(this).val().replace('C:\\fakepath\\', '')); + $('.file-name').text($(this).val().replace('C:\\fakepath\\', '')); $('.file-name-block').show(); chooseBtn.hide(); submitBtn.show(); @@ -145,7 +147,7 @@ define([ var msg = gettext('File format not supported. Please upload a file with a {file_extension} extension.') .replace('{file_extension}', 'tar.gz'); - $('.error-block').html(msg).show(); + $('.error-block').text(msg).show(); } }; diff --git a/cms/static/js/views/export.js b/cms/static/js/views/export.js new file mode 100644 index 0000000000..235edc7e03 --- /dev/null +++ b/cms/static/js/views/export.js @@ -0,0 +1,400 @@ +/** + * Course export-related js. + */ +define([ + 'jquery', 'underscore', 'gettext', 'moment', 'common/js/components/views/feedback_prompt', + 'edx-ui-toolkit/js/utils/html-utils', 'jquery.cookie' +], function($, _, gettext, moment, PromptView, HtmlUtils) { + 'use strict'; + + /** ******** Private properties ****************************************/ + + var COOKIE_NAME = 'lastexport'; + + var STAGE = { + PREPARING: 0, + EXPORTING: 1, + COMPRESSING: 2, + SUCCESS: 3 + }; + + var STATE = { + READY: 1, + IN_PROGRESS: 2, + SUCCESS: 3, + ERROR: 4 + }; + + var courselikeHomeUrl; + var current = {stage: 0, state: STATE.READY, downloadUrl: null}; + var deferred = null; + var isLibrary = false; + var statusUrl = null; + var successUnixDate = null; + var timeout = {id: null, delay: 1000}; + var $dom = { + downloadLink: $('#download-exported-button'), + stages: $('ol.status-progress').children(), + successStage: $('.item-progresspoint-success'), + wrapper: $('div.wrapper-status') + }; + + /** ******** Private functions *****************************************/ + + /** + * Makes Export feedback status list visible + * + */ + var displayFeedbackList = function() { + $dom.wrapper.removeClass('is-hidden'); + }; + + /** + * Updates the Export feedback status list + * + * @param {string} [currStageMsg=''] The message to show on the + * current stage (for now only in case of error) + */ + var updateFeedbackList = function(currStageMsg) { + var $checkmark, $curr, $prev, $next; + var date, stageMsg, time; + + $checkmark = $dom.successStage.find('.icon'); + stageMsg = currStageMsg || ''; + + function completeStage(stage) { + $(stage) + .removeClass('is-not-started is-started') + .addClass('is-complete'); + } + + function errorStage(stage) { + if (!$(stage).hasClass('has-error')) { + stageMsg = HtmlUtils.joinHtml( + HtmlUtils.HTML('

'), + stageMsg, + HtmlUtils.HTML('

') + ); + $(stage) + .removeClass('is-started') + .addClass('has-error') + .find('p.copy') + .hide() + .after(HtmlUtils.ensureHtml(stageMsg).toString()); + } + } + + function resetStage(stage) { + $(stage) + .removeClass('is-complete is-started has-error') + .addClass('is-not-started') + .find('p.error') + .remove() + .end() + .find('p.copy') + .show(); + } + + switch (current.state) { + case STATE.READY: + _.map($dom.stages, resetStage); + + break; + + case STATE.IN_PROGRESS: + $prev = $dom.stages.slice(0, current.stage); + $curr = $dom.stages.eq(current.stage); + + _.map($prev, completeStage); + $curr.removeClass('is-not-started').addClass('is-started'); + + break; + + case STATE.SUCCESS: + date = moment(successUnixDate).utc().format('MM/DD/YYYY'); + time = moment(successUnixDate).utc().format('HH:mm'); + + _.map($dom.stages, completeStage); + + $dom.successStage + .find('.item-progresspoint-success-date') + .text('(' + date + ' at ' + time + ' UTC)'); + + break; + + case STATE.ERROR: + // Make all stages up to, and including, the error stage 'complete'. + $prev = $dom.stages.slice(0, current.stage + 1); + $curr = $dom.stages.eq(current.stage); + $next = $dom.stages.slice(current.stage + 1); + + _.map($prev, completeStage); + _.map($next, resetStage); + errorStage($curr); + + break; + + default: + // Invalid state, don't change anything + return; + } + + if (current.state === STATE.SUCCESS) { + $checkmark.removeClass('fa-square-o').addClass('fa-check-square-o'); + $dom.downloadLink.attr('href', current.downloadUrl); + } else { + $checkmark.removeClass('fa-check-square-o').addClass('fa-square-o'); + $dom.downloadLink.attr('href', '#'); + } + }; + + /** + * Sets the Export in the "error" status. + * + * Immediately stops any further polling from the server. + * Displays the error message at the list element that corresponds + * to the stage where the error occurred. + * + * @param {string} msg Error message to display. + * @param {int} [stage=current.stage] Stage of export process at which error occurred. + */ + var error = function(msg, stage) { + current.stage = Math.abs(stage || current.stage); // Could be negative + current.state = STATE.ERROR; + + clearTimeout(timeout.id); + updateFeedbackList(msg); + + deferred.resolve(); + }; + + /** + * Stores in a cookie the current export data + * + * @param {boolean} [completed=false] If the export has been completed or not + */ + var storeExport = function(completed) { + $.cookie(COOKIE_NAME, JSON.stringify({ + statusUrl: statusUrl, + date: moment().valueOf(), + completed: completed || false + }), {path: window.location.pathname}); + }; + + /** ******** Public functions ******************************************/ + + var CourseExport = { + /** + * Fetches the previous stored export + * + * @param {string} contentHomeUrl the full URL to the course or library being exported + * @return {JSON} the data of the previous export + */ + storedExport: function(contentHomeUrl) { + var storedData = JSON.parse($.cookie(COOKIE_NAME)); + if (storedData) { + successUnixDate = storedData.date; + } + if (contentHomeUrl) { + courselikeHomeUrl = contentHomeUrl; + } + return storedData; + }, + + /** + * Sets the Export on the "success" status + * + * If it wasn't already, marks the stored export as "completed", + * and updates its date timestamp + */ + success: function() { + current.state = STATE.SUCCESS; + + if (this.storedExport().completed !== true) { + storeExport(true); + } + + updateFeedbackList(); + + deferred.resolve(); + }, + + /** + * Entry point for server feedback + * + * Checks for export status updates every `timeout` milliseconds, + * and updates the page accordingly. + * + * @param {int} [stage=0] Starting stage. + */ + pollStatus: function(data) { + var editUnitUrl = null, + msg = data; + if (current.state !== STATE.IN_PROGRESS) { + return; + } + + current.stage = data.ExportStatus || STAGE.PREPARING; + + if (current.stage === STAGE.SUCCESS) { + current.downloadUrl = data.ExportOutput; + this.success(); + } else if (current.stage < STAGE.PREPARING) { // Failed + if (data.ExportError) { + msg = data.ExportError; + } + if (msg.raw_error_msg) { + editUnitUrl = msg.edit_unit_url; + msg = msg.raw_error_msg; + } + error(msg); + this.showError(editUnitUrl, msg); + } else { // In progress + updateFeedbackList(); + + $.getJSON(statusUrl, function(result) { + timeout.id = setTimeout(function() { + this.pollStatus(result); + }.bind(this), timeout.delay); + }.bind(this)); + } + }, + + /** + * Resets the Export internally and visually + * + */ + reset: function(library) { + current.stage = STAGE.PREPARING; + current.state = STATE.READY; + current.downloadUrl = null; + isLibrary = library; + + clearTimeout(timeout.id); + updateFeedbackList(); + }, + + /** + * Show last export status from server and start sending requests + * to the server for status updates + * + * @return {jQuery promise} + */ + resume: function(library) { + deferred = $.Deferred(); + isLibrary = library; + statusUrl = this.storedExport().statusUrl; + + $.getJSON(statusUrl, function(data) { + current.stage = data.ExportStatus; + current.downloadUrl = data.ExportOutput; + + displayFeedbackList(); + current.state = STATE.IN_PROGRESS; + this.pollStatus(data); + }.bind(this)); + + return deferred.promise(); + }, + + /** + * Show a dialog giving further information about the details of an export error. + * + * @param {string} editUnitUrl URL of the unit in which the error occurred, if known + * @param {string} errMsg Detailed error message + */ + showError: function(editUnitUrl, errMsg) { + var action, + dialog, + msg = ''; + if (editUnitUrl) { + dialog = new PromptView({ + title: gettext('There has been an error while exporting.'), + message: gettext('There has been a failure to export to XML at least one component. ' + + 'It is recommended that you go to the edit page and repair the error before attempting ' + + 'another export. Please check that all components on the page are valid and do not display ' + + 'any error messages.'), + intent: 'error', + actions: { + primary: { + text: gettext('Correct failed component'), + click: function(view) { + view.hide(); + document.location = editUnitUrl; + } + }, + secondary: { + text: gettext('Return to Export'), + click: function(view) { + view.hide(); + } + } + } + }); + } else { + if (isLibrary) { + msg += gettext('Your library could not be exported to XML. There is not enough information to ' + + 'identify the failed component. Inspect your library to identify any problematic components ' + + 'and try again.'); + action = gettext('Take me to the main library page'); + } else { + msg += gettext('Your course could not be exported to XML. There is not enough information to ' + + 'identify the failed component. Inspect your course to identify any problematic components ' + + 'and try again.'); + action = gettext('Take me to the main course page'); + } + msg += ' ' + gettext('The raw error message is:') + ' ' + errMsg; + dialog = new PromptView({ + title: gettext('There has been an error with your export.'), + message: msg, + intent: 'error', + actions: { + primary: { + text: action, + click: function(view) { + view.hide(); + document.location = courselikeHomeUrl; + } + }, + secondary: { + text: gettext('Cancel'), + click: function(view) { + view.hide(); + } + } + } + }); + } + + // The CSS animation for the dialog relies on the 'js' class + // being on the body. This happens after this JavaScript is executed, + // causing a 'bouncing' of the dialog after it is initially shown. + // As a workaround, add this class first. + $('body').addClass('js'); + dialog.show(); + }, + + /** + * Starts the exporting process. + * Makes status list visible and starts showing export progress. + * + * @param {string} url The full URL to use to query the server + * about the export status + * @return {jQuery promise} + */ + start: function(url) { + current.state = STATE.IN_PROGRESS; + deferred = $.Deferred(); + + statusUrl = url; + + storeExport(); + displayFeedbackList(); + updateFeedbackList(); + + return deferred.promise(); + } + }; + + return CourseExport; +}); diff --git a/cms/static/js/views/import.js b/cms/static/js/views/import.js index 027f39b323..08179f6bea 100644 --- a/cms/static/js/views/import.js +++ b/cms/static/js/views/import.js @@ -2,8 +2,8 @@ * Course import-related js. */ define( - ['jquery', 'underscore', 'gettext', 'moment', 'jquery.cookie'], - function($, _, gettext, moment) { + ['jquery', 'underscore', 'gettext', 'moment', 'edx-ui-toolkit/js/utils/html-utils', 'jquery.cookie'], + function($, _, gettext, moment, HtmlUtils) { 'use strict'; /** ******** Private properties ****************************************/ @@ -127,10 +127,10 @@ define( */ var updateFeedbackList = function(currStageMsg) { var $checkmark, $curr, $prev, $next; - var date, successUnix, time; + var date, stageMsg, successUnix, time; $checkmark = $dom.successStage.find('.icon'); - currStageMsg = currStageMsg || ''; + stageMsg = currStageMsg || ''; function completeStage(stage) { $(stage) @@ -140,12 +140,17 @@ define( function errorStage(stage) { if (!$(stage).hasClass('has-error')) { + stageMsg = HtmlUtils.joinHtml( + HtmlUtils.HTML('

'), + stageMsg, + HtmlUtils.HTML('

') + ); $(stage) .removeClass('is-started') .addClass('has-error') .find('p.copy') .hide() - .after("

" + currStageMsg + '

'); + .after(HtmlUtils.ensureHtml(stageMsg).toString()); } } @@ -181,7 +186,7 @@ define( $dom.successStage .find('.item-progresspoint-success-date') - .html('(' + date + ' at ' + time + ' UTC)'); + .text('(' + date + ' at ' + time + ' UTC)'); break; diff --git a/cms/static/sass/views/_export.scss b/cms/static/sass/views/_export.scss index 051695802d..385f6adaa5 100644 --- a/cms/static/sass/views/_export.scss +++ b/cms/static/sass/views/_export.scss @@ -86,98 +86,183 @@ } } - // OLD - .description { - @extend %t-copy-sub1; - float: left; - width: 62%; - margin-right: 3%; + // ==================== - h2 { - @extend %t-title5; - @extend %t-strong; + // UI: upload progress + .wrapper-status { + @include transition(opacity $tmg-f2 ease-in-out 0); + opacity: 1.0; + + // STATE: hidden + &.is-hidden { + opacity: 0.0; + display: none; + } + + > .title { + @extend %t-title4; margin-bottom: $baseline; + border-bottom: 1px solid $gray-l3; + padding-bottom: ($baseline/2); } + // elem - progress list + .list-progress { + width: flex-grid(9, 9); - strong { - @extend %t-strong; - } + .status-visual { + position: relative; + float: left; + width: flex-grid(1,9); - p + p { - margin-top: $baseline; - } - - ul { - margin: 20px 0; - list-style: disc inside; - - li { - margin: 0 0 5px 0; - } - } - } - - .export-form-wrapper { - - .export-form { - float: left; - width: 35%; - padding: 25px 30px 35px; - @include box-sizing(border-box); - border: 1px solid $mediumGrey; - border-radius: 3px; - background: $lightGrey; - text-align: center; - - h2 { - @extend %t-title4; - @extend %t-light; - margin-bottom: ($baseline*1.5); - } - - .error-block { - @extend %t-copy-sub1; - display: none; - margin-bottom: ($baseline*0.75); - } - - .error-block { - color: $error-red; - } - - .button-export { - @include green-button(); - @extend %t-action1; - padding: 10px 50px 11px; - } - - .message-status { - @extend %t-copy-sub2; - margin-top: ($baseline/2); - } - - .progress-bar { - display: none; - width: 350px; - height: 30px; - margin: 30px auto 10px; - border: 1px solid $blue; - - &.loaded { - border-color: #66b93d; - - .progress-fill { - background: #66b93d; - } + .icon { + @include transition(opacity $tmg-f1 ease-in-out 0); + @extend %t-icon4; + position: absolute; + top: ($baseline/2); + left: $baseline; } } - .progress-fill { - width: 0%; - height: 30px; - background: $blue; - color: $white; - line-height: 48px; + .status-detail { + float: left; + width: flex-grid(8,9); + margin-left: ($baseline*3); + + .title { + @extend %t-title5; + @extend %t-strong; + } + + .copy { + @extend %t-copy-base; + color: $gray-l2; + } + } + + .item-progresspoint { + @include clearfix(); + @include transition(opacity $tmg-f1 ease-in-out 0); + margin-bottom: $baseline; + border-bottom: 1px solid $gray-l4; + padding-bottom: $baseline; + + &:last-child { + margin-bottom: 0; + border-bottom: none; + padding-bottom: 0; + } + + // CASE: has actions + &.has-actions { + + .list-actions { + display: none; + + .action-primary { + @extend %btn-primary-blue; + } + } + } + + // TYPE: success + &.item-progresspoint-success { + + .item-progresspoint-success-date { + @include margin-left($baseline/4); + display: none; + } + + &.is-complete { + + .item-progresspoint-success-date { + display: inline; + } + } + } + + + // STATE: not started + &.is-not-started { + opacity: 0.5; + + .fa-warning { + visibility: hidden; + opacity: 0.0; + } + + .fa-cog { + visibility: visible; + opacity: 1.0; + } + + .fa-check { + opacity: 0.3; + } + } + + // STATE: started + &.is-started { + + .fa-warning { + visibility: hidden; + opacity: 0.0; + } + + .fa-cog { + @include animation(fa-spin 2s infinite linear); + + visibility: visible; + opacity: 1.0; + } + } + + // STATE: completed + &.is-complete { + + .fa-cog { + visibility: visible; + opacity: 1.0; + } + + .fa-warning { + visibility: hidden; + opacity: 0.0; + } + + .icon { + color: $green; + } + + .status-detail .title { + color: $green; + } + + .list-actions { + display: block; + } + } + + // STATE: error + &.has-error { + + .fa-cog { + visibility: hidden; + opacity: 0.0; + } + + .fa-warning { + visibility: visible; + opacity: 1.0; + } + + .icon { + color: $red; + } + + .status-detail .title, .status-detail .copy { + color: $red; + } + } } } } diff --git a/cms/templates/export.html b/cms/templates/export.html index f6982dabaf..1013fc3aab 100644 --- a/cms/templates/export.html +++ b/cms/templates/export.html @@ -27,17 +27,13 @@ else: <%block name="bodyclass">is-signedin course tools view-export <%block name="requirejs"> -% if in_err: - var hasUnit = ${bool(unit) | n, dump_js_escaped_json}, - editUnitUrl = "${edit_unit_url | n, js_escaped_string}", - courselikeHomeUrl = "${courselike_home_url | n, js_escaped_string}", - is_library = ${library | n, dump_js_escaped_json} - errMsg = "${raw_err_msg | n, js_escaped_string}"; + var courselikeHomeUrl = "${courselike_home_url | n, js_escaped_string}", + is_library = ${library | n, dump_js_escaped_json}, + statusUrl = "${status_url | n, js_escaped_string}"; require(["js/factories/export"], function(ExportFactory) { - ExportFactory(hasUnit, editUnitUrl, courselikeHomeUrl, is_library, errMsg); + ExportFactory(courselikeHomeUrl, is_library, statusUrl); }); -%endif <%block name="content"> @@ -93,7 +89,7 @@ else: + + + %if not library:
diff --git a/cms/templates/import.html b/cms/templates/import.html index 80eb536e16..bd5bf77f98 100644 --- a/cms/templates/import.html +++ b/cms/templates/import.html @@ -1,3 +1,4 @@ +<%page expression_filter="h"/> <%inherit file="base.html" /> <%def name="online_help_token()"> <% @@ -13,6 +14,7 @@ else: from openedx.core.djangolib.js_utils import ( dump_js_escaped_json, js_escaped_string ) + from openedx.core.djangolib.markup import HTML, Text %> <%block name="title"> %if library: @@ -44,11 +46,11 @@ else:
## Translators: ".tar.gz" is a file extension, and files with that extension are called "gzipped tar files": these terms should not be translated %if library: -

${_("Be sure you want to import a library before continuing. The contents of the imported library will replace the contents of the existing library. {em_start}You cannot undo a library import{em_end}. Before you proceed, we recommend that you export the current library, so that you have a backup copy of it.").format(em_start='', em_end="")}

+

${Text(_("Be sure you want to import a library before continuing. The contents of the imported library will replace the contents of the existing library. {em_start}You cannot undo a library import{em_end}. Before you proceed, we recommend that you export the current library, so that you have a backup copy of it.")).format(em_start=HTML(''), em_end=HTML(""))}

${_("The library that you import must be in a .tar.gz file (that is, a .tar file compressed with GNU Zip). This .tar.gz file must contain a library.xml file. It may also contain other files.")}

${_("The import process has five stages. During the first two stages, you must stay on this page. You can leave this page after the Unpacking stage has completed. We recommend, however, that you don't make important changes to your library until the import operation has completed.")}

%else: -

${_("Be sure you want to import a course before continuing. The contents of the imported course will replace the contents of the existing course. {em_start}You cannot undo a course import{em_end}. Before you proceed, we recommend that you export the current course, so that you have a backup copy of it.").format(em_start='', em_end="")}

+

${Text(_("Be sure you want to import a course before continuing. The contents of the imported course will replace the contents of the existing course. {em_start}You cannot undo a course import{em_end}. Before you proceed, we recommend that you export the current course, so that you have a backup copy of it.")).format(em_start=HTML(''), em_end=HTML(""))}

${_("The course that you import must be in a .tar.gz file (that is, a .tar file compressed with GNU Zip). This .tar.gz file must contain a course.xml file. It may also contain other files.")}

${_("The import process has five stages. During the first two stages, you must stay on this page. You can leave this page after the Unpacking stage has completed. We recommend, however, that you don't make important changes to your course until the import operation has completed.")}

%endif diff --git a/cms/urls.py b/cms/urls.py index 130c06cb14..c80c42ebfa 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -96,6 +96,8 @@ urlpatterns += patterns( url(r'^import/{}$'.format(COURSELIKE_KEY_PATTERN), 'import_handler'), url(r'^import_status/{}/(?P.+)$'.format(COURSELIKE_KEY_PATTERN), 'import_status_handler'), url(r'^export/{}$'.format(COURSELIKE_KEY_PATTERN), 'export_handler'), + url(r'^export_output/{}$'.format(COURSELIKE_KEY_PATTERN), 'export_output_handler'), + url(r'^export_status/{}$'.format(COURSELIKE_KEY_PATTERN), 'export_status_handler'), url(r'^xblock/outline/{}$'.format(settings.USAGE_KEY_PATTERN), 'xblock_outline_handler'), url(r'^xblock/container/{}$'.format(settings.USAGE_KEY_PATTERN), 'xblock_container_handler'), url(r'^xblock/{}/(?P[^/]+)$'.format(settings.USAGE_KEY_PATTERN), 'xblock_view_handler'), diff --git a/common/test/acceptance/pages/studio/import_export.py b/common/test/acceptance/pages/studio/import_export.py index 30f583a377..62e9f3ced9 100644 --- a/common/test/acceptance/pages/studio/import_export.py +++ b/common/test/acceptance/pages/studio/import_export.py @@ -29,19 +29,116 @@ class TemplateCheckMixin(object): return self.q(css='h1.page-header')[0].text.split('\n')[-1] -class ExportMixin(object): +class ImportExportMixin(object): + """ + Mixin for functionality common to both the import and export pages + """ + + def is_task_list_showing(self): + """ + The task list shows a series of steps being performed during import or + export. It is normally hidden until the process begins. + + Tell us whether it's currently visible. + """ + return self.q(css='.wrapper-status').visible + + def is_timestamp_visible(self): + """ + Checks if the UTC timestamp of the last successful import/export is visible + """ + return self.q(css='.item-progresspoint-success-date').visible + + @property + def parsed_timestamp(self): + """ + Return python datetime object from the parsed timestamp tuple (date, time) + """ + timestamp = "{0} {1}".format(*self.timestamp) + formatted_timestamp = time.strptime(timestamp, "%m/%d/%Y %H:%M") + return datetime.fromtimestamp(time.mktime(formatted_timestamp)) + + @property + def timestamp(self): + """ + The timestamp is displayed on the page as "(MM/DD/YYYY at HH:mm)" + It parses the timestamp and returns a (date, time) tuple + """ + string = self.q(css='.item-progresspoint-success-date').text[0] + + return re.match(r'\(([^ ]+).+?(\d{2}:\d{2})', string).groups() + + def wait_for_tasks(self, completed=False, fail_on=None): + """ + Wait for all of the items in the task list to be set to the correct state. + """ + if fail_on: + # Makes no sense to include this if the tasks haven't run. + completed = True + + state, desc_template = self._task_properties(completed) + + for desc, css_class in self.task_classes.items(): + desc_text = desc_template.format(desc) + # pylint: disable=cell-var-from-loop + EmptyPromise(lambda: self.q(css='.{}.{}'.format(css_class, state)).present, desc_text, timeout=30) + if fail_on == desc: + EmptyPromise( + lambda: self.q(css='.{}.is-complete.has-error'.format(css_class)).present, + "{} checkpoint marked as failed".format(desc), + timeout=30 + ) + # The rest should never run. + state, desc_template = self._task_properties(False) + + def wait_for_timestamp_visible(self): + """ + Wait for the timestamp of the last successful import/export to be visible. + """ + EmptyPromise(self.is_timestamp_visible, 'Timestamp Visible', timeout=30).fulfill() + + @staticmethod + def _task_properties(completed): + """ + Outputs the CSS class and promise description for task states based on completion. + """ + if completed: + return 'is-complete', "'{}' is marked complete" + else: + return 'is-not-started', "'{}' is in not-yet-started status" + + +class ExportMixin(ImportExportMixin): """ Export page Mixin. """ url_path = "export" + task_classes = { + 'Preparing': 'item-progresspoint-prepare', + 'Exporting': 'item-progresspoint-export', + 'Compressing': 'item-progresspoint-compress', + 'Success': 'item-progresspoint-success' + } + def is_browser_on_page(self): """ Verify this is the export page """ return self.q(css='body.view-export').present + def is_click_handler_registered(self): + """ + Check if the click handler for the export button has been registered yet + """ + script = """ + var $ = require('jquery'), + buttonEvents = $._data($('a.action-primary')[0], 'events'); + return buttonEvents && buttonEvents.hasOwnProperty('click');""" + stripped_script = ''.join([line.strip() for line in script.split('\n')]) + return self.browser.execute_script(stripped_script) + def _get_tarball(self, url): """ Download tarball at `url` @@ -61,14 +158,13 @@ class ExportMixin(object): """ Downloads the course or library in tarball form. """ - tarball_url = self.q(css='a.action-export').attrs('href')[0] + tarball_url = self.q(css='#download-exported-button')[0].get_attribute('href') good_status, headers = self._get_tarball(tarball_url) return good_status, headers['content-type'] == 'application/x-tgz' def click_export(self): """ - Click the export button. Should only be used if expected to fail, as - otherwise a browser dialog for saving the file will be presented. + Click the export button. """ self.q(css='a.action-export').click() @@ -78,6 +174,13 @@ class ExportMixin(object): """ return self.q(css='.prompt.error').visible + def is_export_finished(self): + """ + Checks if the 'Download Exported Course/Library' button is showing. + """ + button = self.q(css='#download-exported-button')[0] + return button.is_displayed() and button.get_attribute('href') + def click_modal_button(self): """ Click the button on the modal dialog that appears when there's a problem. @@ -90,6 +193,18 @@ class ExportMixin(object): """ EmptyPromise(self.is_error_modal_showing, 'Error Modal Displayed', timeout=30).fulfill() + def wait_for_export(self): + """ + Wait for the export process to finish. + """ + EmptyPromise(self.is_export_finished, 'Export Finished', timeout=30).fulfill() + + def wait_for_export_click_handler(self): + """ + Wait for the export button click handler to be registered + """ + EmptyPromise(self.is_click_handler_registered, 'Export Button Click Handler Registered', timeout=30).fulfill() + class LibraryLoader(object): """ @@ -117,31 +232,20 @@ class ExportLibraryPage(ExportMixin, TemplateCheckMixin, LibraryLoader, LibraryP """ -class ImportMixin(object): +class ImportMixin(ImportExportMixin): """ Import page mixin """ url_path = "import" - @property - def timestamp(self): - """ - The timestamp is displayed on the page as "(MM/DD/YYYY at HH:mm)" - It parses the timestamp and returns a (date, time) tuple - """ - string = self.q(css='.item-progresspoint-success-date').text[0] - - return re.match(r'\(([^ ]+).+?(\d{2}:\d{2})', string).groups() - - @property - def parsed_timestamp(self): - """ - Return python datetime object from the parsed timestamp tuple (date, time) - """ - timestamp = "{0} {1}".format(*self.timestamp) - formatted_timestamp = time.strptime(timestamp, "%m/%d/%Y %H:%M") - return datetime.fromtimestamp(time.mktime(formatted_timestamp)) + task_classes = { + 'Uploading': 'item-progresspoint-upload', + 'Unpacking': 'item-progresspoint-unpack', + 'Verifying': 'item-progresspoint-verify', + 'Updating': 'item-progresspoint-import', + 'Success': 'item-progresspoint-success' + } def is_browser_on_page(self): """ @@ -149,6 +253,17 @@ class ImportMixin(object): """ return self.q(css='.choose-file-button').present + def is_click_handler_registered(self): + """ + Check if the click handler for the file selector button has been registered yet + """ + script = """ + var $ = require('jquery'), + buttonEvents = $._data($('a.choose-file-button')[0], 'events'); + return buttonEvents && buttonEvents.hasOwnProperty('click');""" + stripped_script = ''.join([line.strip() for line in script.split('\n')]) + return self.browser.execute_script(stripped_script) + @staticmethod def file_path(filename): """ @@ -196,46 +311,6 @@ class ImportMixin(object): """ return self.q(css='#view-updated-button').visible - @staticmethod - def _task_properties(completed): - """ - Outputs the CSS class and promise description for task states based on completion. - """ - if completed: - return 'is-complete', "'{}' is marked complete" - else: - return 'is-not-started', "'{}' is in not-yet-started status" - - def wait_for_tasks(self, completed=False, fail_on=None): - """ - Wait for all of the items in the task list to be set to the correct state. - """ - classes = { - 'Uploading': 'item-progresspoint-upload', - 'Unpacking': 'item-progresspoint-unpack', - 'Verifying': 'item-progresspoint-verify', - 'Updating': 'item-progresspoint-import', - 'Success': 'item-progresspoint-success' - } - if fail_on: - # Makes no sense to include this if the tasks haven't run. - completed = True - - state, desc_template = self._task_properties(completed) - - for desc, css_class in classes.items(): - desc_text = desc_template.format(desc) - # pylint: disable=cell-var-from-loop - EmptyPromise(lambda: self.q(css='.{}.{}'.format(css_class, state)).present, desc_text, timeout=30) - if fail_on == desc: - EmptyPromise( - lambda: self.q(css='.{}.is-complete.has-error'.format(css_class)).present, - "{} checkpoint marked as failed".format(desc), - timeout=30 - ) - # The rest should never run. - state, desc_template = self._task_properties(False) - def wait_for_upload(self): """ Wait for the upload to be confirmed. @@ -250,26 +325,12 @@ class ImportMixin(object): """ return self.q(css='#fileupload .error-block').visible - def is_task_list_showing(self): + def wait_for_choose_file_click_handler(self): """ - The task list shows a series of steps being performed during import. It is normally - hidden until the upload begins. - - Tell us whether it's currently visible. + Wait for the choose file button click handler to be registered """ - return self.q(css='.wrapper-status').visible - - def is_timestamp_visible(self): - """ - Checks if the UTC timestamp of the last successful import is visible - """ - return self.q(css='.item-progresspoint-success-date').visible - - def wait_for_timestamp_visible(self): - """ - Wait for the timestamp of the last successful import to be visible. - """ - EmptyPromise(self.is_timestamp_visible, 'Timestamp Visible', timeout=30).fulfill() + EmptyPromise(self.is_click_handler_registered, 'Choose File Button Click Handler Registered', + timeout=30).fulfill() def wait_for_filename_error(self): """ diff --git a/common/test/acceptance/tests/studio/test_import_export.py b/common/test/acceptance/tests/studio/test_import_export.py index 547aa37ff1..ef63356fa5 100644 --- a/common/test/acceptance/tests/studio/test_import_export.py +++ b/common/test/acceptance/tests/studio/test_import_export.py @@ -4,8 +4,6 @@ Acceptance tests for the Import and Export pages from nose.plugins.attrib import attr from datetime import datetime -from flaky import flaky - from abc import abstractmethod from common.test.acceptance.tests.studio.base_studio_test import StudioLibraryTest, StudioCourseTest @@ -32,10 +30,70 @@ class ExportTestMixin(object): The download will succeed And the file will be of the right MIME type. """ + self.export_page.wait_for_export_click_handler() + self.export_page.click_export() + self.export_page.wait_for_export() good_status, is_tarball_mimetype = self.export_page.download_tarball() self.assertTrue(good_status) self.assertTrue(is_tarball_mimetype) + def test_export_timestamp(self): + """ + Scenario: I perform a course / library export + On export success, the page displays a UTC timestamp previously not visible + And if I refresh the page, the timestamp is still displayed + """ + self.assertFalse(self.export_page.is_timestamp_visible()) + + # Get the time when the export has started. + # export_page timestamp is in (MM/DD/YYYY at HH:mm) so replacing (second, microsecond) to + # keep the comparison consistent + export_start_time = datetime.utcnow().replace(microsecond=0, second=0) + self.export_page.wait_for_export_click_handler() + self.export_page.click_export() + self.export_page.wait_for_export() + + # Get the time when the export has finished. + # export_page timestamp is in (MM/DD/YYYY at HH:mm) so replacing (second, microsecond) to + # keep the comparison consistent + export_finish_time = datetime.utcnow().replace(microsecond=0, second=0) + + export_timestamp = self.export_page.parsed_timestamp + self.export_page.wait_for_timestamp_visible() + + # Verify that 'export_timestamp' is between start and finish upload time + self.assertLessEqual( + export_start_time, + export_timestamp, + "Course export timestamp should be export_start_time <= export_timestamp <= export_end_time" + ) + self.assertGreaterEqual( + export_finish_time, + export_timestamp, + "Course export timestamp should be export_start_time <= export_timestamp <= export_end_time" + ) + + self.export_page.visit() + self.export_page.wait_for_tasks(completed=True) + self.export_page.wait_for_timestamp_visible() + + def test_task_list(self): + """ + Scenario: I should see feedback checkpoints when exporting a course or library + Given that I am on an export page + No task checkpoint list should be showing + When I export the course or library + Each task in the checklist should be marked confirmed + And the task list should be visible + """ + # The task list shouldn't be visible to start. + self.assertFalse(self.export_page.is_task_list_showing(), "Task list shown too early.") + self.export_page.wait_for_tasks() + self.export_page.wait_for_export_click_handler() + self.export_page.click_export() + self.export_page.wait_for_tasks(completed=True) + self.assertTrue(self.export_page.is_task_list_showing(), "Task list did not display.") + @attr(shard=7) class TestCourseExport(ExportTestMixin, StudioCourseTest): @@ -101,7 +159,6 @@ class ImportTestMixin(object): """ return [] - @flaky # TODO fix this, see PLAT-1186 def test_upload(self): """ Scenario: I want to upload a course or library for import. @@ -110,6 +167,7 @@ class ImportTestMixin(object): I can select the file and upload it And the page will give me confirmation that it uploaded successfully """ + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.import_page.wait_for_upload() @@ -125,6 +183,7 @@ class ImportTestMixin(object): # import_page timestamp is in (MM/DD/YYYY at HH:mm) so replacing (second, microsecond) to # keep the comparison consistent upload_start_time = datetime.utcnow().replace(microsecond=0, second=0) + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.import_page.wait_for_upload() @@ -158,6 +217,7 @@ class ImportTestMixin(object): Given that I upload a library or course A button will appear that contains the URL to the library or course's main page """ + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.assertEqual(self.import_page.finished_target_url(), self.landing_page.url) @@ -167,6 +227,7 @@ class ImportTestMixin(object): Given that I select a file that is an .mp4 for upload An error message will appear """ + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball('funny_cat_video.mp4') self.import_page.wait_for_filename_error() @@ -182,6 +243,7 @@ class ImportTestMixin(object): # The task list shouldn't be visible to start. self.assertFalse(self.import_page.is_task_list_showing(), "Task list shown too early.") self.import_page.wait_for_tasks() + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.import_page.wait_for_tasks(completed=True) self.assertTrue(self.import_page.is_task_list_showing(), "Task list did not display.") @@ -195,6 +257,7 @@ class ImportTestMixin(object): And the 'Updating' task should be marked failed And the remaining tasks should not be marked as started """ + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.bad_tarball_name) self.import_page.wait_for_tasks(fail_on='Updating') @@ -212,7 +275,6 @@ class TestEntranceExamCourseImport(ImportTestMixin, StudioCourseTest): def page_args(self): return [self.browser, self.course_info['org'], self.course_info['number'], self.course_info['run']] - @flaky # TODO fix this, see PLAT-1186 def test_course_updated_with_entrance_exam(self): """ Given that I visit an empty course before import @@ -229,6 +291,7 @@ class TestEntranceExamCourseImport(ImportTestMixin, StudioCourseTest): self.assertRaises(IndexError, self.landing_page.section, "Section") self.assertRaises(IndexError, self.landing_page.section, "Entrance Exam") self.import_page.visit() + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.import_page.wait_for_upload() self.landing_page.visit() @@ -259,7 +322,6 @@ class TestCourseImport(ImportTestMixin, StudioCourseTest): def page_args(self): return [self.browser, self.course_info['org'], self.course_info['number'], self.course_info['run']] - @flaky # TODO fix this, see PLAT-1186 def test_course_updated(self): """ Given that I visit an empty course before import @@ -273,6 +335,7 @@ class TestCourseImport(ImportTestMixin, StudioCourseTest): # Should not exist yet. self.assertRaises(IndexError, self.landing_page.section, "Section") self.import_page.visit() + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.import_page.wait_for_upload() self.landing_page.visit() @@ -299,6 +362,7 @@ class TestCourseImport(ImportTestMixin, StudioCourseTest): Then timestamp is not visible """ self.import_page.visit() + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.import_page.wait_for_upload() self.assertTrue(self.import_page.is_timestamp_visible()) @@ -330,7 +394,6 @@ class TestLibraryImport(ImportTestMixin, StudioLibraryTest): def page_args(self): return [self.browser, self.library_key] - @flaky # TODO fix this, see PLAT-1186 def test_library_updated(self): """ Given that I visit an empty library @@ -345,6 +408,7 @@ class TestLibraryImport(ImportTestMixin, StudioLibraryTest): # No items should be in the library to start. self.assertEqual(len(self.landing_page.xblocks), 0) self.import_page.visit() + self.import_page.wait_for_choose_file_click_handler() self.import_page.upload_tarball(self.tarball_name) self.import_page.wait_for_upload() self.landing_page.visit()