diff --git a/cms/djangoapps/contentstore/management/commands/export.py b/cms/djangoapps/contentstore/management/commands/export.py new file mode 100644 index 0000000000..11b043c2ab --- /dev/null +++ b/cms/djangoapps/contentstore/management/commands/export.py @@ -0,0 +1,35 @@ +### +### Script for exporting courseware from Mongo to a tar.gz file +### +import os + +from django.core.management.base import BaseCommand, CommandError +from xmodule.modulestore.xml_exporter import export_to_xml +from xmodule.modulestore.django import modulestore +from xmodule.contentstore.django import contentstore +from xmodule.modulestore import Location +from xmodule.course_module import CourseDescriptor + + +unnamed_modules = 0 + + +class Command(BaseCommand): + help = \ +'''Import the specified data directory into the default ModuleStore''' + + def handle(self, *args, **options): + if len(args) != 2: + raise CommandError("import requires two arguments: ") + + course_id = args[0] + output_path = args[1] + + print "Exporting course id = {0} to {1}".format(course_id, output_path) + + location = CourseDescriptor.id_to_location(course_id) + + root_dir = os.path.dirname(output_path) + course_dir = os.path.splitext(os.path.basename(output_path))[0] + + export_to_xml(modulestore('direct'), contentstore(), location, root_dir, course_dir) diff --git a/cms/djangoapps/contentstore/tests/tests.py b/cms/djangoapps/contentstore/tests/tests.py index ad4ec8790f..727107cd10 100644 --- a/cms/djangoapps/contentstore/tests/tests.py +++ b/cms/djangoapps/contentstore/tests/tests.py @@ -1,10 +1,12 @@ import json +import shutil from django.test import TestCase from django.test.client import Client from override_settings import override_settings from django.conf import settings from django.core.urlresolvers import reverse from path import path +from tempfile import mkdtemp from student.models import Registration from django.contrib.auth.models import User @@ -18,6 +20,7 @@ from xmodule.modulestore.store_utilities import delete_course from xmodule.modulestore.django import modulestore from xmodule.contentstore.django import contentstore from xmodule.course_module import CourseDescriptor +from xmodule.modulestore.xml_exporter import export_to_xml def parse_json(response): """Parse response, which is assumed to be json""" @@ -385,4 +388,35 @@ class ContentStoreTest(TestCase): items = ms.get_items(Location(['i4x','edX', 'full', 'vertical', None])) self.assertEqual(len(items), 0) + def test_export_course(self): + ms = modulestore('direct') + cs = contentstore() + + import_from_xml(ms, 'common/test/data/', ['full']) + location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012') + + root_dir = path(mkdtemp()) + + print 'Exporting to tempdir = {0}'.format(root_dir) + + # export out to a tempdir + export_to_xml(ms, cs, location, root_dir, 'test_export') + + # remove old course + delete_course(ms, cs, location) + + # reimport + import_from_xml(ms, root_dir, ['test_export']) + + items = ms.get_items(Location(['i4x','edX', 'full', 'vertical', None])) + self.assertGreater(len(items), 0) + for descriptor in items: + print "Checking {0}....".format(descriptor.location.url()) + resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()})) + self.assertEqual(resp.status_code, 200) + + shutil.rmtree(root_dir) + + + diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 29144ce9fb..fb6a40eff3 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -10,6 +10,10 @@ from datetime import datetime from collections import defaultdict from uuid import uuid4 from path import path +from xmodule.modulestore.xml_exporter import export_to_xml +from tempfile import mkdtemp +from django.core.servers.basehttp import FileWrapper +from django.core.files.temp import NamedTemporaryFile # to install PIL on MacOSX: 'easy_install http://dist.repoze.org/PIL-1.1.6.tar.gz' from PIL import Image @@ -1302,3 +1306,55 @@ def import_course(request, org, course, name): course_module.location.course, course_module.location.name]) }) + +@ensure_csrf_cookie +@login_required +def generate_export_course(request, org, course, name): + location = ['i4x', org, course, 'course', name] + course_module = modulestore().get_item(location) + # check that logged in user has permissions to this item + if not has_access(request.user, location): + raise PermissionDenied() + + loc = Location(location) + export_file = NamedTemporaryFile(prefix=name+'.', suffix=".tar.gz") + + root_dir = path(mkdtemp()) + + # export out to a tempdir + + logging.debug('root = {0}'.format(root_dir)) + + export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name) + #filename = root_dir / name + '.tar.gz' + + logging.debug('tar file being generated at {0}'.format(export_file.name)) + tf = tarfile.open(name=export_file.name, mode='w:gz') + tf.add(root_dir/name, arcname=name) + tf.close() + + # remove temp dir + shutil.rmtree(root_dir/name) + + wrapper = FileWrapper(export_file) + response = HttpResponse(wrapper, content_type='application/x-tgz') + response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name) + response['Content-Length'] = os.path.getsize(export_file.name) + return response + + +@ensure_csrf_cookie +@login_required +def export_course(request, org, course, name): + + location = ['i4x', org, course, 'course', name] + course_module = modulestore().get_item(location) + # check that logged in user has permissions to this item + if not has_access(request.user, location): + raise PermissionDenied() + + return render_to_response('export.html', { + 'context_course': course_module, + 'active_tab': 'export', + 'successful_import_redirect_url' : '' + }) diff --git a/cms/static/sass/_export.scss b/cms/static/sass/_export.scss new file mode 100644 index 0000000000..e1ab7eb605 --- /dev/null +++ b/cms/static/sass/_export.scss @@ -0,0 +1,123 @@ +.export { + .export-overview { + @extend .window; + @include clearfix; + padding: 30px 40px; + } + + .description { + float: left; + width: 62%; + margin-right: 3%; + font-size: 14px; + + h2 { + font-weight: 700; + font-size: 19px; + margin-bottom: 20px; + } + + strong { + font-weight: 700; + } + + p + p { + margin-top: 20px; + } + + ul { + margin: 20px 0; + list-style: disc inside; + + li { + margin: 0 0 5px 0; + } + } + } + + .export-form-wrapper { + + .export-form { + float: left; + width: 35%; + padding: 25px 30px 35px; + @include box-sizing(border-box); + border: 1px solid $mediumGrey; + border-radius: 3px; + background: $lightGrey; + text-align: center; + + h2 { + margin-bottom: 30px; + font-size: 26px; + font-weight: 300; + } + + .error-block { + display: none; + margin-bottom: 15px; + font-size: 13px; + } + + .error-block { + color: $error-red; + } + + .button-export { + @include green-button; + padding: 10px 50px 11px; + font-size: 17px; + } + + .message-status { + margin-top: 10px; + font-size: 12px; + } + + .progress-bar { + display: none; + width: 350px; + height: 30px; + margin: 30px auto 10px; + border: 1px solid $blue; + + &.loaded { + border-color: #66b93d; + + .progress-fill { + background: #66b93d; + } + } + } + + .progress-fill { + width: 0%; + height: 30px; + background: $blue; + color: #fff; + line-height: 48px; + } + } + + // downloading state + &.is-downloading { + + .progress-bar { + display: block; + } + + .button-export { + padding: 10px 50px 11px; + font-size: 17px; + + &.disabled { + + pointer-events: none; + cursor: default; + } + } + } + } + + +} \ No newline at end of file diff --git a/cms/static/sass/base-style.scss b/cms/static/sass/base-style.scss index 73812125a8..e3463477c1 100644 --- a/cms/static/sass/base-style.scss +++ b/cms/static/sass/base-style.scss @@ -18,6 +18,7 @@ @import "static-pages"; @import "users"; @import "import"; +@import "export"; @import "settings"; @import "course-info"; @import "landing"; diff --git a/cms/templates/export.html b/cms/templates/export.html new file mode 100644 index 0000000000..2f41ad86df --- /dev/null +++ b/cms/templates/export.html @@ -0,0 +1,93 @@ +<%inherit file="base.html" /> +<%namespace name='static' file='static_content.html'/> + +<%! from django.core.urlresolvers import reverse %> +<%block name="title">Export +<%block name="bodyclass">export + +<%block name="content"> +
+
+
+
+

About Exporting Courses

+

When exporting your course, you will receive a .tar.gz formatted file that contains the following course data:

+ +
    +
  • Course Structure (Sections and sub-section ordering)
    • +
  • Individual Units
    • +
  • Individual Problems
    • +
  • Static Pages
    • +
  • Course Assets
    • +
+ +

Your course export will not include: student data, forum/discussion data, course settings, certificates, grading information, or user data.

+
+ + +
+
+

Export Course:

+ +

+ + Download Files +
+
+ + + <%doc> +
+
+

Export Course:

+ +

+ + Files Downloading +

Download not start? Try again

+
+
+ +
+
+
+ + +<%block name="jsextra"> + + \ No newline at end of file diff --git a/cms/templates/widgets/header.html b/cms/templates/widgets/header.html index f7fc4722e0..5f41452339 100644 --- a/cms/templates/widgets/header.html +++ b/cms/templates/widgets/header.html @@ -33,6 +33,7 @@
  • Assets
  • Settings
  • Import
    • +
  • Export
    • % endif diff --git a/cms/urls.py b/cms/urls.py index ad4b83c923..6f8736551b 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -23,6 +23,11 @@ urlpatterns = ('', url(r'^(?P[^/]+)/(?P[^/]+)/import/(?P[^/]+)$', 'contentstore.views.import_course', name='import_course'), + url(r'^(?P[^/]+)/(?P[^/]+)/export/(?P[^/]+)$', + 'contentstore.views.export_course', name='export_course'), + url(r'^(?P[^/]+)/(?P[^/]+)/generate_export/(?P[^/]+)$', + 'contentstore.views.generate_export_course', name='generate_export_course'), + url(r'^preview/modx/(?P[^/]*)/(?P.*?)/(?P[^/]*)$', 'contentstore.views.preview_dispatch', name='preview_dispatch'), url(r'^(?P[^/]+)/(?P[^/]+)/course/(?P[^/]+)/upload_asset$', diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py index b70ab5058f..b84aa01c54 100644 --- a/common/lib/xmodule/xmodule/contentstore/content.py +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -12,13 +12,16 @@ from .django import contentstore from PIL import Image class StaticContent(object): - def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None): + def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None, import_path=None): self.location = loc self.name = name #a display string which can be edited, and thus not part of the location which needs to be fixed self.content_type = content_type self.data = data self.last_modified_at = last_modified_at self.thumbnail_location = Location(thumbnail_location) + # optional information about where this file was imported from. This is needed to support import/export + # cycles + self.import_path = import_path @property def is_thumbnail(self): diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index aebb6dfd32..01f189a9e4 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -11,6 +11,8 @@ import logging from .content import StaticContent, ContentStore from xmodule.exceptions import NotFoundError +from fs.osfs import OSFS +import os class MongoContentStore(ContentStore): @@ -32,7 +34,7 @@ class MongoContentStore(ContentStore): self.delete(id) with self.fs.new_file(_id = id, filename=content.get_url_path(), content_type=content.content_type, - displayname=content.name, thumbnail_location=content.thumbnail_location) as fp: + displayname=content.name, thumbnail_location=content.thumbnail_location, import_path=content.import_path) as fp: fp.write(content.data) @@ -47,10 +49,32 @@ class MongoContentStore(ContentStore): try: with self.fs.get(id) as fp: return StaticContent(location, fp.displayname, fp.content_type, fp.read(), - fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None) + fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None, + import_path = fp.import_path if hasattr(fp, 'import_path') else None) except NoFile: raise NotFoundError() + def export(self, location, output_directory): + content = self.find(location) + + if content.import_path is not None: + output_directory = output_directory + '/' + os.path.dirname(content.import_path) + + if not os.path.exists(output_directory): + os.makedirs(output_directory) + + disk_fs = OSFS(output_directory) + + with disk_fs.open(content.name, 'wb') as asset_file: + asset_file.write(content.data) + + def export_all_for_course(self, course_location, output_directory): + assets = self.get_all_content_for_course(course_location) + + for asset in assets: + asset_location = Location(asset['_id']) + self.export(asset_location, output_directory) + def get_all_content_thumbnails_for_course(self, location): return self._get_all_content_for_course(location, get_thumbnails = True) diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index cae099845a..709f86bf45 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -160,7 +160,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): filepath = u'{category}/{pathname}.html'.format(category=self.category, pathname=pathname) - resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) + resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True) with resource_fs.open(filepath, 'w') as file: file.write(self.definition['data']) diff --git a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py new file mode 100644 index 0000000000..e0bf0ec1d3 --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py @@ -0,0 +1,20 @@ +import logging +from xmodule.modulestore import Location +from xmodule.modulestore.django import modulestore +from fs.osfs import OSFS + +def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir): + + course = modulestore.get_item(course_location) + + fs = OSFS(root_dir) + export_fs = fs.makeopendir(course_dir) + + xml = course.export_to_xml(export_fs) + with export_fs.open('course.xml', 'w') as course_xml: + course_xml.write(xml) + + # export the static assets + contentstore.export_all_for_course(course_location, root_dir + '/' + course_dir + '/static/') + + \ No newline at end of file diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 3f1229b004..35375d7c51 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -32,7 +32,7 @@ def import_static_content(modules, course_loc, course_data_path, static_content_ with open(content_path, 'rb') as f: data = f.read() - content = StaticContent(content_loc, filename, mime_type, data) + content = StaticContent(content_loc, filename, mime_type, data, import_path = fullname_with_subpath) # first let's save a thumbnail so we can get back a thumbnail location thumbnail_content = static_content_store.generate_thumbnail(content) @@ -66,7 +66,7 @@ def verify_content_links(module, base_dir, static_content_store, link, remap_dic with open(static_pathname, 'rb') as f: data = f.read() - content = StaticContent(content_loc, filename, mime_type, data) + content = StaticContent(content_loc, filename, mime_type, data, import_path = path) # first let's save a thumbnail so we can get back a thumbnail location thumbnail_content = static_content_store.generate_thumbnail(content) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index c56803f3c4..38fcaddd20 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -98,20 +98,30 @@ class XmlDescriptor(XModuleDescriptor): metadata_to_strip = ('data_dir', # cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course - 'tabs', 'grading_policy', + 'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date', + 'discussion_blackouts', # VS[compat] -- remove the below attrs once everything is in the CMS 'course', 'org', 'url_name', 'filename') + metadata_to_export_to_policy = ('discussion_topics') + # A dictionary mapping xml attribute names AttrMaps that describe how # to import and export them # Allow json to specify either the string "true", or the bool True. The string is preferred. to_bool = lambda val: val == 'true' or val == True from_bool = lambda val: str(val).lower() bool_map = AttrMap(to_bool, from_bool) + + to_int = lambda val: int(val) + from_int = lambda val: str(val) + int_map = AttrMap(to_int, from_int) xml_attribute_map = { # type conversion: want True/False in python, "true"/"false" in xml 'graded': bool_map, 'hide_progress_tab': bool_map, + 'allow_anonymous': bool_map, + 'allow_anonymous_to_peers': bool_map, + 'weight':int_map } @@ -359,8 +369,9 @@ class XmlDescriptor(XModuleDescriptor): # Add the non-inherited metadata for attr in sorted(self.own_metadata): # don't want e.g. data_dir - if attr not in self.metadata_to_strip: + if attr not in self.metadata_to_strip and attr not in self.metadata_to_export_to_policy: val = val_for_xml(attr) + #logging.debug('location.category = {0}, attr = {1}'.format(self.location.category, attr)) xml_object.set(attr, val) if self.export_to_file(): diff --git a/lms/djangoapps/courseware/courses.py b/lms/djangoapps/courseware/courses.py index 6de309076d..643382b485 100644 --- a/lms/djangoapps/courseware/courses.py +++ b/lms/djangoapps/courseware/courses.py @@ -148,7 +148,7 @@ def get_course_about_section(course, section_key): request = get_request_for_thread() loc = course.location._replace(category='about', name=section_key) - course_module = get_module(request.user, request, loc, None, course.id, not_found_ok = True, wrap_xmodule_display = False) + course_module = get_module(request.user, request, loc, None, course.id, not_found_ok = True, wrap_xmodule_display = True) html = '' @@ -186,7 +186,7 @@ def get_course_info_section(request, cache, course, section_key): loc = Location(course.location.tag, course.location.org, course.location.course, 'course_info', section_key) - course_module = get_module(request.user, request, loc, cache, course.id, wrap_xmodule_display = False) + course_module = get_module(request.user, request, loc, cache, course.id, wrap_xmodule_display = True) html = '' if course_module is not None: diff --git a/rakefile b/rakefile index 0d5ff5cf82..ca20de9a39 100644 --- a/rakefile +++ b/rakefile @@ -445,6 +445,18 @@ namespace :cms do end end +namespace :cms do + desc "Export course data to a tar.gz file" + task :export do + if ENV['COURSE_ID'] and ENV['OUTPUT_PATH'] + sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH'])) + else + raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" + + "Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`" + end + end +end + desc "Build a properties file used to trigger autodeploy builds" task :autodeploy_properties do File.open("autodeploy.properties", "w") do |file|