Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1211 from MITx/feature/cdodge/export

Browse Source 
Feature/cdodge/export
This commit is contained in:
Don Mitchell
2012-12-27 13:38:28 -08:00
parent a0fa8c98bc 507a1dc073
commit bd8661a8b5
16 changed files with 428 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
cms/djangoapps/contentstore/management/commands/export.py Normal file
View File

@@ -0,0 +1,35 @@
###
### Script for exporting courseware from Mongo to a tar.gz file
###
import os
from django.core.management.base import BaseCommand, CommandError
from xmodule.modulestore.xml_exporter import export_to_xml
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.modulestore import Location
from xmodule.course_module import CourseDescriptor
unnamed_modules = 0
class Command(BaseCommand):
help = \
'''Import the specified data directory into the default ModuleStore'''
def handle(self, *args, **options):
if len(args) != 2:
raise CommandError("import requires two arguments: <course location> <output path>")
course_id = args[0]
output_path = args[1]
print "Exporting course id = {0} to {1}".format(course_id, output_path)
location = CourseDescriptor.id_to_location(course_id)
root_dir = os.path.dirname(output_path)
course_dir = os.path.splitext(os.path.basename(output_path))[0]
export_to_xml(modulestore('direct'), contentstore(), location, root_dir, course_dir)

34
cms/djangoapps/contentstore/tests/tests.py
View File

@@ -1,10 +1,12 @@
import json
import shutil
from django.test import TestCase
from django.test.client import Client
from override_settings import override_settings
from django.conf import settings
from django.core.urlresolvers import reverse
from path import path
from tempfile import mkdtemp
from student.models import Registration
from django.contrib.auth.models import User
@@ -18,6 +20,7 @@ from xmodule.modulestore.store_utilities import delete_course
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.course_module import CourseDescriptor
from xmodule.modulestore.xml_exporter import export_to_xml
def parse_json(response):
"""Parse response, which is assumed to be json"""
@@ -385,4 +388,35 @@ class ContentStoreTest(TestCase):
items = ms.get_items(Location(['i4x','edX', 'full', 'vertical', None]))
self.assertEqual(len(items), 0)
def test_export_course(self):
ms = modulestore('direct')
cs = contentstore()
import_from_xml(ms, 'common/test/data/', ['full'])
location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012')
root_dir = path(mkdtemp())
print 'Exporting to tempdir = {0}'.format(root_dir)
# export out to a tempdir
export_to_xml(ms, cs, location, root_dir, 'test_export')
# remove old course
delete_course(ms, cs, location)
# reimport
import_from_xml(ms, root_dir, ['test_export'])
items = ms.get_items(Location(['i4x','edX', 'full', 'vertical', None]))
self.assertGreater(len(items), 0)
for descriptor in items:
print "Checking {0}....".format(descriptor.location.url())
resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()}))
self.assertEqual(resp.status_code, 200)
shutil.rmtree(root_dir)

56
cms/djangoapps/contentstore/views.py
View File

@@ -10,6 +10,10 @@ from datetime import datetime
from collections import defaultdict
from uuid import uuid4
from path import path
from xmodule.modulestore.xml_exporter import export_to_xml
from tempfile import mkdtemp
from django.core.servers.basehttp import FileWrapper
from django.core.files.temp import NamedTemporaryFile
# to install PIL on MacOSX: 'easy_install http://dist.repoze.org/PIL-1.1.6.tar.gz'
from PIL import Image
@@ -1302,3 +1306,55 @@ def import_course(request, org, course, name):
course_module.location.course,
course_module.location.name])
})
@ensure_csrf_cookie
@login_required
def generate_export_course(request, org, course, name):
location = ['i4x', org, course, 'course', name]
course_module = modulestore().get_item(location)
# check that logged in user has permissions to this item
if not has_access(request.user, location):
raise PermissionDenied()
loc = Location(location)
export_file = NamedTemporaryFile(prefix=name+'.', suffix=".tar.gz")
root_dir = path(mkdtemp())
# export out to a tempdir
logging.debug('root = {0}'.format(root_dir))
export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name)
#filename = root_dir / name + '.tar.gz'
logging.debug('tar file being generated at {0}'.format(export_file.name))
tf = tarfile.open(name=export_file.name, mode='w:gz')
tf.add(root_dir/name, arcname=name)
tf.close()
# remove temp dir
shutil.rmtree(root_dir/name)
wrapper = FileWrapper(export_file)
response = HttpResponse(wrapper, content_type='application/x-tgz')
response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(export_file.name)
response['Content-Length'] = os.path.getsize(export_file.name)
return response
@ensure_csrf_cookie
@login_required
def export_course(request, org, course, name):
location = ['i4x', org, course, 'course', name]
course_module = modulestore().get_item(location)
# check that logged in user has permissions to this item
if not has_access(request.user, location):
raise PermissionDenied()
return render_to_response('export.html', {
'context_course': course_module,
'active_tab': 'export',
'successful_import_redirect_url' : ''
})

123
cms/static/sass/_export.scss Normal file
View File

@@ -0,0 +1,123 @@
.export {
.export-overview {
@extend .window;
@include clearfix;
padding: 30px 40px;
}
.description {
float: left;
width: 62%;
margin-right: 3%;
font-size: 14px;
h2 {
font-weight: 700;
font-size: 19px;
margin-bottom: 20px;
}
strong {
font-weight: 700;
}
p + p {
margin-top: 20px;
}
ul {
margin: 20px 0;
list-style: disc inside;
li {
margin: 0 0 5px 0;
}
}
}
.export-form-wrapper {
.export-form {
float: left;
width: 35%;
padding: 25px 30px 35px;
@include box-sizing(border-box);
border: 1px solid $mediumGrey;
border-radius: 3px;
background: $lightGrey;
text-align: center;
h2 {
margin-bottom: 30px;
font-size: 26px;
font-weight: 300;
}
.error-block {
display: none;
margin-bottom: 15px;
font-size: 13px;
}
.error-block {
color: $error-red;
}
.button-export {
@include green-button;
padding: 10px 50px 11px;
font-size: 17px;
}
.message-status {
margin-top: 10px;
font-size: 12px;
}
.progress-bar {
display: none;
width: 350px;
height: 30px;
margin: 30px auto 10px;
border: 1px solid $blue;
&.loaded {
border-color: #66b93d;
.progress-fill {
background: #66b93d;
}
}
}
.progress-fill {
width: 0%;
height: 30px;
background: $blue;
color: #fff;
line-height: 48px;
}
}
// downloading state
&.is-downloading {
.progress-bar {
display: block;
}
.button-export {
padding: 10px 50px 11px;
font-size: 17px;
&.disabled {
pointer-events: none;
cursor: default;
}
}
}
}
}

1
cms/static/sass/base-style.scss
View File

@@ -18,6 +18,7 @@
@import "static-pages";
@import "users";
@import "import";
@import "export";
@import "settings";
@import "course-info";
@import "landing";

93
cms/templates/export.html Normal file
View File

@@ -0,0 +1,93 @@
<%inherit file="base.html" />
<%namespace name='static' file='static_content.html'/>
<%! from django.core.urlresolvers import reverse %>
<%block name="title">Export</%block>
<%block name="bodyclass">export</%block>
<%block name="content">
<div class="main-wrapper">
<div class="inner-wrapper">
<article class="export-overview">
<div class="description">
<h2>About Exporting Courses</h2>
<p>When exporting your course, you will receive a .tar.gz formatted file that contains the following course data:</p>
<ul>
<li>Course Structure (Sections and sub-section ordering)</li>
<li>Individual Units</li>
<li>Individual Problems</li>
<li>Static Pages</li>
<li>Course Assets</li>
</ul>
<p>Your course export <strong>will not include</strong>: student data, forum/discussion data, course settings, certificates, grading information, or user data.</p>
</div>
<!-- default state -->
<div class="export-form-wrapper">
<form action="${reverse('generate_export_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}" method="post" enctype="multipart/form-data" class="export-form">
<h2>Export Course:</h2>
<p class="error-block"></p>
<a href="${reverse('generate_export_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}" class="button-export">Download Files</a>
</form>
</div>
<!-- download state: after user clicks download buttons -->
<%doc>
<div class="export-form-wrapper is-downloading">
<form action="${reverse('export_course', kwargs=dict(org=context_course.location.org, course=context_course.location.course, name=context_course.location.name))}" method="post" enctype="multipart/form-data" class="export-form">
<h2>Export Course:</h2>
<p class="error-block"></p>
<a href="#" class="button-export disabled">Files Downloading</a>
<p class="message-status">Download not start? <a href="#" class="text-export">Try again</a></p>
</form>
</div>
</%doc>
</article>
</div>
</div>
</%block>
<%block name="jsextra">
<script>
(function() {
var bar = $('.progress-bar');
var fill = $('.progress-fill');
var percent = $('.percent');
var status = $('#status');
var submitBtn = $('.submit-button');
$('form').ajaxForm({
beforeSend: function() {
status.empty();
var percentVal = '0%';
bar.show();
fill.width(percentVal);
percent.html(percentVal);
submitBtn.hide();
},
uploadProgress: function(event, position, total, percentComplete) {
var percentVal = percentComplete + '%';
fill.width(percentVal);
percent.html(percentVal);
},
complete: function(xhr) {
if (xhr.status == 200) {
alert('Your import was successful.');
window.location = '${successful_import_redirect_url}';
}
else
alert('Your import has failed.\n\n' + xhr.responseText);
submitBtn.show();
bar.hide();
}
});
})();
</script>
</%block>

1
cms/templates/widgets/header.html
View File

@@ -33,6 +33,7 @@
<li><a href="${reverse('asset_index', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='assets-tab'>Assets</a></li>
<li><a href="${reverse('course_settings', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='settings-tab'>Settings</a></li>
<li><a href="${reverse('import_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='import-tab'>Import</a></li>
<li><a href="${reverse('export_course', kwargs=dict(org=ctx_loc.org, course=ctx_loc.course, name=ctx_loc.name))}" id='export-tab'>Export</a></li>
</ul>
% endif
</nav>

5
cms/urls.py
View File

@@ -23,6 +23,11 @@ urlpatterns = ('',
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/import/(?P<name>[^/]+)$',
'contentstore.views.import_course', name='import_course'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/export/(?P<name>[^/]+)$',
'contentstore.views.export_course', name='export_course'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/generate_export/(?P<name>[^/]+)$',
'contentstore.views.generate_export_course', name='generate_export_course'),
url(r'^preview/modx/(?P<preview_id>[^/]*)/(?P<location>.*?)/(?P<dispatch>[^/]*)$',
'contentstore.views.preview_dispatch', name='preview_dispatch'),
url(r'^(?P<org>[^/]+)/(?P<course>[^/]+)/course/(?P<coursename>[^/]+)/upload_asset$',

5
common/lib/xmodule/xmodule/contentstore/content.py
View File

@@ -12,13 +12,16 @@ from .django import contentstore
from PIL import Image
class StaticContent(object):
def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None):
def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None, import_path=None):
self.location = loc
self.name = name #a display string which can be edited, and thus not part of the location which needs to be fixed
self.content_type = content_type
self.data = data
self.last_modified_at = last_modified_at
self.thumbnail_location = Location(thumbnail_location)
# optional information about where this file was imported from. This is needed to support import/export
# cycles
self.import_path = import_path
@property
def is_thumbnail(self):

28
common/lib/xmodule/xmodule/contentstore/mongo.py
View File

@@ -11,6 +11,8 @@ import logging
from .content import StaticContent, ContentStore
from xmodule.exceptions import NotFoundError
from fs.osfs import OSFS
import os
class MongoContentStore(ContentStore):
@@ -32,7 +34,7 @@ class MongoContentStore(ContentStore):
self.delete(id)
with self.fs.new_file(_id = id, filename=content.get_url_path(), content_type=content.content_type,
displayname=content.name, thumbnail_location=content.thumbnail_location) as fp:
displayname=content.name, thumbnail_location=content.thumbnail_location, import_path=content.import_path) as fp:
fp.write(content.data)
@@ -47,10 +49,32 @@ class MongoContentStore(ContentStore):
try:
with self.fs.get(id) as fp:
return StaticContent(location, fp.displayname, fp.content_type, fp.read(),
fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None)
fp.uploadDate, thumbnail_location = fp.thumbnail_location if hasattr(fp, 'thumbnail_location') else None,
import_path = fp.import_path if hasattr(fp, 'import_path') else None)
except NoFile:
raise NotFoundError()
def export(self, location, output_directory):
content = self.find(location)
if content.import_path is not None:
output_directory = output_directory + '/' + os.path.dirname(content.import_path)
if not os.path.exists(output_directory):
os.makedirs(output_directory)
disk_fs = OSFS(output_directory)
with disk_fs.open(content.name, 'wb') as asset_file:
asset_file.write(content.data)
def export_all_for_course(self, course_location, output_directory):
assets = self.get_all_content_for_course(course_location)
for asset in assets:
asset_location = Location(asset['_id'])
self.export(asset_location, output_directory)
def get_all_content_thumbnails_for_course(self, location):
return self._get_all_content_for_course(location, get_thumbnails = True)

2
common/lib/xmodule/xmodule/html_module.py
View File

@@ -160,7 +160,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
filepath = u'{category}/{pathname}.html'.format(category=self.category,
pathname=pathname)
resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(self.definition['data'])

20
common/lib/xmodule/xmodule/modulestore/xml_exporter.py Normal file
View File

@@ -0,0 +1,20 @@
import logging
from xmodule.modulestore import Location
from xmodule.modulestore.django import modulestore
from fs.osfs import OSFS
def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir):
course = modulestore.get_item(course_location)
fs = OSFS(root_dir)
export_fs = fs.makeopendir(course_dir)
xml = course.export_to_xml(export_fs)
with export_fs.open('course.xml', 'w') as course_xml:
course_xml.write(xml)
# export the static assets
contentstore.export_all_for_course(course_location, root_dir + '/' + course_dir + '/static/')

4
common/lib/xmodule/xmodule/modulestore/xml_importer.py
View File

@@ -32,7 +32,7 @@ def import_static_content(modules, course_loc, course_data_path, static_content_
with open(content_path, 'rb') as f:
data = f.read()
content = StaticContent(content_loc, filename, mime_type, data)
content = StaticContent(content_loc, filename, mime_type, data, import_path = fullname_with_subpath)
# first let's save a thumbnail so we can get back a thumbnail location
thumbnail_content = static_content_store.generate_thumbnail(content)
@@ -66,7 +66,7 @@ def verify_content_links(module, base_dir, static_content_store, link, remap_dic
with open(static_pathname, 'rb') as f:
data = f.read()
content = StaticContent(content_loc, filename, mime_type, data)
content = StaticContent(content_loc, filename, mime_type, data, import_path = path)
# first let's save a thumbnail so we can get back a thumbnail location
thumbnail_content = static_content_store.generate_thumbnail(content)

15
common/lib/xmodule/xmodule/xml_module.py
View File

@@ -98,20 +98,30 @@ class XmlDescriptor(XModuleDescriptor):
metadata_to_strip = ('data_dir',
# cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course
'tabs', 'grading_policy',
'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date',
'discussion_blackouts',
# VS[compat] -- remove the below attrs once everything is in the CMS
'course', 'org', 'url_name', 'filename')
metadata_to_export_to_policy = ('discussion_topics')
# A dictionary mapping xml attribute names AttrMaps that describe how
# to import and export them
# Allow json to specify either the string "true", or the bool True. The string is preferred.
to_bool = lambda val: val == 'true' or val == True
from_bool = lambda val: str(val).lower()
bool_map = AttrMap(to_bool, from_bool)
to_int = lambda val: int(val)
from_int = lambda val: str(val)
int_map = AttrMap(to_int, from_int)
xml_attribute_map = {
# type conversion: want True/False in python, "true"/"false" in xml
'graded': bool_map,
'hide_progress_tab': bool_map,
'allow_anonymous': bool_map,
'allow_anonymous_to_peers': bool_map,
'weight':int_map
}
@@ -359,8 +369,9 @@ class XmlDescriptor(XModuleDescriptor):
# Add the non-inherited metadata
for attr in sorted(self.own_metadata):
# don't want e.g. data_dir
if attr not in self.metadata_to_strip:
if attr not in self.metadata_to_strip and attr not in self.metadata_to_export_to_policy:
val = val_for_xml(attr)
#logging.debug('location.category = {0}, attr = {1}'.format(self.location.category, attr))
xml_object.set(attr, val)
if self.export_to_file():

4
lms/djangoapps/courseware/courses.py
View File

@@ -148,7 +148,7 @@ def get_course_about_section(course, section_key):
request = get_request_for_thread()
loc = course.location._replace(category='about', name=section_key)
course_module = get_module(request.user, request, loc, None, course.id, not_found_ok = True, wrap_xmodule_display = False)
course_module = get_module(request.user, request, loc, None, course.id, not_found_ok = True, wrap_xmodule_display = True)
html = ''
@@ -186,7 +186,7 @@ def get_course_info_section(request, cache, course, section_key):
loc = Location(course.location.tag, course.location.org, course.location.course, 'course_info', section_key)
course_module = get_module(request.user, request, loc, cache, course.id, wrap_xmodule_display = False)
course_module = get_module(request.user, request, loc, cache, course.id, wrap_xmodule_display = True)
html = ''
if course_module is not None:

12
rakefile
View File

@@ -445,6 +445,18 @@ namespace :cms do
end
end
namespace :cms do
desc "Export course data to a tar.gz file"
task :export do
if ENV['COURSE_ID'] and ENV['OUTPUT_PATH']
sh(django_admin(:cms, :dev, :export, ENV['COURSE_ID'], ENV['OUTPUT_PATH']))
else
raise "Please specify a COURSE_ID and OUTPUT_PATH.\n" +
"Example: \`rake cms:export COURSE_ID=MITx/12345/name OUTPUT_PATH=foo.tar.gz\`"
end
end
end
desc "Build a properties file used to trigger autodeploy builds"
task :autodeploy_properties do
File.open("autodeploy.properties", "w") do |file|