From d1a82f2ac5fa32de531f1e0750ec0f96ca7ba91c Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Fri, 20 Sep 2013 15:31:50 -0400 Subject: [PATCH 1/3] Import/export static content properties Even though xml based courses will ignore. --- common/lib/xmodule/xmodule/contentstore/mongo.py | 10 +++++++++- .../xmodule/xmodule/modulestore/xml_exporter.py | 8 ++++++-- .../xmodule/xmodule/modulestore/xml_importer.py | 16 ++++++++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index 40c0b4bc9f..0322b394ca 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -12,6 +12,7 @@ from .content import StaticContent, ContentStore, StaticContentStream from xmodule.exceptions import NotFoundError from fs.osfs import OSFS import os +import json class MongoContentStore(ContentStore): @@ -103,12 +104,19 @@ class MongoContentStore(ContentStore): with disk_fs.open(content.name, 'wb') as asset_file: asset_file.write(content.data) - def export_all_for_course(self, course_location, output_directory): + def export_all_for_course(self, course_location, output_directory, policy_file): + policy = {} assets = self.get_all_content_for_course(course_location) for asset in assets: asset_location = Location(asset['_id']) self.export(asset_location, output_directory) + for attr, value in asset.iteritems(): + if attr not in ['_id', 'md5', 'uploadDate', 'length', 'chunkSize']: + policy.setdefault(asset_location.url(), {})[attr] = value + + with open(policy_file, 'w') as f: + json.dump(policy, f) def get_all_content_thumbnails_for_course(self, location): return self._get_all_content_for_course(location, get_thumbnails=True) diff --git a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py index c9d6e96761..a54d188b61 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py @@ -55,8 +55,13 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d with export_fs.open('course.xml', 'w') as course_xml: course_xml.write(xml) + policies_dir = export_fs.makeopendir('policies') # export the static assets - contentstore.export_all_for_course(course_location, root_dir + '/' + course_dir + '/static/') + contentstore.export_all_for_course( + course_location, + root_dir + '/' + course_dir + '/static/', + root_dir + '/' + course_dir + '/policies/assets.json', + ) # export the static tabs export_extra_content(export_fs, modulestore, course_location, 'static_tab', 'tabs', '.html') @@ -71,7 +76,6 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d export_extra_content(export_fs, modulestore, course_location, 'about', 'about', '.html') # export the grading policy - policies_dir = export_fs.makeopendir('policies') course_run_policy_dir = policies_dir.makeopendir(course.location.name) with course_run_policy_dir.open('grading_policy.json', 'w') as grading_policy: grading_policy.write(dumps(course.grading_policy, cls=EdxJSONEncoder)) diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 632619e9f7..ad3cbce9df 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -2,6 +2,7 @@ import logging import os import mimetypes from path import path +import json from xblock.fields import Scope @@ -22,6 +23,11 @@ def import_static_content(modules, course_loc, course_data_path, static_content_ # now import all static assets static_dir = course_data_path / subpath + try: + with open(course_data_path / 'policies/assets.json') as f: + policy = json.load(f) + except (IOError, ValueError) as err: + policy = {} verbose = True @@ -46,10 +52,16 @@ def import_static_content(modules, course_loc, course_data_path, static_content_ if fullname_with_subpath.startswith('/'): fullname_with_subpath = fullname_with_subpath[1:] content_loc = StaticContent.compute_location(target_location_namespace.org, target_location_namespace.course, fullname_with_subpath) - mime_type = mimetypes.guess_type(filename)[0] - content = StaticContent(content_loc, filename, mime_type, data, import_path=fullname_with_subpath) + policy_ele = policy.get(content_loc.url(), {}) + displayname = policy_ele.get('displayname', filename) + locked = policy_ele.get('locked', False) + mime_type = policy_ele.get('contentType', mimetypes.guess_type(filename)[0]) + content = StaticContent( + content_loc, displayname, mime_type, data, + import_path=fullname_with_subpath, locked=locked + ) # first let's save a thumbnail so we can get back a thumbnail location (thumbnail_content, thumbnail_location) = static_content_store.generate_thumbnail(content) From 8e7fc1537fc5df134dcd6ab538bf1ea3c3556725 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Mon, 23 Sep 2013 11:19:59 -0400 Subject: [PATCH 2/3] Test asset import/export roundtrip w/ lock setting. --- .../contentstore/tests/test_contentstore.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 32928bb2fd..6f40ac0e53 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -170,6 +170,16 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()})) self.assertEqual(resp.status_code, 200) + def lockAnAsset(self, content_store, course_location): + """ + Lock an arbitrary asset in the course + :param course_location: + """ + course_assets = content_store.get_all_content_for_course(course_location) + self.assertGreater(len(course_assets), 0, "No assets to lock") + content_store.set_attr(course_assets[0]['_id'], 'locked', True) + return course_assets[0]['_id'] + def test_edit_unit_toy(self): self.check_edit_unit('toy') @@ -952,6 +962,11 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.assertIn(private_location_no_draft.url(), sequential.children) + locked_asset = self.lockAnAsset(content_store, location) + locked_asset_attrs = content_store.get_attrs(locked_asset) + # the later import will reupload + del locked_asset_attrs['uploadDate'] + print 'Exporting to tempdir = {0}'.format(root_dir) # export out to a tempdir @@ -1034,6 +1049,10 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.assertGreater(len(course.textbooks), 0) + new_attrs = content_store.get_attrs(locked_asset) + for key, value in locked_asset_attrs.iteritems(): + self.assertEqual(value, new_attrs[key]) + shutil.rmtree(root_dir) def test_export_course_with_metadata_only_video(self): From 20e4d585c85361d6742cc40a9a84ec9afbaf5683 Mon Sep 17 00:00:00 2001 From: Don Mitchell Date: Tue, 24 Sep 2013 16:30:24 -0400 Subject: [PATCH 3/3] Add documentation and comments. --- common/lib/xmodule/xmodule/contentstore/mongo.py | 13 +++++++++++-- .../lib/xmodule/xmodule/modulestore/xml_importer.py | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index 0322b394ca..dcc58b3431 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -104,7 +104,16 @@ class MongoContentStore(ContentStore): with disk_fs.open(content.name, 'wb') as asset_file: asset_file.write(content.data) - def export_all_for_course(self, course_location, output_directory, policy_file): + def export_all_for_course(self, course_location, output_directory, assets_policy_file): + """ + Export all of this course's assets to the output_directory. Export all of the assets' + attributes to the policy file. + + :param course_location: the Location of type 'course' + :param output_directory: the directory under which to put all the asset files + :param assets_policy_file: the filename for the policy file which should be in the same + directory as the other policy files. + """ policy = {} assets = self.get_all_content_for_course(course_location) @@ -115,7 +124,7 @@ class MongoContentStore(ContentStore): if attr not in ['_id', 'md5', 'uploadDate', 'length', 'chunkSize']: policy.setdefault(asset_location.url(), {})[attr] = value - with open(policy_file, 'w') as f: + with open(assets_policy_file, 'w') as f: json.dump(policy, f) def get_all_content_thumbnails_for_course(self, location): diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index ad3cbce9df..64caad0481 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -27,6 +27,8 @@ def import_static_content(modules, course_loc, course_data_path, static_content_ with open(course_data_path / 'policies/assets.json') as f: policy = json.load(f) except (IOError, ValueError) as err: + # xml backed courses won't have this file, only exported courses; so, its absence is not + # really an exception. policy = {} verbose = True