diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 451ab96ca6..ef6ab71b88 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -11,6 +11,7 @@ import json from fs.osfs import OSFS import copy from json import loads +import traceback from django.contrib.auth.models import User from contentstore.utils import get_modulestore @@ -215,13 +216,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): module_store = modulestore('direct') found = False - item = None items = module_store.get_items(['i4x', 'edX', 'full', 'poll_question', None, None]) found = len(items) > 0 self.assertTrue(found) # check that there's actually content in the 'question' field - self.assertGreater(len(items[0].question),0) + self.assertGreater(len(items[0].question), 0) def test_xlint_fails(self): err_cnt = perform_xlint('common/test/data', ['full']) @@ -234,14 +234,14 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): sequential = module_store.get_item(Location(['i4x', 'edX', 'full', 'sequential', 'Administrivia_and_Circuit_Elements', None])) - chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None])) + chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None])) # make sure the parent no longer points to the child object which was deleted self.assertTrue(sequential.location.url() in chapter.children) self.client.post(reverse('delete_item'), json.dumps({'id': sequential.location.url(), 'delete_children': 'true', 'delete_all_versions': 'true'}), - "application/json") + "application/json") found = False try: @@ -252,7 +252,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): self.assertFalse(found) - chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter','Week_1', None])) + chapter = module_store.get_item(Location(['i4x', 'edX', 'full', 'chapter', 'Week_1', None])) # make sure the parent no longer points to the child object which was deleted self.assertFalse(sequential.location.url() in chapter.children) @@ -275,7 +275,6 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): import_from_xml(modulestore(), 'common/test/data/', ['full']) module_store = modulestore('direct') - content_store = contentstore() source_location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012') course = module_store.get_item(source_location) @@ -288,7 +287,7 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): 'org': 'MITx', 'number': '999', 'display_name': 'Robot Super Course', - } + } import_from_xml(modulestore(), 'common/test/data/', ['full']) @@ -347,17 +346,44 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): def test_export_course(self): module_store = modulestore('direct') + draft_store = modulestore('draft') content_store = contentstore() import_from_xml(module_store, 'common/test/data/', ['full']) location = CourseDescriptor.id_to_location('edX/full/6.002_Spring_2012') + # get a vertical (and components in it) to put into 'draft' + vertical = module_store.get_item(Location(['i4x', 'edX', 'full', + 'vertical', 'vertical_66', None]), depth=1) + + draft_store.clone_item(vertical.location, vertical.location) + + for child in vertical.get_children(): + draft_store.clone_item(child.location, child.location) + root_dir = path(mkdtemp_clean()) + # now create a private vertical + private_vertical = draft_store.clone_item(vertical.location, + Location(['i4x', 'edX', 'full', 'vertical', 'a_private_vertical', None])) + + # add private to list of children + sequential = module_store.get_item(Location(['i4x', 'edX', 'full', + 'sequential', 'Administrivia_and_Circuit_Elements', None])) + private_location_no_draft = private_vertical.location._replace(revision=None) + module_store.update_children(sequential.location, sequential.children + + [private_location_no_draft.url()]) + + # read back the sequential, to make sure we have a pointer to + sequential = module_store.get_item(Location(['i4x', 'edX', 'full', + 'sequential', 'Administrivia_and_Circuit_Elements', None])) + + self.assertIn(private_location_no_draft.url(), sequential.children) + print 'Exporting to tempdir = {0}'.format(root_dir) # export out to a tempdir - export_to_xml(module_store, content_store, location, root_dir, 'test_export') + export_to_xml(module_store, content_store, location, root_dir, 'test_export', draft_modulestore=draft_store) # check for static tabs self.verify_content_existence(module_store, root_dir, location, 'tabs', 'static_tab', '.html') @@ -391,20 +417,36 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): delete_course(module_store, content_store, location) # reimport - import_from_xml(module_store, root_dir, ['test_export']) + import_from_xml(module_store, root_dir, ['test_export'], draft_store=draft_store) items = module_store.get_items(Location(['i4x', 'edX', 'full', 'vertical', None])) self.assertGreater(len(items), 0) for descriptor in items: - print "Checking {0}....".format(descriptor.location.url()) - resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()})) - self.assertEqual(resp.status_code, 200) + # don't try to look at private verticals. Right now we're running + # the service in non-draft aware + if getattr(descriptor, 'is_draft', False): + print "Checking {0}....".format(descriptor.location.url()) + resp = self.client.get(reverse('edit_unit', kwargs={'location': descriptor.location.url()})) + self.assertEqual(resp.status_code, 200) + + # verify that we have the content in the draft store as well + vertical = draft_store.get_item(Location(['i4x', 'edX', 'full', + 'vertical', 'vertical_66', None]), depth=1) + + self.assertTrue(getattr(vertical, 'is_draft', False)) + for child in vertical.get_children(): + self.assertTrue(getattr(child, 'is_draft', False)) + + # verify that we have the private vertical + test_private_vertical = draft_store.get_item(Location(['i4x', 'edX', 'full', + 'vertical', 'vertical_66', None])) + + self.assertTrue(getattr(test_private_vertical, 'is_draft', False)) shutil.rmtree(root_dir) def test_course_handouts_rewrites(self): module_store = modulestore('direct') - content_store = contentstore() # import a test course import_from_xml(module_store, 'common/test/data/', ['full']) @@ -437,11 +479,11 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): # make sure we pre-fetched a known sequential which should be at depth=2 self.assertTrue(Location(['i4x', 'edX', 'full', 'sequential', - 'Administrivia_and_Circuit_Elements', None]) in course.system.module_data) + 'Administrivia_and_Circuit_Elements', None]) in course.system.module_data) # make sure we don't have a specific vertical which should be at depth=3 - self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58', - None]) in course.system.module_data) + self.assertFalse(Location(['i4x', 'edX', 'full', 'vertical', 'vertical_58', None]) + in course.system.module_data) def test_export_course_with_unknown_metadata(self): module_store = modulestore('direct') @@ -468,10 +510,12 @@ class ContentStoreToyCourseTest(ModuleStoreTestCase): export_to_xml(module_store, content_store, location, root_dir, 'test_export') exported = True except Exception: + print 'Exception thrown: {0}'.format(traceback.format_exc()) pass self.assertTrue(exported) + class ContentStoreTest(ModuleStoreTestCase): """ Tests for the CMS ContentStore application. @@ -506,7 +550,7 @@ class ContentStoreTest(ModuleStoreTestCase): 'org': 'MITx', 'number': '999', 'display_name': 'Robot Super Course', - } + } def test_create_course(self): """Test new course creation - happy path""" @@ -533,7 +577,7 @@ class ContentStoreTest(ModuleStoreTestCase): self.assertEqual(resp.status_code, 200) self.assertEqual(data['ErrMsg'], - 'There is already a course defined with the same organization and course number.') + 'There is already a course defined with the same organization and course number.') def test_create_course_with_bad_organization(self): """Test new course creation - error path for bad organization name""" @@ -543,7 +587,7 @@ class ContentStoreTest(ModuleStoreTestCase): self.assertEqual(resp.status_code, 200) self.assertEqual(data['ErrMsg'], - "Unable to create course 'Robot Super Course'.\n\nInvalid characters in 'University of California, Berkeley'.") + "Unable to create course 'Robot Super Course'.\n\nInvalid characters in 'University of California, Berkeley'.") def test_course_index_view_with_no_courses(self): """Test viewing the index page with no courses""" @@ -579,10 +623,10 @@ class ContentStoreTest(ModuleStoreTestCase): CourseFactory.create(org='MITx', course='999', display_name='Robot Super Course') data = { - 'org': 'MITx', - 'course': '999', - 'name': Location.clean('Robot Super Course'), - } + 'org': 'MITx', + 'course': '999', + 'name': Location.clean('Robot Super Course'), + } resp = self.client.get(reverse('course_index', kwargs=data)) self.assertContains(resp, @@ -598,7 +642,7 @@ class ContentStoreTest(ModuleStoreTestCase): 'parent_location': 'i4x://MITx/999/course/Robot_Super_Course', 'template': 'i4x://edx/templates/chapter/Empty', 'display_name': 'Section One', - } + } resp = self.client.post(reverse('clone_item'), section_data) @@ -614,7 +658,7 @@ class ContentStoreTest(ModuleStoreTestCase): problem_data = { 'parent_location': 'i4x://MITx/999/course/Robot_Super_Course', 'template': 'i4x://edx/templates/problem/Blank_Common_Problem' - } + } resp = self.client.post(reverse('clone_item'), problem_data) diff --git a/cms/djangoapps/contentstore/views.py b/cms/djangoapps/contentstore/views.py index 227379979e..3169b437ed 100644 --- a/cms/djangoapps/contentstore/views.py +++ b/cms/djangoapps/contentstore/views.py @@ -1586,7 +1586,8 @@ def import_course(request, org, course, name): shutil.move(r / fname, course_dir) module_store, course_items = import_from_xml(modulestore('direct'), settings.GITHUB_REPO_ROOT, - [course_subdir], load_error_modules=False, static_content_store=contentstore(), target_location_namespace=Location(location)) + [course_subdir], load_error_modules=False, static_content_store=contentstore(), + target_location_namespace=Location(location), draft_store=modulestore()) # we can blow this away when we're done importing. shutil.rmtree(course_dir) @@ -1620,8 +1621,8 @@ def generate_export_course(request, org, course, name): logging.debug('root = {0}'.format(root_dir)) - export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name) - # filename = root_dir / name + '.tar.gz' + export_to_xml(modulestore('direct'), contentstore(), loc, root_dir, name, modulestore()) + #filename = root_dir / name + '.tar.gz' logging.debug('tar file being generated at {0}'.format(export_file.name)) tf = tarfile.open(name=export_file.name, mode='w:gz') diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index e9cec32e3e..d901fc5fbe 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -118,8 +118,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor): with system.resources_fs.open(filepath) as file: html = file.read().decode('utf-8') # Log a warning if we can't parse the file, but don't error - if not check_html(html): - msg = "Couldn't parse html in {0}.".format(filepath) + if not check_html(html) and len(html) > 0: + msg = "Couldn't parse html in {0}, content = {1}".format(filepath, html) log.warning(msg) system.error_tracker("Warning: " + msg) @@ -156,7 +156,8 @@ class HtmlDescriptor(HtmlFields, XmlDescriptor, EditingDescriptor): resource_fs.makedir(os.path.dirname(filepath), recursive=True, allow_recreate=True) with resource_fs.open(filepath, 'w') as file: - file.write(self.data.encode('utf-8')) + html_data = self.data.encode('utf-8') + file.write(html_data) # write out the relative name relname = path(pathname).basename() diff --git a/common/lib/xmodule/xmodule/modulestore/draft.py b/common/lib/xmodule/xmodule/modulestore/draft.py index 3682dea55a..43eb050129 100644 --- a/common/lib/xmodule/xmodule/modulestore/draft.py +++ b/common/lib/xmodule/xmodule/modulestore/draft.py @@ -3,7 +3,6 @@ from datetime import datetime from . import ModuleStoreBase, Location, namedtuple_to_son from .exceptions import ItemNotFoundError from .inheritance import own_metadata -import logging DRAFT = 'draft' @@ -107,7 +106,7 @@ class DraftModuleStore(ModuleStoreBase): """ return wrap_draft(super(DraftModuleStore, self).clone_item(source, as_draft(location))) - def update_item(self, location, data): + def update_item(self, location, data, allow_not_found=False): """ Set the data in the item specified by the location to data @@ -116,9 +115,13 @@ class DraftModuleStore(ModuleStoreBase): data: A nested dictionary of problem data """ draft_loc = as_draft(location) - draft_item = self.get_item(location) - if not getattr(draft_item, 'is_draft', False): - self.clone_item(location, draft_loc) + try: + draft_item = self.get_item(location) + if not getattr(draft_item, 'is_draft', False): + self.clone_item(location, draft_loc) + except ItemNotFoundError, e: + if not allow_not_found: + raise e return super(DraftModuleStore, self).update_item(draft_loc, data) @@ -164,7 +167,6 @@ class DraftModuleStore(ModuleStoreBase): """ return super(DraftModuleStore, self).delete_item(as_draft(location)) - def get_parent_locations(self, location, course_id): '''Find all locations that are the parents of this location. Needed for path_to_location(). @@ -178,6 +180,7 @@ class DraftModuleStore(ModuleStoreBase): Save a current draft to the underlying modulestore """ draft = self.get_item(location) + draft.cms.published_date = datetime.utcnow() draft.cms.published_by = published_by_id super(DraftModuleStore, self).update_item(location, draft._model_data._kvs._data) @@ -221,6 +224,6 @@ class DraftModuleStore(ModuleStoreBase): # convert the dict - which is used for look ups - back into a list for key, value in to_process_dict.iteritems(): - queried_children.append(value) + queried_children.append(value) return queried_children diff --git a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py index 0724211ed3..e03c61bb24 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_exporter.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_exporter.py @@ -1,12 +1,11 @@ import logging from xmodule.modulestore import Location -from xmodule.modulestore.django import modulestore from xmodule.modulestore.inheritance import own_metadata from fs.osfs import OSFS from json import dumps -def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir): +def export_to_xml(modulestore, contentstore, course_location, root_dir, course_dir, draft_modulestore=None): course = modulestore.get_item(course_location) @@ -40,6 +39,24 @@ def export_to_xml(modulestore, contentstore, course_location, root_dir, course_d policy = {'course/' + course.location.name: own_metadata(course)} course_policy.write(dumps(policy)) + # export draft content + # NOTE: this code assumes that verticals are the top most draftable container + # should we change the application, then this assumption will no longer + # be valid + if draft_modulestore is not None: + draft_verticals = draft_modulestore.get_items([None, course_location.org, course_location.course, + 'vertical', None, 'draft']) + if len(draft_verticals) > 0: + draft_course_dir = export_fs.makeopendir('drafts') + for draft_vertical in draft_verticals: + parent_locs = draft_modulestore.get_parent_locations(draft_vertical.location, course.location.course_id) + logging.debug('parent_locs = {0}'.format(parent_locs)) + draft_vertical.xml_attributes['parent_sequential_url'] = Location(parent_locs[0]).url() + sequential = modulestore.get_item(Location(parent_locs[0])) + index = sequential.children.index(draft_vertical.location.url()) + draft_vertical.xml_attributes['index_in_children_list'] = str(index) + draft_vertical.export_to_xml(draft_course_dir) + def export_extra_content(export_fs, modulestore, course_location, category_type, dirname, file_suffix=''): query_loc = Location('i4x', course_location.org, course_location.course, category_type, None) diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 97b3396baa..2dea75ef57 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -6,17 +6,17 @@ from path import path from xblock.core import Scope -from .xml import XMLModuleStore -from .exceptions import DuplicateItemError +from .xml import XMLModuleStore, ImportSystem, ParentTracker from xmodule.modulestore import Location -from xmodule.contentstore.content import StaticContent, XASSET_SRCREF_PREFIX +from xmodule.contentstore.content import StaticContent from .inheritance import own_metadata +from xmodule.errortracker import make_error_tracker log = logging.getLogger(__name__) def import_static_content(modules, course_loc, course_data_path, static_content_store, target_location_namespace, - subpath='static', verbose=False): + subpath='static', verbose=False): remap_dict = {} @@ -107,10 +107,10 @@ def import_module_from_xml(modulestore, static_content_store, course_data_path, # the caller passed in if module.location.category != 'course': module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, - course=target_location_namespace.course) + course=target_location_namespace.course) else: module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, - course=target_location_namespace.course, name=target_location_namespace.name) + course=target_location_namespace.course, name=target_location_namespace.name) # then remap children pointers since they too will be re-namespaced if module.has_children: @@ -119,7 +119,7 @@ def import_module_from_xml(modulestore, static_content_store, course_data_path, for child in children_locs: child_loc = Location(child) new_child_loc = child_loc._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, - course=target_location_namespace.course) + course=target_location_namespace.course) new_locs.append(new_child_loc.url()) @@ -139,8 +139,7 @@ def import_module_from_xml(modulestore, static_content_store, course_data_path, # Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's # no good, so we have to do this kludge if isinstance(module.data, str) or isinstance(module.data, unicode): # some module 'data' fields are non strings which blows up the link traversal code - lxml_rewrite_links(module.data, lambda link: verify_content_links(module, course_data_path, - static_content_store, link, remap_dict)) + lxml_rewrite_links(module.data, lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict)) for key in remap_dict.keys(): module.data = module.data.replace(key, remap_dict[key]) @@ -163,9 +162,9 @@ def import_course_from_xml(modulestore, static_content_store, course_data_path, # if there is *any* tabs - then there at least needs to be some predefined ones if module.tabs is None or len(module.tabs) == 0: module.tabs = [{"type": "courseware"}, - {"type": "course_info", "name": "Course Info"}, - {"type": "discussion", "name": "Discussion"}, - {"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge + {"type": "course_info", "name": "Course Info"}, + {"type": "discussion", "name": "Discussion"}, + {"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge # a bit of a hack, but typically the "course image" which is shown on marketing pages is hard coded to /images/course_image.jpg # so let's make sure we import in case there are no other references to it in the modules @@ -175,7 +174,8 @@ def import_course_from_xml(modulestore, static_content_store, course_data_path, def import_from_xml(store, data_dir, course_dirs=None, default_class='xmodule.raw_module.RawDescriptor', - load_error_modules=True, static_content_store=None, target_location_namespace=None, verbose=False): + load_error_modules=True, static_content_store=None, target_location_namespace=None, + verbose=False, draft_store=None): """ Import the specified xml data_dir into the "store" modulestore, using org and course as the location org and course. @@ -190,7 +190,7 @@ def import_from_xml(store, data_dir, course_dirs=None, """ - module_store = XMLModuleStore( + xml_module_store = XMLModuleStore( data_dir, default_class=default_class, course_dirs=course_dirs, @@ -201,7 +201,7 @@ def import_from_xml(store, data_dir, course_dirs=None, # to enumerate the entire collection of course modules. It will be left as a TBD to implement that # method on XmlModuleStore. course_items = [] - for course_id in module_store.modules.keys(): + for course_id in xml_module_store.modules.keys(): if target_location_namespace is not None: pseudo_course_id = '/'.join([target_location_namespace.org, target_location_namespace.course]) @@ -222,7 +222,7 @@ def import_from_xml(store, data_dir, course_dirs=None, # Quick scan to get course module as we need some info from there. Also we need to make sure that the # course module is committed first into the store - for module in module_store.modules[course_id].itervalues(): + for module in xml_module_store.modules[course_id].itervalues(): if module.category == 'course': course_data_path = path(data_dir) / module.data_dir course_location = module.location @@ -235,15 +235,11 @@ def import_from_xml(store, data_dir, course_dirs=None, # if there is *any* tabs - then there at least needs to be some predefined ones if module.tabs is None or len(module.tabs) == 0: module.tabs = [{"type": "courseware"}, - {"type": "course_info", "name": "Course Info"}, - {"type": "discussion", "name": "Discussion"}, - {"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge + {"type": "course_info", "name": "Course Info"}, + {"type": "discussion", "name": "Discussion"}, + {"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge - - if hasattr(module, 'data'): - store.update_item(module.location, module.data) - store.update_children(module.location, module.children) - store.update_metadata(module.location, dict(own_metadata(module))) + import_module(module, store, course_data_path, static_content_store) # a bit of a hack, but typically the "course image" which is shown on marketing pages is hard coded to /images/course_image.jpg # so let's make sure we import in case there are no other references to it in the modules @@ -251,17 +247,16 @@ def import_from_xml(store, data_dir, course_dirs=None, course_items.append(module) - # then import all the static content if static_content_store is not None: _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location # first pass to find everything in /static/ - import_static_content(module_store.modules[course_id], course_location, course_data_path, static_content_store, - _namespace_rename, subpath='static', verbose=verbose) + import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store, + _namespace_rename, subpath='static', verbose=verbose) # finally loop through all the modules - for module in module_store.modules[course_id].itervalues(): + for module in xml_module_store.modules[course_id].itervalues(): if module.category == 'course': # we've already saved the course module up at the top of the loop @@ -275,59 +270,149 @@ def import_from_xml(store, data_dir, course_dirs=None, if verbose: log.debug('importing module location {0}'.format(module.location)) - content = {} - for field in module.fields: - if field.scope != Scope.content: - continue - try: - content[field.name] = module._model_data[field.name] - except KeyError: - # Ignore any missing keys in _model_data - pass + import_module(module, store, course_data_path, static_content_store) - if 'data' in content: - module_data = content['data'] + # now import any 'draft' items + if draft_store is not None: + import_course_draft(xml_module_store, draft_store, course_data_path, + static_content_store, target_location_namespace if target_location_namespace is not None + else course_location) - # cdodge: now go through any link references to '/static/' and make sure we've imported - # it as a StaticContent asset - try: - remap_dict = {} - - # use the rewrite_links as a utility means to enumerate through all links - # in the module data. We use that to load that reference into our asset store - # IMPORTANT: There appears to be a bug in lxml.rewrite_link which makes us not be able to - # do the rewrites natively in that code. - # For example, what I'm seeing is -> - # Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's - # no good, so we have to do this kludge - if isinstance(module_data, str) or isinstance(module_data, unicode): # some module 'data' fields are non strings which blows up the link traversal code - lxml_rewrite_links(module_data, - lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict)) - - for key in remap_dict.keys(): - module_data = module_data.replace(key, remap_dict[key]) - - except Exception: - logging.exception("failed to rewrite links on {0}. Continuing...".format(module.location)) - else: - module_data = content - - store.update_item(module.location, module_data) - - if hasattr(module, 'children') and module.children != []: - store.update_children(module.location, module.children) - - # NOTE: It's important to use own_metadata here to avoid writing - # inherited metadata everywhere. - store.update_metadata(module.location, dict(own_metadata(module))) finally: # turn back on all write signalling - if pseudo_course_id in store.ignore_write_events_on_courses: + if pseudo_course_id in store.ignore_write_events_on_courses: store.ignore_write_events_on_courses.remove(pseudo_course_id) - store.refresh_cached_metadata_inheritance_tree(target_location_namespace if - target_location_namespace is not None else course_location) + store.refresh_cached_metadata_inheritance_tree(target_location_namespace if + target_location_namespace is not None else course_location) + + return xml_module_store, course_items + + +def import_module(module, store, course_data_path, static_content_store, allow_not_found=False): + content = {} + for field in module.fields: + if field.scope != Scope.content: + continue + try: + content[field.name] = module._model_data[field.name] + except KeyError: + # Ignore any missing keys in _model_data + pass + + module_data = {} + if 'data' in content: + module_data = content['data'] + + # cdodge: now go through any link references to '/static/' and make sure we've imported + # it as a StaticContent asset + try: + remap_dict = {} + + # use the rewrite_links as a utility means to enumerate through all links + # in the module data. We use that to load that reference into our asset store + # IMPORTANT: There appears to be a bug in lxml.rewrite_link which makes us not be able to + # do the rewrites natively in that code. + # For example, what I'm seeing is -> + # Note the dropped element closing tag. This causes the LMS to fail when rendering modules - that's + # no good, so we have to do this kludge + if isinstance(module_data, str) or isinstance(module_data, unicode): # some module 'data' fields are non strings which blows up the link traversal code + lxml_rewrite_links(module_data, lambda link: verify_content_links(module, course_data_path, static_content_store, link, remap_dict)) + + for key in remap_dict.keys(): + module_data = module_data.replace(key, remap_dict[key]) + + except Exception: + logging.exception("failed to rewrite links on {0}. Continuing...".format(module.location)) + else: + module_data = content + + if allow_not_found: + store.update_item(module.location, module_data, allow_not_found=allow_not_found) + else: + store.update_item(module.location, module_data) + + if hasattr(module, 'children') and module.children != []: + store.update_children(module.location, module.children) + + # NOTE: It's important to use own_metadata here to avoid writing + # inherited metadata everywhere. + store.update_metadata(module.location, dict(own_metadata(module))) + + +def import_course_draft(xml_module_store, store, course_data_path, static_content_store, target_location_namespace): + ''' + This will import all the content inside of the 'drafts' folder, if it exists + NOTE: This is not a full course import, basically in our current application only verticals (and downwards) + can be in draft. Therefore, we need to use slightly different call points into the import process_xml + as we can't simply call XMLModuleStore() constructor (like we do for importing public content) + ''' + draft_dir = course_data_path + "/drafts" + if not os.path.exists(draft_dir): + return + + # create a new 'System' object which will manage the importing + errorlog = make_error_tracker() + system = ImportSystem( + xml_module_store, + target_location_namespace.course_id, + draft_dir, + {}, + errorlog.tracker, + ParentTracker(), + None, + ) + + # now walk the /vertical directory where each file in there will be a draft copy of the Vertical + for dirname, dirnames, filenames in os.walk(draft_dir + "/vertical"): + for filename in filenames: + module_path = os.path.join(dirname, filename) + with open(module_path) as f: + try: + xml = f.read().decode('utf-8') + descriptor = system.process_xml(xml) + + def _import_module(module): + module.location = module.location._replace(revision='draft') + # make sure our parent has us in its list of children + # this is to make sure private only verticals show up in the list of children since + # they would have been filtered out from the non-draft store export + if module.location.category == 'vertical': + module.location = module.location._replace(revision=None) + sequential_url = module.xml_attributes['parent_sequential_url'] + index = int(module.xml_attributes['index_in_children_list']) + + seq_location = Location(sequential_url) + + # IMPORTANT: Be sure to update the sequential in the NEW namespace + seq_location = seq_location._replace(org=target_location_namespace.org, + course=target_location_namespace.course + ) + sequential = store.get_item(seq_location) + + if module.location.url() not in sequential.children: + sequential.children.insert(index, module.location.url()) + store.update_children(sequential.location, sequential.children) + + del module.xml_attributes['parent_sequential_url'] + del module.xml_attributes['index_in_children_list'] + + import_module(module, store, course_data_path, static_content_store, allow_not_found=True) + for child in module.get_children(): + _import_module(child) + + # HACK: since we are doing partial imports of drafts + # the vertical doesn't have the 'url-name' set in the attributes (they are normally in the parent + # object, aka sequential), so we have to replace the location.name with the XML filename + # that is part of the pack + fn, fileExtension = os.path.splitext(filename) + descriptor.location = descriptor.location._replace(name=fn) + + _import_module(descriptor) + + except Exception, e: + logging.exception('There was an error. {0}'.format(unicode(e))) + pass - return module_store, course_items def remap_namespace(module, target_location_namespace): if target_location_namespace is None: @@ -337,20 +422,20 @@ def remap_namespace(module, target_location_namespace): # the caller passed in if module.location.category != 'course': module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, - course=target_location_namespace.course) + course=target_location_namespace.course) else: module.location = module.location._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, - course=target_location_namespace.course, name=target_location_namespace.name) + course=target_location_namespace.course, name=target_location_namespace.name) # then remap children pointers since they too will be re-namespaced - if hasattr(module,'children'): + if hasattr(module, 'children'): children_locs = module.children if children_locs is not None and children_locs != []: new_locs = [] for child in children_locs: child_loc = Location(child) new_child_loc = child_loc._replace(tag=target_location_namespace.tag, org=target_location_namespace.org, - course=target_location_namespace.course) + course=target_location_namespace.course) new_locs.append(new_child_loc.url()) @@ -365,7 +450,7 @@ def allowed_metadata_by_category(category): 'vertical': [], 'chapter': ['start'], 'sequential': ['due', 'format', 'start', 'graded'] - }.get(category,['*']) + }.get(category, ['*']) def check_module_metadata_editability(module): @@ -380,7 +465,6 @@ def check_module_metadata_editability(module): allowed = allowed + ['xml_attributes', 'display_name'] err_cnt = 0 - my_metadata = dict(own_metadata(module)) illegal_keys = set(own_metadata(module).keys()) - set(allowed) if len(illegal_keys) > 0: @@ -423,7 +507,7 @@ def validate_data_source_path_existence(path, is_err=True, extra_msg=None): _cnt = 0 if not os.path.exists(path): print ("{0}: Expected folder at {1}. {2}".format('ERROR' if is_err == True else 'WARNING', path, extra_msg if - extra_msg is not None else '')) + extra_msg is not None else '')) _cnt = 1 return _cnt @@ -435,13 +519,13 @@ def validate_data_source_paths(data_dir, course_dir): warn_cnt = 0 err_cnt += validate_data_source_path_existence(course_path / 'static') warn_cnt += validate_data_source_path_existence(course_path / 'static/subs', is_err=False, - extra_msg='Video captions (if they are used) will not work unless they are static/subs.') + extra_msg='Video captions (if they are used) will not work unless they are static/subs.') return err_cnt, warn_cnt def perform_xlint(data_dir, course_dirs, - default_class='xmodule.raw_module.RawDescriptor', - load_error_modules=True): + default_class='xmodule.raw_module.RawDescriptor', + load_error_modules=True): err_cnt = 0 warn_cnt = 0 @@ -497,7 +581,6 @@ def perform_xlint(data_dir, course_dirs, print "WARN: Missing course marketing video. It is recommended that every course have a marketing video." warn_cnt += 1 - print "\n\n------------------------------------------\nVALIDATION SUMMARY: {0} Errors {1} Warnings\n".format(err_cnt, warn_cnt) if err_cnt > 0: diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index c8a1ac009c..f9de929c05 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -110,8 +110,7 @@ class XmlDescriptor(XModuleDescriptor): 'name', 'slug') metadata_to_strip = ('data_dir', - # cdodge: @TODO: We need to figure out a way to export out 'tabs' and 'grading_policy' which is on the course - 'tabs', 'grading_policy', 'is_draft', 'published_by', 'published_date', + 'tabs', 'grading_policy', 'published_by', 'published_date', 'discussion_blackouts', 'testcenter_info', # VS[compat] -- remove the below attrs once everything is in the CMS 'course', 'org', 'url_name', 'filename', @@ -135,7 +134,7 @@ class XmlDescriptor(XModuleDescriptor): 'graded': bool_map, 'hide_progress_tab': bool_map, 'allow_anonymous': bool_map, - 'allow_anonymous_to_peers': bool_map + 'allow_anonymous_to_peers': bool_map, }