diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index 46f439b055..520e36f4d2 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -2,7 +2,7 @@ Script for importing courseware from XML format """ -from django.core.management.base import BaseCommand, CommandError +from django.core.management.base import BaseCommand, CommandError, make_option from xmodule.modulestore.xml_importer import import_from_xml from xmodule.modulestore.django import modulestore from xmodule.contentstore.django import contentstore @@ -14,18 +14,26 @@ class Command(BaseCommand): """ help = 'Import the specified data directory into the default ModuleStore' + option_list = BaseCommand.option_list + ( + make_option('--nostatic', + action='store_true', + help='Skip import of static content'), + ) + def handle(self, *args, **options): "Execute the command" if len(args) == 0: - raise CommandError("import requires at least one argument: [...]") + raise CommandError("import requires at least one argument: [--nostatic] [...]") data_dir = args[0] + do_import_static = not (options.get('nostatic', False)) if len(args) > 1: course_dirs = args[1:] else: course_dirs = None print("Importing. Data_dir={data}, course_dirs={courses}".format( data=data_dir, - courses=course_dirs)) + courses=course_dirs, + dis=do_import_static)) import_from_xml(modulestore('direct'), data_dir, course_dirs, load_error_modules=False, - static_content_store=contentstore(), verbose=True) + static_content_store=contentstore(), verbose=True, do_import_static=do_import_static) diff --git a/common/lib/xmodule/xmodule/modulestore/xml_importer.py b/common/lib/xmodule/xmodule/modulestore/xml_importer.py index 0b30a884be..0073863883 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml_importer.py +++ b/common/lib/xmodule/xmodule/modulestore/xml_importer.py @@ -51,7 +51,10 @@ def import_static_content(modules, course_loc, course_data_path, static_content_ content.thumbnail_location = thumbnail_location #then commit the content - static_content_store.save(content) + try: + static_content_store.save(content) + except Exception as err: + log.exception('Error importing {0}'.format(fullname_with_subpath)) #store the remapping information which will be needed to subsitute in the module data remap_dict[fullname_with_subpath] = content_loc.name @@ -64,7 +67,8 @@ def import_static_content(modules, course_loc, course_data_path, static_content_ def import_from_xml(store, data_dir, course_dirs=None, default_class='xmodule.raw_module.RawDescriptor', load_error_modules=True, static_content_store=None, target_location_namespace=None, - verbose=False, draft_store=None): + verbose=False, draft_store=None, + do_import_static=True): """ Import the specified xml data_dir into the "store" modulestore, using org and course as the location org and course. @@ -76,6 +80,10 @@ def import_from_xml(store, data_dir, course_dirs=None, after import off disk. We do this remapping as a post-processing step because there's logic in the importing which expects a 'url_name' as an identifier to where things are on disk e.g. ../policies//policy.json as well as metadata keys in the policy.json. so we need to keep the original url_name during import + + do_import_static: if False, then static files are not imported into the static content store. This can be employed for courses which + have substantial unchanging static content, which is to inefficient to import every time the course is loaded. + Static content for some courses may also be served directly by nginx, instead of going through django. """ @@ -116,8 +124,17 @@ def import_from_xml(store, data_dir, course_dirs=None, course_data_path = path(data_dir) / module.data_dir course_location = module.location + log.debug('======> IMPORTING course to location {0}'.format(course_location)) + module = remap_namespace(module, target_location_namespace) + if not do_import_static: + module.lms.static_asset_path = module.data_dir # for old-style xblock where this was actually linked to kvs + module._model_data['static_asset_path'] = module.data_dir + log.debug('course static_asset_path={0}'.format(module.lms.static_asset_path)) + + log.debug('course data_dir={0}'.format(module.data_dir)) + # cdodge: more hacks (what else). Seems like we have a problem when importing a course (like 6.002) which # does not have any tabs defined in the policy file. The import goes fine and then displays fine in LMS, # but if someone tries to add a new tab in the CMS, then the LMS barfs because it expects that - @@ -129,18 +146,36 @@ def import_from_xml(store, data_dir, course_dirs=None, {"type": "wiki", "name": "Wiki"}] # note, add 'progress' when we can support it on Edge import_module(module, store, course_data_path, static_content_store, course_location, - target_location_namespace or course_location) + target_location_namespace or course_location, do_import_static=do_import_static) course_items.append(module) # then import all the static content - if static_content_store is not None: + if static_content_store is not None and do_import_static: _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location # first pass to find everything in /static/ import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store, _namespace_rename, subpath='static', verbose=verbose) + elif verbose and not do_import_static: + log.debug('Skipping import of static content, since do_import_static={0}'.format(do_import_static)) + + # no matter what do_import_static is, import "static_import" directory + # + # This is needed because the "about" pages (eg "overview") are loaded via load_extra_content, and + # do not inherit the lms metadata from the course module, and thus do not get "static_content_store" + # properly defined. Static content referenced in those extra pages thus need to come through the + # c4x:// contentstore, unfortunately. Tell users to copy that content into the "static_import" subdir. + + simport = 'static_import' + if os.path.exists(course_data_path / simport): + _namespace_rename = target_location_namespace if target_location_namespace is not None else course_location + + import_static_content(xml_module_store.modules[course_id], course_location, course_data_path, static_content_store, + _namespace_rename, subpath=simport, verbose=verbose) + + # finally loop through all the modules for module in xml_module_store.modules[course_id].itervalues(): if module.category == 'course': @@ -156,7 +191,8 @@ def import_from_xml(store, data_dir, course_dirs=None, log.debug('importing module location {0}'.format(module.location)) import_module(module, store, course_data_path, static_content_store, course_location, - target_location_namespace if target_location_namespace else course_location) + target_location_namespace if target_location_namespace else course_location, + do_import_static=do_import_static) # now import any 'draft' items if draft_store is not None: @@ -176,7 +212,8 @@ def import_from_xml(store, data_dir, course_dirs=None, def import_module(module, store, course_data_path, static_content_store, - source_course_location, dest_course_location, allow_not_found=False): + source_course_location, dest_course_location, allow_not_found=False, + do_import_static=True): logging.debug('processing import of module {0}...'.format(module.location.url())) @@ -196,7 +233,7 @@ def import_module(module, store, course_data_path, static_content_store, else: module_data = content - if isinstance(module_data, basestring): + if isinstance(module_data, basestring) and do_import_static: # we want to convert all 'non-portable' links in the module_data (if it is a string) to # portable strings (e.g. /static/) module_data = rewrite_nonportable_content_links(