diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index bb9697d6a1..75f59a0ef6 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -19,11 +19,11 @@ class Command(BaseCommand): org, course, data_dir = args - module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor') + module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True) for module in module_store.modules.itervalues(): keystore().create_item(module.location) if 'data' in module.definition: keystore().update_item(module.location, module.definition['data']) if 'children' in module.definition: keystore().update_children(module.location, module.definition['children']) - keystore().update_metadata(module.url, module.metadata) + keystore().update_metadata(module.location, dict(module.metadata)) diff --git a/common/lib/keystore/xml.py b/common/lib/keystore/xml.py index baa54c4248..52eaf0ce0f 100644 --- a/common/lib/keystore/xml.py +++ b/common/lib/keystore/xml.py @@ -18,7 +18,15 @@ class XMLModuleStore(ModuleStore): """ An XML backed ModuleStore """ - def __init__(self, org, course, data_dir, default_class=None): + def __init__(self, org, course, data_dir, default_class=None, eager=False): + """ + Initialize an XMLModuleStore from data_dir + + org, course: Strings to be used in module keys + data_dir: path to data directory containing course.xml + default_class: dot-separated string defining the default descriptor class to use if non is specified in entry_points + eager: If true, load the modules children immediately to force the entire course tree to be parsed + """ self.data_dir = path(data_dir) self.modules = {} @@ -57,6 +65,9 @@ class XMLModuleStore(ModuleStore): module = XModuleDescriptor.load_from_xml(etree.tostring(xml_data), self, org, course, modulestore.default_class) modulestore.modules[module.location] = module + + if eager: + module.get_children() return module XMLParsingSystem.__init__(self, modulestore.get_item, OSFS(data_dir), process_xml) diff --git a/common/lib/xmodule/xml_module.py b/common/lib/xmodule/xml_module.py index 6639a77d3e..a224e4391d 100644 --- a/common/lib/xmodule/xml_module.py +++ b/common/lib/xmodule/xml_module.py @@ -1,7 +1,56 @@ +from collections import MutableMapping from xmodule.x_module import XModuleDescriptor from lxml import etree +class LazyLoadingDict(MutableMapping): + """ + A dictionary object that lazily loads it's contents from a provided + function on reads (of members that haven't already been set) + """ + + def __init__(self, loader): + self._contents = {} + self._loaded = False + self._loader = loader + self._deleted = set() + + def __getitem__(self, name): + if not (self._loaded or name in self._contents or name in self._deleted): + self.load() + + return self._contents[name] + + def __setitem__(self, name, value): + self._contents[name] = value + self._deleted.discard(name) + + def __delitem__(self, name): + del self._contents[name] + self._deleted.add(name) + + def __contains__(self, name): + self.load() + return name in self._contents + + def __len__(self): + self.load() + return len(self._contents) + + def __iter__(self): + self.load() + return iter(self._contents) + + def load(self): + if self._loaded: + return + + loaded_contents = self._loader() + loaded_contents.update(self._contents) + self._contents = loaded_contents + self._loaded = True + + class XmlDescriptor(XModuleDescriptor): """ Mixin class for standardized parsing of from xml @@ -29,27 +78,30 @@ class XmlDescriptor(XModuleDescriptor): """ xml_object = etree.fromstring(xml_data) - metadata = {} - for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'): - from_xml = xml_object.get(attr) - if from_xml is not None: - metadata[attr] = from_xml + def metadata_loader(): + metadata = {} + for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'): + from_xml = xml_object.get(attr) + if from_xml is not None: + metadata[attr] = from_xml - if xml_object.get('graded') is not None: - metadata['graded'] = xml_object.get('graded') == 'true' + if xml_object.get('graded') is not None: + metadata['graded'] = xml_object.get('graded') == 'true' - if xml_object.get('name') is not None: - metadata['display_name'] = xml_object.get('name') + if xml_object.get('name') is not None: + metadata['display_name'] = xml_object.get('name') + + return metadata return cls( system, - cls.definition_from_xml(xml_object, system), + LazyLoadingDict(lambda: cls.definition_from_xml(xml_object, system)), location=['i4x', org, course, xml_object.tag, xml_object.get('slug')], - metadata=metadata, + metadata=LazyLoadingDict(metadata_loader), ) def export_to_xml(self, resource_fs): diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 5119cc2910..4e5ee62e63 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -273,8 +273,11 @@ def add_histogram(module): module_id = module.id histogram = grade_histogram(module_id) render_histogram = len(histogram) > 0 - staff_context = {'definition': json.dumps(module.definition, indent=4), - 'metadata': json.dumps(module.metadata, indent=4), + + # Cast module.definition and module.metadata to dicts so that json can dump them + # even though they are lazily loaded + staff_context = {'definition': json.dumps(dict(module.definition), indent=4), + 'metadata': json.dumps(dict(module.metadata), indent=4), 'element_id': module.location.html_id(), 'histogram': json.dumps(histogram), 'render_histogram': render_histogram,