Lazily load module definition and metadata as needed, rather than immediately

cms/djangoapps/contentstore/management/commands/import.py

@@ -19,11 +19,11 @@ class Command(BaseCommand):
 
         org, course, data_dir = args
 
-        module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor')
+        module_store = XMLModuleStore(org, course, data_dir, 'xmodule.raw_module.RawDescriptor', eager=True)
         for module in module_store.modules.itervalues():
             keystore().create_item(module.location)
             if 'data' in module.definition:
                 keystore().update_item(module.location, module.definition['data'])
             if 'children' in module.definition:
                 keystore().update_children(module.location, module.definition['children'])
-            keystore().update_metadata(module.url, module.metadata)
+            keystore().update_metadata(module.location, dict(module.metadata))

common/lib/keystore/xml.py

@@ -18,7 +18,15 @@ class XMLModuleStore(ModuleStore):
     """
     An XML backed ModuleStore
     """
-    def __init__(self, org, course, data_dir, default_class=None):
+    def __init__(self, org, course, data_dir, default_class=None, eager=False):
+        """
+        Initialize an XMLModuleStore from data_dir
+
+        org, course: Strings to be used in module keys
+        data_dir: path to data directory containing course.xml
+        default_class: dot-separated string defining the default descriptor class to use if non is specified in entry_points
+        eager: If true, load the modules children immediately to force the entire course tree to be parsed
+        """
         self.data_dir = path(data_dir)
         self.modules = {}
 
@@ -57,6 +65,9 @@ class XMLModuleStore(ModuleStore):
 
                         module = XModuleDescriptor.load_from_xml(etree.tostring(xml_data), self, org, course, modulestore.default_class)
                         modulestore.modules[module.location] = module
+
+                        if eager:
+                            module.get_children()
                         return module
 
                     XMLParsingSystem.__init__(self, modulestore.get_item, OSFS(data_dir), process_xml)

common/lib/xmodule/xml_module.py

@@ -1,7 +1,56 @@
+from collections import MutableMapping
 from xmodule.x_module import XModuleDescriptor
 from lxml import etree
 
 
+class LazyLoadingDict(MutableMapping):
+    """
+    A dictionary object that lazily loads it's contents from a provided
+    function on reads (of members that haven't already been set)
+    """
+
+    def __init__(self, loader):
+        self._contents = {}
+        self._loaded = False
+        self._loader = loader
+        self._deleted = set()
+
+    def __getitem__(self, name):
+        if not (self._loaded or name in self._contents or name in self._deleted):
+            self.load()
+
+        return self._contents[name]
+
+    def __setitem__(self, name, value):
+        self._contents[name] = value
+        self._deleted.discard(name)
+
+    def __delitem__(self, name):
+        del self._contents[name]
+        self._deleted.add(name)
+
+    def __contains__(self, name):
+        self.load()
+        return name in self._contents
+
+    def __len__(self):
+        self.load()
+        return len(self._contents)
+
+    def __iter__(self):
+        self.load()
+        return iter(self._contents)
+
+    def load(self):
+        if self._loaded:
+            return
+
+        loaded_contents = self._loader()
+        loaded_contents.update(self._contents)
+        self._contents = loaded_contents
+        self._loaded = True
+
+
 class XmlDescriptor(XModuleDescriptor):
     """
     Mixin class for standardized parsing of from xml
@@ -29,27 +78,30 @@ class XmlDescriptor(XModuleDescriptor):
         """
         xml_object = etree.fromstring(xml_data)
 
-        metadata = {}
-        for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'):
-            from_xml = xml_object.get(attr)
-            if from_xml is not None:
-                metadata[attr] = from_xml
+        def metadata_loader():
+            metadata = {}
+            for attr in ('format', 'graceperiod', 'showanswer', 'rerandomize', 'due'):
+                from_xml = xml_object.get(attr)
+                if from_xml is not None:
+                    metadata[attr] = from_xml
 
-        if xml_object.get('graded') is not None:
-            metadata['graded'] = xml_object.get('graded') == 'true'
+            if xml_object.get('graded') is not None:
+                metadata['graded'] = xml_object.get('graded') == 'true'
 
-        if xml_object.get('name') is not None:
-            metadata['display_name'] = xml_object.get('name')
+            if xml_object.get('name') is not None:
+                metadata['display_name'] = xml_object.get('name')
+
+            return metadata
 
         return cls(
             system,
-            cls.definition_from_xml(xml_object, system),
+            LazyLoadingDict(lambda: cls.definition_from_xml(xml_object, system)),
             location=['i4x',
                       org,
                       course,
                       xml_object.tag,
                       xml_object.get('slug')],
-            metadata=metadata,
+            metadata=LazyLoadingDict(metadata_loader),
         )
 
     def export_to_xml(self, resource_fs):

lms/djangoapps/courseware/module_render.py

@@ -273,8 +273,11 @@ def add_histogram(module):
         module_id = module.id
         histogram = grade_histogram(module_id)
         render_histogram = len(histogram) > 0
-        staff_context = {'definition': json.dumps(module.definition, indent=4),
-                         'metadata': json.dumps(module.metadata, indent=4),
+
+        # Cast module.definition and module.metadata to dicts so that json can dump them
+        # even though they are lazily loaded
+        staff_context = {'definition': json.dumps(dict(module.definition), indent=4),
+                         'metadata': json.dumps(dict(module.metadata), indent=4),
                          'element_id': module.location.html_id(),
                          'histogram': json.dumps(histogram),
                          'render_histogram': render_histogram,