Decrease the number of queries needed for LMS courseware

This cuts the number of queries in 6.002 courseware loads from ~650 to
~5-40. Still to do: cache CustomTag templates so that we only load them
once per request.

common/lib/xmodule/xmodule/abtest_module.py

@@ -52,9 +52,10 @@ class ABTestModule(XModule):
     def get_shared_state(self):
         return json.dumps({'group': self.group})
     
-    def get_children_locations(self):
-        return self.definition['data']['group_content'][self.group]
-        
+    def get_child_descriptors(self):
+        active_locations = set(self.definition['data']['group_content'][self.group])
+        return [desc for desc in self.descriptor.get_children() if desc.location.url() in active_locations]
+
     def displayable_items(self):
         # Most modules return "self" as the displayable_item. We never display ourself
         # (which is why we don't implement get_html). We only display our children.

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -265,7 +265,7 @@ class ModuleStore(object):
         """
         raise NotImplementedError
 
-    def get_instance(self, course_id, location):
+    def get_instance(self, course_id, location, depth=0):
         """
         Get an instance of this location, with policy for course_id applied.
         TODO (vshnayder): this may want to live outside the modulestore eventually

common/lib/xmodule/xmodule/modulestore/mongo.py

@@ -82,17 +82,26 @@ def location_to_query(location, wildcard=True):
     If `wildcard` is True, then a None in a location is treated as a wildcard
     query. Otherwise, it is searched for literally
     """
-    query = SON()
-    # Location dict is ordered by specificity, and SON
-    # will preserve that order for queries
-    for key, val in Location(location).dict().iteritems():
-        if wildcard and val is None:
-            continue
-        query['_id.{key}'.format(key=key)] = val
+    query = namedtuple_to_son(Location(location), prefix='_id.')
+
+    if wildcard:
+        for key, value in query.items():
+            if value is None:
+                del query[key]
 
     return query
 
 
+def namedtuple_to_son(namedtuple, prefix=''):
+    """
+    Converts a namedtuple into a SON object with the same key order
+    """
+    son = SON()
+    for idx, field_name in enumerate(namedtuple._fields):
+        son[prefix + field_name] = namedtuple[idx]
+    return son
+
+
 class MongoModuleStore(ModuleStoreBase):
     """
     A Mongodb backed ModuleStore
@@ -149,6 +158,7 @@ class MongoModuleStore(ModuleStoreBase):
         If depth is None, will load all the children.
         This will make a number of queries that is linear in the depth.
         """
+
         data = {}
         to_process = list(items)
         while to_process and depth is None or depth >= 0:
@@ -162,8 +172,10 @@ class MongoModuleStore(ModuleStoreBase):
             # http://www.mongodb.org/display/DOCS/Advanced+Queries#AdvancedQueries-%24or
             # for or-query syntax
             if children:
-                to_process = list(self.collection.find(
-                    {'_id': {'$in': [Location(child).dict() for child in children]}}))
+                query = {
+                    '_id': {'$in': [namedtuple_to_son(Location(child)) for child in children]}
+                }
+                to_process = self.collection.find(query)
             else:
                 to_process = []
             # If depth is None, then we just recurse until we hit all the descendents
@@ -255,12 +267,17 @@ class MongoModuleStore(ModuleStoreBase):
         item = self._find_one(location)
         return self._load_items([item], depth)[0]
 
-    def get_instance(self, course_id, location):
+    def get_instance(self, course_id, location, depth=0):
         """
         TODO (vshnayder): implement policy tracking in mongo.
         For now, just delegate to get_item and ignore policy.
+
+        depth (int): An argument that some module stores may use to prefetch
+            descendents of the queried modules for more efficient results later
+            in the request. The depth is counted in the number of
+            calls to get_children() to cache. None indicates to cache all descendents.
         """
-        return self.get_item(location)
+        return self.get_item(location, depth=depth)
 
     def get_items(self, location, course_id=None, depth=0):
         items = self.collection.find(

common/lib/xmodule/xmodule/template_module.py

@@ -61,7 +61,7 @@ class CustomTagDescriptor(RawDescriptor):
         # cdodge: look up the template as a module
         template_loc = self.location._replace(category='custom_tag_template', name=template_name)
 
-        template_module = modulestore().get_instance(system.course_id, template_loc)
+        template_module = self.system.load_item(template_loc)
         template_module_data = template_module.definition['data']
         template = Template(template_module_data)
         return template.render(**params)

common/lib/xmodule/xmodule/x_module.py

@@ -241,17 +241,17 @@ class XModule(HTMLSnippet):
         Return module instances for all the children of this module.
         '''
         if self._loaded_children is None:
-            child_locations = self.get_children_locations()
-            children = [self.system.get_module(loc) for loc in child_locations]
+            child_descriptors = self.get_child_descriptors()
+            children = [self.system.get_module(descriptor) for descriptor in child_descriptors]
             # get_module returns None if the current user doesn't have access
             # to the location.
             self._loaded_children = [c for c in children if c is not None]
 
         return self._loaded_children
 
-    def get_children_locations(self):
+    def get_child_descriptors(self):
         '''
-        Returns the locations of each of child modules.
+        Returns the descriptors of the child modules
 
         Overriding this changes the behavior of get_children and
         anything that uses get_children, such as get_display_items.
@@ -262,7 +262,16 @@ class XModule(HTMLSnippet):
         These children will be the same children returned by the
         descriptor unless descriptor.has_dynamic_children() is true.
         '''
-        return self.definition.get('children', [])
+        return self.descriptor.get_children()
+
+    def get_child_by(self, selector):
+        """
+        Return a child XModuleDescriptor with the specified url_name, if it exists, and None otherwise.
+        """
+        for child in self.get_children():
+            if selector(child):
+                return child
+        return None
 
     def get_display_items(self):
         '''
@@ -577,13 +586,13 @@ class XModuleDescriptor(Plugin, HTMLSnippet, ResourceTemplates):
 
         return self._child_instances
 
-    def get_child_by_url_name(self, url_name):
+    def get_child_by(self, selector):
         """
         Return a child XModuleDescriptor with the specified url_name, if it exists, and None otherwise.
         """
-        for c in self.get_children():
-            if c.url_name == url_name:
-                return c
+        for child in self.get_children():
+            if selector(child):
+                return child
         return None
 
     def xmodule_constructor(self, system):
@@ -847,7 +856,7 @@ class ModuleSystem(object):
                          TODO: Not used, and has inconsistent args in different
                          files.  Update or remove.
 
-        get_module - function that takes (location) and returns a corresponding
+        get_module - function that takes a descriptor and returns a corresponding
                          module instance object.  If the current user does not have
                          access to that location, returns None.