Create indexes before running cross-modulestore import/export tests

common/lib/xmodule/xmodule/contentstore/content.py

@@ -287,3 +287,10 @@ class ContentStore(object):
                 logging.exception(u"Failed to generate thumbnail for {0}. Exception: {1}".format(content.location, str(e)))
 
         return thumbnail_content, thumbnail_file_location
+
+    def ensure_indexes(self):
+        """
+        Ensure that all appropriate indexes are created that are needed by this modulestore, or raise
+        an exception if unable to.
+        """
+        pass

common/lib/xmodule/xmodule/contentstore/mongo.py

@@ -375,6 +375,36 @@ class MongoContentStore(ContentStore):
         fs_entry['_id'] = dbkey
         return dbkey
 
+    def ensure_indexes(self):
+
+        # Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort
+        # which can be `uploadDate`, `display_name`,
+
+        self.fs_files.create_index(
+            [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('_id.name', pymongo.ASCENDING)],
+            sparse=True
+        )
+        self.fs_files.create_index(
+            [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), ('content_son.name', pymongo.ASCENDING)],
+            sparse=True
+        )
+        self.fs_files.create_index(
+            [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('uploadDate', pymongo.ASCENDING)],
+            sparse=True
+        )
+        self.fs_files.create_index(
+            [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('display_name', pymongo.ASCENDING)],
+            sparse=True
+        )
+        self.fs_files.create_index(
+            [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), ('uploadDate', pymongo.ASCENDING)],
+            sparse=True
+        )
+        self.fs_files.create_index(
+            [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), ('display_name', pymongo.ASCENDING)],
+            sparse=True
+        )
+
 
 def query_for_course(course_key, category=None):
     """

common/lib/xmodule/xmodule/modulestore/__init__.py

@@ -486,6 +486,16 @@ class ModuleStoreRead(object):
         """
         yield
 
+    def ensure_indexes(self):
+        """
+        Ensure that all appropriate indexes are created that are needed by this modulestore, or raise
+        an exception if unable to.
+
+        This method is intended for use by tests and administrative commands, and not
+        to be run during server startup.
+        """
+        pass
+
 
 class ModuleStoreWrite(ModuleStoreRead):
     """

common/lib/xmodule/xmodule/modulestore/mixed.py

@@ -653,3 +653,14 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
         store = self._get_modulestore_for_courseid(course_id)
         with store.bulk_operations(course_id):
             yield
+
+    def ensure_indexes(self):
+        """
+        Ensure that all appropriate indexes are created that are needed by this modulestore, or raise
+        an exception if unable to.
+
+        This method is intended for use by tests and administrative commands, and not
+        to be run during server startup.
+        """
+        for store in self.modulestores:
+            store.ensure_indexes()

common/lib/xmodule/xmodule/modulestore/mongo/base.py

@@ -1440,3 +1440,26 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
             return {ModuleStoreEnum.Type.mongo: True}
         else:
             raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo')
+
+    def ensure_indexes(self):
+        """
+        Ensure that all appropriate indexes are created that are needed by this modulestore, or raise
+        an exception if unable to.
+
+        This method is intended for use by tests and administrative commands, and not
+        to be run during server startup.
+        """
+
+        # Because we often query for some subset of the id, we define this index:
+        self.collection.create_index([
+            ('_id.org', pymongo.ASCENDING),
+            ('_id.course', pymongo.ASCENDING),
+            ('_id.category', pymongo.ASCENDING),
+            ('_id.name', pymongo.ASCENDING),
+        ])
+
+        # Because we often scan for all category='course' regardless of the value of the other fields:
+        self.collection.create_index('_id.category')
+
+        # Because lms calls get_parent_locations frequently (for path generation):
+        self.collection.create_index('definition.children', sparse=True)

common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py

@@ -244,4 +244,20 @@ class MongoConnection(object):
         """
         self.definitions.insert(definition)
 
+    def ensure_indexes(self):
+        """
+        Ensure that all appropriate indexes are created that are needed by this modulestore, or raise
+        an exception if unable to.
+
+        This method is intended for use by tests and administrative commands, and not
+        to be run during server startup.
+        """
+        self.course_index.create_index(
+            [
+                ('org', pymongo.ASCENDING),
+                ('course', pymongo.ASCENDING),
+                ('run', pymongo.ASCENDING)
+            ],
+            unique=True
+        )
 

common/lib/xmodule/xmodule/modulestore/split_mongo/split.py

@@ -2360,6 +2360,16 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
             services=self.services,
         )
 
+    def ensure_indexes(self):
+        """
+        Ensure that all appropriate indexes are created that are needed by this modulestore, or raise
+        an exception if unable to.
+
+        This method is intended for use by tests and administrative commands, and not
+        to be run during server startup.
+        """
+        self.db_connection.ensure_indexes()
+
 class SparseList(list):
     """
     Enable inserting items into a list in arbitrary order and then retrieving them.

common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py

@@ -96,6 +96,7 @@ class MongoModulestoreBuilder(object):
             branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
             metadata_inheritance_cache_subsystem=MemoryCache(),
         )
+        modulestore.ensure_indexes()
 
         try:
             yield modulestore
@@ -139,6 +140,7 @@ class VersioningModulestoreBuilder(object):
             fs_root,
             render_template=repr,
         )
+        modulestore.ensure_indexes()
 
         try:
             yield modulestore
@@ -210,6 +212,7 @@ class MongoContentstoreBuilder(object):
             collection='content',
             **COMMON_DOCSTORE_CONFIG
         )
+        contentstore.ensure_indexes()
 
         try:
             yield contentstore