From ce88f61670ecd60fb16eec243ffbc95faa78a006 Mon Sep 17 00:00:00 2001 From: Calen Pennington Date: Wed, 10 Sep 2014 11:37:15 -0400 Subject: [PATCH] Create indexes before running cross-modulestore import/export tests --- .../xmodule/xmodule/contentstore/content.py | 7 +++++ .../lib/xmodule/xmodule/contentstore/mongo.py | 30 +++++++++++++++++++ .../xmodule/xmodule/modulestore/__init__.py | 10 +++++++ .../lib/xmodule/xmodule/modulestore/mixed.py | 11 +++++++ .../xmodule/xmodule/modulestore/mongo/base.py | 23 ++++++++++++++ .../split_mongo/mongo_connection.py | 16 ++++++++++ .../xmodule/modulestore/split_mongo/split.py | 10 +++++++ .../test_cross_modulestore_import_export.py | 3 ++ 8 files changed, 110 insertions(+) diff --git a/common/lib/xmodule/xmodule/contentstore/content.py b/common/lib/xmodule/xmodule/contentstore/content.py index c3c653125f..830d754deb 100644 --- a/common/lib/xmodule/xmodule/contentstore/content.py +++ b/common/lib/xmodule/xmodule/contentstore/content.py @@ -287,3 +287,10 @@ class ContentStore(object): logging.exception(u"Failed to generate thumbnail for {0}. Exception: {1}".format(content.location, str(e))) return thumbnail_content, thumbnail_file_location + + def ensure_indexes(self): + """ + Ensure that all appropriate indexes are created that are needed by this modulestore, or raise + an exception if unable to. + """ + pass diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index 7d8bf949ad..44406c74a8 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -375,6 +375,36 @@ class MongoContentStore(ContentStore): fs_entry['_id'] = dbkey return dbkey + def ensure_indexes(self): + + # Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort + # which can be `uploadDate`, `display_name`, + + self.fs_files.create_index( + [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('_id.name', pymongo.ASCENDING)], + sparse=True + ) + self.fs_files.create_index( + [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), ('content_son.name', pymongo.ASCENDING)], + sparse=True + ) + self.fs_files.create_index( + [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('uploadDate', pymongo.ASCENDING)], + sparse=True + ) + self.fs_files.create_index( + [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('display_name', pymongo.ASCENDING)], + sparse=True + ) + self.fs_files.create_index( + [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), ('uploadDate', pymongo.ASCENDING)], + sparse=True + ) + self.fs_files.create_index( + [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), ('display_name', pymongo.ASCENDING)], + sparse=True + ) + def query_for_course(course_key, category=None): """ diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 592c455e32..72bcf2e169 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -486,6 +486,16 @@ class ModuleStoreRead(object): """ yield + def ensure_indexes(self): + """ + Ensure that all appropriate indexes are created that are needed by this modulestore, or raise + an exception if unable to. + + This method is intended for use by tests and administrative commands, and not + to be run during server startup. + """ + pass + class ModuleStoreWrite(ModuleStoreRead): """ diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 9df05ad219..99c01438a4 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -653,3 +653,14 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): store = self._get_modulestore_for_courseid(course_id) with store.bulk_operations(course_id): yield + + def ensure_indexes(self): + """ + Ensure that all appropriate indexes are created that are needed by this modulestore, or raise + an exception if unable to. + + This method is intended for use by tests and administrative commands, and not + to be run during server startup. + """ + for store in self.modulestores: + store.ensure_indexes() diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 6864134b62..aa136c4bd2 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -1440,3 +1440,26 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo return {ModuleStoreEnum.Type.mongo: True} else: raise HeartbeatFailure("Can't connect to {}".format(self.database.name), 'mongo') + + def ensure_indexes(self): + """ + Ensure that all appropriate indexes are created that are needed by this modulestore, or raise + an exception if unable to. + + This method is intended for use by tests and administrative commands, and not + to be run during server startup. + """ + + # Because we often query for some subset of the id, we define this index: + self.collection.create_index([ + ('_id.org', pymongo.ASCENDING), + ('_id.course', pymongo.ASCENDING), + ('_id.category', pymongo.ASCENDING), + ('_id.name', pymongo.ASCENDING), + ]) + + # Because we often scan for all category='course' regardless of the value of the other fields: + self.collection.create_index('_id.category') + + # Because lms calls get_parent_locations frequently (for path generation): + self.collection.create_index('definition.children', sparse=True) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index 07fdd9af15..f98081e5f6 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -244,4 +244,20 @@ class MongoConnection(object): """ self.definitions.insert(definition) + def ensure_indexes(self): + """ + Ensure that all appropriate indexes are created that are needed by this modulestore, or raise + an exception if unable to. + + This method is intended for use by tests and administrative commands, and not + to be run during server startup. + """ + self.course_index.create_index( + [ + ('org', pymongo.ASCENDING), + ('course', pymongo.ASCENDING), + ('run', pymongo.ASCENDING) + ], + unique=True + ) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 99adc9ef39..45814fa91f 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -2360,6 +2360,16 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): services=self.services, ) + def ensure_indexes(self): + """ + Ensure that all appropriate indexes are created that are needed by this modulestore, or raise + an exception if unable to. + + This method is intended for use by tests and administrative commands, and not + to be run during server startup. + """ + self.db_connection.ensure_indexes() + class SparseList(list): """ Enable inserting items into a list in arbitrary order and then retrieving them. diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py index b63fbb4c4a..226fff8a1e 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py @@ -96,6 +96,7 @@ class MongoModulestoreBuilder(object): branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred, metadata_inheritance_cache_subsystem=MemoryCache(), ) + modulestore.ensure_indexes() try: yield modulestore @@ -139,6 +140,7 @@ class VersioningModulestoreBuilder(object): fs_root, render_template=repr, ) + modulestore.ensure_indexes() try: yield modulestore @@ -210,6 +212,7 @@ class MongoContentstoreBuilder(object): collection='content', **COMMON_DOCSTORE_CONFIG ) + contentstore.ensure_indexes() try: yield contentstore