diff --git a/AUTHORS b/AUTHORS index df8acfeedb..76d80d2ed5 100644 --- a/AUTHORS +++ b/AUTHORS @@ -232,3 +232,4 @@ William Ono Dongwook Yoon Awais Qureshi Eric Fischer +Brian Beggs diff --git a/common/djangoapps/track/backends/mongodb.py b/common/djangoapps/track/backends/mongodb.py index 71845129af..fb891c386b 100644 --- a/common/djangoapps/track/backends/mongodb.py +++ b/common/djangoapps/track/backends/mongodb.py @@ -83,8 +83,8 @@ class MongoBackend(BaseBackend): # TODO: The creation of indexes can be moved to a Django # management command or equivalent. There is also an option to # run the indexing on the background, without locking. - self.collection.ensure_index([('time', pymongo.DESCENDING)]) - self.collection.ensure_index('event_type') + self.collection.ensure_index([('time', pymongo.DESCENDING)], background=True) + self.collection.ensure_index('event_type', background=True) def send(self, event): """Insert the event in to the Mongo collection""" diff --git a/common/djangoapps/xblock_django/management/__init__.py b/common/djangoapps/xblock_django/management/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/xblock_django/management/commands/__init__.py b/common/djangoapps/xblock_django/management/commands/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/common/djangoapps/xblock_django/management/commands/ensure_indexes.py b/common/djangoapps/xblock_django/management/commands/ensure_indexes.py new file mode 100644 index 0000000000..6a42454935 --- /dev/null +++ b/common/djangoapps/xblock_django/management/commands/ensure_indexes.py @@ -0,0 +1,20 @@ +""" +Creates Indexes on contentstore and modulestore databases. +""" +from django.core.management.base import BaseCommand + +from xmodule.contentstore.django import contentstore +from xmodule.modulestore.django import modulestore + + +class Command(BaseCommand): + """ + This command will create indexes on the stores used for both contentstore and modulestore. + """ + args = '' + help = 'Creates the indexes for ContentStore and ModuleStore databases' + + def handle(self, *args, **options): + contentstore().ensure_indexes() + modulestore().ensure_indexes() + print 'contentstore and modulestore indexes created!' diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index 7ff65a6250..6779f246d2 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -393,36 +393,80 @@ class MongoContentStore(ContentStore): return dbkey def ensure_indexes(self): - # Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort # which can be `uploadDate`, `display_name`, - self.fs_files.create_index( - [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('_id.name', pymongo.ASCENDING)], - sparse=True + [ + ('_id.tag', pymongo.ASCENDING), + ('_id.org', pymongo.ASCENDING), + ('_id.course', pymongo.ASCENDING), + ('_id.category', pymongo.ASCENDING) + ], + sparse=True, + background=True ) self.fs_files.create_index( - [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), - ('content_son.name', pymongo.ASCENDING)], - sparse=True + [ + ('content_son.org', pymongo.ASCENDING), + ('content_son.course', pymongo.ASCENDING), + ('uploadDate', pymongo.DESCENDING) + ], + sparse=True, + background=True ) self.fs_files.create_index( - [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('uploadDate', pymongo.ASCENDING)], - sparse=True + [ + ('_id.org', pymongo.ASCENDING), + ('_id.course', pymongo.ASCENDING), + ('_id.name', pymongo.ASCENDING) + ], + sparse=True, + background=True ) self.fs_files.create_index( - [('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('display_name', pymongo.ASCENDING)], - sparse=True + [ + ('content_son.org', pymongo.ASCENDING), + ('content_son.course', pymongo.ASCENDING), + ('content_son.name', pymongo.ASCENDING) + ], + sparse=True, + background=True ) self.fs_files.create_index( - [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), - ('uploadDate', pymongo.ASCENDING)], - sparse=True + [ + ('_id.org', pymongo.ASCENDING), + ('_id.course', pymongo.ASCENDING), + ('uploadDate', pymongo.ASCENDING) + ], + sparse=True, + background=True ) self.fs_files.create_index( - [('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), - ('display_name', pymongo.ASCENDING)], - sparse=True + [ + ('_id.org', pymongo.ASCENDING), + ('_id.course', pymongo.ASCENDING), + ('display_name', pymongo.ASCENDING) + ], + sparse=True, + background=True + ) + self.fs_files.create_index( + [ + ('content_son.org', pymongo.ASCENDING), + ('content_son.course', pymongo.ASCENDING), + ('uploadDate', pymongo.ASCENDING) + ], + sparse=True, + background=True + ) + self.fs_files.create_index( + [ + ('content_son.org', pymongo.ASCENDING), + ('content_son.course', pymongo.ASCENDING), + ('display_name', pymongo.ASCENDING) + ], + sparse=True, + background=True ) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 898db379c0..550bf6961e 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -1914,21 +1914,25 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo """ # Because we often query for some subset of the id, we define this index: - self.collection.create_index([ - ('_id.org', pymongo.ASCENDING), - ('_id.course', pymongo.ASCENDING), - ('_id.category', pymongo.ASCENDING), - ('_id.name', pymongo.ASCENDING), - ]) + self.collection.create_index( + [ + ('_id.tag', pymongo.ASCENDING), + ('_id.org', pymongo.ASCENDING), + ('_id.course', pymongo.ASCENDING), + ('_id.category', pymongo.ASCENDING), + ('_id.name', pymongo.ASCENDING), + ('_id.revision', pymongo.ASCENDING), + ], + background=True) # Because we often scan for all category='course' regardless of the value of the other fields: - self.collection.create_index('_id.category') + self.collection.create_index('_id.category', background=True) # Because lms calls get_parent_locations frequently (for path generation): - self.collection.create_index('definition.children', sparse=True) + self.collection.create_index('definition.children', sparse=True, background=True) # To allow prioritizing draft vs published material - self.collection.create_index('_id.revision') + self.collection.create_index('_id.revision', background=True) # Some overrides that still need to be implemented by subclasses def convert_to_draft(self, location, user_id): diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index f03e9cb8df..9995f0e6be 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -532,5 +532,6 @@ class MongoConnection(object): ('course', pymongo.ASCENDING), ('run', pymongo.ASCENDING) ], - unique=True + unique=True, + background=True )