Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Adding django command to create mongodb indexes

Browse Source
This commit is contained in:
Brian Beggs
2015-08-12 16:57:26 -04:00
parent 19604a4a6e
commit c71d61e533
8 changed files with 99 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
AUTHORS
View File

@@ -232,3 +232,4 @@ William Ono <william.ono@ubc.ca>
Dongwook Yoon <dy252@cornell.edu>
Awais Qureshi <awais.qureshi@arbisoft.com>
Eric Fischer <efischer@edx.org>
Brian Beggs <macdiesel@gmail.com>

4
common/djangoapps/track/backends/mongodb.py
View File

@@ -83,8 +83,8 @@ class MongoBackend(BaseBackend):
# TODO: The creation of indexes can be moved to a Django
# management command or equivalent. There is also an option to
# run the indexing on the background, without locking.
self.collection.ensure_index([('time', pymongo.DESCENDING)])
self.collection.ensure_index('event_type')
self.collection.ensure_index([('time', pymongo.DESCENDING)], background=True)
self.collection.ensure_index('event_type', background=True)
def send(self, event):
"""Insert the event in to the Mongo collection"""

0
common/djangoapps/xblock_django/management/__init__.py Normal file
View File

0
common/djangoapps/xblock_django/management/commands/__init__.py Normal file
View File

20
common/djangoapps/xblock_django/management/commands/ensure_indexes.py Normal file
View File

@@ -0,0 +1,20 @@
"""
Creates Indexes on contentstore and modulestore databases.
"""
from django.core.management.base import BaseCommand
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
class Command(BaseCommand):
"""
This command will create indexes on the stores used for both contentstore and modulestore.
"""
args = ''
help = 'Creates the indexes for ContentStore and ModuleStore databases'
def handle(self, *args, **options):
contentstore().ensure_indexes()
modulestore().ensure_indexes()
print 'contentstore and modulestore indexes created!'

78
common/lib/xmodule/xmodule/contentstore/mongo.py
View File

@@ -393,36 +393,80 @@ class MongoContentStore(ContentStore):
return dbkey
def ensure_indexes(self):
# Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort
# which can be `uploadDate`, `display_name`,
self.fs_files.create_index(
[('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('_id.name', pymongo.ASCENDING)],
sparse=True
[
('_id.tag', pymongo.ASCENDING),
('_id.org', pymongo.ASCENDING),
('_id.course', pymongo.ASCENDING),
('_id.category', pymongo.ASCENDING)
],
sparse=True,
background=True
)
self.fs_files.create_index(
[('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING),
('content_son.name', pymongo.ASCENDING)],
sparse=True
[
('content_son.org', pymongo.ASCENDING),
('content_son.course', pymongo.ASCENDING),
('uploadDate', pymongo.DESCENDING)
],
sparse=True,
background=True
)
self.fs_files.create_index(
[('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('uploadDate', pymongo.ASCENDING)],
sparse=True
[
('_id.org', pymongo.ASCENDING),
('_id.course', pymongo.ASCENDING),
('_id.name', pymongo.ASCENDING)
],
sparse=True,
background=True
)
self.fs_files.create_index(
[('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), ('display_name', pymongo.ASCENDING)],
sparse=True
[
('content_son.org', pymongo.ASCENDING),
('content_son.course', pymongo.ASCENDING),
('content_son.name', pymongo.ASCENDING)
],
sparse=True,
background=True
)
self.fs_files.create_index(
[('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING),
('uploadDate', pymongo.ASCENDING)],
sparse=True
[
('_id.org', pymongo.ASCENDING),
('_id.course', pymongo.ASCENDING),
('uploadDate', pymongo.ASCENDING)
],
sparse=True,
background=True
)
self.fs_files.create_index(
[('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING),
('display_name', pymongo.ASCENDING)],
sparse=True
[
('_id.org', pymongo.ASCENDING),
('_id.course', pymongo.ASCENDING),
('display_name', pymongo.ASCENDING)
],
sparse=True,
background=True
)
self.fs_files.create_index(
[
('content_son.org', pymongo.ASCENDING),
('content_son.course', pymongo.ASCENDING),
('uploadDate', pymongo.ASCENDING)
],
sparse=True,
background=True
)
self.fs_files.create_index(
[
('content_son.org', pymongo.ASCENDING),
('content_son.course', pymongo.ASCENDING),
('display_name', pymongo.ASCENDING)
],
sparse=True,
background=True
)

22
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -1914,21 +1914,25 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
"""
# Because we often query for some subset of the id, we define this index:
self.collection.create_index([
('_id.org', pymongo.ASCENDING),
('_id.course', pymongo.ASCENDING),
('_id.category', pymongo.ASCENDING),
('_id.name', pymongo.ASCENDING),
])
self.collection.create_index(
[
('_id.tag', pymongo.ASCENDING),
('_id.org', pymongo.ASCENDING),
('_id.course', pymongo.ASCENDING),
('_id.category', pymongo.ASCENDING),
('_id.name', pymongo.ASCENDING),
('_id.revision', pymongo.ASCENDING),
],
background=True)
# Because we often scan for all category='course' regardless of the value of the other fields:
self.collection.create_index('_id.category')
self.collection.create_index('_id.category', background=True)
# Because lms calls get_parent_locations frequently (for path generation):
self.collection.create_index('definition.children', sparse=True)
self.collection.create_index('definition.children', sparse=True, background=True)
# To allow prioritizing draft vs published material
self.collection.create_index('_id.revision')
self.collection.create_index('_id.revision', background=True)
# Some overrides that still need to be implemented by subclasses
def convert_to_draft(self, location, user_id):

3
common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py
View File

@@ -532,5 +532,6 @@ class MongoConnection(object):
('course', pymongo.ASCENDING),
('run', pymongo.ASCENDING)
],
unique=True
unique=True,
background=True
)