diff --git a/common/lib/xmodule/xmodule/contentstore/mongo.py b/common/lib/xmodule/xmodule/contentstore/mongo.py index 56ccd039cc..f2cb96497a 100644 --- a/common/lib/xmodule/xmodule/contentstore/mongo.py +++ b/common/lib/xmodule/xmodule/contentstore/mongo.py @@ -15,7 +15,7 @@ from xmodule.contentstore.content import XASSET_LOCATION_TAG from xmodule.exceptions import NotFoundError from xmodule.modulestore.django import ASSET_IGNORE_REGEX from xmodule.util.misc import escape_invalid_characters -from xmodule.mongo_connection import connect_to_mongodb +from xmodule.mongo_utils import connect_to_mongodb, create_collection_index from .content import StaticContent, ContentStore, StaticContentStream @@ -399,7 +399,8 @@ class MongoContentStore(ContentStore): def ensure_indexes(self): # Index needed thru 'category' by `_get_all_content_for_course` and others. That query also takes a sort # which can be `uploadDate`, `display_name`, - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('_id.tag', pymongo.ASCENDING), ('_id.org', pymongo.ASCENDING), @@ -409,7 +410,8 @@ class MongoContentStore(ContentStore): sparse=True, background=True ) - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), @@ -418,7 +420,8 @@ class MongoContentStore(ContentStore): sparse=True, background=True ) - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), @@ -427,7 +430,8 @@ class MongoContentStore(ContentStore): sparse=True, background=True ) - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), @@ -436,7 +440,8 @@ class MongoContentStore(ContentStore): sparse=True, background=True ) - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), @@ -445,7 +450,8 @@ class MongoContentStore(ContentStore): sparse=True, background=True ) - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('_id.org', pymongo.ASCENDING), ('_id.course', pymongo.ASCENDING), @@ -454,7 +460,8 @@ class MongoContentStore(ContentStore): sparse=True, background=True ) - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), @@ -463,7 +470,8 @@ class MongoContentStore(ContentStore): sparse=True, background=True ) - self.fs_files.create_index( + create_collection_index( + self.fs_files, [ ('content_son.org', pymongo.ASCENDING), ('content_son.course', pymongo.ASCENDING), diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index edc2cf1e21..ba82915b26 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -44,7 +44,7 @@ from xmodule.error_module import ErrorDescriptor from xmodule.errortracker import null_error_tracker, exc_info_to_str from xmodule.exceptions import HeartbeatFailure from xmodule.mako_module import MakoDescriptorSystem -from xmodule.mongo_connection import connect_to_mongodb +from xmodule.mongo_utils import connect_to_mongodb, create_collection_index from xmodule.modulestore import ModuleStoreWriteBase, ModuleStoreEnum, BulkOperationsMixin, BulkOpsRecord from xmodule.modulestore.draft_and_published import ModuleStoreDraftAndPublished, DIRECT_ONLY_CATEGORIES from xmodule.modulestore.edit_info import EditInfoRuntimeMixin @@ -1947,9 +1947,9 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo This method is intended for use by tests and administrative commands, and not to be run during server startup. """ - # Because we often query for some subset of the id, we define this index: - self.collection.create_index( + create_collection_index( + self.collection, [ ('_id.tag', pymongo.ASCENDING), ('_id.org', pymongo.ASCENDING), @@ -1958,16 +1958,17 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo ('_id.name', pymongo.ASCENDING), ('_id.revision', pymongo.ASCENDING), ], - background=True) + background=True + ) # Because we often scan for all category='course' regardless of the value of the other fields: - self.collection.create_index('_id.category', background=True) + create_collection_index(self.collection, '_id.category', background=True) # Because lms calls get_parent_locations frequently (for path generation): - self.collection.create_index('definition.children', sparse=True, background=True) + create_collection_index(self.collection, 'definition.children', sparse=True, background=True) # To allow prioritizing draft vs published material - self.collection.create_index('_id.revision', background=True) + create_collection_index(self.collection, '_id.revision', background=True) # Some overrides that still need to be implemented by subclasses def convert_to_draft(self, location, user_id): diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index d675de85ed..4bf908fc93 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -27,7 +27,7 @@ from mongodb_proxy import autoretry_read from xmodule.exceptions import HeartbeatFailure from xmodule.modulestore import BlockData from xmodule.modulestore.split_mongo import BlockKey -from xmodule.mongo_connection import connect_to_mongodb +from xmodule.mongo_utils import connect_to_mongodb, create_collection_index new_contract('BlockData', BlockData) @@ -546,7 +546,8 @@ class MongoConnection(object): This method is intended for use by tests and administrative commands, and not to be run during server startup. """ - self.course_index.create_index( + create_collection_index( + self.course_index, [ ('org', pymongo.ASCENDING), ('course', pymongo.ASCENDING), diff --git a/common/lib/xmodule/xmodule/mongo_connection.py b/common/lib/xmodule/xmodule/mongo_utils.py similarity index 59% rename from common/lib/xmodule/xmodule/mongo_connection.py rename to common/lib/xmodule/xmodule/mongo_utils.py index ea353a60b6..710a556d76 100644 --- a/common/lib/xmodule/xmodule/mongo_connection.py +++ b/common/lib/xmodule/xmodule/mongo_utils.py @@ -3,6 +3,10 @@ Common MongoDB connection functions. """ import pymongo from mongodb_proxy import MongoProxy +import logging + + +logger = logging.getLogger(__name__) # pylint: disable=invalid-name # pylint: disable=bad-continuation @@ -51,3 +55,33 @@ def connect_to_mongodb( mongo_conn.authenticate(user, password) return mongo_conn + + +def create_collection_index( + collection, keys, + ignore_created=True, ignore_created_opts=True, **kwargs +): + """ + Create a MongoDB index in a collection. Optionally, + ignore errors related to the index already existing. + """ + # For an explanation of the error codes: + # https://github.com/mongodb/mongo/blob/v3.0/src/mongo/db/catalog/index_catalog.cpp#L542-L583 + # https://github.com/mongodb/mongo/blob/v3.0/src/mongo/base/error_codes.err#L70-L87 + # pylint: disable=invalid-name + INDEX_ALREADY_EXISTS = 68 + INDEX_OPTIONS_CONFLICT = 85 + try: + collection.create_index(keys, **kwargs) + except pymongo.errors.OperationFailure as exc: + errors_to_ignore = [] + if ignore_created: + errors_to_ignore.append(INDEX_ALREADY_EXISTS) + if ignore_created_opts: + errors_to_ignore.append(INDEX_OPTIONS_CONFLICT) + if exc.code in errors_to_ignore: + logger.warning("Existing index in collection '{}' remained unchanged!: {}".format( + collection.full_name, exc.details['errmsg']) + ) + else: + raise exc