Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #4380 from edx/nimisha/mongo-bwc-config

Browse Source 
auto-migrate old mongo to draft modulestore
This commit is contained in:
Nimisha Asthagiri
2014-07-14 13:46:00 -04:00
parent 95dd2aad0b 23c5febc17
commit bc03fb0881
8 changed files with 169 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cms/envs/aws.py
View File

@@ -15,6 +15,7 @@ import os
from path import path
from dealer.git import git
from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed
# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
# configuration files are read during startup.
@@ -229,7 +230,7 @@ if AWS_SECRET_ACCESS_KEY == "":
AWS_SECRET_ACCESS_KEY = None
DATABASES = AUTH_TOKENS['DATABASES']
MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS['MODULESTORE'])
MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE))
CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
# Datadog for events!

4
cms/envs/common.py
View File

@@ -30,11 +30,11 @@ import sys
import lms.envs.common
# Although this module itself may not use these imported variables, other dependent modules may.
from lms.envs.common import (
USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED, MODULESTORE
USE_TZ, TECH_SUPPORT_EMAIL, PLATFORM_NAME, BUGS_EMAIL, DOC_STORE_CONFIG, ALL_LANGUAGES, WIKI_ENABLED, MODULESTORE,
update_module_store_settings
)
from path import path
from warnings import simplefilter
from lms.envs.modulestore_settings import *
from lms.lib.xblock.mixin import LmsBlockMixin
from dealer.git import git

6
cms/envs/devstack.py
View File

@@ -72,8 +72,12 @@ DEBUG_TOOLBAR_CONFIG = {
DEBUG_TOOLBAR_MONGO_STACKTRACES = False
###############################################################################
# Lastly, see if the developer has any local overrides.
# See if the developer has any local overrides.
try:
from .private import * # pylint: disable=F0401
except ImportError:
pass
#####################################################################
# Lastly, run any migrations, if needed.
MODULESTORE = convert_module_store_setting_if_needed(MODULESTORE)

20
lms/envs/modulestore_settings.py → common/lib/xmodule/xmodule/modulestore/modulestore_settings.py
View File

@@ -2,6 +2,12 @@
This file contains helper functions for configuring module_store_setting settings and support for backward compatibility with older formats.
"""
import warnings
# Python 2.7 by default suppresses DeprecationWarnings. Make sure we show these, always.
warnings.simplefilter('once', DeprecationWarning)
def convert_module_store_setting_if_needed(module_store_setting):
"""
@@ -14,17 +20,26 @@ def convert_module_store_setting_if_needed(module_store_setting):
"""
new_store_list = []
for store_name, store_settings in old_stores.iteritems():
store_settings['NAME'] = store_name
if store_name == 'default':
new_store_list.insert(0, store_settings)
else:
new_store_list.append(store_settings)
# migrate request for the old 'direct' Mongo store to the Draft store
if store_settings['ENGINE'] == 'xmodule.modulestore.mongo.MongoModuleStore':
warnings.warn("MongoModuleStore is deprecated! Please use DraftModuleStore.", DeprecationWarning)
store_settings['ENGINE'] = 'xmodule.modulestore.mongo.draft.DraftModuleStore'
return new_store_list
if module_store_setting is None:
return None
if module_store_setting['default']['ENGINE'] != 'xmodule.modulestore.mixed.MixedModuleStore':
warnings.warn("Direct access to a modulestore is deprecated. Please use MixedModuleStore.", DeprecationWarning)
# convert to using mixed module_store
new_module_store_setting = {
"default": {
@@ -44,6 +59,11 @@ def convert_module_store_setting_if_needed(module_store_setting):
module_store_setting = new_module_store_setting
elif isinstance(module_store_setting['default']['OPTIONS']['stores'], dict):
warnings.warn(
"Using a dict for the Stores option in the MixedModuleStore is deprecated. Please use a list instead.",
DeprecationWarning
)
# convert old-style (unordered) dict to (an ordered) list
module_store_setting['default']['OPTIONS']['stores'] = convert_old_stores_into_list(
module_store_setting['default']['OPTIONS']['stores']

133
common/lib/xmodule/xmodule/modulestore/tests/test_modulestore_settings.py Normal file
View File

@@ -0,0 +1,133 @@
"""
Tests for testing the modulestore settings migration code.
"""
import copy
from unittest import TestCase
from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed
class ModuleStoreSettingsMigration(TestCase):
"""
Tests for the migration code for the module store settings
"""
OLD_CONFIG = {
"default": {
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir": "directory",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
},
"DOC_STORE_CONFIG": {},
}
}
OLD_CONFIG_WITH_DIRECT_MONGO = {
"default": {
"ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "edxapp",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
"fs_root": "/edx/var/edxapp/data",
"host": "localhost",
"password": "password",
"port": 27017,
"render_template": "edxmako.shortcuts.render_to_string",
"user": "edxapp"
},
"DOC_STORE_CONFIG": {},
}
}
OLD_MIXED_CONFIG_WITH_DICT = {
"default": {
"ENGINE": "xmodule.modulestore.mixed.MixedModuleStore",
"OPTIONS": {
"mappings": {},
"reference_type": "Location",
"stores": {
"an_old_mongo_store": {
"DOC_STORE_CONFIG": {},
"ENGINE": "xmodule.modulestore.mongo.MongoModuleStore",
"OPTIONS": {
"collection": "modulestore",
"db": "test",
"default_class": "xmodule.hidden_module.HiddenDescriptor",
}
},
"default": {
"ENGINE": "the_default_store",
"OPTIONS": {
"option1": "value1",
"option2": "value2"
},
"DOC_STORE_CONFIG": {}
},
"xml": {
"ENGINE": "xmodule.modulestore.xml.XMLModuleStore",
"OPTIONS": {
"data_dir": "directory",
"default_class": "xmodule.hidden_module.HiddenDescriptor"
},
"DOC_STORE_CONFIG": {}
}
}
}
}
}
def _get_mixed_stores(self, mixed_setting):
"""
Helper for accessing stores in a configuration setting for the Mixed modulestore
"""
return mixed_setting["default"]["OPTIONS"]["stores"]
def assertStoreValuesEqual(self, store_setting1, store_setting2):
"""
Tests whether the fields in the given store_settings are equal
"""
store_fields = ["OPTIONS", "DOC_STORE_CONFIG"]
for field in store_fields:
self.assertEqual(store_setting1[field], store_setting2[field])
def assertMigrated(self, old_setting):
"""
Migrates the given setting and checks whether it correctly converted
to an ordered list of stores within Mixed.
"""
# pass a copy of the old setting since the migration modifies the given setting
new_mixed_setting = convert_module_store_setting_if_needed(copy.deepcopy(old_setting))
# check whether the configuration is encapsulated within Mixed.
self.assertEqual(new_mixed_setting["default"]["ENGINE"], "xmodule.modulestore.mixed.MixedModuleStore")
# check whether the stores are in an ordered list
new_stores = self._get_mixed_stores(new_mixed_setting)
self.assertIsInstance(new_stores, list)
return new_mixed_setting, new_stores[0]
def test_convert_into_mixed(self):
old_setting = self.OLD_CONFIG
_, new_default_store_setting = self.assertMigrated(old_setting)
self.assertStoreValuesEqual(new_default_store_setting, old_setting["default"])
self.assertEqual(new_default_store_setting["ENGINE"], old_setting["default"]["ENGINE"])
def test_convert_from_old_mongo_to_draft_store(self):
old_setting = self.OLD_CONFIG_WITH_DIRECT_MONGO
_, new_default_store_setting = self.assertMigrated(old_setting)
self.assertStoreValuesEqual(new_default_store_setting, old_setting["default"])
self.assertEqual(new_default_store_setting["ENGINE"], "xmodule.modulestore.mongo.draft.DraftModuleStore")
def test_convert_from_dict_to_list(self):
old_mixed_setting = self.OLD_MIXED_CONFIG_WITH_DICT
new_mixed_setting, new_default_store_setting = self.assertMigrated(old_mixed_setting)
self.assertEqual(new_default_store_setting["ENGINE"], "the_default_store")
# compare each store configured in mixed
old_stores = self._get_mixed_stores(self.OLD_MIXED_CONFIG_WITH_DICT)
new_stores = self._get_mixed_stores(new_mixed_setting)
self.assertEqual(len(new_stores), len(old_stores))
for new_store_setting in self._get_mixed_stores(new_mixed_setting):
self.assertStoreValuesEqual(new_store_setting, old_stores[new_store_setting['NAME']])

1
lms/envs/aws.py
View File

@@ -18,6 +18,7 @@ from logsettings import get_logger_config
import os
from path import path
from xmodule.modulestore.modulestore_settings import convert_module_store_setting_if_needed
# SERVICE_VARIANT specifies name of the variant used, which decides what JSON
# configuration files are read during startup.

2
lms/envs/common.py
View File

@@ -33,7 +33,7 @@ from path import path
from warnings import simplefilter
from .discussionsettings import *
from .modulestore_settings import *
from xmodule.modulestore.modulestore_settings import update_module_store_settings
from lms.lib.xblock.mixin import LmsBlockMixin

6
lms/envs/devstack.py
View File

@@ -82,8 +82,12 @@ FEATURES['ENABLE_PAYMENT_FAKE'] = True
CC_PROCESSOR['CyberSource']['PURCHASE_ENDPOINT'] = '/shoppingcart/payment_fake/'
#####################################################################
# Lastly, see if the developer has any local overrides.
# See if the developer has any local overrides.
try:
from .private import * # pylint: disable=F0401
except ImportError:
pass
#####################################################################
# Lastly, run any migrations, if needed.
MODULESTORE = convert_module_store_setting_if_needed(MODULESTORE)