feat: store split modulestore's course indexes in Django/MySQL
Split modulestore persists data in three MongoDB "collections": course_index (list of courses and the current version of each), structure (outline of the courses, and some XBlock fields), and definition (other XBlock fields). While "structure" and "definition" data can get very large, which is one of the reasons MongoDB was chosen for modulestore, the course index data is very small. By moving course index data to MySQL / a django model, we get these advantages: * Full history of changes to the course index data is now preserved * Includes a django admin view to inspect the list of courses and libraries * It's much easier to "reset" a corrupted course to a known working state, by using the simple-history revert tools from the django admin. * The remaining MongoDB collections (structure and definition) are essentially just used as key-value stores of large JSON data structures. This paves the way for future changes that allow migrating courses one at a time from MongoDB to S3, and thus eliminating any use of MongoDB by split modulestore, simplifying the stack.
This commit is contained in:
committed by
David Ormsbee
parent
d20a769f73
commit
96e5ff8dce
@@ -125,7 +125,7 @@ class CloneCourseTest(CourseTestCase):
|
||||
)
|
||||
|
||||
# try to hit the generic exception catch
|
||||
with patch('xmodule.modulestore.split_mongo.mongo_connection.MongoConnection.insert_course_index', Mock(side_effect=Exception)): # lint-amnesty, pylint: disable=line-too-long
|
||||
with patch('xmodule.modulestore.split_mongo.mongo_connection.MongoPersistenceBackend.insert_course_index', Mock(side_effect=Exception)): # lint-amnesty, pylint: disable=line-too-long
|
||||
split_course4_id = CourseLocator(org="edx3", course="split3", run="rerun_fail")
|
||||
fields = {'display_name': 'total failure'}
|
||||
CourseRerunState.objects.initiated(split_course3_id, split_course4_id, self.user, fields['display_name'])
|
||||
|
||||
@@ -157,8 +157,8 @@ class TestCourseListing(ModuleStoreTestCase):
|
||||
self.assertEqual(len(list(courses_iter)), 0)
|
||||
|
||||
@ddt.data(
|
||||
(ModuleStoreEnum.Type.split, 3),
|
||||
(ModuleStoreEnum.Type.mongo, 2)
|
||||
(ModuleStoreEnum.Type.split, 2),
|
||||
(ModuleStoreEnum.Type.mongo, 1)
|
||||
)
|
||||
@ddt.unpack
|
||||
def test_staff_course_listing(self, default_store, mongo_calls):
|
||||
@@ -239,8 +239,8 @@ class TestCourseListing(ModuleStoreTestCase):
|
||||
)
|
||||
|
||||
@ddt.data(
|
||||
(ModuleStoreEnum.Type.split, 3, 3),
|
||||
(ModuleStoreEnum.Type.mongo, 2, 2)
|
||||
(ModuleStoreEnum.Type.split, 2, 2),
|
||||
(ModuleStoreEnum.Type.mongo, 1, 1)
|
||||
)
|
||||
@ddt.unpack
|
||||
def test_course_listing_performance(self, store, courses_list_from_group_calls, courses_list_calls):
|
||||
@@ -289,7 +289,7 @@ class TestCourseListing(ModuleStoreTestCase):
|
||||
# Calls:
|
||||
# 1) query old mongo
|
||||
# 2) get_more on old mongo
|
||||
# 3) query split (but no courses so no fetching of data)
|
||||
# 3) query split (handled with MySQL only)
|
||||
|
||||
@ddt.data(ModuleStoreEnum.Type.split, ModuleStoreEnum.Type.mongo)
|
||||
def test_course_listing_errored_deleted_courses(self, store):
|
||||
|
||||
@@ -103,7 +103,7 @@ class TestOrphan(TestOrphanBase):
|
||||
self.assertIn(str(location), orphans)
|
||||
|
||||
@ddt.data(
|
||||
(ModuleStoreEnum.Type.split, 9, 5),
|
||||
(ModuleStoreEnum.Type.split, 5, 3),
|
||||
(ModuleStoreEnum.Type.mongo, 34, 12),
|
||||
)
|
||||
@ddt.unpack
|
||||
|
||||
@@ -391,13 +391,13 @@ class TestCourseIndexArchived(CourseTestCase):
|
||||
|
||||
@ddt.data(
|
||||
# Staff user has course staff access
|
||||
(True, 'staff', None, 3, 18),
|
||||
(False, 'staff', None, 3, 18),
|
||||
(True, 'staff', None, 1, 20),
|
||||
(False, 'staff', None, 1, 20),
|
||||
# Base user has global staff access
|
||||
(True, 'user', ORG, 3, 18),
|
||||
(False, 'user', ORG, 3, 18),
|
||||
(True, 'user', None, 3, 18),
|
||||
(False, 'user', None, 3, 18),
|
||||
(True, 'user', ORG, 1, 20),
|
||||
(False, 'user', ORG, 1, 20),
|
||||
(True, 'user', None, 1, 20),
|
||||
(False, 'user', None, 1, 20),
|
||||
)
|
||||
@ddt.unpack
|
||||
def test_separate_archived_courses(self, separate_archived_courses, username, org, mongo_queries, sql_queries):
|
||||
|
||||
@@ -2528,7 +2528,7 @@ class TestXBlockInfo(ItemTest):
|
||||
self.validate_course_xblock_info(json_response, course_outline=True)
|
||||
|
||||
@ddt.data(
|
||||
(ModuleStoreEnum.Type.split, 4, 4),
|
||||
(ModuleStoreEnum.Type.split, 3, 3),
|
||||
(ModuleStoreEnum.Type.mongo, 5, 7),
|
||||
)
|
||||
@ddt.unpack
|
||||
|
||||
@@ -1456,6 +1456,7 @@ INSTALLED_APPS = [
|
||||
|
||||
# For CMS
|
||||
'cms.djangoapps.contentstore.apps.ContentstoreConfig',
|
||||
'common.djangoapps.split_modulestore_django.apps.SplitModulestoreDjangoBackendAppConfig',
|
||||
|
||||
'openedx.core.djangoapps.contentserver',
|
||||
'cms.djangoapps.course_creators',
|
||||
|
||||
Reference in New Issue
Block a user