diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index d616f21efa..17741225e5 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -258,7 +258,7 @@ class ModuleStore(object): An abstract interface for a database backend that stores XModuleDescriptor instances """ - def has_item(self, location): + def has_item(self, course_id, location): """ Returns True if location exists in this ModuleStore. """ diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py index c98e6cadef..7b1ce37d07 100644 --- a/common/lib/xmodule/xmodule/modulestore/django.py +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -25,24 +25,31 @@ def load_function(path): return getattr(import_module(module_path), name) +def create_modulestore_instance(engine, options): + """ + This will return a new instance of a modulestore given an engine and options + """ + class_ = load_function(engine) + + _options = {} + _options.update(options) + + for key in FUNCTION_KEYS: + if key in _options: + _options[key] = load_function(_options[key]) + + return class_( + **_options + ) + + def modulestore(name='default'): + """ + This returns an instance of a modulestore of given name. This will wither return an existing + modulestore or create a new one + """ if name not in _MODULESTORES: - class_ = load_function(settings.MODULESTORE[name]['ENGINE']) - - options = {} - - options.update(settings.MODULESTORE[name]['OPTIONS']) - for key in FUNCTION_KEYS: - if key in options: - options[key] = load_function(options[key]) - - _MODULESTORES[name] = class_( - **options - ) + _MODULESTORES[name] = create_modulestore_instance(settings.MODULESTORE[name]['ENGINE'], + settings.MODULESTORE[name]['OPTIONS']) return _MODULESTORES[name] - -# if 'DJANGO_SETTINGS_MODULE' in environ: -# # Initialize the modulestores immediately -# for store_name in settings.MODULESTORE: -# modulestore(store_name) diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py new file mode 100644 index 0000000000..1ecb12f858 --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -0,0 +1,106 @@ +""" +MixedModuleStore allows for aggregation between multiple modulestores. + +In this way, courses can be served up both - say - XMLModuleStore or MongoModuleStore + +IMPORTANT: This modulestore is experimental AND INCOMPLETE. Therefore this should only be used cautiously +""" + +from . import ModuleStoreBase +from django import create_modulestore_instance + + +class MixedModuleStore(ModuleStoreBase): + """ + ModuleStore that can be backed by either XML or Mongo + """ + def __init__(self, mappings, stores): + """ + Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a + collection of other modulestore configuration informations + """ + super(MixedModuleStore, self).__init__() + + self.modulestores = {} + self.mappings = mappings + for key in stores: + self.modulestores[key] = create_modulestore_instance(stores[key]['ENGINE'], + stores[key]['OPTIONS']) + + def _get_modulestore_for_courseid(self, course_id): + """ + For a given course_id, look in the mapping table and see if it has been pinned + to a particular modulestore + """ + return self.mappings.get(course_id, self.mappings['default']) + + def has_item(self, course_id, location): + return self._get_modulestore_for_courseid(course_id).has_item(course_id, location) + + def get_item(self, location, depth=0): + """ + This method is explicitly not implemented as we need a course_id to disambiguate + We should be able to fix this when the data-model rearchitecting is done + """ + raise NotImplementedError + + def get_instance(self, course_id, location, depth=0): + return self._get_modulestore_for_courseid(course_id).get_instance(course_id, location, depth) + + def get_items(self, location, course_id=None, depth=0): + """ + Returns a list of XModuleDescriptor instances for the items + that match location. Any element of location that is None is treated + as a wildcard that matches any value + + location: Something that can be passed to Location + + depth: An argument that some module stores may use to prefetch + descendents of the queried modules for more efficient results later + in the request. The depth is counted in the number of calls to + get_children() to cache. None indicates to cache all descendents + """ + if not course_id: + raise Exception("Must pass in a course_id when calling get_items() with MixedModuleStore") + + return self._get_modulestore_for_courseid(course_id).get_items(location, course_id, depth) + + def update_item(self, location, data, allow_not_found=False): + """ + MixedModuleStore is for read-only (aka LMS) + """ + raise NotImplementedError + + def update_children(self, location, children): + """ + MixedModuleStore is for read-only (aka LMS) + """ + raise NotImplementedError + + def update_metadata(self, location, metadata): + """ + MixedModuleStore is for read-only (aka LMS) + """ + raise NotImplementedError + + def delete_item(self, location): + """ + MixedModuleStore is for read-only (aka LMS) + """ + raise NotImplementedError + + def get_courses(self): + ''' + Returns a list containing the top level XModuleDescriptors of the courses + in this modulestore. + ''' + courses = [] + for key in self.modulestores: + courses.append(self.modulestores[key].get_courses) + return courses + + def get_course(self, course_id): + return self._get_modulestore_for_courseid(course_id).get_course(course_id) + + def get_parent_locations(self, location, course_id): + return self._get_modulestore_for_courseid(course_id).get_parent_locations(location, course_id) diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 21daff1875..8b4ce23ba7 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -547,7 +547,7 @@ class MongoModuleStore(ModuleStoreBase): raise ItemNotFoundError(location) return item - def has_item(self, location): + def has_item(self, course_id, location): """ Returns True if location exists in this ModuleStore. """ diff --git a/common/lib/xmodule/xmodule/modulestore/search.py b/common/lib/xmodule/xmodule/modulestore/search.py index 25ebc7e89c..804cdb0194 100644 --- a/common/lib/xmodule/xmodule/modulestore/search.py +++ b/common/lib/xmodule/xmodule/modulestore/search.py @@ -81,7 +81,7 @@ def path_to_location(modulestore, course_id, location): # If we're here, there is no path return None - if not modulestore.has_item(location): + if not modulestore.has_item(course_id, location): raise ItemNotFoundError path = find_path_to_course() diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 74c7e7241a..52f5539bae 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -275,7 +275,14 @@ class SplitMongoModuleStore(ModuleStoreBase): result = self._load_items(course_entry, [root], 0, lazy=True) return result[0] - def has_item(self, block_location): + def get_course_for_item(self, location): + ''' + Provided for backward compatibility. Is equivalent to calling get_course + :param location: + ''' + return self.get_course(location) + + def has_item(self, course_id, block_location): """ Returns True if location exists in its course. Returns false if the course or the block w/in the course do not exist for the given version. diff --git a/common/lib/xmodule/xmodule/modulestore/store_utilities.py b/common/lib/xmodule/xmodule/modulestore/store_utilities.py index e0f3db6810..19d1cac988 100644 --- a/common/lib/xmodule/xmodule/modulestore/store_utilities.py +++ b/common/lib/xmodule/xmodule/modulestore/store_utilities.py @@ -152,7 +152,7 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele # check to see if the dest_location exists as an empty course # we need an empty course because the app layers manage the permissions and users - if not modulestore.has_item(dest_location): + if not modulestore.has_item(dest_location.course_id, dest_location): raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location)) # verify that the dest_location really is an empty course, which means only one with an optional 'overview' @@ -171,7 +171,7 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location)) # check to see if the source course is actually there - if not modulestore.has_item(source_location): + if not modulestore.has_item(source_location.course_id, source_location): raise Exception("Cannot find a course at {0}. Aborting".format(source_location)) # Get all modules under this namespace which is (tag, org, course) tuple @@ -250,7 +250,7 @@ def delete_course(modulestore, contentstore, source_location, commit=False): """ # check to see if the source course is actually there - if not modulestore.has_item(source_location): + if not modulestore.has_item(source_location.course_id, source_location): raise Exception("Cannot find a course at {0}. Aborting".format(source_location)) # first delete all of the thumbnails diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py index 9976a33a00..ca5eb72a26 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py @@ -257,18 +257,19 @@ class SplitModuleItemTests(SplitModuleTest): ''' has_item(BlockUsageLocator) ''' + course_id = 'GreekHero' # positive tests of various forms locator = BlockUsageLocator(version_guid=self.GUID_D1, usage_id='head12345') - self.assertTrue(modulestore().has_item(locator), + self.assertTrue(modulestore().has_item(course_id, locator), "couldn't find in %s" % self.GUID_D1) locator = BlockUsageLocator(course_id='GreekHero', usage_id='head12345', branch='draft') self.assertTrue( - modulestore().has_item(locator), + modulestore().has_item(course_id, locator), "couldn't find in 12345" ) self.assertTrue( - modulestore().has_item(BlockUsageLocator( + modulestore().has_item(course_id, BlockUsageLocator( course_id=locator.course_id, branch='draft', usage_id=locator.usage_id @@ -276,7 +277,7 @@ class SplitModuleItemTests(SplitModuleTest): "couldn't find in draft 12345" ) self.assertFalse( - modulestore().has_item(BlockUsageLocator( + modulestore().has_item(course_id, BlockUsageLocator( course_id=locator.course_id, branch='published', usage_id=locator.usage_id)), @@ -284,40 +285,41 @@ class SplitModuleItemTests(SplitModuleTest): ) locator.branch = 'draft' self.assertTrue( - modulestore().has_item(locator), + modulestore().has_item(course_id, locator), "not found in draft 12345" ) # not a course obj locator = BlockUsageLocator(course_id='GreekHero', usage_id='chapter1', branch='draft') self.assertTrue( - modulestore().has_item(locator), + modulestore().has_item(course_id, locator), "couldn't find chapter1" ) # in published course - locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", branch='draft') - self.assertTrue(modulestore().has_item(BlockUsageLocator(course_id=locator.course_id, - usage_id=locator.usage_id, - branch='published')), + locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", revision='draft') + self.assertTrue(modulestore().has_item(course_id, BlockUsageLocator(course_id=locator.course_id, + usage_id=locator.usage_id, + revision='published')), "couldn't find in 23456") locator.branch = 'published' - self.assertTrue(modulestore().has_item(locator), "couldn't find in 23456") + self.assertTrue(modulestore().has_item(course_id, locator), "couldn't find in 23456") def test_negative_has_item(self): # negative tests--not found # no such course or block + course_id = 'GreekHero' locator = BlockUsageLocator(course_id="doesnotexist", usage_id="head23456", branch='draft') - self.assertFalse(modulestore().has_item(locator)) + self.assertFalse(modulestore().has_item(course_id, locator)) locator = BlockUsageLocator(course_id="wonderful", usage_id="doesnotexist", branch='draft') - self.assertFalse(modulestore().has_item(locator)) + self.assertFalse(modulestore().has_item(course_id, locator)) # negative tests--insufficient specification self.assertRaises(InsufficientSpecificationError, BlockUsageLocator) self.assertRaises(InsufficientSpecificationError, - modulestore().has_item, BlockUsageLocator(version_guid=self.GUID_D1)) + modulestore().has_item, None, BlockUsageLocator(version_guid=self.GUID_D1)) self.assertRaises(InsufficientSpecificationError, - modulestore().has_item, BlockUsageLocator(course_id='GreekHero')) + modulestore().has_item, None, BlockUsageLocator(course_id='GreekHero')) def test_get_item(self): ''' @@ -737,13 +739,13 @@ class TestItemCrud(SplitModuleTest): deleted = BlockUsageLocator(course_id=reusable_location.course_id, branch=reusable_location.branch, usage_id=locn_to_del.usage_id) - self.assertFalse(modulestore().has_item(deleted)) - self.assertRaises(VersionConflictError, modulestore().has_item, locn_to_del) + self.assertFalse(modulestore().has_item(reusable_location.course_id, deleted)) + self.assertRaises(VersionConflictError, modulestore().has_item, reusable_location.course_id, locn_to_del) locator = BlockUsageLocator( version_guid=locn_to_del.version_guid, usage_id=locn_to_del.usage_id ) - self.assertTrue(modulestore().has_item(locator)) + self.assertTrue(modulestore().has_item(reusable_location.course_id, locator)) self.assertNotEqual(new_course_loc.version_guid, course.location.version_guid) # delete a subtree @@ -754,7 +756,7 @@ class TestItemCrud(SplitModuleTest): def check_subtree(node): if node: node_loc = node.location - self.assertFalse(modulestore().has_item( + self.assertFalse(modulestore().has_item(reusable_location.course_id, BlockUsageLocator( course_id=node_loc.course_id, branch=node_loc.branch, @@ -762,7 +764,7 @@ class TestItemCrud(SplitModuleTest): locator = BlockUsageLocator( version_guid=node.location.version_guid, usage_id=node.location.usage_id) - self.assertTrue(modulestore().has_item(locator)) + self.assertTrue(modulestore().has_item(reusable_location.course_id, locator)) if node.has_children: for sub in node.get_children(): check_subtree(sub) @@ -873,7 +875,7 @@ class TestCourseCreation(SplitModuleTest): original_course = modulestore().get_course(original_locator) self.assertEqual(original_course.location.version_guid, original_index['versions']['draft']) self.assertFalse( - modulestore().has_item(BlockUsageLocator( + modulestore().has_item(new_draft_locator.course_id, BlockUsageLocator( original_locator, usage_id=new_item.location.usage_id )) diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 8bc3142c77..ae726041ab 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -505,12 +505,12 @@ class XMLModuleStore(ModuleStoreBase): except KeyError: raise ItemNotFoundError(location) - def has_item(self, location): + def has_item(self, course_id, location): """ Returns True if location exists in this ModuleStore. """ location = Location(location) - return any(location in course_modules for course_modules in self.modules.values()) + return location in self.modules[course_id] def get_item(self, location, depth=0): """ diff --git a/lms/envs/cms/mixed_dev.py b/lms/envs/cms/mixed_dev.py new file mode 100644 index 0000000000..22fd5eeb0e --- /dev/null +++ b/lms/envs/cms/mixed_dev.py @@ -0,0 +1,37 @@ +""" +This configuration is to run the MixedModuleStore on a localdev environment +""" + +from .dev import * + +MODULESTORE = { + 'default': { + 'ENGINE': 'xmodule.modulestore.mixed.MixedModuleStore', + 'OPTIONS': { + 'mappings': { + '6.002/a/a': 'xml', + '6.002/b/b': 'xml' + }, + 'stores': { + 'xml': { + 'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore', + 'OPTIONS': { + 'data_dir': DATA_DIR, + 'default_class': 'xmodule.hidden_module.HiddenDescriptor', + } + }, + 'default': { + 'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore', + 'OPTIONS': { + 'default_class': 'xmodule.raw_module.RawDescriptor', + 'host': 'localhost', + 'db': 'xmodule', + 'collection': 'modulestore', + 'fs_root': DATA_DIR, + 'render_template': 'mitxmako.shortcuts.render_to_string', + } + } + }, + } + } +}