Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit for a mixed module store which can interoperate with both XML and Mongo module stores

Browse Source
This commit is contained in:
Chris Dodge
2013-07-22 10:26:12 -04:00
parent a260115b01
commit 6f11b98b4f
10 changed files with 206 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -258,7 +258,7 @@ class ModuleStore(object):
An abstract interface for a database backend that stores XModuleDescriptor
instances
"""
def has_item(self, location):
def has_item(self, course_id, location):
"""
Returns True if location exists in this ModuleStore.
"""

41
common/lib/xmodule/xmodule/modulestore/django.py
View File

@@ -25,24 +25,31 @@ def load_function(path):
return getattr(import_module(module_path), name)
def create_modulestore_instance(engine, options):
"""
This will return a new instance of a modulestore given an engine and options
"""
class_ = load_function(engine)
_options = {}
_options.update(options)
for key in FUNCTION_KEYS:
if key in _options:
_options[key] = load_function(_options[key])
return class_(
**_options
)
def modulestore(name='default'):
"""
This returns an instance of a modulestore of given name. This will wither return an existing
modulestore or create a new one
"""
if name not in _MODULESTORES:
class_ = load_function(settings.MODULESTORE[name]['ENGINE'])
options = {}
options.update(settings.MODULESTORE[name]['OPTIONS'])
for key in FUNCTION_KEYS:
if key in options:
options[key] = load_function(options[key])
_MODULESTORES[name] = class_(
**options
)
_MODULESTORES[name] = create_modulestore_instance(settings.MODULESTORE[name]['ENGINE'],
settings.MODULESTORE[name]['OPTIONS'])
return _MODULESTORES[name]
# if 'DJANGO_SETTINGS_MODULE' in environ:
# # Initialize the modulestores immediately
# for store_name in settings.MODULESTORE:
# modulestore(store_name)

106
common/lib/xmodule/xmodule/modulestore/mixed.py Normal file
View File

@@ -0,0 +1,106 @@
"""
MixedModuleStore allows for aggregation between multiple modulestores.
In this way, courses can be served up both - say - XMLModuleStore or MongoModuleStore
IMPORTANT: This modulestore is experimental AND INCOMPLETE. Therefore this should only be used cautiously
"""
from . import ModuleStoreBase
from django import create_modulestore_instance
class MixedModuleStore(ModuleStoreBase):
"""
ModuleStore that can be backed by either XML or Mongo
"""
def __init__(self, mappings, stores):
"""
Initialize a MixedModuleStore. Here we look into our passed in kwargs which should be a
collection of other modulestore configuration informations
"""
super(MixedModuleStore, self).__init__()
self.modulestores = {}
self.mappings = mappings
for key in stores:
self.modulestores[key] = create_modulestore_instance(stores[key]['ENGINE'],
stores[key]['OPTIONS'])
def _get_modulestore_for_courseid(self, course_id):
"""
For a given course_id, look in the mapping table and see if it has been pinned
to a particular modulestore
"""
return self.mappings.get(course_id, self.mappings['default'])
def has_item(self, course_id, location):
return self._get_modulestore_for_courseid(course_id).has_item(course_id, location)
def get_item(self, location, depth=0):
"""
This method is explicitly not implemented as we need a course_id to disambiguate
We should be able to fix this when the data-model rearchitecting is done
"""
raise NotImplementedError
def get_instance(self, course_id, location, depth=0):
return self._get_modulestore_for_courseid(course_id).get_instance(course_id, location, depth)
def get_items(self, location, course_id=None, depth=0):
"""
Returns a list of XModuleDescriptor instances for the items
that match location. Any element of location that is None is treated
as a wildcard that matches any value
location: Something that can be passed to Location
depth: An argument that some module stores may use to prefetch
descendents of the queried modules for more efficient results later
in the request. The depth is counted in the number of calls to
get_children() to cache. None indicates to cache all descendents
"""
if not course_id:
raise Exception("Must pass in a course_id when calling get_items() with MixedModuleStore")
return self._get_modulestore_for_courseid(course_id).get_items(location, course_id, depth)
def update_item(self, location, data, allow_not_found=False):
"""
MixedModuleStore is for read-only (aka LMS)
"""
raise NotImplementedError
def update_children(self, location, children):
"""
MixedModuleStore is for read-only (aka LMS)
"""
raise NotImplementedError
def update_metadata(self, location, metadata):
"""
MixedModuleStore is for read-only (aka LMS)
"""
raise NotImplementedError
def delete_item(self, location):
"""
MixedModuleStore is for read-only (aka LMS)
"""
raise NotImplementedError
def get_courses(self):
'''
Returns a list containing the top level XModuleDescriptors of the courses
in this modulestore.
'''
courses = []
for key in self.modulestores:
courses.append(self.modulestores[key].get_courses)
return courses
def get_course(self, course_id):
return self._get_modulestore_for_courseid(course_id).get_course(course_id)
def get_parent_locations(self, location, course_id):
return self._get_modulestore_for_courseid(course_id).get_parent_locations(location, course_id)

2
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -547,7 +547,7 @@ class MongoModuleStore(ModuleStoreBase):
raise ItemNotFoundError(location)
return item
def has_item(self, location):
def has_item(self, course_id, location):
"""
Returns True if location exists in this ModuleStore.
"""

2
common/lib/xmodule/xmodule/modulestore/search.py
View File

@@ -81,7 +81,7 @@ def path_to_location(modulestore, course_id, location):
# If we're here, there is no path
return None
if not modulestore.has_item(location):
if not modulestore.has_item(course_id, location):
raise ItemNotFoundError
path = find_path_to_course()

9
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -275,7 +275,14 @@ class SplitMongoModuleStore(ModuleStoreBase):
result = self._load_items(course_entry, [root], 0, lazy=True)
return result[0]
def has_item(self, block_location):
def get_course_for_item(self, location):
'''
Provided for backward compatibility. Is equivalent to calling get_course
:param location:
'''
return self.get_course(location)
def has_item(self, course_id, block_location):
"""
Returns True if location exists in its course. Returns false if
the course or the block w/in the course do not exist for the given version.

6
common/lib/xmodule/xmodule/modulestore/store_utilities.py
View File

@@ -152,7 +152,7 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele
# check to see if the dest_location exists as an empty course
# we need an empty course because the app layers manage the permissions and users
if not modulestore.has_item(dest_location):
if not modulestore.has_item(dest_location.course_id, dest_location):
raise Exception("An empty course at {0} must have already been created. Aborting...".format(dest_location))
# verify that the dest_location really is an empty course, which means only one with an optional 'overview'
@@ -171,7 +171,7 @@ def clone_course(modulestore, contentstore, source_location, dest_location, dele
raise Exception("Course at destination {0} is not an empty course. You can only clone into an empty course. Aborting...".format(dest_location))
# check to see if the source course is actually there
if not modulestore.has_item(source_location):
if not modulestore.has_item(source_location.course_id, source_location):
raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
# Get all modules under this namespace which is (tag, org, course) tuple
@@ -250,7 +250,7 @@ def delete_course(modulestore, contentstore, source_location, commit=False):
"""
# check to see if the source course is actually there
if not modulestore.has_item(source_location):
if not modulestore.has_item(source_location.course_id, source_location):
raise Exception("Cannot find a course at {0}. Aborting".format(source_location))
# first delete all of the thumbnails

44
common/lib/xmodule/xmodule/modulestore/tests/test_split_modulestore.py
View File

@@ -257,18 +257,19 @@ class SplitModuleItemTests(SplitModuleTest):
'''
has_item(BlockUsageLocator)
'''
course_id = 'GreekHero'
# positive tests of various forms
locator = BlockUsageLocator(version_guid=self.GUID_D1, usage_id='head12345')
self.assertTrue(modulestore().has_item(locator),
self.assertTrue(modulestore().has_item(course_id, locator),
"couldn't find in %s" % self.GUID_D1)
locator = BlockUsageLocator(course_id='GreekHero', usage_id='head12345', branch='draft')
self.assertTrue(
modulestore().has_item(locator),
modulestore().has_item(course_id, locator),
"couldn't find in 12345"
)
self.assertTrue(
modulestore().has_item(BlockUsageLocator(
modulestore().has_item(course_id, BlockUsageLocator(
course_id=locator.course_id,
branch='draft',
usage_id=locator.usage_id
@@ -276,7 +277,7 @@ class SplitModuleItemTests(SplitModuleTest):
"couldn't find in draft 12345"
)
self.assertFalse(
modulestore().has_item(BlockUsageLocator(
modulestore().has_item(course_id, BlockUsageLocator(
course_id=locator.course_id,
branch='published',
usage_id=locator.usage_id)),
@@ -284,40 +285,41 @@ class SplitModuleItemTests(SplitModuleTest):
)
locator.branch = 'draft'
self.assertTrue(
modulestore().has_item(locator),
modulestore().has_item(course_id, locator),
"not found in draft 12345"
)
# not a course obj
locator = BlockUsageLocator(course_id='GreekHero', usage_id='chapter1', branch='draft')
self.assertTrue(
modulestore().has_item(locator),
modulestore().has_item(course_id, locator),
"couldn't find chapter1"
)
# in published course
locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", branch='draft')
self.assertTrue(modulestore().has_item(BlockUsageLocator(course_id=locator.course_id,
usage_id=locator.usage_id,
branch='published')),
locator = BlockUsageLocator(course_id="wonderful", usage_id="head23456", revision='draft')
self.assertTrue(modulestore().has_item(course_id, BlockUsageLocator(course_id=locator.course_id,
usage_id=locator.usage_id,
revision='published')),
"couldn't find in 23456")
locator.branch = 'published'
self.assertTrue(modulestore().has_item(locator), "couldn't find in 23456")
self.assertTrue(modulestore().has_item(course_id, locator), "couldn't find in 23456")
def test_negative_has_item(self):
# negative tests--not found
# no such course or block
course_id = 'GreekHero'
locator = BlockUsageLocator(course_id="doesnotexist", usage_id="head23456", branch='draft')
self.assertFalse(modulestore().has_item(locator))
self.assertFalse(modulestore().has_item(course_id, locator))
locator = BlockUsageLocator(course_id="wonderful", usage_id="doesnotexist", branch='draft')
self.assertFalse(modulestore().has_item(locator))
self.assertFalse(modulestore().has_item(course_id, locator))
# negative tests--insufficient specification
self.assertRaises(InsufficientSpecificationError, BlockUsageLocator)
self.assertRaises(InsufficientSpecificationError,
modulestore().has_item, BlockUsageLocator(version_guid=self.GUID_D1))
modulestore().has_item, None, BlockUsageLocator(version_guid=self.GUID_D1))
self.assertRaises(InsufficientSpecificationError,
modulestore().has_item, BlockUsageLocator(course_id='GreekHero'))
modulestore().has_item, None, BlockUsageLocator(course_id='GreekHero'))
def test_get_item(self):
'''
@@ -737,13 +739,13 @@ class TestItemCrud(SplitModuleTest):
deleted = BlockUsageLocator(course_id=reusable_location.course_id,
branch=reusable_location.branch,
usage_id=locn_to_del.usage_id)
self.assertFalse(modulestore().has_item(deleted))
self.assertRaises(VersionConflictError, modulestore().has_item, locn_to_del)
self.assertFalse(modulestore().has_item(reusable_location.course_id, deleted))
self.assertRaises(VersionConflictError, modulestore().has_item, reusable_location.course_id, locn_to_del)
locator = BlockUsageLocator(
version_guid=locn_to_del.version_guid,
usage_id=locn_to_del.usage_id
)
self.assertTrue(modulestore().has_item(locator))
self.assertTrue(modulestore().has_item(reusable_location.course_id, locator))
self.assertNotEqual(new_course_loc.version_guid, course.location.version_guid)
# delete a subtree
@@ -754,7 +756,7 @@ class TestItemCrud(SplitModuleTest):
def check_subtree(node):
if node:
node_loc = node.location
self.assertFalse(modulestore().has_item(
self.assertFalse(modulestore().has_item(reusable_location.course_id,
BlockUsageLocator(
course_id=node_loc.course_id,
branch=node_loc.branch,
@@ -762,7 +764,7 @@ class TestItemCrud(SplitModuleTest):
locator = BlockUsageLocator(
version_guid=node.location.version_guid,
usage_id=node.location.usage_id)
self.assertTrue(modulestore().has_item(locator))
self.assertTrue(modulestore().has_item(reusable_location.course_id, locator))
if node.has_children:
for sub in node.get_children():
check_subtree(sub)
@@ -873,7 +875,7 @@ class TestCourseCreation(SplitModuleTest):
original_course = modulestore().get_course(original_locator)
self.assertEqual(original_course.location.version_guid, original_index['versions']['draft'])
self.assertFalse(
modulestore().has_item(BlockUsageLocator(
modulestore().has_item(new_draft_locator.course_id, BlockUsageLocator(
original_locator,
usage_id=new_item.location.usage_id
))

4
common/lib/xmodule/xmodule/modulestore/xml.py
View File

@@ -505,12 +505,12 @@ class XMLModuleStore(ModuleStoreBase):
except KeyError:
raise ItemNotFoundError(location)
def has_item(self, location):
def has_item(self, course_id, location):
"""
Returns True if location exists in this ModuleStore.
"""
location = Location(location)
return any(location in course_modules for course_modules in self.modules.values())
return location in self.modules[course_id]
def get_item(self, location, depth=0):
"""

37
lms/envs/cms/mixed_dev.py Normal file
View File

@@ -0,0 +1,37 @@
"""
This configuration is to run the MixedModuleStore on a localdev environment
"""
from .dev import *
MODULESTORE = {
'default': {
'ENGINE': 'xmodule.modulestore.mixed.MixedModuleStore',
'OPTIONS': {
'mappings': {
'6.002/a/a': 'xml',
'6.002/b/b': 'xml'
},
'stores': {
'xml': {
'ENGINE': 'xmodule.modulestore.xml.XMLModuleStore',
'OPTIONS': {
'data_dir': DATA_DIR,
'default_class': 'xmodule.hidden_module.HiddenDescriptor',
}
},
'default': {
'ENGINE': 'xmodule.modulestore.mongo.MongoModuleStore',
'OPTIONS': {
'default_class': 'xmodule.raw_module.RawDescriptor',
'host': 'localhost',
'db': 'xmodule',
'collection': 'modulestore',
'fs_root': DATA_DIR,
'render_template': 'mitxmako.shortcuts.render_to_string',
}
}
},
}
}
}