466 lines
15 KiB
Python
466 lines
15 KiB
Python
"""
|
|
Helper classes and methods for running modulestore tests without Django.
|
|
"""
|
|
import os
|
|
from contextlib import contextmanager, ExitStack
|
|
from importlib import import_module
|
|
from shutil import rmtree
|
|
from tempfile import mkdtemp
|
|
from uuid import uuid4
|
|
|
|
from path import Path as path
|
|
|
|
from xmodule.contentstore.mongo import MongoContentStore
|
|
from xmodule.modulestore.draft_and_published import ModuleStoreDraftAndPublished
|
|
from xmodule.modulestore.inheritance import InheritanceMixin
|
|
from xmodule.modulestore.mixed import MixedModuleStore
|
|
from xmodule.modulestore.mongo.base import ModuleStoreEnum
|
|
from xmodule.modulestore.mongo.draft import DraftModuleStore
|
|
from xmodule.modulestore.split_mongo.split_draft import DraftVersioningModuleStore
|
|
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, TEST_DATA_ONLY_SPLIT_MODULESTORE_DRAFT_PREFERRED
|
|
from xmodule.modulestore.tests.factories import BlockFactory
|
|
from xmodule.modulestore.tests.mongo_connection import MONGO_HOST, MONGO_PORT_NUM
|
|
from xmodule.modulestore.xml import XMLModuleStore
|
|
from xmodule.tests import DATA_DIR
|
|
from xmodule.x_module import XModuleMixin
|
|
|
|
|
|
def load_function(path): # lint-amnesty, pylint: disable=redefined-outer-name
|
|
"""
|
|
Load a function by name.
|
|
|
|
path is a string of the form "path.to.module.function"
|
|
returns the imported python object `function` from `path.to.module`
|
|
"""
|
|
module_path, _, name = path.rpartition('.')
|
|
return getattr(import_module(module_path), name)
|
|
|
|
|
|
# pylint: disable=unused-argument
|
|
def create_modulestore_instance(
|
|
engine,
|
|
contentstore,
|
|
doc_store_config,
|
|
options,
|
|
i18n_service=None,
|
|
fs_service=None,
|
|
user_service=None,
|
|
signal_handler=None,
|
|
):
|
|
"""
|
|
This will return a new instance of a modulestore given an engine and options
|
|
"""
|
|
class_ = load_function(engine)
|
|
|
|
if issubclass(class_, ModuleStoreDraftAndPublished):
|
|
options['branch_setting_func'] = lambda: ModuleStoreEnum.Branch.draft_preferred
|
|
|
|
return class_(
|
|
doc_store_config=doc_store_config,
|
|
contentstore=contentstore,
|
|
signal_handler=signal_handler,
|
|
**options
|
|
)
|
|
|
|
|
|
def add_temp_files_from_dict(file_dict, dir): # lint-amnesty, pylint: disable=redefined-builtin
|
|
"""
|
|
Takes in a dict formatted as: { file_name: content }, and adds files to directory
|
|
"""
|
|
for file_name in file_dict:
|
|
with open(f"{dir}/{file_name}", "w") as opened_file:
|
|
content = file_dict[file_name]
|
|
if content:
|
|
opened_file.write(str(content))
|
|
|
|
|
|
def remove_temp_files_from_list(file_list, dir): # lint-amnesty, pylint: disable=redefined-builtin
|
|
"""
|
|
Takes in a list of file names and removes them from dir if they exist
|
|
"""
|
|
for file_name in file_list:
|
|
file_path = f"{dir}/{file_name}"
|
|
if os.path.exists(file_path):
|
|
os.remove(file_path)
|
|
|
|
|
|
class MixedSplitTestCase(ModuleStoreTestCase):
|
|
"""
|
|
A minimal version of ModuleStoreTestCase for testing in xmodule/modulestore that sets up MixedModuleStore
|
|
and Split (only). It also enables "draft preferred" mode, like Studio uses.
|
|
|
|
Draft/old mongo modulestore is not initialized.
|
|
"""
|
|
CREATE_USER = False
|
|
MODULESTORE = TEST_DATA_ONLY_SPLIT_MODULESTORE_DRAFT_PREFERRED
|
|
|
|
def setUp(self):
|
|
"""
|
|
Set up requirements for testing: a user ID and a modulestore
|
|
"""
|
|
super().setUp()
|
|
self.user_id = ModuleStoreEnum.UserID.test
|
|
|
|
def make_block(self, category, parent_block, **kwargs):
|
|
"""
|
|
Create a block of type `category` as a child of `parent_block`, in any
|
|
course or library. You can pass any field values as kwargs.
|
|
"""
|
|
extra = {"publish_item": False, "user_id": self.user_id}
|
|
extra.update(kwargs)
|
|
return BlockFactory.create(
|
|
category=category,
|
|
parent=parent_block,
|
|
parent_location=parent_block.location,
|
|
modulestore=self.store,
|
|
**extra
|
|
)
|
|
|
|
|
|
class ProceduralCourseTestMixin:
|
|
"""
|
|
Contains methods for testing courses generated procedurally
|
|
"""
|
|
def populate_course(self, branching=2, emit_signals=False):
|
|
"""
|
|
Add k chapters, k^2 sections, k^3 verticals, k^4 problems to self.course (where k = branching)
|
|
"""
|
|
user_id = self.user.id
|
|
self.populated_usage_keys = {}
|
|
|
|
def descend(parent, stack):
|
|
if not stack:
|
|
return
|
|
|
|
xblock_type = stack[0]
|
|
for _ in range(branching):
|
|
child = BlockFactory.create(
|
|
category=xblock_type,
|
|
parent=parent,
|
|
user_id=user_id
|
|
)
|
|
self.populated_usage_keys.setdefault(xblock_type, []).append(
|
|
child.location
|
|
)
|
|
descend(child, stack[1:])
|
|
|
|
with self.store.bulk_operations(self.course.id, emit_signals=emit_signals):
|
|
descend(self.course, ['chapter', 'sequential', 'vertical', 'problem'])
|
|
|
|
|
|
class MemoryCache:
|
|
"""
|
|
This fits the metadata_inheritance_cache_subsystem interface used by
|
|
the modulestore, and stores the data in a dictionary in memory.
|
|
"""
|
|
def __init__(self):
|
|
self.data = {}
|
|
|
|
def get(self, key, default=None):
|
|
"""
|
|
Get a key from the cache.
|
|
|
|
Args:
|
|
key: The key to update.
|
|
default: The value to return if the key hasn't been set previously.
|
|
"""
|
|
return self.data.get(key, default)
|
|
|
|
def set(self, key, value):
|
|
"""
|
|
Set a key in the cache.
|
|
|
|
Args:
|
|
key: The key to update.
|
|
value: The value change the key to.
|
|
"""
|
|
self.data[key] = value
|
|
|
|
|
|
class MongoContentstoreBuilder:
|
|
"""
|
|
A builder class for a MongoContentStore.
|
|
"""
|
|
@contextmanager
|
|
def build(self):
|
|
"""
|
|
A contextmanager that returns a MongoContentStore, and deletes its contents
|
|
when the context closes.
|
|
"""
|
|
contentstore = MongoContentStore(
|
|
db=f'contentstore{THIS_UUID}',
|
|
collection='content',
|
|
**COMMON_DOCSTORE_CONFIG
|
|
)
|
|
contentstore.ensure_indexes()
|
|
|
|
try:
|
|
yield contentstore
|
|
finally:
|
|
# Delete the created database
|
|
contentstore._drop_database() # pylint: disable=protected-access
|
|
|
|
def __repr__(self):
|
|
return 'MongoContentstoreBuilder()'
|
|
|
|
|
|
class StoreBuilderBase:
|
|
"""
|
|
Base class for all modulestore builders.
|
|
"""
|
|
@contextmanager
|
|
def build(self, **kwargs):
|
|
"""
|
|
Build the modulestore, optionally building the contentstore as well.
|
|
"""
|
|
contentstore = kwargs.pop('contentstore', None)
|
|
if not contentstore:
|
|
with self.build_without_contentstore(**kwargs) as (contentstore, modulestore):
|
|
yield contentstore, modulestore
|
|
else:
|
|
with self.build_with_contentstore(contentstore, **kwargs) as modulestore:
|
|
yield modulestore
|
|
|
|
@contextmanager
|
|
def build_without_contentstore(self, **kwargs):
|
|
"""
|
|
Build both the contentstore and the modulestore.
|
|
"""
|
|
with MongoContentstoreBuilder().build() as contentstore:
|
|
with self.build_with_contentstore(contentstore, **kwargs) as modulestore:
|
|
yield contentstore, modulestore
|
|
|
|
|
|
class MongoModulestoreBuilder(StoreBuilderBase):
|
|
"""
|
|
A builder class for a DraftModuleStore.
|
|
"""
|
|
@contextmanager
|
|
def build_with_contentstore(self, contentstore, **kwargs):
|
|
"""
|
|
A contextmanager that returns an isolated mongo modulestore, and then deletes
|
|
all of its data at the end of the context.
|
|
|
|
Args:
|
|
contentstore: The contentstore that this modulestore should use to store
|
|
all of its assets.
|
|
"""
|
|
doc_store_config = dict(
|
|
db=f'modulestore{THIS_UUID}',
|
|
collection='xmodule',
|
|
asset_collection='asset_metadata',
|
|
**COMMON_DOCSTORE_CONFIG
|
|
)
|
|
|
|
# Set up a temp directory for storing filesystem content created during import
|
|
fs_root = mkdtemp()
|
|
|
|
modulestore = DraftModuleStore(
|
|
contentstore,
|
|
doc_store_config,
|
|
fs_root,
|
|
render_template=repr,
|
|
branch_setting_func=lambda: ModuleStoreEnum.Branch.draft_preferred,
|
|
metadata_inheritance_cache_subsystem=MemoryCache(),
|
|
xblock_mixins=XBLOCK_MIXINS,
|
|
)
|
|
modulestore.ensure_indexes()
|
|
|
|
try:
|
|
yield modulestore
|
|
finally:
|
|
# Delete the created database
|
|
modulestore._drop_database() # pylint: disable=protected-access
|
|
|
|
# Delete the created directory on the filesystem
|
|
rmtree(fs_root, ignore_errors=True)
|
|
|
|
def __repr__(self):
|
|
return 'MongoModulestoreBuilder()'
|
|
|
|
|
|
class VersioningModulestoreBuilder(StoreBuilderBase):
|
|
"""
|
|
A builder class for a VersioningModuleStore.
|
|
"""
|
|
@contextmanager
|
|
def build_with_contentstore(self, contentstore, **kwargs):
|
|
"""
|
|
A contextmanager that returns an isolated versioning modulestore, and then deletes
|
|
all of its data at the end of the context.
|
|
|
|
Args:
|
|
contentstore: The contentstore that this modulestore should use to store
|
|
all of its assets.
|
|
"""
|
|
doc_store_config = dict(
|
|
db=f'modulestore{THIS_UUID}',
|
|
collection='split_module',
|
|
**COMMON_DOCSTORE_CONFIG
|
|
)
|
|
# Set up a temp directory for storing filesystem content created during import
|
|
fs_root = mkdtemp()
|
|
|
|
modulestore = DraftVersioningModuleStore(
|
|
contentstore,
|
|
doc_store_config,
|
|
fs_root,
|
|
render_template=repr,
|
|
xblock_mixins=XBLOCK_MIXINS,
|
|
**kwargs
|
|
)
|
|
modulestore.ensure_indexes()
|
|
|
|
try:
|
|
yield modulestore
|
|
finally:
|
|
# Delete the created database
|
|
modulestore._drop_database() # pylint: disable=protected-access
|
|
|
|
# Delete the created directory on the filesystem
|
|
rmtree(fs_root, ignore_errors=True)
|
|
|
|
def __repr__(self):
|
|
return 'SplitModulestoreBuilder()'
|
|
|
|
|
|
class XmlModulestoreBuilder(StoreBuilderBase):
|
|
"""
|
|
A builder class for a XMLModuleStore.
|
|
"""
|
|
# pylint: disable=unused-argument
|
|
@contextmanager
|
|
def build_with_contentstore(self, contentstore=None, course_ids=None, **kwargs):
|
|
"""
|
|
A contextmanager that returns an isolated xml modulestore
|
|
|
|
Args:
|
|
contentstore: The contentstore that this modulestore should use to store
|
|
all of its assets.
|
|
"""
|
|
modulestore = XMLModuleStore(
|
|
DATA_DIR,
|
|
course_ids=course_ids,
|
|
default_class='xmodule.hidden_block.HiddenBlock',
|
|
xblock_mixins=XBLOCK_MIXINS,
|
|
)
|
|
|
|
yield modulestore
|
|
|
|
|
|
class MixedModulestoreBuilder(StoreBuilderBase):
|
|
"""
|
|
A builder class for a MixedModuleStore.
|
|
"""
|
|
def __init__(self, store_builders, mappings=None):
|
|
"""
|
|
Args:
|
|
store_builders: A list of modulestore builder objects. These will be instantiated, in order,
|
|
as the backing stores for the MixedModuleStore.
|
|
mappings: Any course mappings to pass to the MixedModuleStore on instantiation.
|
|
"""
|
|
self.store_builders = store_builders
|
|
self.mappings = mappings or {}
|
|
self.mixed_modulestore = None
|
|
|
|
@contextmanager
|
|
def build_with_contentstore(self, contentstore, **kwargs):
|
|
"""
|
|
A contextmanager that returns a mixed modulestore built on top of modulestores
|
|
generated by other builder classes.
|
|
|
|
Args:
|
|
contentstore: The contentstore that this modulestore should use to store
|
|
all of its assets.
|
|
"""
|
|
names, generators = list(zip(*self.store_builders))
|
|
|
|
with ExitStack() as stack:
|
|
modulestores = [stack.enter_context(gen.build_with_contentstore(contentstore, **kwargs)) for gen in generators] # lint-amnesty, pylint: disable=line-too-long
|
|
# Make the modulestore creation function just return the already-created modulestores
|
|
store_iterator = iter(modulestores)
|
|
next_modulestore = lambda *args, **kwargs: next(store_iterator)
|
|
|
|
# Generate a fake list of stores to give the already generated stores appropriate names
|
|
stores = [{'NAME': name, 'ENGINE': 'This space deliberately left blank'} for name in names]
|
|
|
|
self.mixed_modulestore = MixedModuleStore(
|
|
contentstore,
|
|
self.mappings,
|
|
stores,
|
|
create_modulestore_instance=next_modulestore,
|
|
xblock_mixins=XBLOCK_MIXINS,
|
|
)
|
|
|
|
yield self.mixed_modulestore
|
|
|
|
def __repr__(self):
|
|
return f'MixedModulestoreBuilder({self.store_builders!r}, {self.mappings!r})'
|
|
|
|
def asset_collection(self):
|
|
"""
|
|
Returns the collection storing the asset metadata.
|
|
"""
|
|
all_stores = self.mixed_modulestore.modulestores
|
|
if len(all_stores) > 1:
|
|
return None
|
|
|
|
store = all_stores[0]
|
|
if hasattr(store, 'asset_collection'):
|
|
# Mongo modulestore beneath mixed.
|
|
# Returns the entire collection with *all* courses' asset metadata.
|
|
return store.asset_collection
|
|
else:
|
|
# Split modulestore beneath mixed.
|
|
# Split stores all asset metadata in the structure collection.
|
|
return store.db_connection.structures
|
|
|
|
|
|
THIS_UUID = uuid4().hex
|
|
|
|
COMMON_DOCSTORE_CONFIG = {
|
|
'host': MONGO_HOST,
|
|
'port': MONGO_PORT_NUM,
|
|
}
|
|
DATA_DIR = path(__file__).dirname().parent.parent / "tests" / "data" / "xml-course-root"
|
|
TEST_DATA_DIR = 'common/test/data/'
|
|
|
|
XBLOCK_MIXINS = (InheritanceMixin, XModuleMixin)
|
|
|
|
|
|
MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
|
|
('draft', MongoModulestoreBuilder()),
|
|
('split', VersioningModulestoreBuilder())
|
|
])
|
|
DRAFT_MODULESTORE_SETUP = MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())])
|
|
SPLIT_MODULESTORE_SETUP = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())])
|
|
MIXED_MODULESTORE_SETUPS = (
|
|
DRAFT_MODULESTORE_SETUP,
|
|
SPLIT_MODULESTORE_SETUP,
|
|
)
|
|
MIXED_MS_SETUPS_SHORT = (
|
|
'mixed_mongo',
|
|
'mixed_split',
|
|
)
|
|
DIRECT_MODULESTORE_SETUPS = (
|
|
MongoModulestoreBuilder(),
|
|
# VersioningModulestoreBuilder(), # FUTUREDO: LMS-11227
|
|
)
|
|
DIRECT_MS_SETUPS_SHORT = (
|
|
'mongo',
|
|
#'split',
|
|
)
|
|
MODULESTORE_SETUPS = DIRECT_MODULESTORE_SETUPS + MIXED_MODULESTORE_SETUPS
|
|
MODULESTORE_SHORTNAMES = DIRECT_MS_SETUPS_SHORT + MIXED_MS_SETUPS_SHORT
|
|
SHORT_NAME_MAP = dict(list(zip(MODULESTORE_SETUPS, MODULESTORE_SHORTNAMES)))
|
|
|
|
CONTENTSTORE_SETUPS = (MongoContentstoreBuilder(),)
|
|
|
|
DOT_FILES_DICT = {
|
|
".DS_Store": None,
|
|
".example.txt": "BLUE",
|
|
}
|
|
TILDA_FILES_DICT = {
|
|
"example.txt~": "RED"
|
|
}
|