Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Consolidate Block Structure folders

Browse Source 
TNL-6518
This commit is contained in:
Nimisha Asthagiri
2017-03-01 14:09:37 -05:00
parent 11dce770c4
commit 00d466224e
51 changed files with 442 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
lms/djangoapps/course_api/blocks/api.py
View File

@@ -4,7 +4,7 @@ API function for retrieving course blocks data
from lms.djangoapps.course_blocks.api import get_course_blocks, COURSE_BLOCK_ACCESS_TRANSFORMERS
from lms.djangoapps.course_blocks.transformers.hidden_content import HiddenContentTransformer
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from .transformers.blocks_api import BlocksAPITransformer
from .transformers.milestones import MilestonesTransformer

2
lms/djangoapps/course_api/blocks/tests/test_serializers.py
View File

@@ -3,7 +3,7 @@ Tests for Course Blocks serializers
"""
from mock import MagicMock
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from student.tests.factories import UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase

2
lms/djangoapps/course_api/blocks/transformers/block_counts.py
View File

@@ -1,7 +1,7 @@
"""
Block Counts Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
class BlockCountsTransformer(BlockStructureTransformer):

2
lms/djangoapps/course_api/blocks/transformers/block_depth.py
View File

@@ -1,7 +1,7 @@
"""
Block Depth Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
class BlockDepthTransformer(BlockStructureTransformer):

2
lms/djangoapps/course_api/blocks/transformers/blocks_api.py
View File

@@ -1,7 +1,7 @@
"""
Blocks API Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
from .block_counts import BlockCountsTransformer
from .block_depth import BlockDepthTransformer
from .navigation import BlockNavigationTransformer

5
lms/djangoapps/course_api/blocks/transformers/milestones.py
View File

@@ -4,7 +4,10 @@ Milestones Transformer
from django.conf import settings
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from util import milestones_helpers

2
lms/djangoapps/course_api/blocks/transformers/navigation.py
View File

@@ -1,7 +1,7 @@
"""
TODO
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
from .block_depth import BlockDepthTransformer

2
lms/djangoapps/course_api/blocks/transformers/student_view.py
View File

@@ -1,7 +1,7 @@
"""
Student View Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
class StudentViewTransformer(BlockStructureTransformer):

2
lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py
View File

@@ -3,7 +3,7 @@ Tests for BlockCountsTransformer.
"""
# pylint: disable=protected-access
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import SampleCourseFactory

4
lms/djangoapps/course_api/blocks/transformers/tests/test_block_depth.py
View File

@@ -7,8 +7,8 @@ Tests for BlockDepthTransformer.
import ddt
from unittest import TestCase
from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData
from ..block_depth import BlockDepthTransformer

6
lms/djangoapps/course_api/blocks/transformers/tests/test_navigation.py
View File

@@ -7,9 +7,9 @@ from unittest import TestCase
from lms.djangoapps.course_api.blocks.transformers.block_depth import BlockDepthTransformer
from lms.djangoapps.course_api.blocks.transformers.navigation import BlockNavigationTransformer
from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin
from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import SampleCourseFactory
from xmodule.modulestore import ModuleStoreEnum

2
lms/djangoapps/course_api/blocks/transformers/tests/test_student_view.py
View File

@@ -4,7 +4,7 @@ Tests for StudentViewTransformer.
# pylint: disable=protected-access
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
from xmodule.modulestore.tests.factories import ToyCourseFactory

2
lms/djangoapps/course_blocks/__init__.py
View File

@@ -1,6 +1,6 @@
"""
The Course Blocks app, built upon the Block Cache framework in
openedx.core.lib.block_structure, is a higher layer django app in LMS that
openedx.core.djangoapps.content.block_structure, is a higher layer django app in LMS that
provides additional context of Courses and Users (via usage_info.py) with
implementations for Block Structure Transformers that are related to
block structure course access.

2
lms/djangoapps/course_blocks/api.py
View File

@@ -3,7 +3,7 @@ API entry point to the course_blocks app with top-level
get_course_blocks function.
"""
from openedx.core.djangoapps.content.block_structure.api import get_block_structure_manager
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from .transformers import (
library_content,

2
lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py
View File

@@ -8,7 +8,7 @@ from xmodule.modulestore.django import modulestore
import openedx.core.djangoapps.content.block_structure.api as api
import openedx.core.djangoapps.content.block_structure.tasks as tasks
import openedx.core.lib.block_structure.store as store
import openedx.core.djangoapps.content.block_structure.store as store
from openedx.core.lib.command_utils import (
get_mutually_exclusive_required_option,
validate_dependent_option,

2
lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py
View File

@@ -59,7 +59,7 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase):
self.command.handle(all_courses=True)
self._assert_courses_in_block_cache(*self.course_keys)
with patch(
'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_modulestore'
'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore'
) as mock_update_from_store:
self.command.handle(all_courses=True, force_update=force_update)
self.assertEqual(mock_update_from_store.call_count, self.num_courses if force_update else 0)

5
lms/djangoapps/course_blocks/transformers/hidden_content.py
View File

@@ -4,7 +4,10 @@ Visibility Transformer implementation.
from datetime import datetime
from pytz import utc
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from xmodule.seq_module import SequenceModule
from .utils import collect_merged_boolean_field, collect_merged_date_field

5
lms/djangoapps/course_blocks/transformers/library_content.py
View File

@@ -3,7 +3,10 @@ Content Library Transformer.
"""
import json
from courseware.models import StudentModule
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from xmodule.library_content_module import LibraryContentModule
from xmodule.modulestore.django import modulestore
from eventtracking import tracker

5
lms/djangoapps/course_blocks/transformers/split_test.py
View File

@@ -1,7 +1,10 @@
"""
Split Test Block Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
class SplitTestTransformer(FilteringTransformerMixin, BlockStructureTransformer):

5
lms/djangoapps/course_blocks/transformers/start_date.py
View File

@@ -1,7 +1,10 @@
"""
Start Date Transformer implementation.
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from lms.djangoapps.courseware.access_utils import check_start_date
from xmodule.course_metadata_utils import DEFAULT_START_DATE

7
lms/djangoapps/course_blocks/transformers/tests/helpers.py
View File

@@ -4,8 +4,8 @@ Test helpers for testing course block transformers.
from mock import patch
from course_modes.models import CourseMode
from lms.djangoapps.courseware.access import has_access
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.lib.block_structure.tests.helpers import clear_registered_transformers_cache
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.tests.helpers import clear_registered_transformers_cache
from student.tests.factories import CourseEnrollmentFactory, UserFactory
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.django import modulestore
@@ -23,7 +23,8 @@ class TransformerRegistryTestMixin(object):
def setUp(self):
super(TransformerRegistryTestMixin, self).setUp()
self.patcher = patch(
'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers'
'openedx.core.djangoapps.content.block_structure.transformer_registry.'
'TransformerRegistry.get_registered_transformers'
)
mock_registry = self.patcher.start()
mock_registry.return_value = {self.TRANSFORMER_CLASS_TO_TEST}

2
lms/djangoapps/course_blocks/transformers/tests/test_library_content.py
View File

@@ -5,7 +5,7 @@ Tests for ContentLibraryTransformer.
from student.tests.factories import CourseEnrollmentFactory
from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache
from openedx.core.lib.block_structure.transformers import BlockStructureTransformers
from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers
from ...api import get_course_blocks
from ..library_content import ContentLibraryTransformer

5
lms/djangoapps/course_blocks/transformers/user_partitions.py
View File

@@ -1,7 +1,10 @@
"""
User Partitions Transformer
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from .split_test import SplitTestTransformer
from .utils import get_field_on_block

5
lms/djangoapps/course_blocks/transformers/visibility.py
View File

@@ -1,7 +1,10 @@
"""
Visibility Transformer implementation.
"""
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin
from openedx.core.djangoapps.content.block_structure.transformer import (
BlockStructureTransformer,
FilteringTransformerMixin,
)
from .utils import collect_merged_boolean_field

2
lms/djangoapps/grades/tests/test_grades.py
View File

@@ -12,7 +12,7 @@ from courseware.model_data import set_score
from courseware.tests.helpers import LoginEnrollmentTestCase
from lms.djangoapps.course_blocks.api import get_course_blocks
from openedx.core.lib.block_structure.factory import BlockStructureFactory
from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory
from openedx.core.djangolib.testing.utils import get_mock_request
from student.tests.factories import UserFactory
from student.models import CourseEnrollment

2
lms/djangoapps/grades/tests/test_scores.py
View File

@@ -11,7 +11,7 @@ from lms.djangoapps.grades.models import BlockRecord
import lms.djangoapps.grades.scores as scores
from lms.djangoapps.grades.transformer import GradesTransformer
from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator
from openedx.core.lib.block_structure.block_structure import BlockData
from openedx.core.djangoapps.content.block_structure.block_structure import BlockData
from xmodule.graders import ProblemScore

4
lms/djangoapps/grades/tests/test_tasks.py
View File

@@ -13,7 +13,7 @@ from mock import patch, MagicMock
import pytz
from util.date_utils import to_timestamp
from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
from openedx.core.djangoapps.content.block_structure.exceptions import BlockStructureNotFound
from student.models import anonymous_id_for_user
from student.tests.factories import UserFactory
from track.event_transaction_utils import (
@@ -136,7 +136,7 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase):
self.set_up_course()
self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id))
with patch(
'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_store',
'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_store',
side_effect=BlockStructureNotFound(self.course.location),
) as mock_block_structure_create:
self._apply_recalculate_subsection_grade()

2
lms/djangoapps/grades/transformer.py
View File

@@ -8,7 +8,7 @@ from logging import getLogger
import json
from lms.djangoapps.course_blocks.transformers.utils import collect_unioned_set_field, get_field_on_block
from openedx.core.lib.block_structure.transformer import BlockStructureTransformer
from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer
log = getLogger(__name__)

61
openedx/core/djangoapps/content/block_structure/__init__.py
View File

@@ -1,5 +1,62 @@
"""
This code exists in openedx/core/djangoapp because it needs access to django signaling mechanisms
The block_structure django app provides an extensible framework for caching
data of block structures from the modulestore.
Most of the underlying functionality is implemented in openedx/core/lib/block_structure/
Dual-Phase. The framework is meant to be used in 2 phases.
* Collect Phase (for expensive and full-tree traversals) - In the
first phase, the "collect" phase, any and all data from the
modulestore should be collected and cached for later access to
the block structure. Instantiating any and all xBlocks in the block
structure is also done at this phase, since that is also (currently)
a costly operation.
Any full tree traversals should also be done during this phase. For
example, if data for a block depends on its parents, the traversal
should happen during the collection phase and any required data
for the block should be percolated down the tree and stored as
aggregate values on the descendants. This allows for faster and
direct access to blocks in the Transform phase.
* Transform Phase (for fast access to blocks) - In the second
phase, the "transform" phase, only the previously collected and
cached data should be accessed. There should be no access to the
modulestore or instantiation of xBlocks in this phase.
To make this framework extensible, the Transformer and
Extensibility design patterns are used. This django app only
provides the underlying framework for Block Structure Transformers
and a Transformer Registry. Clients are expected to provide actual
implementations of Transformers or add them to the extensible Registry.
Transformers. As inspired by
http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf,
a Block Structure Transformer takes in a block structure (or tree) and
manipulates the structure and the data of its blocks according to its
own requirements. Its output can then be used for further
transformations by other transformers down the pipeline.
Note: For performance and space optimization, our implementation
differs from the paper in that our transformers mutate the block
structure in-place rather than returning a modified copy of it.
Block Structure. The BlockStructure and its family of classes
provided with this framework are the base data types for accessing
and manipulating block structures. BlockStructures are constructed
using the BlockStructureFactory and then used as the currency across
Transformers.
Registry. Transformers are registered using the platform's
PluginManager (e.g., Stevedore). This is currently done by updating
setup.py. Only registered transformers are called during the Collect
Phase. And only registered transformers can be used during the
Transform phase. Exceptions to this rule are any nested transformers
that are contained within higher-order transformers - as long as the
higher-order transformers are registered and appropriately call the
contained transformers within them.
Note: A partial subset (as an ordered list) of the registered
transformers can be requested during the Transform phase, allowing
the client to manipulate exactly which transformers to call.
"""

3
openedx/core/djangoapps/content/block_structure/api.py
View File

@@ -2,9 +2,10 @@
Higher order functions built on the BlockStructureManager to interact with a django cache.
"""
from django.core.cache import cache
from openedx.core.lib.block_structure.manager import BlockStructureManager
from xmodule.modulestore.django import modulestore
from .manager import BlockStructureManager
def get_course_in_cache(course_key):
"""

0
openedx/core/lib/block_structure/block_structure.py → openedx/core/djangoapps/content/block_structure/block_structure.py
View File

0
openedx/core/lib/block_structure/exceptions.py → openedx/core/djangoapps/content/block_structure/exceptions.py
View File

0
openedx/core/lib/block_structure/factory.py → openedx/core/djangoapps/content/block_structure/factory.py
View File

3
openedx/core/lib/block_structure/manager.py → openedx/core/djangoapps/content/block_structure/manager.py
View File

@@ -4,8 +4,7 @@ BlockStructures.
"""
from contextlib import contextmanager
from openedx.core.djangoapps.content.block_structure import config
from . import config
from .exceptions import UsageKeyNotInBlockStructure, TransformerDataIncompatible, BlockStructureNotFound
from .factory import BlockStructureFactory
from .store import BlockStructureStore

4
openedx/core/djangoapps/content/block_structure/models.py
View File

@@ -10,10 +10,10 @@ from logging import getLogger
from model_utils.models import TimeStampedModel
from openedx.core.djangoapps.xmodule_django.models import UsageKeyField
from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
from openedx.core.storage import get_storage
import openedx.core.djangoapps.content.block_structure.config as config
from . import config
from .exceptions import BlockStructureNotFound
log = getLogger(__name__)

5
openedx/core/lib/block_structure/store.py → openedx/core/djangoapps/content/block_structure/store.py
View File

@@ -4,14 +4,13 @@ Module for the Storage of BlockStructure objects.
# pylint: disable=protected-access
from logging import getLogger
import openedx.core.djangoapps.content.block_structure.config as config
from openedx.core.djangoapps.content.block_structure.models import BlockStructureModel
from openedx.core.lib.cache_utils import zpickle, zunpickle
from . import config
from .block_structure import BlockStructureBlockData
from .exceptions import BlockStructureNotFound
from .factory import BlockStructureFactory
from .models import BlockStructureModel
from .transformer_registry import TransformerRegistry

313
openedx/core/djangoapps/content/block_structure/tests/helpers.py
View File

@@ -1,12 +1,21 @@
"""
Helpers for Course Blocks tests.
Common utilities for tests in block_structure module
"""
from contextlib import contextmanager
from mock import patch
from xmodule.modulestore.exceptions import ItemNotFoundError
from uuid import uuid4
from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from openedx.core.djangolib.testing.waffle_utils import override_switch
from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
from openedx.core.lib.block_structure.store import BlockStructureStore
from ..api import get_cache
from ..block_structure import BlockStructureBlockData
from ..config import _bs_waffle_switch_name
from ..exceptions import BlockStructureNotFound
from ..store import BlockStructureStore
from ..transformer import BlockStructureTransformer, FilteringTransformerMixin
from ..transformer_registry import TransformerRegistry
def is_course_in_block_structure_cache(course_key, store):
@@ -31,3 +40,301 @@ class override_config_setting(override_switch): # pylint:disable=invalid-name
_bs_waffle_switch_name(name),
active
)
class MockXBlock(object):
"""
A mock XBlock to be used in unit tests, thereby decoupling the
implementation of the block cache framework from the xBlock
implementation. This class provides only the minimum xBlock
capabilities needed by the block cache framework.
"""
def __init__(self, location, field_map=None, children=None, modulestore=None):
self.location = location
self.field_map = field_map or {}
self.children = children or []
self.modulestore = modulestore
def __getattr__(self, attr):
try:
return self.field_map[attr]
except KeyError:
raise AttributeError
def get_children(self):
"""
Returns the children of the mock XBlock.
"""
return [self.modulestore.get_item(child) for child in self.children]
class MockModulestore(object):
"""
A mock Modulestore to be used in unit tests, providing only the
minimum methods needed by the block cache framework.
"""
def __init__(self):
self.get_items_call_count = 0
self.blocks = None
def set_blocks(self, blocks):
"""
Updates the mock modulestore with a dictionary of blocks.
Arguments:
blocks ({block key, MockXBlock}) - A map of block_key
to its mock xBlock.
"""
self.blocks = blocks
def get_item(self, block_key, depth=None, lazy=False): # pylint: disable=unused-argument
"""
Returns the mock XBlock (MockXBlock) associated with the
given block_key.
Raises ItemNotFoundError if the item is not found.
"""
self.get_items_call_count += 1
item = self.blocks.get(block_key)
if not item:
raise ItemNotFoundError
return item
@contextmanager
def bulk_operations(self, ignore): # pylint: disable=unused-argument
"""
A context manager for notifying the store of bulk operations.
"""
yield
class MockCache(object):
"""
A mock Cache object, providing only the minimum features needed
by the block cache framework.
"""
def __init__(self):
# An in-memory map of cache keys to cache values.
self.map = {}
self.set_call_count = 0
self.timeout_from_last_call = 0
def set(self, key, val, timeout):
"""
Associates the given key with the given value in the cache.
"""
self.set_call_count += 1
self.map[key] = val
self.timeout_from_last_call = timeout
def get(self, key, default=None):
"""
Returns the value associated with the given key in the cache;
returns default if not found.
"""
return self.map.get(key, default)
def delete(self, key):
"""
Deletes the given key from the cache.
"""
del self.map[key]
class MockModulestoreFactory(object):
"""
A factory for creating MockModulestore objects.
"""
@classmethod
def create(cls, children_map, block_key_factory):
"""
Creates and returns a MockModulestore from the given
children_map.
Arguments:
children_map ({block_key: [block_key]}) - A dictionary
mapping a block key to a list of block keys of the
block's corresponding children.
"""
modulestore = MockModulestore()
modulestore.set_blocks({
block_key_factory(block_key): MockXBlock(
block_key_factory(block_key),
children=[block_key_factory(child) for child in children],
modulestore=modulestore,
)
for block_key, children in enumerate(children_map)
})
return modulestore
class MockTransformer(BlockStructureTransformer):
"""
A mock BlockStructureTransformer class.
"""
WRITE_VERSION = 1
READ_VERSION = 1
@classmethod
def name(cls):
# Use the class' name for Mock transformers.
return cls.__name__
def transform(self, usage_info, block_structure):
pass
class MockFilteringTransformer(FilteringTransformerMixin, BlockStructureTransformer):
"""
A mock FilteringTransformerMixin class.
"""
WRITE_VERSION = 1
READ_VERSION = 1
@classmethod
def name(cls):
# Use the class' name for Mock transformers.
return cls.__name__
def transform_block_filters(self, usage_info, block_structure):
return [block_structure.create_universal_filter()]
def clear_registered_transformers_cache():
"""
Test helper to clear out any cached values of registered transformers.
"""
TransformerRegistry.get_write_version_hash.cache.clear()
@contextmanager
def mock_registered_transformers(transformers):
"""
Context manager for mocking the transformer registry to return the given transformers.
"""
clear_registered_transformers_cache()
with patch(
'openedx.core.djangoapps.content.block_structure.transformer_registry.'
'TransformerRegistry.get_registered_transformers'
) as mock_available_transforms:
mock_available_transforms.return_value = {transformer for transformer in transformers}
yield
class ChildrenMapTestMixin(object):
"""
A Test Mixin with utility methods for testing with block structures
created and manipulated using children_map and parents_map.
"""
# 0
# / \
# 1 2
# / \
# 3 4
SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
# 0
# /
# 1
# /
# 2
# /
# 3
LINEAR_CHILDREN_MAP = [[1], [2], [3], []]
# 0
# / \
# 1 2
# \ / \
# 3 4
# / \
# 5 6
DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []]
def block_key_factory(self, block_id):
"""
Returns a block key object for the given block_id.
Override this method if the block_key should be anything
different from the index integer values in the Children Maps.
"""
return block_id
def create_block_structure(self, children_map, block_structure_cls=BlockStructureBlockData):
"""
Factory method for creating and returning a block structure
for the given children_map.
"""
# create empty block structure
block_structure = block_structure_cls(root_block_usage_key=self.block_key_factory(0))
# _add_relation
for parent, children in enumerate(children_map):
for child in children:
block_structure._add_relation(self.block_key_factory(parent), self.block_key_factory(child)) # pylint: disable=protected-access
return block_structure
def get_parents_map(self, children_map):
"""
Converts and returns the given children_map to a parents_map.
"""
parent_map = [[] for _ in children_map]
for parent, children in enumerate(children_map):
for child in children:
parent_map[child].append(parent)
return parent_map
def assert_block_structure(self, block_structure, children_map, missing_blocks=None):
"""
Verifies that the relations in the given block structure
equate the relations described in the children_map. Use the
missing_blocks parameter to pass in any blocks that were removed
from the block structure but still have a positional entry in
the children_map.
"""
if not missing_blocks:
missing_blocks = []
for block_key, children in enumerate(children_map):
# Verify presence
self.assertEqual(
self.block_key_factory(block_key) in block_structure,
block_key not in missing_blocks,
'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'.format(
unicode(block_key)
),
)
# Verify children
if block_key not in missing_blocks:
self.assertEqual(
set(block_structure.get_children(self.block_key_factory(block_key))),
set(self.block_key_factory(child) for child in children),
)
# Verify parents
parents_map = self.get_parents_map(children_map)
for block_key, parents in enumerate(parents_map):
if block_key not in missing_blocks:
self.assertEqual(
set(block_structure.get_parents(self.block_key_factory(block_key))),
set(self.block_key_factory(parent) for parent in parents),
)
class UsageKeyFactoryMixin(object):
"""
Test Mixin that provides a block_key_factory to create OpaqueKey objects
for block_ids rather than simple integers. By default, the children maps in
ChildrenMapTestMixin use integers for block_ids.
"""
def setUp(self):
super(UsageKeyFactoryMixin, self).setUp()
self.course_key = CourseLocator('org', 'course', unicode(uuid4()))
def block_key_factory(self, block_id):
"""
Returns a block key object for the given block_id.
"""
return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=unicode(block_id))

0
openedx/core/lib/block_structure/tests/test_block_structure.py → openedx/core/djangoapps/content/block_structure/tests/test_block_structure.py
View File

0
openedx/core/lib/block_structure/tests/test_factory.py → openedx/core/djangoapps/content/block_structure/tests/test_factory.py
View File

5
openedx/core/lib/block_structure/tests/test_manager.py → openedx/core/djangoapps/content/block_structure/tests/test_manager.py
View File

@@ -5,10 +5,8 @@ import ddt
from nose.plugins.attrib import attr
from unittest import TestCase
from openedx.core.djangoapps.content.block_structure.config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE
from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting
from ..block_structure import BlockStructureBlockData
from ..config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE
from ..exceptions import UsageKeyNotInBlockStructure, BlockStructureNotFound
from ..manager import BlockStructureManager
from ..transformers import BlockStructureTransformers
@@ -16,6 +14,7 @@ from .helpers import (
MockModulestoreFactory, MockCache, MockTransformer,
ChildrenMapTestMixin, UsageKeyFactoryMixin,
mock_registered_transformers,
override_config_setting,
)

2
openedx/core/djangoapps/content/block_structure/tests/test_models.py
View File

@@ -10,9 +10,9 @@ from mock import patch, Mock
from uuid import uuid4
from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound
from ..config import PRUNE_OLD_VERSIONS
from ..exceptions import BlockStructureNotFound
from ..models import BlockStructureModel
from .helpers import override_config_setting

2
openedx/core/djangoapps/content/block_structure/tests/test_signals.py
View File

@@ -50,7 +50,7 @@ class CourseBlocksSignalTest(ModuleStoreTestCase):
)
@ddt.data(True, False)
@patch('openedx.core.lib.block_structure.manager.BlockStructureManager.clear')
@patch('openedx.core.djangoapps.content.block_structure.manager.BlockStructureManager.clear')
def test_cache_invalidation(self, invalidate_cache_enabled, mock_bs_manager_clear):
test_display_name = "Jedi 101"

9
openedx/core/lib/block_structure/tests/test_store.py → openedx/core/djangoapps/content/block_structure/tests/test_store.py
View File

@@ -4,14 +4,13 @@ Tests for block_structure/cache.py
import ddt
from nose.plugins.attrib import attr
from openedx.core.djangoapps.content.block_structure.config import STORAGE_BACKING_FOR_CACHE
from openedx.core.djangoapps.content.block_structure.config.models import BlockStructureConfiguration
from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting
from openedx.core.djangolib.testing.utils import CacheIsolationTestCase
from ..store import BlockStructureStore
from ..config import STORAGE_BACKING_FOR_CACHE
from ..config.models import BlockStructureConfiguration
from ..exceptions import BlockStructureNotFound
from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer
from ..store import BlockStructureStore
from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer, override_config_setting
@attr(shard=2)

0
openedx/core/lib/block_structure/tests/test_transformer_registry.py → openedx/core/djangoapps/content/block_structure/tests/test_transformer_registry.py
View File

4
openedx/core/lib/block_structure/tests/test_transformers.py → openedx/core/djangoapps/content/block_structure/tests/test_transformers.py
View File

@@ -57,7 +57,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
def test_collect(self):
with mock_registered_transformers(self.registered_transformers):
with patch(
'openedx.core.lib.block_structure.tests.helpers.MockTransformer.collect'
'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.collect'
) as mock_collect_call:
BlockStructureTransformers.collect(block_structure=MagicMock())
self.assertTrue(mock_collect_call.called)
@@ -66,7 +66,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase):
self.add_mock_transformer()
with patch(
'openedx.core.lib.block_structure.tests.helpers.MockTransformer.transform'
'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.transform'
) as mock_transform_call:
self.transformers.transform(block_structure=MagicMock())
self.assertTrue(mock_transform_call.called)

0
openedx/core/lib/block_structure/transformer.py → openedx/core/djangoapps/content/block_structure/transformer.py
View File

0
openedx/core/lib/block_structure/transformer_registry.py → openedx/core/djangoapps/content/block_structure/transformer_registry.py
View File

0
openedx/core/lib/block_structure/transformers.py → openedx/core/djangoapps/content/block_structure/transformers.py
View File

62
openedx/core/lib/block_structure/__init__.py
View File

@@ -1,62 +0,0 @@
"""
The block_structure django app provides an extensible framework for caching
data of block structures from the modulestore.
Dual-Phase. The framework is meant to be used in 2 phases.
* Collect Phase (for expensive and full-tree traversals) - In the
first phase, the "collect" phase, any and all data from the
modulestore should be collected and cached for later access to
the block structure. Instantiating any and all xBlocks in the block
structure is also done at this phase, since that is also (currently)
a costly operation.
Any full tree traversals should also be done during this phase. For
example, if data for a block depends on its parents, the traversal
should happen during the collection phase and any required data
for the block should be percolated down the tree and stored as
aggregate values on the descendants. This allows for faster and
direct access to blocks in the Transform phase.
* Transform Phase (for fast access to blocks) - In the second
phase, the "transform" phase, only the previously collected and
cached data should be accessed. There should be no access to the
modulestore or instantiation of xBlocks in this phase.
To make this framework extensible, the Transformer and
Extensibility design patterns are used. This django app only
provides the underlying framework for Block Structure Transformers
and a Transformer Registry. Clients are expected to provide actual
implementations of Transformers or add them to the extensible Registry.
Transformers. As inspired by
http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf,
a Block Structure Transformer takes in a block structure (or tree) and
manipulates the structure and the data of its blocks according to its
own requirements. Its output can then be used for further
transformations by other transformers down the pipeline.
Note: For performance and space optimization, our implementation
differs from the paper in that our transformers mutate the block
structure in-place rather than returning a modified copy of it.
Block Structure. The BlockStructure and its family of classes
provided with this framework are the base data types for accessing
and manipulating block structures. BlockStructures are constructed
using the BlockStructureFactory and then used as the currency across
Transformers.
Registry. Transformers are registered using the platform's
PluginManager (e.g., Stevedore). This is currently done by updating
setup.py. Only registered transformers are called during the Collect
Phase. And only registered transformers can be used during the
Transform phase. Exceptions to this rule are any nested transformers
that are contained within higher-order transformers - as long as the
higher-order transformers are registered and appropriately call the
contained transformers within them.
Note: A partial subset (as an ordered list) of the registered
transformers can be requested during the Transform phase, allowing
the client to manipulate exactly which transformers to call.
"""

0
openedx/core/lib/block_structure/tests/__init__.py
View File

310
openedx/core/lib/block_structure/tests/helpers.py
View File

@@ -1,310 +0,0 @@
"""
Common utilities for tests in block_structure module
"""
from contextlib import contextmanager
from mock import patch
from xmodule.modulestore.exceptions import ItemNotFoundError
from uuid import uuid4
from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator
from ..block_structure import BlockStructureBlockData
from ..transformer import BlockStructureTransformer, FilteringTransformerMixin
from ..transformer_registry import TransformerRegistry
class MockXBlock(object):
"""
A mock XBlock to be used in unit tests, thereby decoupling the
implementation of the block cache framework from the xBlock
implementation. This class provides only the minimum xBlock
capabilities needed by the block cache framework.
"""
def __init__(self, location, field_map=None, children=None, modulestore=None):
self.location = location
self.field_map = field_map or {}
self.children = children or []
self.modulestore = modulestore
def __getattr__(self, attr):
try:
return self.field_map[attr]
except KeyError:
raise AttributeError
def get_children(self):
"""
Returns the children of the mock XBlock.
"""
return [self.modulestore.get_item(child) for child in self.children]
class MockModulestore(object):
"""
A mock Modulestore to be used in unit tests, providing only the
minimum methods needed by the block cache framework.
"""
def __init__(self):
self.get_items_call_count = 0
self.blocks = None
def set_blocks(self, blocks):
"""
Updates the mock modulestore with a dictionary of blocks.
Arguments:
blocks ({block key, MockXBlock}) - A map of block_key
to its mock xBlock.
"""
self.blocks = blocks
def get_item(self, block_key, depth=None, lazy=False): # pylint: disable=unused-argument
"""
Returns the mock XBlock (MockXBlock) associated with the
given block_key.
Raises ItemNotFoundError if the item is not found.
"""
self.get_items_call_count += 1
item = self.blocks.get(block_key)
if not item:
raise ItemNotFoundError
return item
@contextmanager
def bulk_operations(self, ignore): # pylint: disable=unused-argument
"""
A context manager for notifying the store of bulk operations.
"""
yield
class MockCache(object):
"""
A mock Cache object, providing only the minimum features needed
by the block cache framework.
"""
def __init__(self):
# An in-memory map of cache keys to cache values.
self.map = {}
self.set_call_count = 0
self.timeout_from_last_call = 0
def set(self, key, val, timeout):
"""
Associates the given key with the given value in the cache.
"""
self.set_call_count += 1
self.map[key] = val
self.timeout_from_last_call = timeout
def get(self, key, default=None):
"""
Returns the value associated with the given key in the cache;
returns default if not found.
"""
return self.map.get(key, default)
def delete(self, key):
"""
Deletes the given key from the cache.
"""
del self.map[key]
class MockModulestoreFactory(object):
"""
A factory for creating MockModulestore objects.
"""
@classmethod
def create(cls, children_map, block_key_factory):
"""
Creates and returns a MockModulestore from the given
children_map.
Arguments:
children_map ({block_key: [block_key]}) - A dictionary
mapping a block key to a list of block keys of the
block's corresponding children.
"""
modulestore = MockModulestore()
modulestore.set_blocks({
block_key_factory(block_key): MockXBlock(
block_key_factory(block_key),
children=[block_key_factory(child) for child in children],
modulestore=modulestore,
)
for block_key, children in enumerate(children_map)
})
return modulestore
class MockTransformer(BlockStructureTransformer):
"""
A mock BlockStructureTransformer class.
"""
WRITE_VERSION = 1
READ_VERSION = 1
@classmethod
def name(cls):
# Use the class' name for Mock transformers.
return cls.__name__
def transform(self, usage_info, block_structure):
pass
class MockFilteringTransformer(FilteringTransformerMixin, BlockStructureTransformer):
"""
A mock FilteringTransformerMixin class.
"""
WRITE_VERSION = 1
READ_VERSION = 1
@classmethod
def name(cls):
# Use the class' name for Mock transformers.
return cls.__name__
def transform_block_filters(self, usage_info, block_structure):
return [block_structure.create_universal_filter()]
def clear_registered_transformers_cache():
"""
Test helper to clear out any cached values of registered transformers.
"""
TransformerRegistry.get_write_version_hash.cache.clear()
@contextmanager
def mock_registered_transformers(transformers):
"""
Context manager for mocking the transformer registry to return the given transformers.
"""
clear_registered_transformers_cache()
with patch(
'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers'
) as mock_available_transforms:
mock_available_transforms.return_value = {transformer for transformer in transformers}
yield
class ChildrenMapTestMixin(object):
"""
A Test Mixin with utility methods for testing with block structures
created and manipulated using children_map and parents_map.
"""
# 0
# / \
# 1 2
# / \
# 3 4
SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []]
# 0
# /
# 1
# /
# 2
# /
# 3
LINEAR_CHILDREN_MAP = [[1], [2], [3], []]
# 0
# / \
# 1 2
# \ / \
# 3 4
# / \
# 5 6
DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []]
def block_key_factory(self, block_id):
"""
Returns a block key object for the given block_id.
Override this method if the block_key should be anything
different from the index integer values in the Children Maps.
"""
return block_id
def create_block_structure(self, children_map, block_structure_cls=BlockStructureBlockData):
"""
Factory method for creating and returning a block structure
for the given children_map.
"""
# create empty block structure
block_structure = block_structure_cls(root_block_usage_key=self.block_key_factory(0))
# _add_relation
for parent, children in enumerate(children_map):
for child in children:
block_structure._add_relation(self.block_key_factory(parent), self.block_key_factory(child)) # pylint: disable=protected-access
return block_structure
def get_parents_map(self, children_map):
"""
Converts and returns the given children_map to a parents_map.
"""
parent_map = [[] for _ in children_map]
for parent, children in enumerate(children_map):
for child in children:
parent_map[child].append(parent)
return parent_map
def assert_block_structure(self, block_structure, children_map, missing_blocks=None):
"""
Verifies that the relations in the given block structure
equate the relations described in the children_map. Use the
missing_blocks parameter to pass in any blocks that were removed
from the block structure but still have a positional entry in
the children_map.
"""
if not missing_blocks:
missing_blocks = []
for block_key, children in enumerate(children_map):
# Verify presence
self.assertEqual(
self.block_key_factory(block_key) in block_structure,
block_key not in missing_blocks,
'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'.format(
unicode(block_key)
),
)
# Verify children
if block_key not in missing_blocks:
self.assertEqual(
set(block_structure.get_children(self.block_key_factory(block_key))),
set(self.block_key_factory(child) for child in children),
)
# Verify parents
parents_map = self.get_parents_map(children_map)
for block_key, parents in enumerate(parents_map):
if block_key not in missing_blocks:
self.assertEqual(
set(block_structure.get_parents(self.block_key_factory(block_key))),
set(self.block_key_factory(parent) for parent in parents),
)
class UsageKeyFactoryMixin(object):
"""
Test Mixin that provides a block_key_factory to create OpaqueKey objects
for block_ids rather than simple integers. By default, the children maps in
ChildrenMapTestMixin use integers for block_ids.
"""
def setUp(self):
super(UsageKeyFactoryMixin, self).setUp()
self.course_key = CourseLocator('org', 'course', unicode(uuid4()))
def block_key_factory(self, block_id):
"""
Returns a block key object for the given block_id.
"""
return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=unicode(block_id))