diff --git a/lms/djangoapps/course_api/blocks/api.py b/lms/djangoapps/course_api/blocks/api.py index eb7ffb4fd7..68a6454804 100644 --- a/lms/djangoapps/course_api/blocks/api.py +++ b/lms/djangoapps/course_api/blocks/api.py @@ -4,7 +4,7 @@ API function for retrieving course blocks data from lms.djangoapps.course_blocks.api import get_course_blocks, COURSE_BLOCK_ACCESS_TRANSFORMERS from lms.djangoapps.course_blocks.transformers.hidden_content import HiddenContentTransformer -from openedx.core.lib.block_structure.transformers import BlockStructureTransformers +from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers from .transformers.blocks_api import BlocksAPITransformer from .transformers.milestones import MilestonesTransformer diff --git a/lms/djangoapps/course_api/blocks/tests/test_serializers.py b/lms/djangoapps/course_api/blocks/tests/test_serializers.py index 6c118ed3e6..d5a3be825f 100644 --- a/lms/djangoapps/course_api/blocks/tests/test_serializers.py +++ b/lms/djangoapps/course_api/blocks/tests/test_serializers.py @@ -3,7 +3,7 @@ Tests for Course Blocks serializers """ from mock import MagicMock -from openedx.core.lib.block_structure.transformers import BlockStructureTransformers +from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers from student.tests.factories import UserFactory from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase diff --git a/lms/djangoapps/course_api/blocks/transformers/block_counts.py b/lms/djangoapps/course_api/blocks/transformers/block_counts.py index ea38c3ed13..35108eb29b 100644 --- a/lms/djangoapps/course_api/blocks/transformers/block_counts.py +++ b/lms/djangoapps/course_api/blocks/transformers/block_counts.py @@ -1,7 +1,7 @@ """ Block Counts Transformer """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer +from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer class BlockCountsTransformer(BlockStructureTransformer): diff --git a/lms/djangoapps/course_api/blocks/transformers/block_depth.py b/lms/djangoapps/course_api/blocks/transformers/block_depth.py index 889bf1f987..78d9a935ce 100644 --- a/lms/djangoapps/course_api/blocks/transformers/block_depth.py +++ b/lms/djangoapps/course_api/blocks/transformers/block_depth.py @@ -1,7 +1,7 @@ """ Block Depth Transformer """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer +from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer class BlockDepthTransformer(BlockStructureTransformer): diff --git a/lms/djangoapps/course_api/blocks/transformers/blocks_api.py b/lms/djangoapps/course_api/blocks/transformers/blocks_api.py index 1f99fc94da..ce988d39e3 100644 --- a/lms/djangoapps/course_api/blocks/transformers/blocks_api.py +++ b/lms/djangoapps/course_api/blocks/transformers/blocks_api.py @@ -1,7 +1,7 @@ """ Blocks API Transformer """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer +from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer from .block_counts import BlockCountsTransformer from .block_depth import BlockDepthTransformer from .navigation import BlockNavigationTransformer diff --git a/lms/djangoapps/course_api/blocks/transformers/milestones.py b/lms/djangoapps/course_api/blocks/transformers/milestones.py index e05c123228..aaa93ec551 100644 --- a/lms/djangoapps/course_api/blocks/transformers/milestones.py +++ b/lms/djangoapps/course_api/blocks/transformers/milestones.py @@ -4,7 +4,10 @@ Milestones Transformer from django.conf import settings -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin +from openedx.core.djangoapps.content.block_structure.transformer import ( + BlockStructureTransformer, + FilteringTransformerMixin, +) from util import milestones_helpers diff --git a/lms/djangoapps/course_api/blocks/transformers/navigation.py b/lms/djangoapps/course_api/blocks/transformers/navigation.py index 643199adb5..4a9e2b9697 100644 --- a/lms/djangoapps/course_api/blocks/transformers/navigation.py +++ b/lms/djangoapps/course_api/blocks/transformers/navigation.py @@ -1,7 +1,7 @@ """ TODO """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer +from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer from .block_depth import BlockDepthTransformer diff --git a/lms/djangoapps/course_api/blocks/transformers/student_view.py b/lms/djangoapps/course_api/blocks/transformers/student_view.py index aea7ce5ef4..1b88ef5013 100644 --- a/lms/djangoapps/course_api/blocks/transformers/student_view.py +++ b/lms/djangoapps/course_api/blocks/transformers/student_view.py @@ -1,7 +1,7 @@ """ Student View Transformer """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer +from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer class StudentViewTransformer(BlockStructureTransformer): diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py index f512ca10d4..d8834d3617 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_block_counts.py @@ -3,7 +3,7 @@ Tests for BlockCountsTransformer. """ # pylint: disable=protected-access -from openedx.core.lib.block_structure.factory import BlockStructureFactory +from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import SampleCourseFactory diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_block_depth.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_block_depth.py index 6c4f4c8695..60ee26ae9a 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_block_depth.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_block_depth.py @@ -7,8 +7,8 @@ Tests for BlockDepthTransformer. import ddt from unittest import TestCase -from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin -from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData +from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin +from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData from ..block_depth import BlockDepthTransformer diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_navigation.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_navigation.py index 973303ca85..8cf73593cd 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_navigation.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_navigation.py @@ -7,9 +7,9 @@ from unittest import TestCase from lms.djangoapps.course_api.blocks.transformers.block_depth import BlockDepthTransformer from lms.djangoapps.course_api.blocks.transformers.navigation import BlockNavigationTransformer -from openedx.core.lib.block_structure.tests.helpers import ChildrenMapTestMixin -from openedx.core.lib.block_structure.block_structure import BlockStructureModulestoreData -from openedx.core.lib.block_structure.factory import BlockStructureFactory +from openedx.core.djangoapps.content.block_structure.tests.helpers import ChildrenMapTestMixin +from openedx.core.djangoapps.content.block_structure.block_structure import BlockStructureModulestoreData +from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import SampleCourseFactory from xmodule.modulestore import ModuleStoreEnum diff --git a/lms/djangoapps/course_api/blocks/transformers/tests/test_student_view.py b/lms/djangoapps/course_api/blocks/transformers/tests/test_student_view.py index 63665bcf14..b37ec88e80 100644 --- a/lms/djangoapps/course_api/blocks/transformers/tests/test_student_view.py +++ b/lms/djangoapps/course_api/blocks/transformers/tests/test_student_view.py @@ -4,7 +4,7 @@ Tests for StudentViewTransformer. # pylint: disable=protected-access -from openedx.core.lib.block_structure.factory import BlockStructureFactory +from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import ToyCourseFactory diff --git a/lms/djangoapps/course_blocks/__init__.py b/lms/djangoapps/course_blocks/__init__.py index fc4d282098..891764f2d8 100644 --- a/lms/djangoapps/course_blocks/__init__.py +++ b/lms/djangoapps/course_blocks/__init__.py @@ -1,6 +1,6 @@ """ The Course Blocks app, built upon the Block Cache framework in -openedx.core.lib.block_structure, is a higher layer django app in LMS that +openedx.core.djangoapps.content.block_structure, is a higher layer django app in LMS that provides additional context of Courses and Users (via usage_info.py) with implementations for Block Structure Transformers that are related to block structure course access. diff --git a/lms/djangoapps/course_blocks/api.py b/lms/djangoapps/course_blocks/api.py index 849527915d..6c37624887 100644 --- a/lms/djangoapps/course_blocks/api.py +++ b/lms/djangoapps/course_blocks/api.py @@ -3,7 +3,7 @@ API entry point to the course_blocks app with top-level get_course_blocks function. """ from openedx.core.djangoapps.content.block_structure.api import get_block_structure_manager -from openedx.core.lib.block_structure.transformers import BlockStructureTransformers +from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers from .transformers import ( library_content, diff --git a/lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py b/lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py index d7d24552da..6dd9c895d7 100644 --- a/lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py +++ b/lms/djangoapps/course_blocks/management/commands/generate_course_blocks.py @@ -8,7 +8,7 @@ from xmodule.modulestore.django import modulestore import openedx.core.djangoapps.content.block_structure.api as api import openedx.core.djangoapps.content.block_structure.tasks as tasks -import openedx.core.lib.block_structure.store as store +import openedx.core.djangoapps.content.block_structure.store as store from openedx.core.lib.command_utils import ( get_mutually_exclusive_required_option, validate_dependent_option, diff --git a/lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py b/lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py index 6e320f329f..25b49da7c7 100644 --- a/lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py +++ b/lms/djangoapps/course_blocks/management/commands/tests/test_generate_course_blocks.py @@ -59,7 +59,7 @@ class TestGenerateCourseBlocks(ModuleStoreTestCase): self.command.handle(all_courses=True) self._assert_courses_in_block_cache(*self.course_keys) with patch( - 'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_modulestore' + 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_modulestore' ) as mock_update_from_store: self.command.handle(all_courses=True, force_update=force_update) self.assertEqual(mock_update_from_store.call_count, self.num_courses if force_update else 0) diff --git a/lms/djangoapps/course_blocks/transformers/hidden_content.py b/lms/djangoapps/course_blocks/transformers/hidden_content.py index 559f2b1d35..0627390ef2 100644 --- a/lms/djangoapps/course_blocks/transformers/hidden_content.py +++ b/lms/djangoapps/course_blocks/transformers/hidden_content.py @@ -4,7 +4,10 @@ Visibility Transformer implementation. from datetime import datetime from pytz import utc -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin +from openedx.core.djangoapps.content.block_structure.transformer import ( + BlockStructureTransformer, + FilteringTransformerMixin, +) from xmodule.seq_module import SequenceModule from .utils import collect_merged_boolean_field, collect_merged_date_field diff --git a/lms/djangoapps/course_blocks/transformers/library_content.py b/lms/djangoapps/course_blocks/transformers/library_content.py index 943b0e1503..44f7958aea 100644 --- a/lms/djangoapps/course_blocks/transformers/library_content.py +++ b/lms/djangoapps/course_blocks/transformers/library_content.py @@ -3,7 +3,10 @@ Content Library Transformer. """ import json from courseware.models import StudentModule -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin +from openedx.core.djangoapps.content.block_structure.transformer import ( + BlockStructureTransformer, + FilteringTransformerMixin, +) from xmodule.library_content_module import LibraryContentModule from xmodule.modulestore.django import modulestore from eventtracking import tracker diff --git a/lms/djangoapps/course_blocks/transformers/split_test.py b/lms/djangoapps/course_blocks/transformers/split_test.py index 9a3dd1231c..4db261a6e1 100644 --- a/lms/djangoapps/course_blocks/transformers/split_test.py +++ b/lms/djangoapps/course_blocks/transformers/split_test.py @@ -1,7 +1,10 @@ """ Split Test Block Transformer """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin +from openedx.core.djangoapps.content.block_structure.transformer import ( + BlockStructureTransformer, + FilteringTransformerMixin, +) class SplitTestTransformer(FilteringTransformerMixin, BlockStructureTransformer): diff --git a/lms/djangoapps/course_blocks/transformers/start_date.py b/lms/djangoapps/course_blocks/transformers/start_date.py index 90822e72ba..561d3c3153 100644 --- a/lms/djangoapps/course_blocks/transformers/start_date.py +++ b/lms/djangoapps/course_blocks/transformers/start_date.py @@ -1,7 +1,10 @@ """ Start Date Transformer implementation. """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin +from openedx.core.djangoapps.content.block_structure.transformer import ( + BlockStructureTransformer, + FilteringTransformerMixin, +) from lms.djangoapps.courseware.access_utils import check_start_date from xmodule.course_metadata_utils import DEFAULT_START_DATE diff --git a/lms/djangoapps/course_blocks/transformers/tests/helpers.py b/lms/djangoapps/course_blocks/transformers/tests/helpers.py index 2ae2123ad1..073dd08b5b 100644 --- a/lms/djangoapps/course_blocks/transformers/tests/helpers.py +++ b/lms/djangoapps/course_blocks/transformers/tests/helpers.py @@ -4,8 +4,8 @@ Test helpers for testing course block transformers. from mock import patch from course_modes.models import CourseMode from lms.djangoapps.courseware.access import has_access -from openedx.core.lib.block_structure.transformers import BlockStructureTransformers -from openedx.core.lib.block_structure.tests.helpers import clear_registered_transformers_cache +from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers +from openedx.core.djangoapps.content.block_structure.tests.helpers import clear_registered_transformers_cache from student.tests.factories import CourseEnrollmentFactory, UserFactory from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore @@ -23,7 +23,8 @@ class TransformerRegistryTestMixin(object): def setUp(self): super(TransformerRegistryTestMixin, self).setUp() self.patcher = patch( - 'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers' + 'openedx.core.djangoapps.content.block_structure.transformer_registry.' + 'TransformerRegistry.get_registered_transformers' ) mock_registry = self.patcher.start() mock_registry.return_value = {self.TRANSFORMER_CLASS_TO_TEST} diff --git a/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py b/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py index 846081b69f..2d36175832 100644 --- a/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py +++ b/lms/djangoapps/course_blocks/transformers/tests/test_library_content.py @@ -5,7 +5,7 @@ Tests for ContentLibraryTransformer. from student.tests.factories import CourseEnrollmentFactory from openedx.core.djangoapps.content.block_structure.api import clear_course_from_cache -from openedx.core.lib.block_structure.transformers import BlockStructureTransformers +from openedx.core.djangoapps.content.block_structure.transformers import BlockStructureTransformers from ...api import get_course_blocks from ..library_content import ContentLibraryTransformer diff --git a/lms/djangoapps/course_blocks/transformers/user_partitions.py b/lms/djangoapps/course_blocks/transformers/user_partitions.py index b3635c43f1..78930ae879 100644 --- a/lms/djangoapps/course_blocks/transformers/user_partitions.py +++ b/lms/djangoapps/course_blocks/transformers/user_partitions.py @@ -1,7 +1,10 @@ """ User Partitions Transformer """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin +from openedx.core.djangoapps.content.block_structure.transformer import ( + BlockStructureTransformer, + FilteringTransformerMixin, +) from .split_test import SplitTestTransformer from .utils import get_field_on_block diff --git a/lms/djangoapps/course_blocks/transformers/visibility.py b/lms/djangoapps/course_blocks/transformers/visibility.py index 162541cdaa..bd87260428 100644 --- a/lms/djangoapps/course_blocks/transformers/visibility.py +++ b/lms/djangoapps/course_blocks/transformers/visibility.py @@ -1,7 +1,10 @@ """ Visibility Transformer implementation. """ -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer, FilteringTransformerMixin +from openedx.core.djangoapps.content.block_structure.transformer import ( + BlockStructureTransformer, + FilteringTransformerMixin, +) from .utils import collect_merged_boolean_field diff --git a/lms/djangoapps/grades/tests/test_grades.py b/lms/djangoapps/grades/tests/test_grades.py index 6f3194b421..1b6975e825 100644 --- a/lms/djangoapps/grades/tests/test_grades.py +++ b/lms/djangoapps/grades/tests/test_grades.py @@ -12,7 +12,7 @@ from courseware.model_data import set_score from courseware.tests.helpers import LoginEnrollmentTestCase from lms.djangoapps.course_blocks.api import get_course_blocks -from openedx.core.lib.block_structure.factory import BlockStructureFactory +from openedx.core.djangoapps.content.block_structure.factory import BlockStructureFactory from openedx.core.djangolib.testing.utils import get_mock_request from student.tests.factories import UserFactory from student.models import CourseEnrollment diff --git a/lms/djangoapps/grades/tests/test_scores.py b/lms/djangoapps/grades/tests/test_scores.py index aeefd4e279..50805ecb7f 100644 --- a/lms/djangoapps/grades/tests/test_scores.py +++ b/lms/djangoapps/grades/tests/test_scores.py @@ -11,7 +11,7 @@ from lms.djangoapps.grades.models import BlockRecord import lms.djangoapps.grades.scores as scores from lms.djangoapps.grades.transformer import GradesTransformer from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator -from openedx.core.lib.block_structure.block_structure import BlockData +from openedx.core.djangoapps.content.block_structure.block_structure import BlockData from xmodule.graders import ProblemScore diff --git a/lms/djangoapps/grades/tests/test_tasks.py b/lms/djangoapps/grades/tests/test_tasks.py index 1ddf4efc8c..bb1c99b6fa 100644 --- a/lms/djangoapps/grades/tests/test_tasks.py +++ b/lms/djangoapps/grades/tests/test_tasks.py @@ -13,7 +13,7 @@ from mock import patch, MagicMock import pytz from util.date_utils import to_timestamp -from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound +from openedx.core.djangoapps.content.block_structure.exceptions import BlockStructureNotFound from student.models import anonymous_id_for_user from student.tests.factories import UserFactory from track.event_transaction_utils import ( @@ -136,7 +136,7 @@ class RecalculateSubsectionGradeTest(ModuleStoreTestCase): self.set_up_course() self.assertTrue(PersistentGradesEnabledFlag.feature_enabled(self.course.id)) with patch( - 'openedx.core.lib.block_structure.factory.BlockStructureFactory.create_from_store', + 'openedx.core.djangoapps.content.block_structure.factory.BlockStructureFactory.create_from_store', side_effect=BlockStructureNotFound(self.course.location), ) as mock_block_structure_create: self._apply_recalculate_subsection_grade() diff --git a/lms/djangoapps/grades/transformer.py b/lms/djangoapps/grades/transformer.py index f481bd2665..8353eac136 100644 --- a/lms/djangoapps/grades/transformer.py +++ b/lms/djangoapps/grades/transformer.py @@ -8,7 +8,7 @@ from logging import getLogger import json from lms.djangoapps.course_blocks.transformers.utils import collect_unioned_set_field, get_field_on_block -from openedx.core.lib.block_structure.transformer import BlockStructureTransformer +from openedx.core.djangoapps.content.block_structure.transformer import BlockStructureTransformer log = getLogger(__name__) diff --git a/openedx/core/djangoapps/content/block_structure/__init__.py b/openedx/core/djangoapps/content/block_structure/__init__.py index 3b81c1b750..f77cf27f04 100644 --- a/openedx/core/djangoapps/content/block_structure/__init__.py +++ b/openedx/core/djangoapps/content/block_structure/__init__.py @@ -1,5 +1,62 @@ """ -This code exists in openedx/core/djangoapp because it needs access to django signaling mechanisms +The block_structure django app provides an extensible framework for caching +data of block structures from the modulestore. -Most of the underlying functionality is implemented in openedx/core/lib/block_structure/ +Dual-Phase. The framework is meant to be used in 2 phases. + + * Collect Phase (for expensive and full-tree traversals) - In the + first phase, the "collect" phase, any and all data from the + modulestore should be collected and cached for later access to + the block structure. Instantiating any and all xBlocks in the block + structure is also done at this phase, since that is also (currently) + a costly operation. + + Any full tree traversals should also be done during this phase. For + example, if data for a block depends on its parents, the traversal + should happen during the collection phase and any required data + for the block should be percolated down the tree and stored as + aggregate values on the descendants. This allows for faster and + direct access to blocks in the Transform phase. + + * Transform Phase (for fast access to blocks) - In the second + phase, the "transform" phase, only the previously collected and + cached data should be accessed. There should be no access to the + modulestore or instantiation of xBlocks in this phase. + + +To make this framework extensible, the Transformer and +Extensibility design patterns are used. This django app only +provides the underlying framework for Block Structure Transformers +and a Transformer Registry. Clients are expected to provide actual +implementations of Transformers or add them to the extensible Registry. + +Transformers. As inspired by +http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf, +a Block Structure Transformer takes in a block structure (or tree) and +manipulates the structure and the data of its blocks according to its +own requirements. Its output can then be used for further +transformations by other transformers down the pipeline. + +Note: For performance and space optimization, our implementation +differs from the paper in that our transformers mutate the block +structure in-place rather than returning a modified copy of it. + +Block Structure. The BlockStructure and its family of classes +provided with this framework are the base data types for accessing +and manipulating block structures. BlockStructures are constructed +using the BlockStructureFactory and then used as the currency across +Transformers. + +Registry. Transformers are registered using the platform's +PluginManager (e.g., Stevedore). This is currently done by updating +setup.py. Only registered transformers are called during the Collect +Phase. And only registered transformers can be used during the +Transform phase. Exceptions to this rule are any nested transformers +that are contained within higher-order transformers - as long as the +higher-order transformers are registered and appropriately call the +contained transformers within them. + +Note: A partial subset (as an ordered list) of the registered +transformers can be requested during the Transform phase, allowing +the client to manipulate exactly which transformers to call. """ diff --git a/openedx/core/djangoapps/content/block_structure/api.py b/openedx/core/djangoapps/content/block_structure/api.py index b4cb22512b..a86b5bc242 100644 --- a/openedx/core/djangoapps/content/block_structure/api.py +++ b/openedx/core/djangoapps/content/block_structure/api.py @@ -2,9 +2,10 @@ Higher order functions built on the BlockStructureManager to interact with a django cache. """ from django.core.cache import cache -from openedx.core.lib.block_structure.manager import BlockStructureManager from xmodule.modulestore.django import modulestore +from .manager import BlockStructureManager + def get_course_in_cache(course_key): """ diff --git a/openedx/core/lib/block_structure/block_structure.py b/openedx/core/djangoapps/content/block_structure/block_structure.py similarity index 100% rename from openedx/core/lib/block_structure/block_structure.py rename to openedx/core/djangoapps/content/block_structure/block_structure.py diff --git a/openedx/core/lib/block_structure/exceptions.py b/openedx/core/djangoapps/content/block_structure/exceptions.py similarity index 100% rename from openedx/core/lib/block_structure/exceptions.py rename to openedx/core/djangoapps/content/block_structure/exceptions.py diff --git a/openedx/core/lib/block_structure/factory.py b/openedx/core/djangoapps/content/block_structure/factory.py similarity index 100% rename from openedx/core/lib/block_structure/factory.py rename to openedx/core/djangoapps/content/block_structure/factory.py diff --git a/openedx/core/lib/block_structure/manager.py b/openedx/core/djangoapps/content/block_structure/manager.py similarity index 98% rename from openedx/core/lib/block_structure/manager.py rename to openedx/core/djangoapps/content/block_structure/manager.py index 3f9338359a..a5a4c59f5b 100644 --- a/openedx/core/lib/block_structure/manager.py +++ b/openedx/core/djangoapps/content/block_structure/manager.py @@ -4,8 +4,7 @@ BlockStructures. """ from contextlib import contextmanager -from openedx.core.djangoapps.content.block_structure import config - +from . import config from .exceptions import UsageKeyNotInBlockStructure, TransformerDataIncompatible, BlockStructureNotFound from .factory import BlockStructureFactory from .store import BlockStructureStore diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index a0ad5927de..d402873d4f 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -10,10 +10,10 @@ from logging import getLogger from model_utils.models import TimeStampedModel from openedx.core.djangoapps.xmodule_django.models import UsageKeyField -from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound from openedx.core.storage import get_storage -import openedx.core.djangoapps.content.block_structure.config as config +from . import config +from .exceptions import BlockStructureNotFound log = getLogger(__name__) diff --git a/openedx/core/lib/block_structure/store.py b/openedx/core/djangoapps/content/block_structure/store.py similarity index 98% rename from openedx/core/lib/block_structure/store.py rename to openedx/core/djangoapps/content/block_structure/store.py index 04e6e06c32..16203fec67 100644 --- a/openedx/core/lib/block_structure/store.py +++ b/openedx/core/djangoapps/content/block_structure/store.py @@ -4,14 +4,13 @@ Module for the Storage of BlockStructure objects. # pylint: disable=protected-access from logging import getLogger -import openedx.core.djangoapps.content.block_structure.config as config -from openedx.core.djangoapps.content.block_structure.models import BlockStructureModel - from openedx.core.lib.cache_utils import zpickle, zunpickle +from . import config from .block_structure import BlockStructureBlockData from .exceptions import BlockStructureNotFound from .factory import BlockStructureFactory +from .models import BlockStructureModel from .transformer_registry import TransformerRegistry diff --git a/openedx/core/djangoapps/content/block_structure/tests/helpers.py b/openedx/core/djangoapps/content/block_structure/tests/helpers.py index bd90484b31..07c3dba4bd 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/helpers.py +++ b/openedx/core/djangoapps/content/block_structure/tests/helpers.py @@ -1,12 +1,21 @@ """ -Helpers for Course Blocks tests. +Common utilities for tests in block_structure module """ +from contextlib import contextmanager +from mock import patch +from xmodule.modulestore.exceptions import ItemNotFoundError +from uuid import uuid4 + +from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator from openedx.core.djangolib.testing.waffle_utils import override_switch -from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound -from openedx.core.lib.block_structure.store import BlockStructureStore from ..api import get_cache +from ..block_structure import BlockStructureBlockData from ..config import _bs_waffle_switch_name +from ..exceptions import BlockStructureNotFound +from ..store import BlockStructureStore +from ..transformer import BlockStructureTransformer, FilteringTransformerMixin +from ..transformer_registry import TransformerRegistry def is_course_in_block_structure_cache(course_key, store): @@ -31,3 +40,301 @@ class override_config_setting(override_switch): # pylint:disable=invalid-name _bs_waffle_switch_name(name), active ) + + +class MockXBlock(object): + """ + A mock XBlock to be used in unit tests, thereby decoupling the + implementation of the block cache framework from the xBlock + implementation. This class provides only the minimum xBlock + capabilities needed by the block cache framework. + """ + def __init__(self, location, field_map=None, children=None, modulestore=None): + self.location = location + self.field_map = field_map or {} + + self.children = children or [] + self.modulestore = modulestore + + def __getattr__(self, attr): + try: + return self.field_map[attr] + except KeyError: + raise AttributeError + + def get_children(self): + """ + Returns the children of the mock XBlock. + """ + return [self.modulestore.get_item(child) for child in self.children] + + +class MockModulestore(object): + """ + A mock Modulestore to be used in unit tests, providing only the + minimum methods needed by the block cache framework. + """ + def __init__(self): + self.get_items_call_count = 0 + self.blocks = None + + def set_blocks(self, blocks): + """ + Updates the mock modulestore with a dictionary of blocks. + + Arguments: + blocks ({block key, MockXBlock}) - A map of block_key + to its mock xBlock. + """ + self.blocks = blocks + + def get_item(self, block_key, depth=None, lazy=False): # pylint: disable=unused-argument + """ + Returns the mock XBlock (MockXBlock) associated with the + given block_key. + + Raises ItemNotFoundError if the item is not found. + """ + self.get_items_call_count += 1 + item = self.blocks.get(block_key) + if not item: + raise ItemNotFoundError + return item + + @contextmanager + def bulk_operations(self, ignore): # pylint: disable=unused-argument + """ + A context manager for notifying the store of bulk operations. + """ + yield + + +class MockCache(object): + """ + A mock Cache object, providing only the minimum features needed + by the block cache framework. + """ + def __init__(self): + # An in-memory map of cache keys to cache values. + self.map = {} + self.set_call_count = 0 + self.timeout_from_last_call = 0 + + def set(self, key, val, timeout): + """ + Associates the given key with the given value in the cache. + """ + self.set_call_count += 1 + self.map[key] = val + self.timeout_from_last_call = timeout + + def get(self, key, default=None): + """ + Returns the value associated with the given key in the cache; + returns default if not found. + """ + return self.map.get(key, default) + + def delete(self, key): + """ + Deletes the given key from the cache. + """ + del self.map[key] + + +class MockModulestoreFactory(object): + """ + A factory for creating MockModulestore objects. + """ + @classmethod + def create(cls, children_map, block_key_factory): + """ + Creates and returns a MockModulestore from the given + children_map. + + Arguments: + children_map ({block_key: [block_key]}) - A dictionary + mapping a block key to a list of block keys of the + block's corresponding children. + """ + modulestore = MockModulestore() + modulestore.set_blocks({ + block_key_factory(block_key): MockXBlock( + block_key_factory(block_key), + children=[block_key_factory(child) for child in children], + modulestore=modulestore, + ) + for block_key, children in enumerate(children_map) + }) + return modulestore + + +class MockTransformer(BlockStructureTransformer): + """ + A mock BlockStructureTransformer class. + """ + WRITE_VERSION = 1 + READ_VERSION = 1 + + @classmethod + def name(cls): + # Use the class' name for Mock transformers. + return cls.__name__ + + def transform(self, usage_info, block_structure): + pass + + +class MockFilteringTransformer(FilteringTransformerMixin, BlockStructureTransformer): + """ + A mock FilteringTransformerMixin class. + """ + WRITE_VERSION = 1 + READ_VERSION = 1 + + @classmethod + def name(cls): + # Use the class' name for Mock transformers. + return cls.__name__ + + def transform_block_filters(self, usage_info, block_structure): + return [block_structure.create_universal_filter()] + + +def clear_registered_transformers_cache(): + """ + Test helper to clear out any cached values of registered transformers. + """ + TransformerRegistry.get_write_version_hash.cache.clear() + + +@contextmanager +def mock_registered_transformers(transformers): + """ + Context manager for mocking the transformer registry to return the given transformers. + """ + clear_registered_transformers_cache() + with patch( + 'openedx.core.djangoapps.content.block_structure.transformer_registry.' + 'TransformerRegistry.get_registered_transformers' + ) as mock_available_transforms: + mock_available_transforms.return_value = {transformer for transformer in transformers} + yield + + +class ChildrenMapTestMixin(object): + """ + A Test Mixin with utility methods for testing with block structures + created and manipulated using children_map and parents_map. + """ + + # 0 + # / \ + # 1 2 + # / \ + # 3 4 + SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []] + + # 0 + # / + # 1 + # / + # 2 + # / + # 3 + LINEAR_CHILDREN_MAP = [[1], [2], [3], []] + + # 0 + # / \ + # 1 2 + # \ / \ + # 3 4 + # / \ + # 5 6 + DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []] + + def block_key_factory(self, block_id): + """ + Returns a block key object for the given block_id. + Override this method if the block_key should be anything + different from the index integer values in the Children Maps. + """ + return block_id + + def create_block_structure(self, children_map, block_structure_cls=BlockStructureBlockData): + """ + Factory method for creating and returning a block structure + for the given children_map. + """ + # create empty block structure + block_structure = block_structure_cls(root_block_usage_key=self.block_key_factory(0)) + + # _add_relation + for parent, children in enumerate(children_map): + for child in children: + block_structure._add_relation(self.block_key_factory(parent), self.block_key_factory(child)) # pylint: disable=protected-access + return block_structure + + def get_parents_map(self, children_map): + """ + Converts and returns the given children_map to a parents_map. + """ + parent_map = [[] for _ in children_map] + for parent, children in enumerate(children_map): + for child in children: + parent_map[child].append(parent) + return parent_map + + def assert_block_structure(self, block_structure, children_map, missing_blocks=None): + """ + Verifies that the relations in the given block structure + equate the relations described in the children_map. Use the + missing_blocks parameter to pass in any blocks that were removed + from the block structure but still have a positional entry in + the children_map. + """ + if not missing_blocks: + missing_blocks = [] + + for block_key, children in enumerate(children_map): + # Verify presence + self.assertEqual( + self.block_key_factory(block_key) in block_structure, + block_key not in missing_blocks, + 'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'.format( + unicode(block_key) + ), + ) + + # Verify children + if block_key not in missing_blocks: + self.assertEqual( + set(block_structure.get_children(self.block_key_factory(block_key))), + set(self.block_key_factory(child) for child in children), + ) + + # Verify parents + parents_map = self.get_parents_map(children_map) + for block_key, parents in enumerate(parents_map): + if block_key not in missing_blocks: + self.assertEqual( + set(block_structure.get_parents(self.block_key_factory(block_key))), + set(self.block_key_factory(parent) for parent in parents), + ) + + +class UsageKeyFactoryMixin(object): + """ + Test Mixin that provides a block_key_factory to create OpaqueKey objects + for block_ids rather than simple integers. By default, the children maps in + ChildrenMapTestMixin use integers for block_ids. + """ + def setUp(self): + super(UsageKeyFactoryMixin, self).setUp() + self.course_key = CourseLocator('org', 'course', unicode(uuid4())) + + def block_key_factory(self, block_id): + """ + Returns a block key object for the given block_id. + """ + return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=unicode(block_id)) diff --git a/openedx/core/lib/block_structure/tests/test_block_structure.py b/openedx/core/djangoapps/content/block_structure/tests/test_block_structure.py similarity index 100% rename from openedx/core/lib/block_structure/tests/test_block_structure.py rename to openedx/core/djangoapps/content/block_structure/tests/test_block_structure.py diff --git a/openedx/core/lib/block_structure/tests/test_factory.py b/openedx/core/djangoapps/content/block_structure/tests/test_factory.py similarity index 100% rename from openedx/core/lib/block_structure/tests/test_factory.py rename to openedx/core/djangoapps/content/block_structure/tests/test_factory.py diff --git a/openedx/core/lib/block_structure/tests/test_manager.py b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py similarity index 97% rename from openedx/core/lib/block_structure/tests/test_manager.py rename to openedx/core/djangoapps/content/block_structure/tests/test_manager.py index 23f654b1bd..edb9193446 100644 --- a/openedx/core/lib/block_structure/tests/test_manager.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_manager.py @@ -5,10 +5,8 @@ import ddt from nose.plugins.attrib import attr from unittest import TestCase -from openedx.core.djangoapps.content.block_structure.config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE -from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting - from ..block_structure import BlockStructureBlockData +from ..config import RAISE_ERROR_WHEN_NOT_FOUND, STORAGE_BACKING_FOR_CACHE from ..exceptions import UsageKeyNotInBlockStructure, BlockStructureNotFound from ..manager import BlockStructureManager from ..transformers import BlockStructureTransformers @@ -16,6 +14,7 @@ from .helpers import ( MockModulestoreFactory, MockCache, MockTransformer, ChildrenMapTestMixin, UsageKeyFactoryMixin, mock_registered_transformers, + override_config_setting, ) diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_models.py b/openedx/core/djangoapps/content/block_structure/tests/test_models.py index 74d41fc2d2..5d42a7a879 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_models.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_models.py @@ -10,9 +10,9 @@ from mock import patch, Mock from uuid import uuid4 from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator -from openedx.core.lib.block_structure.exceptions import BlockStructureNotFound from ..config import PRUNE_OLD_VERSIONS +from ..exceptions import BlockStructureNotFound from ..models import BlockStructureModel from .helpers import override_config_setting diff --git a/openedx/core/djangoapps/content/block_structure/tests/test_signals.py b/openedx/core/djangoapps/content/block_structure/tests/test_signals.py index 1dbcd516fa..ae53af7c2d 100644 --- a/openedx/core/djangoapps/content/block_structure/tests/test_signals.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_signals.py @@ -50,7 +50,7 @@ class CourseBlocksSignalTest(ModuleStoreTestCase): ) @ddt.data(True, False) - @patch('openedx.core.lib.block_structure.manager.BlockStructureManager.clear') + @patch('openedx.core.djangoapps.content.block_structure.manager.BlockStructureManager.clear') def test_cache_invalidation(self, invalidate_cache_enabled, mock_bs_manager_clear): test_display_name = "Jedi 101" diff --git a/openedx/core/lib/block_structure/tests/test_store.py b/openedx/core/djangoapps/content/block_structure/tests/test_store.py similarity index 92% rename from openedx/core/lib/block_structure/tests/test_store.py rename to openedx/core/djangoapps/content/block_structure/tests/test_store.py index b598059f3e..6610489b81 100644 --- a/openedx/core/lib/block_structure/tests/test_store.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_store.py @@ -4,14 +4,13 @@ Tests for block_structure/cache.py import ddt from nose.plugins.attrib import attr -from openedx.core.djangoapps.content.block_structure.config import STORAGE_BACKING_FOR_CACHE -from openedx.core.djangoapps.content.block_structure.config.models import BlockStructureConfiguration -from openedx.core.djangoapps.content.block_structure.tests.helpers import override_config_setting from openedx.core.djangolib.testing.utils import CacheIsolationTestCase -from ..store import BlockStructureStore +from ..config import STORAGE_BACKING_FOR_CACHE +from ..config.models import BlockStructureConfiguration from ..exceptions import BlockStructureNotFound -from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer +from ..store import BlockStructureStore +from .helpers import ChildrenMapTestMixin, UsageKeyFactoryMixin, MockCache, MockTransformer, override_config_setting @attr(shard=2) diff --git a/openedx/core/lib/block_structure/tests/test_transformer_registry.py b/openedx/core/djangoapps/content/block_structure/tests/test_transformer_registry.py similarity index 100% rename from openedx/core/lib/block_structure/tests/test_transformer_registry.py rename to openedx/core/djangoapps/content/block_structure/tests/test_transformer_registry.py diff --git a/openedx/core/lib/block_structure/tests/test_transformers.py b/openedx/core/djangoapps/content/block_structure/tests/test_transformers.py similarity index 93% rename from openedx/core/lib/block_structure/tests/test_transformers.py rename to openedx/core/djangoapps/content/block_structure/tests/test_transformers.py index dda1adba25..02f89ddbc0 100644 --- a/openedx/core/lib/block_structure/tests/test_transformers.py +++ b/openedx/core/djangoapps/content/block_structure/tests/test_transformers.py @@ -57,7 +57,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase): def test_collect(self): with mock_registered_transformers(self.registered_transformers): with patch( - 'openedx.core.lib.block_structure.tests.helpers.MockTransformer.collect' + 'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.collect' ) as mock_collect_call: BlockStructureTransformers.collect(block_structure=MagicMock()) self.assertTrue(mock_collect_call.called) @@ -66,7 +66,7 @@ class TestBlockStructureTransformers(ChildrenMapTestMixin, TestCase): self.add_mock_transformer() with patch( - 'openedx.core.lib.block_structure.tests.helpers.MockTransformer.transform' + 'openedx.core.djangoapps.content.block_structure.tests.helpers.MockTransformer.transform' ) as mock_transform_call: self.transformers.transform(block_structure=MagicMock()) self.assertTrue(mock_transform_call.called) diff --git a/openedx/core/lib/block_structure/transformer.py b/openedx/core/djangoapps/content/block_structure/transformer.py similarity index 100% rename from openedx/core/lib/block_structure/transformer.py rename to openedx/core/djangoapps/content/block_structure/transformer.py diff --git a/openedx/core/lib/block_structure/transformer_registry.py b/openedx/core/djangoapps/content/block_structure/transformer_registry.py similarity index 100% rename from openedx/core/lib/block_structure/transformer_registry.py rename to openedx/core/djangoapps/content/block_structure/transformer_registry.py diff --git a/openedx/core/lib/block_structure/transformers.py b/openedx/core/djangoapps/content/block_structure/transformers.py similarity index 100% rename from openedx/core/lib/block_structure/transformers.py rename to openedx/core/djangoapps/content/block_structure/transformers.py diff --git a/openedx/core/lib/block_structure/__init__.py b/openedx/core/lib/block_structure/__init__.py deleted file mode 100644 index f77cf27f04..0000000000 --- a/openedx/core/lib/block_structure/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -""" -The block_structure django app provides an extensible framework for caching -data of block structures from the modulestore. - -Dual-Phase. The framework is meant to be used in 2 phases. - - * Collect Phase (for expensive and full-tree traversals) - In the - first phase, the "collect" phase, any and all data from the - modulestore should be collected and cached for later access to - the block structure. Instantiating any and all xBlocks in the block - structure is also done at this phase, since that is also (currently) - a costly operation. - - Any full tree traversals should also be done during this phase. For - example, if data for a block depends on its parents, the traversal - should happen during the collection phase and any required data - for the block should be percolated down the tree and stored as - aggregate values on the descendants. This allows for faster and - direct access to blocks in the Transform phase. - - * Transform Phase (for fast access to blocks) - In the second - phase, the "transform" phase, only the previously collected and - cached data should be accessed. There should be no access to the - modulestore or instantiation of xBlocks in this phase. - - -To make this framework extensible, the Transformer and -Extensibility design patterns are used. This django app only -provides the underlying framework for Block Structure Transformers -and a Transformer Registry. Clients are expected to provide actual -implementations of Transformers or add them to the extensible Registry. - -Transformers. As inspired by -http://www.ccs.neu.edu/home/riccardo/courses/csu370-fa07/lect18.pdf, -a Block Structure Transformer takes in a block structure (or tree) and -manipulates the structure and the data of its blocks according to its -own requirements. Its output can then be used for further -transformations by other transformers down the pipeline. - -Note: For performance and space optimization, our implementation -differs from the paper in that our transformers mutate the block -structure in-place rather than returning a modified copy of it. - -Block Structure. The BlockStructure and its family of classes -provided with this framework are the base data types for accessing -and manipulating block structures. BlockStructures are constructed -using the BlockStructureFactory and then used as the currency across -Transformers. - -Registry. Transformers are registered using the platform's -PluginManager (e.g., Stevedore). This is currently done by updating -setup.py. Only registered transformers are called during the Collect -Phase. And only registered transformers can be used during the -Transform phase. Exceptions to this rule are any nested transformers -that are contained within higher-order transformers - as long as the -higher-order transformers are registered and appropriately call the -contained transformers within them. - -Note: A partial subset (as an ordered list) of the registered -transformers can be requested during the Transform phase, allowing -the client to manipulate exactly which transformers to call. -""" diff --git a/openedx/core/lib/block_structure/tests/__init__.py b/openedx/core/lib/block_structure/tests/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/openedx/core/lib/block_structure/tests/helpers.py b/openedx/core/lib/block_structure/tests/helpers.py deleted file mode 100644 index d820843f1f..0000000000 --- a/openedx/core/lib/block_structure/tests/helpers.py +++ /dev/null @@ -1,310 +0,0 @@ -""" -Common utilities for tests in block_structure module -""" -from contextlib import contextmanager -from mock import patch -from xmodule.modulestore.exceptions import ItemNotFoundError -from uuid import uuid4 - -from opaque_keys.edx.locator import CourseLocator, BlockUsageLocator - -from ..block_structure import BlockStructureBlockData -from ..transformer import BlockStructureTransformer, FilteringTransformerMixin -from ..transformer_registry import TransformerRegistry - - -class MockXBlock(object): - """ - A mock XBlock to be used in unit tests, thereby decoupling the - implementation of the block cache framework from the xBlock - implementation. This class provides only the minimum xBlock - capabilities needed by the block cache framework. - """ - def __init__(self, location, field_map=None, children=None, modulestore=None): - self.location = location - self.field_map = field_map or {} - - self.children = children or [] - self.modulestore = modulestore - - def __getattr__(self, attr): - try: - return self.field_map[attr] - except KeyError: - raise AttributeError - - def get_children(self): - """ - Returns the children of the mock XBlock. - """ - return [self.modulestore.get_item(child) for child in self.children] - - -class MockModulestore(object): - """ - A mock Modulestore to be used in unit tests, providing only the - minimum methods needed by the block cache framework. - """ - def __init__(self): - self.get_items_call_count = 0 - self.blocks = None - - def set_blocks(self, blocks): - """ - Updates the mock modulestore with a dictionary of blocks. - - Arguments: - blocks ({block key, MockXBlock}) - A map of block_key - to its mock xBlock. - """ - self.blocks = blocks - - def get_item(self, block_key, depth=None, lazy=False): # pylint: disable=unused-argument - """ - Returns the mock XBlock (MockXBlock) associated with the - given block_key. - - Raises ItemNotFoundError if the item is not found. - """ - self.get_items_call_count += 1 - item = self.blocks.get(block_key) - if not item: - raise ItemNotFoundError - return item - - @contextmanager - def bulk_operations(self, ignore): # pylint: disable=unused-argument - """ - A context manager for notifying the store of bulk operations. - """ - yield - - -class MockCache(object): - """ - A mock Cache object, providing only the minimum features needed - by the block cache framework. - """ - def __init__(self): - # An in-memory map of cache keys to cache values. - self.map = {} - self.set_call_count = 0 - self.timeout_from_last_call = 0 - - def set(self, key, val, timeout): - """ - Associates the given key with the given value in the cache. - """ - self.set_call_count += 1 - self.map[key] = val - self.timeout_from_last_call = timeout - - def get(self, key, default=None): - """ - Returns the value associated with the given key in the cache; - returns default if not found. - """ - return self.map.get(key, default) - - def delete(self, key): - """ - Deletes the given key from the cache. - """ - del self.map[key] - - -class MockModulestoreFactory(object): - """ - A factory for creating MockModulestore objects. - """ - @classmethod - def create(cls, children_map, block_key_factory): - """ - Creates and returns a MockModulestore from the given - children_map. - - Arguments: - children_map ({block_key: [block_key]}) - A dictionary - mapping a block key to a list of block keys of the - block's corresponding children. - """ - modulestore = MockModulestore() - modulestore.set_blocks({ - block_key_factory(block_key): MockXBlock( - block_key_factory(block_key), - children=[block_key_factory(child) for child in children], - modulestore=modulestore, - ) - for block_key, children in enumerate(children_map) - }) - return modulestore - - -class MockTransformer(BlockStructureTransformer): - """ - A mock BlockStructureTransformer class. - """ - WRITE_VERSION = 1 - READ_VERSION = 1 - - @classmethod - def name(cls): - # Use the class' name for Mock transformers. - return cls.__name__ - - def transform(self, usage_info, block_structure): - pass - - -class MockFilteringTransformer(FilteringTransformerMixin, BlockStructureTransformer): - """ - A mock FilteringTransformerMixin class. - """ - WRITE_VERSION = 1 - READ_VERSION = 1 - - @classmethod - def name(cls): - # Use the class' name for Mock transformers. - return cls.__name__ - - def transform_block_filters(self, usage_info, block_structure): - return [block_structure.create_universal_filter()] - - -def clear_registered_transformers_cache(): - """ - Test helper to clear out any cached values of registered transformers. - """ - TransformerRegistry.get_write_version_hash.cache.clear() - - -@contextmanager -def mock_registered_transformers(transformers): - """ - Context manager for mocking the transformer registry to return the given transformers. - """ - clear_registered_transformers_cache() - with patch( - 'openedx.core.lib.block_structure.transformer_registry.TransformerRegistry.get_registered_transformers' - ) as mock_available_transforms: - mock_available_transforms.return_value = {transformer for transformer in transformers} - yield - - -class ChildrenMapTestMixin(object): - """ - A Test Mixin with utility methods for testing with block structures - created and manipulated using children_map and parents_map. - """ - - # 0 - # / \ - # 1 2 - # / \ - # 3 4 - SIMPLE_CHILDREN_MAP = [[1, 2], [3, 4], [], [], []] - - # 0 - # / - # 1 - # / - # 2 - # / - # 3 - LINEAR_CHILDREN_MAP = [[1], [2], [3], []] - - # 0 - # / \ - # 1 2 - # \ / \ - # 3 4 - # / \ - # 5 6 - DAG_CHILDREN_MAP = [[1, 2], [3], [3, 4], [5, 6], [], [], []] - - def block_key_factory(self, block_id): - """ - Returns a block key object for the given block_id. - Override this method if the block_key should be anything - different from the index integer values in the Children Maps. - """ - return block_id - - def create_block_structure(self, children_map, block_structure_cls=BlockStructureBlockData): - """ - Factory method for creating and returning a block structure - for the given children_map. - """ - # create empty block structure - block_structure = block_structure_cls(root_block_usage_key=self.block_key_factory(0)) - - # _add_relation - for parent, children in enumerate(children_map): - for child in children: - block_structure._add_relation(self.block_key_factory(parent), self.block_key_factory(child)) # pylint: disable=protected-access - return block_structure - - def get_parents_map(self, children_map): - """ - Converts and returns the given children_map to a parents_map. - """ - parent_map = [[] for _ in children_map] - for parent, children in enumerate(children_map): - for child in children: - parent_map[child].append(parent) - return parent_map - - def assert_block_structure(self, block_structure, children_map, missing_blocks=None): - """ - Verifies that the relations in the given block structure - equate the relations described in the children_map. Use the - missing_blocks parameter to pass in any blocks that were removed - from the block structure but still have a positional entry in - the children_map. - """ - if not missing_blocks: - missing_blocks = [] - - for block_key, children in enumerate(children_map): - # Verify presence - self.assertEqual( - self.block_key_factory(block_key) in block_structure, - block_key not in missing_blocks, - 'Expected presence in block_structure for block_key {} to match absence in missing_blocks.'.format( - unicode(block_key) - ), - ) - - # Verify children - if block_key not in missing_blocks: - self.assertEqual( - set(block_structure.get_children(self.block_key_factory(block_key))), - set(self.block_key_factory(child) for child in children), - ) - - # Verify parents - parents_map = self.get_parents_map(children_map) - for block_key, parents in enumerate(parents_map): - if block_key not in missing_blocks: - self.assertEqual( - set(block_structure.get_parents(self.block_key_factory(block_key))), - set(self.block_key_factory(parent) for parent in parents), - ) - - -class UsageKeyFactoryMixin(object): - """ - Test Mixin that provides a block_key_factory to create OpaqueKey objects - for block_ids rather than simple integers. By default, the children maps in - ChildrenMapTestMixin use integers for block_ids. - """ - def setUp(self): - super(UsageKeyFactoryMixin, self).setUp() - self.course_key = CourseLocator('org', 'course', unicode(uuid4())) - - def block_key_factory(self, block_id): - """ - Returns a block key object for the given block_id. - """ - return BlockUsageLocator(course_key=self.course_key, block_type='course', block_id=unicode(block_id))