Learning Contexts, New XBlock Runtime, Blockstore API Client + Content Libraries

https://github.com/edx/edx-platform/pull/20645

This introduces:
* A new XBlock runtime that can read and write XBlocks that are persisted using
  Blockstore instead of Modulestore. The new runtime is currently isolated so
  that it can be tested without risk to the current courseware/runtime.
* Content Libraries v2, which store XBlocks in Blockstore not modulestore
* An API Client for Blockstore
* "Learning Context" plugin API. A learning context is a more abstract concept
  than a course; it's a collection of XBlocks that serves some learning purpose.

common/lib/xmodule/setup.py

@@ -35,6 +35,7 @@ XBLOCKS = [
     "library = xmodule.library_root_xblock:LibraryRoot",
     "problem = xmodule.capa_module:ProblemBlock",
     "static_tab = xmodule.html_module:StaticTabBlock",
+    "unit = xmodule.unit_block:UnitBlock",
     "vertical = xmodule.vertical_block:VerticalBlock",
     "video = xmodule.video_module:VideoBlock",
     "videoalpha = xmodule.video_module:VideoBlock",

common/lib/xmodule/xmodule/errortracker.py

@@ -1,3 +1,35 @@
+"""
+error_tracker: A hook for tracking errors in loading XBlocks.
+
+Used for example to get a list of all non-fatal problems on course
+load, and display them to the user.
+
+Patterns for using the error handler:
+    try:
+        x = access_some_resource()
+        check_some_format(x)
+    except SomeProblem as err:
+        msg = 'Grommet {0} is broken: {1}'.format(x, str(err))
+        log.warning(msg)  # don't rely on tracker to log
+        # NOTE: we generally don't want content errors logged as errors
+        error_tracker = self.runtime.service(self, 'error_tracker')
+        if error_tracker:
+            error_tracker(msg)
+        # work around
+        return 'Oops, couldn't load grommet'
+
+    OR, if not in an exception context:
+
+    if not check_something(thingy):
+        msg = "thingy {0} is broken".format(thingy)
+        log.critical(msg)
+        error_tracker = self.runtime.service(self, 'error_tracker')
+        if error_tracker:
+            error_tracker(msg)
+
+    NOTE: To avoid duplication, do not call the tracker on errors
+    that you're about to re-raise---let the caller track them.
+"""
 from __future__ import absolute_import
 
 import logging

common/lib/xmodule/xmodule/html_module.py

@@ -293,6 +293,20 @@ class HtmlBlock(
                 # add more info and re-raise
                 six.reraise(Exception(msg), None, sys.exc_info()[2])
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        Parse XML in the new blockstore-based runtime. Since it doesn't yet
+        support loading separate .html files, the HTML data is assumed to be in
+        a CDATA child or otherwise just inline in the OLX.
+        """
+        block = runtime.construct_xblock_from_class(cls, keys)
+        block.data = stringify_children(node)
+        # Attributes become fields.
+        for name, value in node.items():
+            cls._set_field_if_present(block, name, value, {})
+        return block
+
     # TODO (vshnayder): make export put things in the right places.
 
     def definition_to_xml(self, resource_fs):

common/lib/xmodule/xmodule/raw_module.py

@@ -60,6 +60,28 @@ class RawMixin(object):
             )
             raise SerializationError(self.location, msg)
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        Interpret the parsed XML in `node`, creating a new instance of this
+        module.
+        """
+        # In the new/blockstore-based runtime, XModule parsing (from
+        # XmlMixin) is disabled, so definition_from_xml will not be
+        # called, and instead the "normal" XBlock parse_xml will be used.
+        # However, it's not compatible with RawMixin, so we implement
+        # support here.
+        data_field_value = cls.definition_from_xml(node, None)[0]["data"]
+        for child in node.getchildren():
+            node.remove(child)
+        # Get attributes, if any, via normal parse_xml.
+        try:
+            block = super(RawMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            block = super(RawMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+        block.data = data_field_value
+        return block
+
 
 class RawDescriptor(RawMixin, XmlDescriptor, XMLEditingDescriptor):
     """

common/lib/xmodule/xmodule/tests/test_unit_block.py

@@ -0,0 +1,87 @@
+"""
+Tests for the Unit XBlock
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import re
+import unittest
+from xml.dom import minidom
+
+from mock import patch
+from web_fragments.fragment import Fragment
+from xblock.core import XBlock
+from xblock.completable import XBlockCompletionMode
+from xblock.test.test_parsing import XmlTest
+
+from xmodule.unit_block import UnitBlock
+
+
+class FakeHTMLBlock(XBlock):
+    """ An HTML block for use in tests """
+    def student_view(self, context=None):  # pylint: disable=unused-argument
+        """Provide simple HTML student view."""
+        return Fragment("This is some HTML.")
+
+
+class FakeVideoBlock(XBlock):
+    """ A video block for use in tests """
+    def student_view(self, context=None):  # pylint: disable=unused-argument
+        """Provide simple Video student view."""
+        return Fragment(
+            '<iframe width="560" height="315" src="https://www.youtube.com/embed/B-EFayAA5_0"'
+            ' frameborder="0" allow="autoplay; encrypted-media"></iframe>'
+        )
+
+
+class UnitBlockTests(XmlTest, unittest.TestCase):
+    """
+    Tests of the Unit XBlock.
+
+    There's not much to this block, so we keep it simple.
+    """
+    maxDiff = None
+
+    @XBlock.register_temp_plugin(FakeHTMLBlock, identifier='fake-html')
+    @XBlock.register_temp_plugin(FakeVideoBlock, identifier='fake-video')
+    def test_unit_html(self):
+        block = self.parse_xml_to_block("""\
+            <unit>
+                <fake-html/>
+                <fake-video/>
+            </unit>
+        """)
+
+        with patch.object(block.runtime, 'applicable_aside_types', return_value=[]):  # Disable problematic Acid aside
+            html = block.runtime.render(block, 'student_view').content
+
+        self.assertXmlEqual(html, (
+            '<div class="xblock-v1 xblock-v1-student_view" data-usage="u_1" data-block-type="unit">'
+            '<div class="unit-xblock vertical">'
+            '<div class="xblock-v1 xblock-v1-student_view" data-usage="u_3" data-block-type="fake-html">'
+            'This is some HTML.'
+            '</div>'
+            '<div class="xblock-v1 xblock-v1-student_view" data-usage="u_5" data-block-type="fake-video">'
+            '<iframe width="560" height="315" src="https://www.youtube.com/embed/B-EFayAA5_0"'
+            ' frameborder="0" allow="autoplay; encrypted-media"></iframe>'
+            '</div>'
+            '</div>'
+            '</div>'
+        ))
+
+    def test_is_aggregator(self):
+        """
+        The unit XBlock is designed to hold other XBlocks, so check that its
+        completion status is defined as the aggregation of its child blocks.
+        """
+        self.assertEqual(XBlockCompletionMode.get_mode(UnitBlock), XBlockCompletionMode.AGGREGATOR)
+
+    def assertXmlEqual(self, xml_str_a, xml_str_b):
+        """
+        Assert that the given XML strings are equal,
+        ignoring attribute order and some whitespace variations.
+        """
+        def clean(xml_str):
+            # Collapse repeated whitespace:
+            xml_str = re.sub(r'(\s)\s+', r'\1', xml_str)
+            xml_bytes = xml_str.encode('utf8')
+            return minidom.parseString(xml_bytes).toprettyxml()
+        self.assertEqual(clean(xml_str_a), clean(xml_str_b))

common/lib/xmodule/xmodule/unit_block.py

@@ -0,0 +1,73 @@
+"""
+An XBlock which groups related XBlocks together.
+
+This is like the "vertical" block, but without that block's UI code, JavaScript,
+and other legacy features.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from web_fragments.fragment import Fragment
+from xblock.completable import XBlockCompletionMode
+from xblock.core import XBlock
+from xblock.fields import Scope, String
+
+# Make '_' a no-op so we can scrape strings.
+_ = lambda text: text
+
+
+class UnitBlock(XBlock):
+    """
+    Unit XBlock: An XBlock which groups related XBlocks together.
+
+    This is like the "vertical" block in principle, but this version is
+    explicitly designed to not contain LMS-related logic, like vertical does.
+
+    The application which renders XBlocks and/or the runtime should manage
+    things like bookmarks, completion tracking, etc.
+    This version also avoids any XModule mixins and has no JavaScript code.
+    """
+    has_children = True
+    # This is a block containing other blocks, so its completion is defined by
+    # the completion of its child blocks:
+    completion_mode = XBlockCompletionMode.AGGREGATOR
+    # Define a non-existent resources dir because we don't have resources, but
+    # the default will pull in all files in this folder.
+    resources_dir = 'assets/unit'
+
+    display_name = String(
+        display_name=_("Display Name"),
+        help=_("The display name for this component."),
+        scope=Scope.settings,
+        default=_("Unit"),
+    )
+
+    def student_view(self, context=None):
+        """Provide default student view."""
+        result = Fragment()
+        child_frags = self.runtime.render_children(self, context=context)
+        result.add_resources(child_frags)
+        result.add_content('<div class="unit-xblock vertical">')
+        for frag in child_frags:
+            result.add_content(frag.content)
+        result.add_content('</div>')
+        return result
+
+    def index_dictionary(self):
+        """
+        Return dictionary prepared with module content and type for indexing, so
+        that the contents of this block can be found in free-text searches.
+        """
+        # return key/value fields in a Python dict object
+        # values may be numeric / string or dict
+        xblock_body = super(UnitBlock, self).index_dictionary()
+        index_body = {
+            "display_name": self.display_name,
+        }
+        if "content" in xblock_body:
+            xblock_body["content"].update(index_body)
+        else:
+            xblock_body["content"] = index_body
+        # We use "Sequence" for sequentials and units/verticals
+        xblock_body["content_type"] = "Sequence"
+
+        return xblock_body

common/lib/xmodule/xmodule/video_module/transcripts_utils.py

@@ -20,6 +20,7 @@ from six import text_type
 from six.moves import range, zip
 from six.moves.html_parser import HTMLParser  # pylint: disable=import-error
 
+from opaque_keys.edx.locator import CourseLocator, LibraryLocator
 from xmodule.contentstore.content import StaticContent
 from xmodule.contentstore.django import contentstore
 from xmodule.exceptions import NotFoundError
@@ -1016,6 +1017,12 @@ def get_transcript(video, lang=None, output_format=Transcript.SRT, youtube_id=No
             raise NotFoundError
         return get_transcript_from_val(edx_video_id, lang, output_format)
     except NotFoundError:
+        # If this is not in a modulestore course or library, don't try loading from contentstore:
+        if not isinstance(video.scope_ids.usage_id.course_key, (CourseLocator, LibraryLocator)):
+            raise NotFoundError(
+                u'Video transcripts cannot yet be loaded from Blockstore (block: {})'.format(video.scope_ids.usage_id),
+            )
+
         return get_transcript_from_contentstore(
             video,
             lang,

common/lib/xmodule/xmodule/video_module/video_module.py

@@ -597,6 +597,25 @@ class VideoBlock(
 
         return editable_fields
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        Implement the video block's special XML parsing requirements for the
+        new runtime only. For all other runtimes, use the existing XModule-style
+        methods like .from_xml().
+        """
+        video_block = runtime.construct_xblock_from_class(cls, keys)
+        field_data = cls.parse_video_xml(node)
+        for key, val in field_data.items():
+            setattr(video_block, key, cls.fields[key].from_json(val))
+        # Update VAL with info extracted from `xml_object`
+        video_block.edx_video_id = video_block.import_video_info_into_val(
+            node,
+            runtime.resources_fs,
+            keys.usage_id.context_key,
+        )
+        return video_block
+
     @classmethod
     def from_xml(cls, xml_data, system, id_generator):
         """

common/lib/xmodule/xmodule/x_module.py

@@ -1125,6 +1125,17 @@ class XModuleDescriptorToXBlockMixin(object):
         block = cls.from_xml(xml, runtime, id_generator)
         return block
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        This XML lives within Blockstore and the new runtime doesn't need this
+        legacy XModule code. Use the "normal" XBlock parsing code.
+        """
+        try:
+            return super(XModuleDescriptorToXBlockMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            return super(XModuleDescriptorToXBlockMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+
     @classmethod
     def from_xml(cls, xml_data, system, id_generator):
         """
@@ -1241,6 +1252,9 @@ class XModuleDescriptor(XModuleDescriptorToXBlockMixin, HTMLSnippet, ResourceTem
 
     # =============================== BUILTIN METHODS ==========================
     def __eq__(self, other):
+        """
+        Is this XModule effectively equal to the other instance?
+        """
         return (hasattr(other, 'scope_ids') and
                 self.scope_ids == other.scope_ids and
                 list(self.fields.keys()) == list(other.fields.keys()) and
@@ -1469,30 +1483,7 @@ class DescriptorSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):
             Used for example to get a list of all non-fatal problems on course
             load, and display them to the user.
 
-            A function of (error_msg). errortracker.py provides a
-            handy make_error_tracker() function.
-
-            Patterns for using the error handler:
-               try:
-                  x = access_some_resource()
-                  check_some_format(x)
-               except SomeProblem as err:
-                  msg = 'Grommet {0} is broken: {1}'.format(x, str(err))
-                  log.warning(msg)  # don't rely on tracker to log
-                        # NOTE: we generally don't want content errors logged as errors
-                  self.system.error_tracker(msg)
-                  # work around
-                  return 'Oops, couldn't load grommet'
-
-               OR, if not in an exception context:
-
-               if not check_something(thingy):
-                  msg = "thingy {0} is broken".format(thingy)
-                  log.critical(msg)
-                  self.system.error_tracker(msg)
-
-               NOTE: To avoid duplication, do not call the tracker on errors
-               that you're about to re-raise---let the caller track them.
+            See errortracker.py for more documentation
 
         get_policy: a function that takes a usage id and returns a dict of
             policy to apply.

common/lib/xmodule/xmodule/xml_module.py

@@ -393,6 +393,17 @@ class XmlParserMixin(object):
 
         return xblock
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        This XML lives within Blockstore and the new runtime doesn't need this
+        legacy XModule code. Use the "normal" XBlock parsing code.
+        """
+        try:
+            return super(XmlParserMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            return super(XmlParserMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+
     @classmethod
     def _get_url_name(cls, node):
         """
@@ -559,6 +570,17 @@ class XmlMixin(XmlParserMixin):
         else:
             return super(XmlMixin, cls).parse_xml(node, runtime, keys, id_generator)
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        This XML lives within Blockstore and the new runtime doesn't need this
+        legacy XModule code. Use the "normal" XBlock parsing code.
+        """
+        try:
+            return super(XmlMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            return super(XmlMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+
     def export_to_xml(self, resource_fs):
         """
         Returns an xml string representing this module, and all modules