Merge pull request #20645 from open-craft/new-runtime

Foundation for Learning Contexts and Blockstore XBlock Runtime

cms/envs/common.py

@@ -1201,6 +1201,9 @@ INSTALLED_APPS = [
     'openedx.core.djangoapps.course_groups',  # not used in cms (yet), but tests run
     'xblock_config.apps.XBlockConfig',
 
+    # New (Blockstore-based) XBlock runtime
+    'openedx.core.djangoapps.xblock.apps.StudioXBlockAppConfig',
+
     # Maintenance tools
     'maintenance',
     'openedx.core.djangoapps.util.apps.UtilConfig',
@@ -1683,6 +1686,13 @@ DATABASE_ROUTERS = [
 ############################ Cache Configuration ###############################
 
 CACHES = {
+    'blockstore': {
+        'KEY_PREFIX': 'blockstore',
+        'KEY_FUNCTION': 'util.memcache.safe_key',
+        'LOCATION': ['localhost:11211'],
+        'TIMEOUT': '86400',  # This data should be long-lived for performance, BundleCache handles invalidation
+        'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
+    },
     'course_structure_cache': {
         'KEY_PREFIX': 'course_structure',
         'KEY_FUNCTION': 'util.memcache.safe_key',

cms/envs/devstack.py

@@ -181,6 +181,9 @@ IDA_LOGOUT_URI_LIST = [
     'http://localhost:18150/logout/',  # credentials
 ]
 
+############################### BLOCKSTORE #####################################
+BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/"
+
 #####################################################################
 
 # pylint: disable=wrong-import-order, wrong-import-position

cms/envs/production.py

@@ -404,6 +404,12 @@ XBLOCK_FIELD_DATA_WRAPPERS = ENV_TOKENS.get(
 
 CONTENTSTORE = AUTH_TOKENS['CONTENTSTORE']
 DOC_STORE_CONFIG = AUTH_TOKENS['DOC_STORE_CONFIG']
+
+############################### BLOCKSTORE #####################################
+BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None)  # e.g. "https://blockstore.example.com/api/v1/"
+# Configure an API auth token at (blockstore URL)/admin/authtoken/token/
+BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None)
+
 # Datadog for events!
 DATADOG = AUTH_TOKENS.get("DATADOG", {})
 DATADOG.update(ENV_TOKENS.get("DATADOG", {}))

cms/envs/test.py

@@ -182,6 +182,12 @@ CACHES = {
     },
 }
 
+############################### BLOCKSTORE #####################################
+# Blockstore tests
+RUN_BLOCKSTORE_TESTS = os.environ.get('EDXAPP_RUN_BLOCKSTORE_TESTS', 'no').lower() in ('true', 'yes', '1')
+BLOCKSTORE_API_URL = os.environ.get('EDXAPP_BLOCKSTORE_API_URL', "http://edx.devstack.blockstore-test:18251/api/v1/")
+BLOCKSTORE_API_AUTH_TOKEN = os.environ.get('EDXAPP_BLOCKSTORE_API_AUTH_TOKEN', 'edxapp-test-key')
+
 ################################# CELERY ######################################
 
 CELERY_ALWAYS_EAGER = True

cms/urls.py

@@ -56,6 +56,7 @@ urlpatterns = [
         contentstore.views.component_handler, name='component_handler'),
     url(r'^xblock/resource/(?P<block_type>[^/]*)/(?P<uri>.*)$',
         openedx.core.djangoapps.common_views.xblock.xblock_resource, name='xblock_resource_url'),
+    url(r'', include('openedx.core.djangoapps.xblock.rest_api.urls', namespace='xblock_api')),
     url(r'^not_found$', contentstore.views.not_found, name='not_found'),
     url(r'^server_error$', contentstore.views.server_error, name='server_error'),
     url(r'^organizations$', OrganizationListView.as_view(), name='organizations'),

common/lib/xmodule/setup.py

@@ -35,6 +35,7 @@ XBLOCKS = [
     "library = xmodule.library_root_xblock:LibraryRoot",
     "problem = xmodule.capa_module:ProblemBlock",
     "static_tab = xmodule.html_module:StaticTabBlock",
+    "unit = xmodule.unit_block:UnitBlock",
     "vertical = xmodule.vertical_block:VerticalBlock",
     "video = xmodule.video_module:VideoBlock",
     "videoalpha = xmodule.video_module:VideoBlock",

common/lib/xmodule/xmodule/errortracker.py

@@ -1,3 +1,35 @@
+"""
+error_tracker: A hook for tracking errors in loading XBlocks.
+
+Used for example to get a list of all non-fatal problems on course
+load, and display them to the user.
+
+Patterns for using the error handler:
+    try:
+        x = access_some_resource()
+        check_some_format(x)
+    except SomeProblem as err:
+        msg = 'Grommet {0} is broken: {1}'.format(x, str(err))
+        log.warning(msg)  # don't rely on tracker to log
+        # NOTE: we generally don't want content errors logged as errors
+        error_tracker = self.runtime.service(self, 'error_tracker')
+        if error_tracker:
+            error_tracker(msg)
+        # work around
+        return 'Oops, couldn't load grommet'
+
+    OR, if not in an exception context:
+
+    if not check_something(thingy):
+        msg = "thingy {0} is broken".format(thingy)
+        log.critical(msg)
+        error_tracker = self.runtime.service(self, 'error_tracker')
+        if error_tracker:
+            error_tracker(msg)
+
+    NOTE: To avoid duplication, do not call the tracker on errors
+    that you're about to re-raise---let the caller track them.
+"""
 from __future__ import absolute_import
 
 import logging

common/lib/xmodule/xmodule/html_module.py

@@ -293,6 +293,20 @@ class HtmlBlock(
                 # add more info and re-raise
                 six.reraise(Exception(msg), None, sys.exc_info()[2])
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        Parse XML in the new blockstore-based runtime. Since it doesn't yet
+        support loading separate .html files, the HTML data is assumed to be in
+        a CDATA child or otherwise just inline in the OLX.
+        """
+        block = runtime.construct_xblock_from_class(cls, keys)
+        block.data = stringify_children(node)
+        # Attributes become fields.
+        for name, value in node.items():
+            cls._set_field_if_present(block, name, value, {})
+        return block
+
     # TODO (vshnayder): make export put things in the right places.
 
     def definition_to_xml(self, resource_fs):

common/lib/xmodule/xmodule/raw_module.py

@@ -60,6 +60,28 @@ class RawMixin(object):
             )
             raise SerializationError(self.location, msg)
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        Interpret the parsed XML in `node`, creating a new instance of this
+        module.
+        """
+        # In the new/blockstore-based runtime, XModule parsing (from
+        # XmlMixin) is disabled, so definition_from_xml will not be
+        # called, and instead the "normal" XBlock parse_xml will be used.
+        # However, it's not compatible with RawMixin, so we implement
+        # support here.
+        data_field_value = cls.definition_from_xml(node, None)[0]["data"]
+        for child in node.getchildren():
+            node.remove(child)
+        # Get attributes, if any, via normal parse_xml.
+        try:
+            block = super(RawMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            block = super(RawMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+        block.data = data_field_value
+        return block
+
 
 class RawDescriptor(RawMixin, XmlDescriptor, XMLEditingDescriptor):
     """

common/lib/xmodule/xmodule/tests/test_unit_block.py

@@ -0,0 +1,87 @@
+"""
+Tests for the Unit XBlock
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import re
+import unittest
+from xml.dom import minidom
+
+from mock import patch
+from web_fragments.fragment import Fragment
+from xblock.core import XBlock
+from xblock.completable import XBlockCompletionMode
+from xblock.test.test_parsing import XmlTest
+
+from xmodule.unit_block import UnitBlock
+
+
+class FakeHTMLBlock(XBlock):
+    """ An HTML block for use in tests """
+    def student_view(self, context=None):  # pylint: disable=unused-argument
+        """Provide simple HTML student view."""
+        return Fragment("This is some HTML.")
+
+
+class FakeVideoBlock(XBlock):
+    """ A video block for use in tests """
+    def student_view(self, context=None):  # pylint: disable=unused-argument
+        """Provide simple Video student view."""
+        return Fragment(
+            '<iframe width="560" height="315" src="https://www.youtube.com/embed/B-EFayAA5_0"'
+            ' frameborder="0" allow="autoplay; encrypted-media"></iframe>'
+        )
+
+
+class UnitBlockTests(XmlTest, unittest.TestCase):
+    """
+    Tests of the Unit XBlock.
+
+    There's not much to this block, so we keep it simple.
+    """
+    maxDiff = None
+
+    @XBlock.register_temp_plugin(FakeHTMLBlock, identifier='fake-html')
+    @XBlock.register_temp_plugin(FakeVideoBlock, identifier='fake-video')
+    def test_unit_html(self):
+        block = self.parse_xml_to_block("""\
+            <unit>
+                <fake-html/>
+                <fake-video/>
+            </unit>
+        """)
+
+        with patch.object(block.runtime, 'applicable_aside_types', return_value=[]):  # Disable problematic Acid aside
+            html = block.runtime.render(block, 'student_view').content
+
+        self.assertXmlEqual(html, (
+            '<div class="xblock-v1 xblock-v1-student_view" data-usage="u_1" data-block-type="unit">'
+            '<div class="unit-xblock vertical">'
+            '<div class="xblock-v1 xblock-v1-student_view" data-usage="u_3" data-block-type="fake-html">'
+            'This is some HTML.'
+            '</div>'
+            '<div class="xblock-v1 xblock-v1-student_view" data-usage="u_5" data-block-type="fake-video">'
+            '<iframe width="560" height="315" src="https://www.youtube.com/embed/B-EFayAA5_0"'
+            ' frameborder="0" allow="autoplay; encrypted-media"></iframe>'
+            '</div>'
+            '</div>'
+            '</div>'
+        ))
+
+    def test_is_aggregator(self):
+        """
+        The unit XBlock is designed to hold other XBlocks, so check that its
+        completion status is defined as the aggregation of its child blocks.
+        """
+        self.assertEqual(XBlockCompletionMode.get_mode(UnitBlock), XBlockCompletionMode.AGGREGATOR)
+
+    def assertXmlEqual(self, xml_str_a, xml_str_b):
+        """
+        Assert that the given XML strings are equal,
+        ignoring attribute order and some whitespace variations.
+        """
+        def clean(xml_str):
+            # Collapse repeated whitespace:
+            xml_str = re.sub(r'(\s)\s+', r'\1', xml_str)
+            xml_bytes = xml_str.encode('utf8')
+            return minidom.parseString(xml_bytes).toprettyxml()
+        self.assertEqual(clean(xml_str_a), clean(xml_str_b))

common/lib/xmodule/xmodule/unit_block.py

@@ -0,0 +1,73 @@
+"""
+An XBlock which groups related XBlocks together.
+
+This is like the "vertical" block, but without that block's UI code, JavaScript,
+and other legacy features.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from web_fragments.fragment import Fragment
+from xblock.completable import XBlockCompletionMode
+from xblock.core import XBlock
+from xblock.fields import Scope, String
+
+# Make '_' a no-op so we can scrape strings.
+_ = lambda text: text
+
+
+class UnitBlock(XBlock):
+    """
+    Unit XBlock: An XBlock which groups related XBlocks together.
+
+    This is like the "vertical" block in principle, but this version is
+    explicitly designed to not contain LMS-related logic, like vertical does.
+
+    The application which renders XBlocks and/or the runtime should manage
+    things like bookmarks, completion tracking, etc.
+    This version also avoids any XModule mixins and has no JavaScript code.
+    """
+    has_children = True
+    # This is a block containing other blocks, so its completion is defined by
+    # the completion of its child blocks:
+    completion_mode = XBlockCompletionMode.AGGREGATOR
+    # Define a non-existent resources dir because we don't have resources, but
+    # the default will pull in all files in this folder.
+    resources_dir = 'assets/unit'
+
+    display_name = String(
+        display_name=_("Display Name"),
+        help=_("The display name for this component."),
+        scope=Scope.settings,
+        default=_("Unit"),
+    )
+
+    def student_view(self, context=None):
+        """Provide default student view."""
+        result = Fragment()
+        child_frags = self.runtime.render_children(self, context=context)
+        result.add_resources(child_frags)
+        result.add_content('<div class="unit-xblock vertical">')
+        for frag in child_frags:
+            result.add_content(frag.content)
+        result.add_content('</div>')
+        return result
+
+    def index_dictionary(self):
+        """
+        Return dictionary prepared with module content and type for indexing, so
+        that the contents of this block can be found in free-text searches.
+        """
+        # return key/value fields in a Python dict object
+        # values may be numeric / string or dict
+        xblock_body = super(UnitBlock, self).index_dictionary()
+        index_body = {
+            "display_name": self.display_name,
+        }
+        if "content" in xblock_body:
+            xblock_body["content"].update(index_body)
+        else:
+            xblock_body["content"] = index_body
+        # We use "Sequence" for sequentials and units/verticals
+        xblock_body["content_type"] = "Sequence"
+
+        return xblock_body

common/lib/xmodule/xmodule/video_module/transcripts_utils.py

@@ -20,6 +20,7 @@ from six import text_type
 from six.moves import range, zip
 from six.moves.html_parser import HTMLParser  # pylint: disable=import-error
 
+from opaque_keys.edx.locator import CourseLocator, LibraryLocator
 from xmodule.contentstore.content import StaticContent
 from xmodule.contentstore.django import contentstore
 from xmodule.exceptions import NotFoundError
@@ -1016,6 +1017,12 @@ def get_transcript(video, lang=None, output_format=Transcript.SRT, youtube_id=No
             raise NotFoundError
         return get_transcript_from_val(edx_video_id, lang, output_format)
     except NotFoundError:
+        # If this is not in a modulestore course or library, don't try loading from contentstore:
+        if not isinstance(video.scope_ids.usage_id.course_key, (CourseLocator, LibraryLocator)):
+            raise NotFoundError(
+                u'Video transcripts cannot yet be loaded from Blockstore (block: {})'.format(video.scope_ids.usage_id),
+            )
+
         return get_transcript_from_contentstore(
             video,
             lang,

common/lib/xmodule/xmodule/video_module/video_module.py

@@ -597,6 +597,25 @@ class VideoBlock(
 
         return editable_fields
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        Implement the video block's special XML parsing requirements for the
+        new runtime only. For all other runtimes, use the existing XModule-style
+        methods like .from_xml().
+        """
+        video_block = runtime.construct_xblock_from_class(cls, keys)
+        field_data = cls.parse_video_xml(node)
+        for key, val in field_data.items():
+            setattr(video_block, key, cls.fields[key].from_json(val))
+        # Update VAL with info extracted from `xml_object`
+        video_block.edx_video_id = video_block.import_video_info_into_val(
+            node,
+            runtime.resources_fs,
+            keys.usage_id.context_key,
+        )
+        return video_block
+
     @classmethod
     def from_xml(cls, xml_data, system, id_generator):
         """

common/lib/xmodule/xmodule/x_module.py

@@ -1125,6 +1125,17 @@ class XModuleDescriptorToXBlockMixin(object):
         block = cls.from_xml(xml, runtime, id_generator)
         return block
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        This XML lives within Blockstore and the new runtime doesn't need this
+        legacy XModule code. Use the "normal" XBlock parsing code.
+        """
+        try:
+            return super(XModuleDescriptorToXBlockMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            return super(XModuleDescriptorToXBlockMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+
     @classmethod
     def from_xml(cls, xml_data, system, id_generator):
         """
@@ -1241,6 +1252,9 @@ class XModuleDescriptor(XModuleDescriptorToXBlockMixin, HTMLSnippet, ResourceTem
 
     # =============================== BUILTIN METHODS ==========================
     def __eq__(self, other):
+        """
+        Is this XModule effectively equal to the other instance?
+        """
         return (hasattr(other, 'scope_ids') and
                 self.scope_ids == other.scope_ids and
                 list(self.fields.keys()) == list(other.fields.keys()) and
@@ -1469,30 +1483,7 @@ class DescriptorSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime):
             Used for example to get a list of all non-fatal problems on course
             load, and display them to the user.
 
-            A function of (error_msg). errortracker.py provides a
-            handy make_error_tracker() function.
-
-            Patterns for using the error handler:
-               try:
-                  x = access_some_resource()
-                  check_some_format(x)
-               except SomeProblem as err:
-                  msg = 'Grommet {0} is broken: {1}'.format(x, str(err))
-                  log.warning(msg)  # don't rely on tracker to log
-                        # NOTE: we generally don't want content errors logged as errors
-                  self.system.error_tracker(msg)
-                  # work around
-                  return 'Oops, couldn't load grommet'
-
-               OR, if not in an exception context:
-
-               if not check_something(thingy):
-                  msg = "thingy {0} is broken".format(thingy)
-                  log.critical(msg)
-                  self.system.error_tracker(msg)
-
-               NOTE: To avoid duplication, do not call the tracker on errors
-               that you're about to re-raise---let the caller track them.
+            See errortracker.py for more documentation
 
         get_policy: a function that takes a usage id and returns a dict of
             policy to apply.

common/lib/xmodule/xmodule/xml_module.py

@@ -393,6 +393,17 @@ class XmlParserMixin(object):
 
         return xblock
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        This XML lives within Blockstore and the new runtime doesn't need this
+        legacy XModule code. Use the "normal" XBlock parsing code.
+        """
+        try:
+            return super(XmlParserMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            return super(XmlParserMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+
     @classmethod
     def _get_url_name(cls, node):
         """
@@ -559,6 +570,17 @@ class XmlMixin(XmlParserMixin):
         else:
             return super(XmlMixin, cls).parse_xml(node, runtime, keys, id_generator)
 
+    @classmethod
+    def parse_xml_new_runtime(cls, node, runtime, keys):
+        """
+        This XML lives within Blockstore and the new runtime doesn't need this
+        legacy XModule code. Use the "normal" XBlock parsing code.
+        """
+        try:
+            return super(XmlMixin, cls).parse_xml_new_runtime(node, runtime, keys)
+        except AttributeError:
+            return super(XmlMixin, cls).parse_xml(node, runtime, keys, id_generator=None)
+
     def export_to_xml(self, resource_fs):
         """
         Returns an xml string representing this module, and all modules

lms/envs/common.py

@@ -2285,6 +2285,9 @@ INSTALLED_APPS = [
     'bulk_email',
     'branding',
 
+    # New (Blockstore-based) XBlock runtime
+    'openedx.core.djangoapps.xblock.apps.LmsXBlockAppConfig',
+
     # Student support tools
     'support',
 

lms/envs/devstack.py

@@ -233,6 +233,9 @@ if FEATURES.get('ENABLE_THIRD_PARTY_AUTH') and 'third_party_auth.dummy.DummyBack
 ############## ECOMMERCE API CONFIGURATION SETTINGS ###############
 ECOMMERCE_PUBLIC_URL_ROOT = "http://localhost:8002"
 
+############################### BLOCKSTORE #####################################
+BLOCKSTORE_API_URL = "http://edx.devstack.blockstore:18250/api/v1/"
+
 ###################### Cross-domain requests ######################
 FEATURES['ENABLE_CORS_HEADERS'] = True
 CORS_ALLOW_CREDENTIALS = True

lms/envs/production.py

@@ -572,6 +572,11 @@ MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {})
 EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '')  # django default is ''
 EMAIL_HOST_PASSWORD = AUTH_TOKENS.get('EMAIL_HOST_PASSWORD', '')  # django default is ''
 
+############################### BLOCKSTORE #####################################
+BLOCKSTORE_API_URL = ENV_TOKENS.get('BLOCKSTORE_API_URL', None)  # e.g. "https://blockstore.example.com/api/v1/"
+# Configure an API auth token at (blockstore URL)/admin/authtoken/token/
+BLOCKSTORE_API_AUTH_TOKEN = AUTH_TOKENS.get('BLOCKSTORE_API_AUTH_TOKEN', None)
+
 # Datadog for events!
 DATADOG = AUTH_TOKENS.get("DATADOG", {})
 DATADOG.update(ENV_TOKENS.get("DATADOG", {}))

lms/envs/test.py

@@ -235,6 +235,12 @@ CACHES = {
     },
 }
 
+############################### BLOCKSTORE #####################################
+# Blockstore tests
+RUN_BLOCKSTORE_TESTS = os.environ.get('EDXAPP_RUN_BLOCKSTORE_TESTS', 'no').lower() in ('true', 'yes', '1')
+BLOCKSTORE_API_URL = os.environ.get('EDXAPP_BLOCKSTORE_API_URL', "http://edx.devstack.blockstore-test:18251/api/v1/")
+BLOCKSTORE_API_AUTH_TOKEN = os.environ.get('EDXAPP_BLOCKSTORE_API_AUTH_TOKEN', 'edxapp-test-key')
+
 # Dummy secret key for dev
 SECRET_KEY = '85920908f28904ed733fe576320db18cabd7b6cd'
 

lms/urls.py

@@ -270,6 +270,9 @@ urlpatterns += [
         name='xblock_resource_url',
     ),
 
+    # New (Blockstore-based) XBlock REST API
+    url(r'', include('openedx.core.djangoapps.xblock.rest_api.urls', namespace='xblock_api')),
+
     url(
         r'^courses/{}/xqueue/(?P<userid>[^/]*)/(?P<mod_id>.*?)/(?P<dispatch>[^/]*)$'.format(
             settings.COURSE_ID_PATTERN,

openedx/core/djangoapps/content_libraries/admin.py

@@ -0,0 +1,33 @@
+"""
+Admin site for content libraries
+"""
+from django.contrib import admin
+from .models import ContentLibrary, ContentLibraryPermission
+
+
+class ContentLibraryPermissionInline(admin.TabularInline):
+    """
+    Inline form for a content library's permissions
+    """
+    model = ContentLibraryPermission
+    raw_id_fields = ("user", )
+    extra = 0
+
+
+@admin.register(ContentLibrary)
+class ContentLibraryAdmin(admin.ModelAdmin):
+    """
+    Definition of django admin UI for Content Libraries
+    """
+    fields = ("library_key", "org", "slug", "bundle_uuid", "allow_public_learning", "allow_public_read")
+    list_display = ("slug", "org", "bundle_uuid")
+    inlines = (ContentLibraryPermissionInline, )
+
+    def get_readonly_fields(self, request, obj=None):
+        """
+        Ensure that 'slug' and 'uuid' cannot be edited after creation.
+        """
+        if obj:
+            return ["library_key", "org", "slug", "bundle_uuid"]
+        else:
+            return ["library_key", ]

openedx/core/djangoapps/content_libraries/api.py

@@ -0,0 +1,485 @@
+"""
+Python API for content libraries
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+from uuid import UUID
+import logging
+
+import attr
+from django.core.validators import validate_unicode_slug
+from django.db import IntegrityError
+from lxml import etree
+from organizations.models import Organization
+import six
+from xblock.core import XBlock
+from xblock.exceptions import XBlockNotFoundError
+
+from cms.djangoapps.contentstore.views.helpers import xblock_type_display_name
+from openedx.core.djangoapps.content_libraries.library_bundle import LibraryBundle
+from openedx.core.djangoapps.xblock.api import get_block_display_name, load_block
+from openedx.core.djangoapps.xblock.learning_context.keys import BundleDefinitionLocator
+from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl
+from openedx.core.djangoapps.xblock.runtime.olx_parsing import XBlockInclude
+from openedx.core.lib.blockstore_api import (
+    get_bundle,
+    get_bundle_file_data,
+    get_bundle_files,
+    get_or_create_bundle_draft,
+    create_bundle,
+    update_bundle,
+    delete_bundle,
+    write_draft_file,
+    commit_draft,
+    delete_draft,
+)
+from openedx.core.djangolib.blockstore_cache import BundleCache
+from .keys import LibraryLocatorV2, LibraryUsageLocatorV2
+from .models import ContentLibrary, ContentLibraryPermission
+
+log = logging.getLogger(__name__)
+
+# This API is only used in Studio, so we always work with this draft of any
+# content library bundle:
+DRAFT_NAME = 'studio_draft'
+
+# Exceptions:
+ContentLibraryNotFound = ContentLibrary.DoesNotExist
+
+
+class ContentLibraryBlockNotFound(XBlockNotFoundError):
+    """ XBlock not found in the content library """
+
+
+class LibraryAlreadyExists(KeyError):
+    """ A library with the specified slug already exists """
+
+
+class LibraryBlockAlreadyExists(KeyError):
+    """ An XBlock with that ID already exists in the library """
+
+
+# Models:
+
+@attr.s
+class ContentLibraryMetadata(object):
+    """
+    Class that represents the metadata about a content library.
+    """
+    key = attr.ib(type=LibraryLocatorV2)
+    bundle_uuid = attr.ib(type=UUID)
+    title = attr.ib("")
+    description = attr.ib("")
+    version = attr.ib(0)
+    has_unpublished_changes = attr.ib(False)
+    # has_unpublished_deletes will be true when the draft version of the library's bundle
+    # contains deletes of any XBlocks that were in the most recently published version
+    has_unpublished_deletes = attr.ib(False)
+
+
+@attr.s
+class LibraryXBlockMetadata(object):
+    """
+    Class that represents the metadata about an XBlock in a content library.
+    """
+    usage_key = attr.ib(type=LibraryUsageLocatorV2)
+    def_key = attr.ib(type=BundleDefinitionLocator)
+    display_name = attr.ib("")
+    has_unpublished_changes = attr.ib(False)
+
+
+@attr.s
+class LibraryXBlockType(object):
+    """
+    An XBlock type that can be added to a content library
+    """
+    block_type = attr.ib("")
+    display_name = attr.ib("")
+
+
+class AccessLevel(object):
+    """ Enum defining library access levels/permissions """
+    ADMIN_LEVEL = ContentLibraryPermission.ADMIN_LEVEL
+    AUTHOR_LEVEL = ContentLibraryPermission.AUTHOR_LEVEL
+    READ_LEVEL = ContentLibraryPermission.READ_LEVEL
+    NO_ACCESS = None
+
+
+def list_libraries():
+    """
+    TEMPORARY method for testing. Lists all content libraries.
+    This should be replaced with a method for listing all libraries that belong
+    to a particular user, and/or has permission to view. This method makes at
+    least one HTTP call per library so should only be used for development.
+    """
+    refs = ContentLibrary.objects.all()[:1000]
+    return [get_library(ref.library_key) for ref in refs]
+
+
+def get_library(library_key):
+    """
+    Get the library with the specified key. Does not check permissions.
+    returns a ContentLibraryMetadata instance.
+
+    Raises ContentLibraryNotFound if the library doesn't exist.
+    """
+    assert isinstance(library_key, LibraryLocatorV2)
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    bundle_metadata = get_bundle(ref.bundle_uuid)
+    lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME)
+    (has_unpublished_changes, has_unpublished_deletes) = lib_bundle.has_changes()
+    return ContentLibraryMetadata(
+        key=library_key,
+        bundle_uuid=ref.bundle_uuid,
+        title=bundle_metadata.title,
+        description=bundle_metadata.description,
+        version=bundle_metadata.latest_version,
+        has_unpublished_changes=has_unpublished_changes,
+        has_unpublished_deletes=has_unpublished_deletes,
+    )
+
+
+def create_library(collection_uuid, org, slug, title, description):
+    """
+    Create a new content library.
+
+    org: an organizations.models.Organization instance
+
+    slug: a slug for this library like 'physics-problems'
+
+    title: title for this library
+
+    description: description of this library
+
+    Returns a ContentLibraryMetadata instance.
+    """
+    assert isinstance(collection_uuid, UUID)
+    assert isinstance(org, Organization)
+    validate_unicode_slug(slug)
+    # First, create the blockstore bundle:
+    bundle = create_bundle(
+        collection_uuid,
+        slug=slug,
+        title=title,
+        description=description,
+    )
+    # Now create the library reference in our database:
+    try:
+        ref = ContentLibrary.objects.create(
+            org=org,
+            slug=slug,
+            bundle_uuid=bundle.uuid,
+            allow_public_learning=True,
+            allow_public_read=True,
+        )
+    except IntegrityError:
+        delete_bundle(bundle.uuid)
+        raise LibraryAlreadyExists(slug)
+    return ContentLibraryMetadata(
+        key=ref.library_key,
+        bundle_uuid=bundle.uuid,
+        title=title,
+        description=description,
+        version=0,
+    )
+
+
+def set_library_user_permissions(library_key, user, access_level):
+    """
+    Change the specified user's level of access to this library.
+
+    access_level should be one of the AccessLevel values defined above.
+    """
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    if access_level is None:
+        ref.authorized_users.filter(user=user).delete()
+    else:
+        ContentLibraryPermission.objects.update_or_create(user=user, library=ref, access_level=access_level)
+
+
+def update_library(library_key, title=None, description=None):
+    """
+    Update a library's title or description.
+    (Slug cannot be changed as it would break IDs throughout the system.)
+
+    A value of None means "don't change".
+    """
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    fields = {
+        # We don't ever read the "slug" value from the Blockstore bundle, but
+        # we might as well always do our best to keep it in sync with the "slug"
+        # value in the LMS that we do use.
+        "slug": ref.slug,
+    }
+    if title is not None:
+        assert isinstance(title, six.string_types)
+        fields["title"] = title
+    if description is not None:
+        assert isinstance(description, six.string_types)
+        fields["description"] = description
+    update_bundle(ref.bundle_uuid, **fields)
+
+
+def delete_library(library_key):
+    """
+    Delete a content library
+    """
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    bundle_uuid = ref.bundle_uuid
+    # We can't atomically delete the ref and bundle at the same time.
+    # Delete the ref first, then the bundle. An error may cause the bundle not
+    # to get deleted, but the library will still be effectively gone from the
+    # system, which is a better state than having a reference to a library with
+    # no backing blockstore bundle.
+    ref.delete()
+    try:
+        delete_bundle(bundle_uuid)
+    except:
+        log.exception("Failed to delete blockstore bundle %s when deleting library. Delete it manually.", bundle_uuid)
+        raise
+
+
+def get_library_blocks(library_key):
+    """
+    Get the list of top-level XBlocks in the specified library.
+
+    Returns a list of LibraryXBlockMetadata objects
+    """
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    lib_bundle = LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME)
+    usages = lib_bundle.get_top_level_usages()
+    blocks = []
+    for usage_key in usages:
+        # For top-level definitions, we can go from definition key to usage key using the following, but this would not
+        # work for non-top-level blocks as they may have multiple usages. Top level blocks are guaranteed to have only
+        # a single usage in the library, which is part of the definition of top level block.
+        def_key = lib_bundle.definition_for_usage(usage_key)
+        blocks.append(LibraryXBlockMetadata(
+            usage_key=usage_key,
+            def_key=def_key,
+            display_name=get_block_display_name(def_key),
+            has_unpublished_changes=lib_bundle.does_definition_have_unpublished_changes(def_key),
+        ))
+    return blocks
+
+
+def get_library_block(usage_key):
+    """
+    Get metadata (LibraryXBlockMetadata) about one specific XBlock in a library
+
+    To load the actual XBlock instance, use
+        openedx.core.djangoapps.xblock.api.load_block()
+    instead.
+    """
+    assert isinstance(usage_key, LibraryUsageLocatorV2)
+    lib_context = get_learning_context_impl(usage_key)
+    def_key = lib_context.definition_for_usage(usage_key)
+    if def_key is None:
+        raise ContentLibraryBlockNotFound(usage_key)
+    lib_bundle = LibraryBundle(usage_key.library_slug, def_key.bundle_uuid, draft_name=DRAFT_NAME)
+    return LibraryXBlockMetadata(
+        usage_key=usage_key,
+        def_key=def_key,
+        display_name=get_block_display_name(def_key),
+        has_unpublished_changes=lib_bundle.does_definition_have_unpublished_changes(def_key),
+    )
+
+
+def get_library_block_olx(usage_key):
+    """
+    Get the OLX source of the given XBlock.
+    """
+    assert isinstance(usage_key, LibraryUsageLocatorV2)
+    definition_key = get_library_block(usage_key).def_key
+    xml_str = get_bundle_file_data(
+        bundle_uuid=definition_key.bundle_uuid,  # pylint: disable=no-member
+        path=definition_key.olx_path,  # pylint: disable=no-member
+        use_draft=DRAFT_NAME,
+    )
+    return xml_str
+
+
+def set_library_block_olx(usage_key, new_olx_str):
+    """
+    Replace the OLX source of the given XBlock.
+    This is only meant for use by developers or API client applications, as
+    very little validation is done and this can easily result in a broken XBlock
+    that won't load.
+    """
+    # because this old pylint can't understand attr.ib() objects, pylint: disable=no-member
+    assert isinstance(usage_key, LibraryUsageLocatorV2)
+    # Make sure the block exists:
+    metadata = get_library_block(usage_key)
+    block_type = usage_key.block_type
+    # Verify that the OLX parses, at least as generic XML:
+    node = etree.fromstring(new_olx_str)
+    if node.tag != block_type:
+        raise ValueError("Invalid root tag in OLX, expected {}".format(block_type))
+    # Write the new XML/OLX file into the library bundle's draft
+    draft = get_or_create_bundle_draft(metadata.def_key.bundle_uuid, DRAFT_NAME)
+    write_draft_file(draft.uuid, metadata.def_key.olx_path, new_olx_str)
+    # Clear the bundle cache so everyone sees the new block immediately:
+    BundleCache(metadata.def_key.bundle_uuid, draft_name=DRAFT_NAME).clear()
+
+
+def create_library_block(library_key, block_type, definition_id):
+    """
+    Create a new XBlock in this library of the specified type (e.g. "html").
+
+    The 'definition_id' value (which should be a string like "problem1") will be
+    used as both the definition_id and the usage_id.
+    """
+    assert isinstance(library_key, LibraryLocatorV2)
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    # Make sure the proposed ID will be valid:
+    validate_unicode_slug(definition_id)
+    # Ensure the XBlock type is valid and installed:
+    XBlock.load_class(block_type)  # Will raise an exception if invalid
+    # Make sure the new ID is not taken already:
+    new_usage_id = definition_id  # Since this is a top level XBlock, usage_id == definition_id
+    usage_key = LibraryUsageLocatorV2(
+        library_org=library_key.org,
+        library_slug=library_key.slug,
+        block_type=block_type,
+        usage_id=new_usage_id,
+    )
+    library_context = get_learning_context_impl(usage_key)
+    if library_context.definition_for_usage(usage_key) is not None:
+        raise LibraryBlockAlreadyExists("An XBlock with ID '{}' already exists".format(new_usage_id))
+
+    new_definition_xml = '<{}/>'.format(block_type)  # xss-lint: disable=python-wrap-html
+    path = "{}/{}/definition.xml".format(block_type, definition_id)
+    # Write the new XML/OLX file into the library bundle's draft
+    draft = get_or_create_bundle_draft(ref.bundle_uuid, DRAFT_NAME)
+    write_draft_file(draft.uuid, path, new_definition_xml)
+    # Clear the bundle cache so everyone sees the new block immediately:
+    BundleCache(ref.bundle_uuid, draft_name=DRAFT_NAME).clear()
+    # Now return the metadata about the new block:
+    return get_library_block(usage_key)
+
+
+def delete_library_block(usage_key, remove_from_parent=True):
+    """
+    Delete the specified block from this library (and any children it has).
+
+    If the block's definition (OLX file) is within this same library as the
+    usage key, both the definition and the usage will be deleted.
+
+    If the usage points to a definition in a linked bundle, the usage will be
+    deleted but the link and the linked bundle will be unaffected.
+
+    If the block is in use by some other bundle that links to this one, that
+    will not prevent deletion of the definition.
+
+    remove_from_parent: modify the parent to remove the reference to this
+        delete block. This should always be true except when this function
+        calls itself recursively.
+    """
+    assert isinstance(usage_key, LibraryUsageLocatorV2)
+    library_context = get_learning_context_impl(usage_key)
+    library_ref = ContentLibrary.objects.get_by_key(usage_key.context_key)
+    def_key = library_context.definition_for_usage(usage_key)
+    if def_key is None:
+        raise ContentLibraryBlockNotFound(usage_key)
+    lib_bundle = LibraryBundle(usage_key.context_key, library_ref.bundle_uuid, draft_name=DRAFT_NAME)
+    # Create a draft:
+    draft_uuid = get_or_create_bundle_draft(def_key.bundle_uuid, DRAFT_NAME).uuid
+    # Does this block have a parent?
+    if usage_key not in lib_bundle.get_top_level_usages() and remove_from_parent:
+        # Yes: this is not a top-level block.
+        # First need to modify the parent to remove this block as a child.
+        raise NotImplementedError
+    # Does this block have children?
+    block = load_block(usage_key, user=None)
+    if block.has_children:
+        # Next, recursively call delete_library_block(...) on each child usage
+        for child_usage in block.children:
+            # Specify remove_from_parent=False to avoid unnecessary work to
+            # modify this block's children list when deleting each child, since
+            # we're going to delete this block anyways.
+            delete_library_block(child_usage, remove_from_parent=False)
+    # Delete the definition:
+    if def_key.bundle_uuid == library_ref.bundle_uuid:
+        # This definition is in the library, so delete it:
+        path_prefix = lib_bundle.olx_prefix(def_key)
+        for bundle_file in get_bundle_files(def_key.bundle_uuid, use_draft=DRAFT_NAME):
+            if bundle_file.path.startswith(path_prefix):
+                # Delete this file, within this definition's "folder"
+                write_draft_file(draft_uuid, bundle_file.path, contents=None)
+    else:
+        # The definition must be in a linked bundle, so we don't want to delete
+        # it; just the <xblock-include /> in the parent, which was already
+        # deleted above.
+        pass
+    # Clear the bundle cache so everyone sees the deleted block immediately:
+    lib_bundle.cache.clear()
+
+
+def create_library_block_child(parent_usage_key, block_type, definition_id):
+    """
+    Create a new XBlock definition in this library of the specified type (e.g.
+    "html"), and add it as a child of the specified existing block.
+
+    The 'definition_id' value (which should be a string like "problem1") will be
+    used as both the definition_id and the usage_id of the child.
+    """
+    assert isinstance(parent_usage_key, LibraryUsageLocatorV2)
+    # Load the parent block to make sure it exists and so we can modify its 'children' field:
+    parent_block = load_block(parent_usage_key, user=None)
+    if not parent_block.has_children:
+        raise ValueError("The specified parent XBlock does not allow child XBlocks.")
+    # Create the new block in the library:
+    metadata = create_library_block(parent_usage_key.context_key, block_type, definition_id)
+    # Set the block as a child.
+    # This will effectively "move" the newly created block from being a top-level block in the library to a child.
+    include_data = XBlockInclude(link_id=None, block_type=block_type, definition_id=definition_id, usage_hint=None)
+    parent_block.runtime.add_child_include(parent_block, include_data)
+    parent_block.save()
+    return metadata
+
+
+def get_allowed_block_types(library_key):  # pylint: disable=unused-argument
+    """
+    Get a list of XBlock types that can be added to the specified content
+    library. For now, the result is the same regardless of which library is
+    specified, but that may change in the future.
+    """
+    # TODO: return support status and template options
+    # See cms/djangoapps/contentstore/views/component.py
+    block_types = sorted(name for name, class_ in XBlock.load_classes())
+    info = []
+    for block_type in block_types:
+        display_name = xblock_type_display_name(block_type, None)
+        # For now as a crude heuristic, we exclude blocks that don't have a display_name
+        if display_name:
+            info.append(LibraryXBlockType(block_type=block_type, display_name=display_name))
+    return info
+
+
+def publish_changes(library_key):
+    """
+    Publish all pending changes to the specified library.
+    """
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    bundle = get_bundle(ref.bundle_uuid)
+    if DRAFT_NAME in bundle.drafts:  # pylint: disable=unsupported-membership-test
+        draft_uuid = bundle.drafts[DRAFT_NAME]  # pylint: disable=unsubscriptable-object
+        commit_draft(draft_uuid)
+    else:
+        return  # If there is no draft, no action is needed.
+    LibraryBundle(library_key, ref.bundle_uuid).cache.clear()
+    LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME).cache.clear()
+
+
+def revert_changes(library_key):
+    """
+    Revert all pending changes to the specified library, restoring it to the
+    last published version.
+    """
+    ref = ContentLibrary.objects.get_by_key(library_key)
+    bundle = get_bundle(ref.bundle_uuid)
+    if DRAFT_NAME in bundle.drafts:  # pylint: disable=unsupported-membership-test
+        draft_uuid = bundle.drafts[DRAFT_NAME]  # pylint: disable=unsubscriptable-object
+        delete_draft(draft_uuid)
+    else:
+        return  # If there is no draft, no action is needed.
+    LibraryBundle(library_key, ref.bundle_uuid, draft_name=DRAFT_NAME).cache.clear()

openedx/core/djangoapps/content_libraries/apps.py

@@ -0,0 +1,32 @@
+"""
+Django AppConfig for Content Libraries Implementation
+"""
+# -*- coding: utf-8 -*-
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.apps import AppConfig
+
+from openedx.core.djangoapps.plugins.constants import ProjectType, PluginURLs, PluginSettings
+
+
+class ContentLibrariesConfig(AppConfig):
+    """
+    Django AppConfig for Content Libraries Implementation
+    """
+
+    name = 'openedx.core.djangoapps.content_libraries'
+    verbose_name = 'Content Libraries (Blockstore-based)'
+    # This is designed as a plugin for now so that
+    # the whole thing is self-contained and can easily be enabled/disabled
+    plugin_app = {
+        PluginURLs.CONFIG: {
+            ProjectType.CMS: {
+                # The namespace to provide to django's urls.include.
+                PluginURLs.NAMESPACE: u'content_libraries',
+            },
+        },
+        PluginSettings.CONFIG: {
+            ProjectType.CMS: {
+            },
+        },
+    }

openedx/core/djangoapps/content_libraries/keys.py

@@ -0,0 +1,132 @@
+"""
+Key/locator types for Blockstore-based content libraries
+"""
+# Disable warnings about _to_deprecated_string etc. which we don't want to implement:
+# pylint: disable=abstract-method, no-member
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from opaque_keys import InvalidKeyError
+
+from openedx.core.djangoapps.xblock.learning_context.keys import (
+    check_key_string_field,
+    BlockUsageKeyV2,
+    LearningContextKey,
+)
+
+
+class LibraryLocatorV2(LearningContextKey):
+    """
+    A key that represents a Blockstore-based content library.
+
+    When serialized, these keys look like:
+        lib:MITx:reallyhardproblems
+        lib:hogwarts:p300-potions-exercises
+    """
+    CANONICAL_NAMESPACE = 'lib'
+    KEY_FIELDS = ('org', 'slug')
+    __slots__ = KEY_FIELDS
+    CHECKED_INIT = False
+
+    def __init__(self, org, slug):
+        """
+        Construct a GlobalUsageLocator
+        """
+        check_key_string_field(org)
+        check_key_string_field(slug)
+        super(LibraryLocatorV2, self).__init__(org=org, slug=slug)
+
+    def _to_string(self):
+        """
+        Serialize this key as a string
+        """
+        return ":".join((self.org, self.slug))
+
+    @classmethod
+    def _from_string(cls, serialized):
+        """
+        Instantiate this key from a serialized string
+        """
+        try:
+            (org, slug) = serialized.split(':')
+        except ValueError:
+            raise InvalidKeyError(cls, serialized)
+        return cls(org=org, slug=slug)
+
+    def make_definition_usage(self, definition_key, usage_id=None):
+        """
+        Return a usage key, given the given the specified definition key and
+        usage_id.
+        """
+        return LibraryUsageLocatorV2(
+            library_org=self.org,
+            library_slug=self.slug,
+            block_type=definition_key.block_type,
+            usage_id=usage_id,
+        )
+
+    def for_branch(self, branch):
+        """
+        Compatibility helper.
+        Some code calls .for_branch(None) on course keys. By implementing this,
+        it improves backwards compatibility between library keys and course
+        keys.
+        """
+        if branch is not None:
+            raise ValueError("Cannot call for_branch on a content library key, except for_branch(None).")
+        return self
+
+
+class LibraryUsageLocatorV2(BlockUsageKeyV2):
+    """
+    An XBlock in a Blockstore-based content library.
+
+    When serialized, these keys look like:
+        lb:MITx:reallyhardproblems:problem:problem1
+    """
+    CANONICAL_NAMESPACE = 'lb'  # "Library Block"
+    KEY_FIELDS = ('library_org', 'library_slug', 'block_type', 'usage_id')
+    __slots__ = KEY_FIELDS
+    CHECKED_INIT = False
+
+    def __init__(self, library_org, library_slug, block_type, usage_id):
+        """
+        Construct a LibraryUsageLocatorV2
+        """
+        check_key_string_field(library_org)
+        check_key_string_field(library_slug)
+        check_key_string_field(block_type)
+        check_key_string_field(usage_id)
+        super(LibraryUsageLocatorV2, self).__init__(
+            library_org=library_org,
+            library_slug=library_slug,
+            block_type=block_type,
+            usage_id=usage_id,
+        )
+
+    @property
+    def context_key(self):
+        return LibraryLocatorV2(org=self.library_org, slug=self.library_slug)
+
+    @property
+    def block_id(self):
+        """
+        Get the 'block ID' which is another name for the usage ID.
+        """
+        return self.usage_id
+
+    def _to_string(self):
+        """
+        Serialize this key as a string
+        """
+        return ":".join((self.library_org, self.library_slug, self.block_type, self.usage_id))
+
+    @classmethod
+    def _from_string(cls, serialized):
+        """
+        Instantiate this key from a serialized string
+        """
+        try:
+            (library_org, library_slug, block_type, usage_id) = serialized.split(':')
+        except ValueError:
+            raise InvalidKeyError(cls, serialized)
+        return cls(library_org=library_org, library_slug=library_slug, block_type=block_type, usage_id=usage_id)

openedx/core/djangoapps/content_libraries/library_bundle.py

@@ -0,0 +1,346 @@
+"""
+Helper code for working with Blockstore bundles that contain OLX
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import logging
+
+from django.utils.lru_cache import lru_cache
+from xblock.core import XBlock
+from xblock.plugin import PluginMissingError
+
+from openedx.core.djangoapps.content_libraries.keys import LibraryUsageLocatorV2
+from openedx.core.djangoapps.content_libraries.models import ContentLibrary
+from openedx.core.djangoapps.xblock.learning_context.keys import BundleDefinitionLocator
+from openedx.core.djangoapps.xblock.runtime.blockstore_runtime import xml_for_definition
+from openedx.core.djangoapps.xblock.runtime.olx_parsing import (
+    BundleFormatException,
+    definition_for_include,
+    parse_xblock_include,
+)
+from openedx.core.djangolib.blockstore_cache import (
+    BundleCache,
+    get_bundle_files_cached,
+    get_bundle_file_metadata_with_cache,
+    get_bundle_version_number,
+)
+from openedx.core.lib import blockstore_api
+
+log = logging.getLogger(__name__)
+
+
+@lru_cache()
+def bundle_uuid_for_library_key(library_key):
+    """
+    Given a library slug, look up its bundle UUID.
+    Can be cached aggressively since bundle UUID is immutable.
+
+    May raise ContentLibrary.DoesNotExist
+    """
+    library_metadata = ContentLibrary.objects.get_by_key(library_key)
+    return library_metadata.bundle_uuid
+
+
+def usage_for_child_include(parent_usage, parent_definition, parsed_include):
+    """
+    Get the usage ID for a child XBlock, given the parent's keys and the
+    <xblock-include /> element that specifies the child.
+
+    Consider two bundles, one with three definitions:
+        main-unit, html1, subunit1
+    And a second bundle with two definitions:
+        unit1, html1
+    Note that both bundles have a definition called "html1". Now, with the
+    following tree structure, where "unit/unit1" and the second "html/html1"
+    are in a linked bundle:
+
+    <unit> in unit/main-unit/definition.xml
+        <xblock-include definition="html/html1" />
+        <xblock-include definition="unit/subunit1" />
+            <xblock-include source="linked_bundle" definition="unit/unit1" usage="alias1" />
+                <xblock-include definition="html/html1" />
+
+    The following usage IDs would result:
+
+    main-unit
+        html1
+        subunit1
+            alias1
+                alias1-html1
+
+    Notice that "html1" in the linked bundle is prefixed so its ID stays
+    unique from the "html1" in the original library.
+    """
+    assert isinstance(parent_usage, LibraryUsageLocatorV2)
+    usage_id = parsed_include.usage_hint if parsed_include.usage_hint else parsed_include.definition_id
+    library_bundle_uuid = bundle_uuid_for_library_key(parent_usage.context_key)
+    # Is the parent usage from the same bundle as the library?
+    parent_usage_from_library_bundle = parent_definition.bundle_uuid == library_bundle_uuid
+    if not parent_usage_from_library_bundle:
+        # This XBlock has been linked in to the library via a chain of one
+        # or more bundle links. In order to keep usage_id collisions from
+        # happening, any descdenants of the first linked block must have
+        # their usage_id prefixed with the parent usage's usage_id.
+        # (It would be possible to only change the prefix when the block is
+        # a child of a block with an explicit usage="" attribute on its
+        # <xblock-include> but that requires much more complex logic.)
+        usage_id = parent_usage.usage_id + "-" + usage_id
+    return LibraryUsageLocatorV2(
+        library_org=parent_usage.library_org,
+        library_slug=parent_usage.library_slug,
+        block_type=parsed_include.block_type,
+        usage_id=usage_id,
+    )
+
+
+class LibraryBundle(object):
+    """
+    Wrapper around a Content Library Blockstore bundle that contains OLX.
+    """
+
+    def __init__(self, library_key, bundle_uuid, draft_name=None):
+        """
+        Instantiate this wrapper for the bundle with the specified library_key,
+        UUID, and optionally the specified draft name.
+        """
+        self.library_key = library_key
+        self.bundle_uuid = bundle_uuid
+        self.draft_name = draft_name
+        self.cache = BundleCache(bundle_uuid, draft_name)
+
+    def get_olx_files(self):
+        """
+        Get the list of OLX files in this bundle (using a heuristic)
+
+        Because this uses a heuristic, it will only return files with filenames
+        that seem like OLX files that are in the expected locations of OLX
+        files. They are not guaranteed to be valid OLX nor will OLX files in
+        nonstandard locations be returned.
+
+        Example return value: [
+            'html/intro/definition.xml',
+            'unit/unit1/definition.xml',
+        ]
+        """
+        bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name)
+        return [f.path for f in bundle_files if f.path.endswith("/definition.xml")]
+
+    def definition_for_usage(self, usage_key):
+        """
+        Given the usage key for an XBlock in this library bundle, return the
+        BundleDefinitionLocator which specifies the actual XBlock definition (as
+        a path to an OLX in a specific blockstore bundle).
+
+        Must return a BundleDefinitionLocator if the XBlock exists in this
+        context, or None otherwise.
+
+        For a content library, the rules are simple:
+        * If the usage key points to a block in this library, the filename
+          (definition) of the OLX file is always
+            {block_type}/{usage_id}/definition.xml
+          Each library has exactly one usage per definition for its own blocks.
+        * However, block definitions from other content libraries may be linked
+          into this library via <xblock-include ... /> directives. In that case,
+          it's necessary to inspect every OLX file in this library that might
+          have an <xblock-include /> directive in order to find what external
+          block the usage ID refers to.
+        """
+        # Now that we know the library/bundle, find the block's definition
+        if self.draft_name:
+            version_arg = {"draft_name": self.draft_name}
+        else:
+            version_arg = {"bundle_version": get_bundle_version_number(self.bundle_uuid)}
+        olx_path = "{}/{}/definition.xml".format(usage_key.block_type, usage_key.usage_id)
+        try:
+            get_bundle_file_metadata_with_cache(self.bundle_uuid, olx_path, **version_arg)
+            return BundleDefinitionLocator(self.bundle_uuid, usage_key.block_type, olx_path, **version_arg)
+        except blockstore_api.BundleFileNotFound:
+            # This must be a usage of a block from a linked bundle. One of the
+            # OLX files in this bundle contains an <xblock-include usage="..."/>
+            bundle_includes = self.get_bundle_includes()
+            try:
+                return bundle_includes[usage_key]
+            except KeyError:
+                return None
+
+    def get_top_level_usages(self):
+        """
+        Get the set of usage keys in this bundle that have no parent.
+        """
+        own_usage_keys = []
+        for olx_file_path in self.get_olx_files():
+            block_type, usage_id, _unused = olx_file_path.split('/')
+            usage_key = LibraryUsageLocatorV2(self.library_key.org, self.library_key.slug, block_type, usage_id)
+            own_usage_keys.append(usage_key)
+
+        usage_keys_with_parents = self.get_bundle_includes().keys()
+        return [usage_key for usage_key in own_usage_keys if usage_key not in usage_keys_with_parents]
+
+    def get_bundle_includes(self):
+        """
+        Scan through the bundle and all linked bundles as needed to generate
+        a complete list of all the blocks that are included as
+        child/grandchild/... blocks of the blocks in this bundle.
+
+        Returns a dict of {usage_key -> BundleDefinitionLocator}
+
+        Blocks in the bundle that have no parent are not included.
+        """
+        cache_key = ("bundle_includes", )
+        usages_found = self.cache.get(cache_key)
+        if usages_found is not None:
+            return usages_found
+
+        usages_found = {}
+
+        def add_definitions_children(usage_key, def_key):
+            """
+            Recursively add any children of the given XBlock usage+definition to
+            usages_found.
+            """
+            if not does_block_type_support_children(def_key.block_type):
+                return
+            try:
+                xml_node = xml_for_definition(def_key)
+            except:  # pylint:disable=bare-except
+                log.exception("Unable to load definition {}".format(def_key))
+                return
+
+            for child in xml_node:
+                if child.tag != 'xblock-include':
+                    continue
+                try:
+                    parsed_include = parse_xblock_include(child)
+                    child_usage = usage_for_child_include(usage_key, def_key, parsed_include)
+                    child_def_key = definition_for_include(parsed_include, def_key)
+                except BundleFormatException:
+                    log.exception("Unable to parse a child of {}".format(def_key))
+                    continue
+                usages_found[child_usage] = child_def_key
+                add_definitions_children(child_usage, child_def_key)
+
+        # Find all the definitions in this bundle and recursively add all their descendants:
+        bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name)
+        if self.draft_name:
+            version_arg = {"draft_name": self.draft_name}
+        else:
+            version_arg = {"bundle_version": get_bundle_version_number(self.bundle_uuid)}
+        for bfile in bundle_files:
+            if not bfile.path.endswith("/definition.xml") or bfile.path.count('/') != 2:
+                continue  # Not an OLX file.
+            block_type, usage_id, _unused = bfile.path.split('/')
+            def_key = BundleDefinitionLocator(
+                bundle_uuid=self.bundle_uuid,
+                block_type=block_type,
+                olx_path=bfile.path,
+                **version_arg
+            )
+            usage_key = LibraryUsageLocatorV2(self.library_key.org, self.library_key.slug, block_type, usage_id)
+            add_definitions_children(usage_key, def_key)
+
+        self.cache.set(cache_key, usages_found)
+        return usages_found
+
+    def does_definition_have_unpublished_changes(self, definition_key):
+        """
+        Given the defnition key of an XBlock, which exists in an OLX file like
+            problem/quiz1/definition.xml
+        Check if the bundle's draft has _any_ unpublished changes in the
+            problem/quiz1/
+        directory.
+        """
+        if self.draft_name is None:
+            return False  # No active draft so can't be changes
+        prefix = self.olx_prefix(definition_key)
+        return prefix in self._get_changed_definitions()
+
+    def _get_changed_definitions(self):
+        """
+        Helper method to get a list of all paths with changes, where a path is
+            problem/quiz1/
+        Or similar (a type and an ID), excluding 'definition.xml'
+        """
+        cached_result = self.cache.get(('changed_definition_prefixes', ))
+        if cached_result is not None:
+            return cached_result
+        changed = []
+        bundle_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name)
+        for file_ in bundle_files:
+            if getattr(file_, 'modified', False) and file_.path.count('/') >= 2:
+                (type_part, id_part, _rest) = file_.path.split('/', 2)
+                prefix = type_part + '/' + id_part + '/'
+                if prefix not in changed:
+                    changed.append(prefix)
+        self.cache.set(('changed_definition_prefixes', ), changed)
+        return changed
+
+    def has_changes(self):
+        """
+        Helper method to check if this OLX bundle has any pending changes,
+        including any deleted blocks.
+
+        Returns a tuple of (
+            has_unpublished_changes,
+            has_unpublished_deletes,
+        )
+        Where has_unpublished_changes is true if there is any type of change,
+        including deletes, and has_unpublished_deletes is only true if one or
+        more blocks has been deleted since the last publish.
+        """
+        if not self.draft_name:
+            return (False, False)
+        cached_result = self.cache.get(('has_changes', ))
+        if cached_result is not None:
+            return cached_result
+        draft_files = get_bundle_files_cached(self.bundle_uuid, draft_name=self.draft_name)
+
+        has_unpublished_changes = False
+        has_unpublished_deletes = False
+
+        for file_ in draft_files:
+            if getattr(file_, 'modified', False):
+                has_unpublished_changes = True
+                break
+
+        published_file_paths = set(f.path for f in get_bundle_files_cached(self.bundle_uuid))
+        draft_file_paths = set(f.path for f in draft_files)
+        for file_path in published_file_paths:
+            if file_path not in draft_file_paths:
+                has_unpublished_changes = True
+                if file_path.endswith('/definition.xml'):
+                    # only set 'has_unpublished_deletes' if the actual main definition XML
+                    # file was deleted, not if only some asset file was deleted, etc.
+                    has_unpublished_deletes = True
+                    break
+
+        result = (has_unpublished_changes, has_unpublished_deletes)
+        self.cache.set(('has_changes', ), result)
+        return result
+
+    @staticmethod
+    def olx_prefix(definition_key):
+        """
+        Given a definition key in a compatible bundle, whose olx_path refers to
+            block_type/some_id/definition.xml
+        Return the "folder name" / "path prefix"
+            block-type/some_id/
+
+        This method is here rather than a method of BundleDefinitionLocator
+        because BundleDefinitionLocator is more generic and doesn't require
+        that its olx_path always ends in /definition.xml
+        """
+        if not definition_key.olx_path.endswith('/definition.xml'):
+            raise ValueError
+        return definition_key.olx_path[:-14]  # Remove 'definition.xml', keep trailing slash
+
+
+def does_block_type_support_children(block_type):
+    """
+    Does the specified block type (e.g. "html", "vertical") support child
+    blocks?
+    """
+    try:
+        return XBlock.load_class(block_type).has_children
+    except PluginMissingError:
+        # We don't know if this now-uninstalled block type had children
+        # but to be conservative, assume it may have.
+        return True

openedx/core/djangoapps/content_libraries/library_context.py

@@ -0,0 +1,82 @@
+"""
+Definition of "Library" as a learning context.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import logging
+
+from openedx.core.djangoapps.content_libraries.library_bundle import (
+    LibraryBundle,
+    bundle_uuid_for_library_key,
+    usage_for_child_include,
+)
+from openedx.core.djangoapps.content_libraries.models import ContentLibrary
+from openedx.core.djangoapps.xblock.learning_context import LearningContext
+
+log = logging.getLogger(__name__)
+
+
+class LibraryContextImpl(LearningContext):
+    """
+    Implements content libraries as a learning context.
+
+    This is the *new* content libraries based on Blockstore, not the old content
+    libraries based on modulestore.
+    """
+
+    def __init__(self, **kwargs):
+        super(LibraryContextImpl, self).__init__(**kwargs)
+        self.use_draft = kwargs.get('use_draft', None)
+
+    def can_edit_block(self, user, usage_key):
+        """
+        Does the specified usage key exist in its context, and if so, does the
+        specified user (which may be an AnonymousUser) have permission to edit
+        it?
+
+        Must return a boolean.
+        """
+        def_key = self.definition_for_usage(usage_key)
+        if not def_key:
+            return False
+        # TODO: implement permissions
+        return True
+
+    def can_view_block(self, user, usage_key):
+        """
+        Does the specified usage key exist in its context, and if so, does the
+        specified user (which may be an AnonymousUser) have permission to view
+        it and interact with it (call handlers, save user state, etc.)?
+
+        Must return a boolean.
+        """
+        def_key = self.definition_for_usage(usage_key)
+        if not def_key:
+            return False
+        # TODO: implement permissions
+        return True
+
+    def definition_for_usage(self, usage_key):
+        """
+        Given a usage key for an XBlock in this context, return the
+        BundleDefinitionLocator which specifies the actual XBlock definition
+        (as a path to an OLX in a specific blockstore bundle).
+
+        Must return a BundleDefinitionLocator if the XBlock exists in this
+        context, or None otherwise.
+        """
+        library_key = usage_key.context_key
+        try:
+            bundle_uuid = bundle_uuid_for_library_key(library_key)
+        except ContentLibrary.DoesNotExist:
+            return None
+        bundle = LibraryBundle(library_key, bundle_uuid, self.use_draft)
+        return bundle.definition_for_usage(usage_key)
+
+    def usage_for_child_include(self, parent_usage, parent_definition, parsed_include):
+        """
+        Method that the runtime uses when loading a block's child, to get the
+        ID of the child.
+
+        The child is always from an <xblock-include /> element.
+        """
+        return usage_for_child_include(parent_usage, parent_definition, parsed_include)

openedx/core/djangoapps/content_libraries/migrations/0001_initial.py

@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.23 on 2019-08-28 20:27
+from __future__ import unicode_literals
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('organizations', '0007_historicalorganization'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ContentLibrary',
+            fields=[
+                ('id', models.AutoField(primary_key=True, serialize=False)),
+                ('slug', models.SlugField()),
+                ('bundle_uuid', models.UUIDField(unique=True)),
+                ('allow_public_learning', models.BooleanField(default=False, help_text='\n            Allow any user (even unregistered users) to view and interact with\n            content in this library (in the LMS; not in Studio). If this is not\n            enabled, then the content in this library is not directly accessible\n            in the LMS, and learners will only ever see this content if it is\n            explicitly added to a course. If in doubt, leave this unchecked.\n        ')),
+                ('allow_public_read', models.BooleanField(default=False, help_text="\n            Allow any user with Studio access to view this library's content in\n            Studio, use it in their courses, and copy content out of this\n            library. If in doubt, leave this unchecked.\n        ")),
+            ],
+            options={
+                'verbose_name_plural': 'Content Libraries',
+            },
+        ),
+        migrations.CreateModel(
+            name='ContentLibraryPermission',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('access_level', models.CharField(choices=[('admin', 'Administer users and author content'), ('author', 'Author content'), ('read', 'Read-only')], max_length=30)),
+                ('library', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='content_libraries.ContentLibrary')),
+                ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.AddField(
+            model_name='contentlibrary',
+            name='authorized_users',
+            field=models.ManyToManyField(through='content_libraries.ContentLibraryPermission', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AddField(
+            model_name='contentlibrary',
+            name='org',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='organizations.Organization'),
+        ),
+        migrations.AlterUniqueTogether(
+            name='contentlibrary',
+            unique_together=set([('org', 'slug')]),
+        ),
+    ]

openedx/core/djangoapps/content_libraries/models.py

@@ -0,0 +1,107 @@
+"""
+Models for new Content Libraries
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.contrib.auth import get_user_model
+from django.db import models
+from django.utils.translation import ugettext_lazy as _
+from organizations.models import Organization
+import six
+
+from openedx.core.djangoapps.content_libraries.keys import LibraryLocatorV2
+
+User = get_user_model()
+
+
+class ContentLibraryManager(models.Manager):
+    """
+    Custom manager for ContentLibrary class.
+    """
+    def get_by_key(self, library_key):
+        """
+        Get the ContentLibrary for the given LibraryLocatorV2 key.
+        """
+        assert isinstance(library_key, LibraryLocatorV2)
+        return self.get(org__short_name=library_key.org, slug=library_key.slug)
+
+
+@six.python_2_unicode_compatible  # pylint: disable=model-missing-unicode
+class ContentLibrary(models.Model):
+    """
+    A Content Library is a collection of content (XBlocks and/or static assets)
+
+    All actual content is stored in Blockstore, and any data that we'd want to
+    transfer to another instance if this library were exported and then
+    re-imported on another Open edX instance should be kept in Blockstore. This
+    model in the LMS should only be used to track settings specific to this Open
+    edX instance, like who has permission to edit this content library.
+    """
+    objects = ContentLibraryManager()
+
+    id = models.AutoField(primary_key=True)
+    # Every Library is uniquely and permanently identified by an 'org' and a
+    # 'slug' that are set during creation/import. Both will appear in the
+    # library's opaque key:
+    # e.g. "lib:org:slug" is the opaque key for a library.
+    org = models.ForeignKey(Organization, on_delete=models.PROTECT, null=False)
+    slug = models.SlugField()
+    bundle_uuid = models.UUIDField(unique=True, null=False)
+
+    # How is this library going to be used?
+    allow_public_learning = models.BooleanField(
+        default=False,
+        help_text=("""
+            Allow any user (even unregistered users) to view and interact with
+            content in this library (in the LMS; not in Studio). If this is not
+            enabled, then the content in this library is not directly accessible
+            in the LMS, and learners will only ever see this content if it is
+            explicitly added to a course. If in doubt, leave this unchecked.
+        """),
+    )
+    allow_public_read = models.BooleanField(
+        default=False,
+        help_text=("""
+            Allow any user with Studio access to view this library's content in
+            Studio, use it in their courses, and copy content out of this
+            library. If in doubt, leave this unchecked.
+        """),
+    )
+
+    authorized_users = models.ManyToManyField(User, through='ContentLibraryPermission')
+
+    class Meta:
+        verbose_name_plural = "Content Libraries"
+        unique_together = ("org", "slug")
+
+    @property
+    def library_key(self):
+        """
+        Get the LibraryLocatorV2 opaque key for this library
+        """
+        return LibraryLocatorV2(org=self.org.short_name, slug=self.slug)
+
+    def __str__(self):
+        return "ContentLibrary ({})".format(six.text_type(self.library_key))
+
+
+@six.python_2_unicode_compatible  # pylint: disable=model-missing-unicode
+class ContentLibraryPermission(models.Model):
+    """
+    Row recording permissions for a content library
+    """
+    library = models.ForeignKey(ContentLibrary, on_delete=models.CASCADE)
+    user = models.ForeignKey(User, on_delete=models.CASCADE)
+    # TODO: allow permissions to be assign to a group, not just a user
+    ADMIN_LEVEL = 'admin'
+    AUTHOR_LEVEL = 'author'
+    READ_LEVEL = 'read'
+    ACCESS_LEVEL_CHOICES = (
+        (ADMIN_LEVEL, _("Administer users and author content")),
+        (AUTHOR_LEVEL, _("Author content")),
+        (READ_LEVEL, _("Read-only")),
+    )
+    access_level = models.CharField(max_length=30, choices=ACCESS_LEVEL_CHOICES)
+
+    def __str__(self):
+        return "ContentLibraryPermission ({} for {})".format(self.access_level, self.user.username)

openedx/core/djangoapps/content_libraries/serializers.py

@@ -0,0 +1,75 @@
+"""
+Serializers for the content libraries REST API
+"""
+# pylint: disable=abstract-method
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from rest_framework import serializers
+
+
+class ContentLibraryMetadataSerializer(serializers.Serializer):
+    """
+    Serializer for ContentLibraryMetadata
+    """
+    # We rename the primary key field to "id" in the REST API since API clients
+    # often implement magic functionality for fields with that name, and "key"
+    # is a reserved prop name in React
+    id = serializers.CharField(source="key", read_only=True)
+    org = serializers.SlugField(source="key.org")
+    slug = serializers.SlugField(source="key.slug")
+    bundle_uuid = serializers.UUIDField(format='hex_verbose', read_only=True)
+    collection_uuid = serializers.UUIDField(format='hex_verbose', write_only=True)
+    title = serializers.CharField()
+    description = serializers.CharField(allow_blank=True)
+    version = serializers.IntegerField(read_only=True)
+    has_unpublished_changes = serializers.BooleanField(read_only=True)
+    has_unpublished_deletes = serializers.BooleanField(read_only=True)
+
+
+class ContentLibraryUpdateSerializer(serializers.Serializer):
+    """
+    Serializer for updating an existing content library
+    """
+    # These are the only fields that support changes:
+    title = serializers.CharField()
+    description = serializers.CharField()
+
+
+class LibraryXBlockMetadataSerializer(serializers.Serializer):
+    """
+    Serializer for LibraryXBlockMetadata
+    """
+    id = serializers.CharField(source="usage_key", read_only=True)
+    def_key = serializers.CharField(read_only=True)
+    block_type = serializers.CharField(source="def_key.block_type")
+    display_name = serializers.CharField(read_only=True)
+    has_unpublished_changes = serializers.BooleanField(read_only=True)
+    # When creating a new XBlock in a library, the slug becomes the ID part of
+    # the definition key and usage key:
+    slug = serializers.CharField(write_only=True)
+
+
+class LibraryXBlockTypeSerializer(serializers.Serializer):
+    """
+    Serializer for LibraryXBlockType
+    """
+    block_type = serializers.CharField()
+    display_name = serializers.CharField()
+
+
+class LibraryXBlockCreationSerializer(serializers.Serializer):
+    """
+    Serializer for adding a new XBlock to a content library
+    """
+    # Parent block: optional usage key of an existing block to add this child
+    # block to.
+    parent_block = serializers.CharField(required=False)
+    block_type = serializers.CharField()
+    definition_id = serializers.SlugField()
+
+
+class LibraryXBlockOlxSerializer(serializers.Serializer):
+    """
+    Serializer for representing an XBlock's OLX
+    """
+    olx = serializers.CharField()

openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py

@@ -0,0 +1,341 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for Blockstore-based Content Libraries
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import unittest
+from uuid import UUID
+
+from django.conf import settings
+from organizations.models import Organization
+from rest_framework.test import APITestCase
+
+from student.tests.factories import UserFactory
+from openedx.core.lib import blockstore_api
+
+# Define the URLs here - don't use reverse() because we want to detect
+# backwards-incompatible changes like changed URLs.
+URL_PREFIX = '/api/libraries/v2/'
+URL_LIB_CREATE = URL_PREFIX
+URL_LIB_DETAIL = URL_PREFIX + '{lib_key}/'  # Get data about a library, update or delete library
+URL_LIB_BLOCK_TYPES = URL_LIB_DETAIL + 'block_types/'  # Get the list of XBlock types that can be added to this library
+URL_LIB_COMMIT = URL_LIB_DETAIL + 'commit/'  # Commit (POST) or revert (DELETE) all pending changes to this library
+URL_LIB_BLOCKS = URL_LIB_DETAIL + 'blocks/'  # Get the list of XBlocks in this library, or add a new one
+URL_LIB_BLOCK = URL_PREFIX + 'blocks/{block_key}/'  # Get data about a block, or delete it
+URL_LIB_BLOCK_OLX = URL_LIB_BLOCK + 'olx/'  # Get or set the OLX of the specified XBlock
+
+URL_BLOCK_RENDER_VIEW = '/api/xblock/v2/xblocks/{block_key}/view/{view_name}/'
+URL_BLOCK_GET_HANDLER_URL = '/api/xblock/v2/xblocks/{block_key}/handler_url/{handler_name}/'
+
+
+@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server")
+class ContentLibrariesTest(APITestCase):
+    """
+    Test for Blockstore-based Content Libraries
+
+    These tests use the REST API, which in turn relies on the Python API.
+    Some tests may use the python API directly if necessary to provide
+    coverage of any code paths not accessible via the REST API.
+
+    In general, these tests should
+    (1) Use public APIs only - don't directly create data using other methods,
+        which results in a less realistic test and ties the test suite too
+        closely to specific implementation details.
+        (Exception: users can be provisioned using a user factory)
+    (2) Assert that fields are present in responses, but don't assert that the
+        entire response has some specific shape. That way, things like adding
+        new fields to an API response, which are backwards compatible, won't
+        break any tests, but backwards-incompatible API changes will.
+
+    WARNING: every test should have a unique library slug, because even though
+    the django/mysql database gets reset for each test case, the lookup between
+    library slug and bundle UUID does not because it's assumed to be immutable
+    and cached forever.
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        super(ContentLibrariesTest, cls).setUpClass()
+        cls.user = UserFactory.create(username="Bob", email="bob@example.com", password="edx")
+        # Create a collection using Blockstore API directly only because there
+        # is not yet any Studio REST API for doing so:
+        cls.collection = blockstore_api.create_collection("Content Library Test Collection")
+        # Create an organization
+        cls.organization = Organization.objects.create(
+            name="Content Libraries Tachyon Exploration & Survey Team",
+            short_name="CL-TEST",
+        )
+
+    def setUp(self):
+        super(ContentLibrariesTest, self).setUp()
+        self.client.login(username=self.user.username, password="edx")
+
+    # API helpers
+
+    def _api(self, method, url, data, expect_response):
+        """
+        Call a REST API
+        """
+        response = getattr(self.client, method)(url, data, format="json")
+        self.assertEqual(
+            response.status_code, expect_response,
+            "Unexpected response code {}:\n{}".format(response.status_code, getattr(response, 'data', '(no data)')),
+        )
+        return response.data
+
+    def _create_library(self, slug, title, description="", expect_response=200):
+        """ Create a library """
+        return self._api('post', URL_LIB_CREATE, {
+            "org": self.organization.short_name,
+            "slug": slug,
+            "title": title,
+            "description": description,
+            "collection_uuid": str(self.collection.uuid),
+        }, expect_response)
+
+    def _get_library(self, lib_key, expect_response=200):
+        """ Get a library """
+        return self._api('get', URL_LIB_DETAIL.format(lib_key=lib_key), None, expect_response)
+
+    def _update_library(self, lib_key, **data):
+        """ Update an existing library """
+        return self._api('patch', URL_LIB_DETAIL.format(lib_key=lib_key), data=data, expect_response=200)
+
+    def _delete_library(self, lib_key, expect_response=200):
+        """ Delete an existing library """
+        return self._api('delete', URL_LIB_DETAIL.format(lib_key=lib_key), None, expect_response)
+
+    def _commit_library_changes(self, lib_key):
+        """ Commit changes to an existing library """
+        return self._api('post', URL_LIB_COMMIT.format(lib_key=lib_key), None, expect_response=200)
+
+    def _revert_library_changes(self, lib_key):
+        """ Revert pending changes to an existing library """
+        return self._api('delete', URL_LIB_COMMIT.format(lib_key=lib_key), None, expect_response=200)
+
+    def _get_library_blocks(self, lib_key):
+        """ Get the list of XBlocks in the library """
+        return self._api('get', URL_LIB_BLOCKS.format(lib_key=lib_key), None, expect_response=200)
+
+    def _add_block_to_library(self, lib_key, block_type, slug, parent_block=None, expect_response=200):
+        """ Add a new XBlock to the library """
+        data = {"block_type": block_type, "definition_id": slug}
+        if parent_block:
+            data["parent_block"] = parent_block
+        return self._api('post', URL_LIB_BLOCKS.format(lib_key=lib_key), data, expect_response)
+
+    def _get_library_block(self, block_key, expect_response=200):
+        """ Get a specific block in the library """
+        return self._api('get', URL_LIB_BLOCK.format(block_key=block_key), None, expect_response)
+
+    def _delete_library_block(self, block_key, expect_response=200):
+        """ Delete a specific block from the library """
+        self._api('delete', URL_LIB_BLOCK.format(block_key=block_key), None, expect_response)
+
+    def _get_library_block_olx(self, block_key, expect_response=200):
+        """ Get the OLX of a specific block in the library """
+        result = self._api('get', URL_LIB_BLOCK_OLX.format(block_key=block_key), None, expect_response)
+        if expect_response == 200:
+            return result["olx"]
+        return result
+
+    def _set_library_block_olx(self, block_key, new_olx, expect_response=200):
+        """ Overwrite the OLX of a specific block in the library """
+        return self._api('post', URL_LIB_BLOCK_OLX.format(block_key=block_key), {"olx": new_olx}, expect_response)
+
+    def _render_block_view(self, block_key, view_name, expect_response=200):
+        """
+        Render an XBlock's view in the active application's runtime.
+        Note that this endpoint has different behavior in Studio (draft mode)
+        vs. the LMS (published version only).
+        """
+        url = URL_BLOCK_RENDER_VIEW.format(block_key=block_key, view_name=view_name)
+        return self._api('get', url, None, expect_response)
+
+    def _get_block_handler_url(self, block_key, handler_name):
+        """
+        Get the URL to call a specific XBlock's handler.
+        The URL itself encodes authentication information so can be called
+        without session authentication or any other kind of authentication.
+        """
+        url = URL_BLOCK_GET_HANDLER_URL.format(block_key=block_key, handler_name=handler_name)
+        return self._api('get', url, None, expect_response=200)["handler_url"]
+
+    # General Content Library tests
+
+    def test_library_crud(self):
+        """
+        Test Create, Read, Update, and Delete of a Content Library
+        """
+        # Create:
+        lib = self._create_library(slug="lib-crud", title="A Test Library", description="Just Testing")
+        expected_data = {
+            "id": "lib:CL-TEST:lib-crud",
+            "org": "CL-TEST",
+            "slug": "lib-crud",
+            "title": "A Test Library",
+            "description": "Just Testing",
+            "version": 0,
+            "has_unpublished_changes": False,
+            "has_unpublished_deletes": False,
+        }
+        self.assertDictContainsSubset(expected_data, lib)
+        # Check that bundle_uuid looks like a valid UUID
+        UUID(lib["bundle_uuid"])  # will raise an exception if not valid
+
+        # Read:
+        lib2 = self._get_library(lib["id"])
+        self.assertDictContainsSubset(expected_data, lib2)
+
+        # Update:
+        lib3 = self._update_library(lib["id"], title="New Title")
+        expected_data["title"] = "New Title"
+        self.assertDictContainsSubset(expected_data, lib3)
+
+        # Delete:
+        self._delete_library(lib["id"])
+        # And confirm it is deleted:
+        self._get_library(lib["id"], expect_response=404)
+        self._delete_library(lib["id"], expect_response=404)
+
+    def test_library_validation(self):
+        """
+        You can't create a library with the same slug as an existing library,
+        or an invalid slug.
+        """
+        self._create_library(slug="some-slug", title="Existing Library")
+        self._create_library(slug="some-slug", title="Duplicate Library", expect_response=400)
+
+        self._create_library(slug="Invalid Slug!", title="Library with Bad Slug", expect_response=400)
+
+    # General Content Library XBlock tests:
+
+    def test_library_blocks(self):
+        """
+        Test the happy path of creating and working with XBlocks in a content
+        library.
+        """
+        lib = self._create_library(slug="testlib1", title="A Test Library", description="Testing XBlocks")
+        lib_id = lib["id"]
+        self.assertEqual(lib["has_unpublished_changes"], False)
+
+        # A library starts out empty:
+        self.assertEqual(self._get_library_blocks(lib_id), [])
+
+        # Add a 'problem' XBlock to the library:
+        block_data = self._add_block_to_library(lib_id, "problem", "problem1")
+        self.assertDictContainsSubset({
+            "id": "lb:CL-TEST:testlib1:problem:problem1",
+            "display_name": "Blank Advanced Problem",
+            "block_type": "problem",
+            "has_unpublished_changes": True,
+        }, block_data)
+        block_id = block_data["id"]
+        # Confirm that the result contains a definition key, but don't check its value,
+        # which for the purposes of these tests is an implementation detail.
+        self.assertIn("def_key", block_data)
+
+        # now the library should contain one block and have unpublished changes:
+        self.assertEqual(self._get_library_blocks(lib_id), [block_data])
+        self.assertEqual(self._get_library(lib_id)["has_unpublished_changes"], True)
+
+        # Publish the changes:
+        self._commit_library_changes(lib_id)
+        self.assertEqual(self._get_library(lib_id)["has_unpublished_changes"], False)
+        # And now the block information should also show that block has no unpublished changes:
+        block_data["has_unpublished_changes"] = False
+        self.assertDictContainsSubset(block_data, self._get_library_block(block_id))
+        self.assertEqual(self._get_library_blocks(lib_id), [block_data])
+
+        # Now update the block's OLX:
+        orig_olx = self._get_library_block_olx(block_id)
+        self.assertIn("<problem", orig_olx)
+        new_olx = """
+        <problem display_name="New Multi Choice Question" max_attempts="5">
+            <multiplechoiceresponse>
+                <p>This is a normal capa problem. It has "maximum attempts" set to **5**.</p>
+                <label>Blockstore is designed to store.</label>
+                <choicegroup type="MultipleChoice">
+                    <choice correct="false">XBlock metadata only</choice>
+                    <choice correct="true">XBlock data/metadata and associated static asset files</choice>
+                    <choice correct="false">Static asset files for XBlocks and courseware</choice>
+                    <choice correct="false">XModule metadata only</choice>
+                </choicegroup>
+            </multiplechoiceresponse>
+        </problem>
+        """.strip()
+        self._set_library_block_olx(block_id, new_olx)
+        # now reading it back, we should get that exact OLX (no change to whitespace etc.):
+        self.assertEqual(self._get_library_block_olx(block_id), new_olx)
+        # And the display name and "unpublished changes" status of the block should be updated:
+        self.assertDictContainsSubset({
+            "display_name": "New Multi Choice Question",
+            "has_unpublished_changes": True,
+        }, self._get_library_block(block_id))
+
+        # Now view the XBlock's student_view (including draft changes):
+        fragment = self._render_block_view(block_id, "student_view")
+        self.assertIn("resources", fragment)
+        self.assertIn("Blockstore is designed to store.", fragment["content"])
+
+        # Also call a handler to make sure that's working:
+        handler_url = self._get_block_handler_url(block_id, "xmodule_handler") + "problem_get"
+        problem_get_response = self.client.get(handler_url)
+        self.assertEqual(problem_get_response.status_code, 200)
+        self.assertIn("You have used 0 of 5 attempts", problem_get_response.content)
+
+        # Now delete the block:
+        self.assertEqual(self._get_library(lib_id)["has_unpublished_deletes"], False)
+        self._delete_library_block(block_id)
+        # Confirm it's deleted:
+        self._render_block_view(block_id, "student_view", expect_response=404)
+        self._get_library_block(block_id, expect_response=404)
+        self.assertEqual(self._get_library(lib_id)["has_unpublished_deletes"], True)
+
+        # Now revert all the changes back until the last publish:
+        self._revert_library_changes(lib_id)
+        self.assertEqual(self._get_library(lib_id)["has_unpublished_deletes"], False)
+        self.assertEqual(self._get_library_block_olx(block_id), orig_olx)
+
+        # fin
+
+    def test_library_blocks_with_hierarchy(self):
+        """
+        Test library blocks with children
+        """
+        lib = self._create_library(slug="hierarchy_test_lib", title="A Test Library")
+        lib_id = lib["id"]
+
+        # Add a 'unit' XBlock to the library:
+        unit_block = self._add_block_to_library(lib_id, "unit", "unit1")
+        # Add an HTML child block:
+        child1 = self._add_block_to_library(lib_id, "html", "html1", parent_block=unit_block["id"])
+        self._set_library_block_olx(child1["id"], "<html>Hello world</html>")
+        # Add a problem child block:
+        child2 = self._add_block_to_library(lib_id, "problem", "problem1", parent_block=unit_block["id"])
+        self._set_library_block_olx(child2["id"], """
+            <problem><multiplechoiceresponse>
+                    <p>What is an even number?</p>
+                    <choicegroup type="MultipleChoice">
+                        <choice correct="false">3</choice>
+                        <choice correct="true">2</choice>
+                    </choicegroup>
+            </multiplechoiceresponse></problem>
+        """)
+
+        # Check the resulting OLX of the unit:
+        self.assertEqual(self._get_library_block_olx(unit_block["id"]), (
+            '<unit xblock-family="xblock.v1">\n'
+            '  <xblock-include definition="html/html1"/>\n'
+            '  <xblock-include definition="problem/problem1"/>\n'
+            '</unit>\n'
+        ))
+
+        # The unit can see and render its children:
+        fragment = self._render_block_view(unit_block["id"], "student_view")
+        self.assertIn("Hello world", fragment["content"])
+        self.assertIn("What is an even number?", fragment["content"])
+
+        # We cannot add a duplicate ID to the library, either at the top level or as a child:
+        self._add_block_to_library(lib_id, "problem", "problem1", expect_response=400)
+        self._add_block_to_library(lib_id, "problem", "problem1", parent_block=unit_block["id"], expect_response=400)

openedx/core/djangoapps/content_libraries/urls.py

@@ -0,0 +1,38 @@
+"""
+URL configuration for Studio's Content Libraries REST API
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.conf.urls import include, url
+
+from . import views
+
+# These URLs are only used in Studio. The LMS already provides all the
+# API endpoints needed to serve XBlocks from content libraries using the
+# standard XBlock REST API (see openedx.core.django_apps.xblock.rest_api.urls)
+urlpatterns = [
+    url(r'^api/libraries/v2/', include([
+        # list of libraries / create a library:
+        url(r'^$', views.LibraryRootView.as_view()),
+        url(r'^(?P<lib_key_str>[^/]+)/', include([
+            # get data about a library, update a library, or delete a library:
+            url(r'^$', views.LibraryDetailsView.as_view()),
+            # Get the list of XBlock types that can be added to this library
+            url(r'^block_types/$', views.LibraryBlockTypesView.as_view()),
+            # Get the list of XBlocks in this library, or add a new one:
+            url(r'^blocks/$', views.LibraryBlocksView.as_view()),
+            # Commit (POST) or revert (DELETE) all pending changes to this library:
+            url(r'^commit/$', views.LibraryCommitView.as_view()),
+        ])),
+        url(r'^blocks/(?P<usage_key_str>[^/]+)/', include([
+            # Get metadata about a specific XBlock in this library, or delete the block:
+            url(r'^$', views.LibraryBlockView.as_view()),
+            # Get the OLX source code of the specified block:
+            url(r'^olx/$', views.LibraryBlockOlxView.as_view()),
+            # TODO: Publish the draft changes made to this block:
+            # url(r'^commit/$', views.LibraryBlockCommitView.as_view()),
+            # View todo: discard draft changes
+            # Future: set a block's tags (tags are stored in a Tag bundle and linked in)
+        ])),
+    ])),
+]

openedx/core/djangoapps/content_libraries/views.py

@@ -0,0 +1,263 @@
+"""
+REST API for Blockstore-based content libraries
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+from functools import wraps
+import logging
+
+from organizations.models import Organization
+from rest_framework.exceptions import NotFound, ValidationError
+from rest_framework.views import APIView
+from rest_framework.response import Response
+#from rest_framework import authentication, permissions
+
+from openedx.core.lib.api.view_utils import view_auth_classes
+from . import api
+from .keys import LibraryLocatorV2, LibraryUsageLocatorV2
+from .serializers import (
+    ContentLibraryMetadataSerializer,
+    ContentLibraryUpdateSerializer,
+    LibraryXBlockCreationSerializer,
+    LibraryXBlockMetadataSerializer,
+    LibraryXBlockTypeSerializer,
+    LibraryXBlockOlxSerializer,
+)
+
+log = logging.getLogger(__name__)
+
+
+def convert_exceptions(fn):
+    """
+    Catch any Content Library API exceptions that occur and convert them to
+    DRF exceptions so DRF will return an appropriate HTTP response
+    """
+
+    @wraps(fn)
+    def wrapped_fn(*args, **kwargs):
+        try:
+            return fn(*args, **kwargs)
+        except api.ContentLibraryNotFound:
+            log.exception("Content library not found")
+            raise NotFound
+        except api.ContentLibraryBlockNotFound:
+            log.exception("XBlock not found in content library")
+            raise NotFound
+        except api.LibraryBlockAlreadyExists as exc:
+            log.exception(exc.message)
+            raise ValidationError(exc.message)
+    return wrapped_fn
+
+
+@view_auth_classes()
+class LibraryRootView(APIView):
+    """
+    Views to list, search for, and create content libraries.
+    """
+
+    def get(self, request):
+        """
+        Return a list of all content libraries. This is a temporary view for
+        development.
+        """
+        result = api.list_libraries()
+        return Response(ContentLibraryMetadataSerializer(result, many=True).data)
+
+    def post(self, request):
+        """
+        Create a new content library.
+        """
+        serializer = ContentLibraryMetadataSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        data = serializer.validated_data
+        # Get the organization short_name out of the "key.org" pseudo-field that the serializer added:
+        org_name = data["key"]["org"]
+        # Move "slug" out of the "key.slug" pseudo-field that the serializer added:
+        data["slug"] = data.pop("key")["slug"]
+        try:
+            org = Organization.objects.get(short_name=org_name)
+        except Organization.DoesNotExist:
+            raise ValidationError(detail={"org": "No such organization '{}' found.".format(org_name)})
+        try:
+            result = api.create_library(org=org, **data)
+        except api.LibraryAlreadyExists:
+            raise ValidationError(detail={"slug": "A library with that ID already exists."})
+        # Grant the current user admin permissions on the library:
+        api.set_library_user_permissions(result.key, request.user, api.AccessLevel.ADMIN_LEVEL)
+        return Response(ContentLibraryMetadataSerializer(result).data)
+
+
+@view_auth_classes()
+class LibraryDetailsView(APIView):
+    """
+    Views to work with a specific content library
+    """
+    @convert_exceptions
+    def get(self, request, lib_key_str):
+        """
+        Get a specific content library
+        """
+        key = LibraryLocatorV2.from_string(lib_key_str)
+        result = api.get_library(key)
+        return Response(ContentLibraryMetadataSerializer(result).data)
+
+    @convert_exceptions
+    def patch(self, request, lib_key_str):
+        """
+        Update a content library
+        """
+        key = LibraryLocatorV2.from_string(lib_key_str)
+        serializer = ContentLibraryUpdateSerializer(data=request.data, partial=True)
+        serializer.is_valid(raise_exception=True)
+        api.update_library(key, **serializer.validated_data)
+        result = api.get_library(key)
+        return Response(ContentLibraryMetadataSerializer(result).data)
+
+    @convert_exceptions
+    def delete(self, request, lib_key_str):  # pylint: disable=unused-argument
+        """
+        Delete a content library
+        """
+        key = LibraryLocatorV2.from_string(lib_key_str)
+        api.delete_library(key)
+        return Response({})
+
+
+@view_auth_classes()
+class LibraryBlockTypesView(APIView):
+    """
+    View to get the list of XBlock types that can be added to this library
+    """
+    @convert_exceptions
+    def get(self, request, lib_key_str):
+        """
+        Get the list of XBlock types that can be added to this library
+        """
+        key = LibraryLocatorV2.from_string(lib_key_str)
+        result = api.get_allowed_block_types(key)
+        return Response(LibraryXBlockTypeSerializer(result, many=True).data)
+
+
+@view_auth_classes()
+class LibraryCommitView(APIView):
+    """
+    Commit/publish or revert all of the draft changes made to the library.
+    """
+    @convert_exceptions
+    def post(self, request, lib_key_str):
+        """
+        Commit the draft changes made to the specified block and its
+        descendants.
+        """
+        key = LibraryLocatorV2.from_string(lib_key_str)
+        api.publish_changes(key)
+        return Response({})
+
+    @convert_exceptions
+    def delete(self, request, lib_key_str):  # pylint: disable=unused-argument
+        """
+        Revent the draft changes made to the specified block and its
+        descendants. Restore it to the last published version
+        """
+        key = LibraryLocatorV2.from_string(lib_key_str)
+        api.revert_changes(key)
+        return Response({})
+
+
+@view_auth_classes()
+class LibraryBlocksView(APIView):
+    """
+    Views to work with XBlocks in a specific content library.
+    """
+    @convert_exceptions
+    def get(self, request, lib_key_str):
+        """
+        Get the list of all top-level blocks in this content library
+        """
+        key = LibraryLocatorV2.from_string(lib_key_str)
+        result = api.get_library_blocks(key)
+        return Response(LibraryXBlockMetadataSerializer(result, many=True).data)
+
+    @convert_exceptions
+    def post(self, request, lib_key_str):
+        """
+        Add a new XBlock to this content library
+        """
+        library_key = LibraryLocatorV2.from_string(lib_key_str)
+        serializer = LibraryXBlockCreationSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        parent_block_usage_str = serializer.validated_data.pop("parent_block", None)
+        if parent_block_usage_str:
+            # Add this as a child of an existing block:
+            parent_block_usage = LibraryUsageLocatorV2.from_string(parent_block_usage_str)
+            if parent_block_usage.context_key != library_key:
+                raise ValidationError(detail={"parent_block": "Usage ID doesn't match library ID in the URL."})
+            result = api.create_library_block_child(parent_block_usage, **serializer.validated_data)
+        else:
+            # Create a new regular top-level block:
+            result = api.create_library_block(library_key, **serializer.validated_data)
+        return Response(LibraryXBlockMetadataSerializer(result).data)
+
+
+@view_auth_classes()
+class LibraryBlockView(APIView):
+    """
+    Views to work with an existing XBlock in a content library.
+    """
+    @convert_exceptions
+    def get(self, request, usage_key_str):
+        """
+        Get metadata about an existing XBlock in the content library
+        """
+        key = LibraryUsageLocatorV2.from_string(usage_key_str)
+        result = api.get_library_block(key)
+        return Response(LibraryXBlockMetadataSerializer(result).data)
+
+    @convert_exceptions
+    def delete(self, request, usage_key_str):  # pylint: disable=unused-argument
+        """
+        Delete a usage of a block from the library (and any children it has).
+
+        If this is the only usage of the block's definition within this library,
+        both the definition and the usage will be deleted. If this is only one
+        of several usages, the definition will be kept. Usages by linked bundles
+        are ignored and will not prevent deletion of the definition.
+
+        If the usage points to a definition in a linked bundle, the usage will
+        be deleted but the link and the linked bundle will be unaffected.
+        """
+        key = LibraryUsageLocatorV2.from_string(usage_key_str)
+        api.delete_library_block(key)
+        return Response({})
+
+
+@view_auth_classes()
+class LibraryBlockOlxView(APIView):
+    """
+    Views to work with an existing XBlock's OLX
+    """
+    @convert_exceptions
+    def get(self, request, usage_key_str):
+        """
+        Get the block's OLX
+        """
+        key = LibraryUsageLocatorV2.from_string(usage_key_str)
+        xml_str = api.get_library_block_olx(key)
+        return Response(LibraryXBlockOlxSerializer({"olx": xml_str}).data)
+
+    @convert_exceptions
+    def post(self, request, usage_key_str):
+        """
+        Replace the block's OLX.
+
+        This API is only meant for use by developers or API client applications.
+        Very little validation is done.
+        """
+        key = LibraryUsageLocatorV2.from_string(usage_key_str)
+        serializer = LibraryXBlockOlxSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        new_olx_str = serializer.validated_data["olx"]
+        try:
+            api.set_library_block_olx(key, new_olx_str)
+        except ValueError as err:
+            raise ValidationError(detail=str(err))
+        return Response(LibraryXBlockOlxSerializer({"olx": new_olx_str}).data)

openedx/core/djangoapps/xblock/README.rst

@@ -0,0 +1,94 @@
+XBlock App Suite (New)
+======================
+
+This django app and its sub-apps provide a python and REST API for the new XBlock runtime.
+
+The API can be accessed in both the LMS and Studio, though it has different behavior in each.
+
+The new runtime loads content out of **Blockstore** and is based on **Learning Contexts**, a more generic concept than courses. (The old runtime loads content out of modulestore/contentstore and is tied to courses.)
+
+This runtime cannot currently be used for courses or course content in modulestore. It is currently being developed to support content libraries and (if the "pathways" plugin is installed) pathways (short linear playlists of XBlock content).
+
+Ideally this API and the XBlock runtime it contains would run in a separate process from the LMS/Studio. However, that is currently too difficult to achieve. We should nevertheless strive to keep dependencies on other parts of edxapp to an absolute minimum, so that this can eventually be extracted to an independent app.
+
+High level design goals
+-----------------------
+
+The goals for this code are:
+
+* To exist and operate independently of the existing runtime
+   - We want to build, test, and iterate on this design and the overall integration with Blockstore without fear of breaking existing content or LMS features. To that end, this new runtime is designed to work in parallel and share minimal core code nor data with the existing runtime(s)/modulestore/etc.
+* To provide a complete runtime environment for XBlocks, with little or no support for legacy XModules
+   - Avoiding XModule support as much as possible keeps the code cleaner
+   - All XModule compatibility code that we do include must be isolated within the ``shims.py`` file for eventual removal
+   - As a firm rule, any block with code split into separate ``Module`` and ``Descriptor`` classes will not be supported.
+* To support learning experiences other than courses
+   - The new runtime is built around the concept of "learning contexts," which are more generic and flexible than courses. See `Learning Contexts`_ below.
+   - Additional learning contexts and related functionality can be added via plugins.
+   - The new runtime makes as few assumptions as possible about how XBlocks are structured (doesn't expect a rooted tree hierarchy like every course and library in the old runtime must have).
+* To avoid any frontend code
+   - This new runtime is API-first, designed to be consumed by SPA frontends or mobile apps rather than rendering HTML into templates and sending them to the browser. Since it is based on XBlocks, there are some circumstances where it returns HTML generated by the XBlock views themselves, but it returns that HTML as an API response only, wrapped in a JSON "Fragment" object. In no other circumstances does this new runtime produce HTML.
+   - This is in contrast to the old runtime in which key parts of the LMS UI were added to HTML output by the runtime and/or the XBlocks themselves (e.g. the ``sequential`` XBlock rendered next/prev navigation buttons).
+* To provide first-class support for anonymous usage of XBlocks (use by unregistered users)
+   - If the user is unregistered, field data will be saved into the user's django session rather than the database.
+* To support sandboxed execution of XBlocks
+   - The runtime API provides a mechanism for calling XBlock handlers without session authentication (using a secure token in the URL), so that XBlocks can be run in sandboxed IFrames that cannot access the user's LMS cookies or interact with the LMS other than through the XBlock JavaScript API.
+
+Code Layout and Separation of Concerns
+--------------------------------------
+
+All of this code is found in this ``openedx.core.djangoapps.xblock`` folder:
+
+* `api.py <./api.py>`_ : **Python API** for this new XBlock runtime. Provides methods for loading and interacting with XBlocks given a user and a usage ID, such as: ``load_block()``, ``get_block_metadata()``, ``render_block_view()``, and ``get_handler_url()``
+
+  - Implementation differences in how Studio and the LMS implement these APIs are encapsulated in ``lms_impl.py`` and ``studio_impl.py``
+* `rest_api <./rest_api/>`_ : **REST API** for this new XBlock runtime. Provides `endpoints <./rest_api/urls.py>`_ for things like getting an XBlock's metadata, rendering an XBlock's view, calling an XBlock's handler, etc.
+  - An example URL is https://studio.example.com/api/xblock/v1/xblocks/lb:library1:html:introduction/view/student_view/
+* `runtime <./runtime/>`_ : **Blockstore-based XBlock runtime** which implements the complete XBlock runtime API and many Open edX-specific additions and runtime services.
+
+  - ``Scope.content`` and ``Scope.settings`` field data is handled by ``BlockstoreFieldData`` in `blockstore_field_data.py <./runtime/blockstore_field_data.py>`_ which knows how to read OLX files out of Blockstore given a ``BundleDefinitionLocator`` key. Since reading OLX out of Blockstore may be "slow", reads are cached per bundle (per bundle so that the cache is easily invalidated if Blockstore notifies us that the bundle has changed in any way). In the future, the LMS implementation of this runtime will likely load field data from a more optimized store, i.e. a more persistent version of the same cache.
+  - ``Scope.user_*`` and ``Scope.preferences`` field data is stored in the user's session (in Studio) or in new database tables (for the LMS) analogous to the existing runtime's ``StudentModule`` table.
+  - Each instance of the new runtime is designed to act as the runtime for multiple XBlocks at once (as long as they are requested by the same user), unlike the previous runtime which had to be instantiated separately for each XBlock. This is intended to be a memory and performance improvement, though it does require changes to some of the Open edX-specific runtime API extensions.
+  - Runtime code and services which can be shared among all instances of the runtime are encapsulated in a new ``XBlockRuntimeSystem`` singleton, again for memory and performance reasons.
+* `learning_context <./learning_context/>`_ : **Learning Context** management code: defines the abstract base class for learning contexts, the base opaque keys that learning contexts use, and provides a plugin-based API to get the learning context given any usage key. See `Learning Contexts`_ below.
+
+Learning Contexts
+-----------------
+
+A "Learning Context" is a course, a library, a program, or some other collection of content where learning happens.
+
+To work with this runtime, all learnable XBlock content for a learning context must ultimately be stored in a Blockstore bundle, but the runtime does not impose any restrictions beyond that. The relationship between a learning context and a bundle will usually be 1:1, but some future types of learning context may not stick to that, and the runtime doesn't require it.
+
+Currently only `the "Content Library" learning context <./learning_context/content_library/>`_ is implemented. Additional learning contexts can be implemented as plugins, by subclassing the ``LearningContext`` class and registering in the ``openedx.learning_context`` entry point.
+
+A learning context is responsible for:
+
+* Definining whether or not a given XBlock [usage key] exists in the learning context (e.g. "Does the content library X contain block Y?")
+
+  - This is done via its ``definition_for_usage()`` method which must return ``None`` if the usage key doesn't exist in the context.
+  - There is **no** general method to "list all XBlocks" in the learning context, because it's expected that learning contexts may be dynamic, e.g. with content assigned just in time via adaptive learning. However, if a learning context is static it can certainly implement an API to list all the blocks it contains.
+* Determining whether or not a given user has **permission to view/use** a given XBlock and/or to **edit** that XBlock.
+
+  - This is done by via its ``can_view_block()`` and ``can_edit_block()`` methods.
+  - For example, "pathways" might allow any user to view any XBlock, but "courses" require enrollment, cohort, and due date checks as part of this permissions logic.
+
+* Mapping usage keys to ``BundleDefinitionLocator`` keys.
+
+  - The XBlock runtime and other parts of the system do not know nor prescribe how a usage locator like ``lb:library15:html:introduction`` (HTML block with usage ID "introduction" in library with slug "library15") maps to an OLX file in Blockstore like "``html/introduction/definition.xml` in bundle with UUID `11111111-1111-1111-1111-111111111111`" - that logic is left to the learning context.
+  - This is done via its ``definition_for_usage()`` method.
+
+* Definining field overrides: Learning contexts may optionally implement some mechanism for overriding field data found in the Blockstore definitions based on arbitrary criteria.
+
+  - For example, a course may specify a list of XBlock field override rules, such as:
+
+    + "All ``problem`` XBlocks in this course override the ``num_attempts`` field to have a value of ``5``" or
+    + "Users in the ``class B`` group have the ``due_date`` field of all XBlocks adjusted by ``+2 weeks``"
+
+* Implementing other useful Studio or LMS APIs: Each learning context may also be a django app plugin that implements any additional python/REST APIs it deems useful.
+
+  - For example, the Content Libraries learning context implements Studio python/REST API methods to:
+
+    + Add/remove an XBlock to/from the content library
+    + Set/get metadata of an XBlock in Studio (this refers to metadata like tags; setting XBlock fields is done via standard XBlock view/handler APIs).
+    + Publish draft changes
+    + Discard draft changes

openedx/core/djangoapps/xblock/__init__.py

@@ -0,0 +1,3 @@
+"""
+The new XBlock runtime and related code.
+"""

openedx/core/djangoapps/xblock/api.py

@@ -0,0 +1,228 @@
+"""
+Python API for interacting with edx-platform's new XBlock Runtime.
+
+For content in modulestore (currently all course content), you'll need to use
+the older runtime.
+
+Note that these views are only for interacting with existing blocks. Other
+Studio APIs cover use cases like adding/deleting/editing blocks.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import logging
+
+from django.urls import reverse
+from django.utils.translation import ugettext as _
+from rest_framework.exceptions import NotFound
+import six
+from xblock.core import XBlock
+from xblock.exceptions import NoSuchViewError
+
+from openedx.core.djangoapps.xblock.apps import get_xblock_app_config
+from openedx.core.djangoapps.xblock.learning_context.keys import BundleDefinitionLocator, BlockUsageKeyV2
+from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl
+from openedx.core.djangoapps.xblock.runtime.blockstore_runtime import BlockstoreXBlockRuntime, xml_for_definition
+from openedx.core.djangoapps.xblock.runtime.runtime import XBlockRuntimeSystem
+from openedx.core.djangolib.blockstore_cache import BundleCache
+from .utils import get_secure_token_for_xblock_handler
+
+log = logging.getLogger(__name__)
+
+
+def get_runtime_system():
+    """
+    Get the XBlockRuntimeSystem, which is a single long-lived factory that can
+    create user-specific runtimes.
+
+    The Runtime System isn't always needed (e.g. for management commands), so to
+    keep application startup faster, it's only initialized when first accessed
+    via this method.
+    """
+    # pylint: disable=protected-access
+    if not hasattr(get_runtime_system, '_system'):
+        params = dict(
+            handler_url=get_handler_url,
+            runtime_class=BlockstoreXBlockRuntime,
+        )
+        params.update(get_xblock_app_config().get_runtime_system_params())
+        get_runtime_system._system = XBlockRuntimeSystem(**params)
+    return get_runtime_system._system
+
+
+def load_block(usage_key, user):
+    """
+    Load the specified XBlock for the given user.
+
+    Returns an instantiated XBlock.
+
+    Exceptions:
+        NotFound - if the XBlock doesn't exist or if the user doesn't have the
+                   necessary permissions
+    """
+    # Is this block part of a course, a library, or what?
+    # Get the Learning Context Implementation based on the usage key
+    context_impl = get_learning_context_impl(usage_key)
+    # Now, using the LearningContext and the Studio/LMS-specific logic, check if
+    # the block exists in this context and if the user has permission to render
+    # this XBlock view:
+    if get_xblock_app_config().require_edit_permission:
+        authorized = context_impl.can_edit_block(user, usage_key)
+    else:
+        authorized = context_impl.can_view_block(user, usage_key)
+    if not authorized:
+        # We do not know if the block was not found or if the user doesn't have
+        # permission, but we want to return the same result in either case:
+        raise NotFound("XBlock {} does not exist, or you don't have permission to view it.".format(usage_key))
+
+    # TODO: load field overrides from the context
+    # e.g. a course might specify that all 'problem' XBlocks have 'max_attempts'
+    # set to 3.
+    # field_overrides = context_impl.get_field_overrides(usage_key)
+
+    runtime = get_runtime_system().get_runtime(user_id=user.id if user else None)
+
+    return runtime.get_block(usage_key)
+
+
+def get_block_metadata(block):
+    """
+    Get metadata about the specified XBlock
+    """
+    return {
+        "block_id": six.text_type(block.scope_ids.usage_id),
+        "block_type": block.scope_ids.block_type,
+        "display_name": get_block_display_name(block),
+    }
+
+
+def resolve_definition(block_or_key):
+    """
+    Given an XBlock, definition key, or usage key, return the definition key.
+    """
+    if isinstance(block_or_key, BundleDefinitionLocator):
+        return block_or_key
+    elif isinstance(block_or_key, BlockUsageKeyV2):
+        context_impl = get_learning_context_impl(block_or_key)
+        return context_impl.definition_for_usage(block_or_key)
+    elif isinstance(block_or_key, XBlock):
+        return block_or_key.scope_ids.def_id
+    else:
+        raise TypeError(block_or_key)
+
+
+def xblock_type_display_name(block_type):
+    """
+    Get the display name for the specified XBlock class.
+    """
+    block_class = XBlock.load_class(block_type)
+    if hasattr(block_class, 'display_name') and block_class.display_name.default:
+        return _(block_class.display_name.default)  # pylint: disable=translation-of-non-string
+    else:
+        return block_type  # Just use the block type as the name
+
+
+def get_block_display_name(block_or_key):
+    """
+    Efficiently get the display name of the specified block. This is done in a
+    way that avoids having to load and parse the block's entire XML field data
+    using its parse_xml() method, which may be very expensive (e.g. the video
+    XBlock parse_xml leads to various slow edxval API calls in some cases).
+
+    This method also defines and implements various fallback mechanisms in case
+    the ID can't be loaded.
+
+    block_or_key can be an XBlock instance, a usage key or a definition key.
+
+    Returns the display name as a string
+    """
+    def_key = resolve_definition(block_or_key)
+    use_draft = get_xblock_app_config().get_learning_context_params().get('use_draft')
+    cache = BundleCache(def_key.bundle_uuid, draft_name=use_draft)
+    cache_key = ('block_display_name', six.text_type(def_key))
+    display_name = cache.get(cache_key)
+    if display_name is None:
+        # Instead of loading the block, just load its XML and parse it
+        try:
+            olx_node = xml_for_definition(def_key)
+        except Exception:  # pylint: disable=broad-except
+            log.exception("Error when trying to get display_name for block definition %s", def_key)
+            # Return now so we don't cache the error result
+            return xblock_type_display_name(def_key.block_type)
+        try:
+            display_name = olx_node.attrib['display_name']
+        except KeyError:
+            display_name = xblock_type_display_name(def_key.block_type)
+        cache.set(cache_key, display_name)
+    return display_name
+
+
+def render_block_view(block, view_name, user):  # pylint: disable=unused-argument
+    """
+    Get the HTML, JS, and CSS needed to render the given XBlock view.
+
+    The difference between this method and calling
+        load_block().render(view_name)
+    is that this method will automatically save any changes to field data that
+    resulted from rendering the view. If you don't want that, call .render()
+    directly.
+
+    Returns a Fragment.
+    """
+    try:
+        fragment = block.render(view_name)
+    except NoSuchViewError:
+        fallback_view = None
+        if view_name == 'author_view':
+            fallback_view = 'student_view'
+        if fallback_view:
+            fragment = block.render(fallback_view)
+        else:
+            raise
+
+    # TODO: save any changed user state fields
+    # TODO: if the view is anything other than student_view and we're not in the LMS, save any changed
+    # content/settings/children fields.
+
+    return fragment
+
+
+def get_handler_url(usage_key, handler_name, user_id):
+    """
+    A method for getting the URL to any XBlock handler. The URL must be usable
+    without any authentication (no cookie, no OAuth/JWT), and may expire. (So
+    that we can render the XBlock in a secure IFrame without any access to
+    existing cookies.)
+
+    The returned URL will contain the provided handler_name, but is valid for
+    any other handler on the same XBlock. Callers may replace any occurrences of
+    the handler name in the resulting URL with the name of any other handler and
+    the URL will still work. (This greatly reduces the number of calls to this
+    API endpoint that are needed to interact with any given XBlock.)
+
+    Params:
+        usage_key       - Usage Key (Opaque Key object or string)
+        handler_name    - Name of the handler or a dummy name like 'any_handler'
+        user_id         - User ID or XBlockRuntimeSystem.ANONYMOUS_USER
+
+    This view does not check/care if the XBlock actually exists.
+    """
+    usage_key_str = six.text_type(usage_key)
+    site_root_url = get_xblock_app_config().get_site_root_url()
+    if user_id is None:
+        raise TypeError("Cannot get handler URLs without specifying a specific user ID.")
+    elif user_id == XBlockRuntimeSystem.ANONYMOUS_USER:
+        raise NotImplementedError("handler links for anonymous users are not yet implemented")  # TODO: implement
+    else:
+        # Normal case: generate a token-secured URL for this handler, specific
+        # to this user and this XBlock.
+        secure_token = get_secure_token_for_xblock_handler(user_id, usage_key_str)
+        path = reverse('xblock_api:xblock_handler', kwargs={
+            'usage_key_str': usage_key_str,
+            'user_id': user_id,
+            'secure_token': secure_token,
+            'handler_name': handler_name,
+        })
+    # We must return an absolute URL. We can't just use
+    # rest_framework.reverse.reverse to get the absolute URL because this method
+    # can be called by the XBlock from python as well and in that case we don't
+    # have access to the request.
+    return site_root_url + path

openedx/core/djangoapps/xblock/apps.py

@@ -0,0 +1,119 @@
+"""
+Django app configuration for the XBlock Runtime django app
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.apps import AppConfig, apps
+from django.conf import settings
+from xblock.runtime import DictKeyValueStore, KvsFieldData
+
+from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers
+from openedx.core.djangoapps.xblock.runtime.blockstore_field_data import BlockstoreFieldData
+
+
+class XBlockAppConfig(AppConfig):
+    """
+    Django app configuration for the new XBlock Runtime django app
+    """
+    name = 'openedx.core.djangoapps.xblock'
+    verbose_name = 'New XBlock Runtime'
+    label = 'xblock_new'  # The name 'xblock' is already taken by ORA2's 'openassessment.xblock' app :/
+
+    # If this is True, users must have 'edit' permission to be allowed even to
+    # view content. (It's only true in Studio)
+    require_edit_permission = False
+
+    def get_runtime_system_params(self):
+        """
+        Get the XBlockRuntimeSystem parameters appropriate for viewing and/or
+        editing XBlock content.
+        """
+        raise NotImplementedError
+
+    def get_site_root_url(self):
+        """
+        Get the absolute root URL to this site, e.g. 'https://courses.example.com'
+        Should not have any trailing slash.
+        """
+        raise NotImplementedError
+
+    def get_learning_context_params(self):
+        """
+        Get additional kwargs that are passed to learning context implementations
+        (LearningContext subclass constructors). For example, this can be used to
+        specify that the course learning context should load the course's list of
+        blocks from the _draft_ version of the course in studio, but from the
+        published version of the course in the LMS.
+        """
+        return {}
+
+
+class LmsXBlockAppConfig(XBlockAppConfig):
+    """
+    LMS-specific configuration of the XBlock Runtime django app.
+    """
+
+    def get_runtime_system_params(self):
+        """
+        Get the XBlockRuntimeSystem parameters appropriate for viewing and/or
+        editing XBlock content in the LMS
+        """
+        return dict(
+            authored_data_store=BlockstoreFieldData(),
+            student_data_store=KvsFieldData(kvs=DictKeyValueStore()),
+        )
+
+    def get_site_root_url(self):
+        """
+        Get the absolute root URL to this site, e.g. 'https://courses.example.com'
+        Should not have any trailing slash.
+        """
+        return configuration_helpers.get_value('LMS_ROOT_URL', settings.LMS_ROOT_URL)
+
+
+class StudioXBlockAppConfig(XBlockAppConfig):
+    """
+    Studio-specific configuration of the XBlock Runtime django app.
+    """
+    # In Studio, users must have 'edit' permission to be allowed even to view content
+    require_edit_permission = True
+
+    BLOCKSTORE_DRAFT_NAME = "studio_draft"
+
+    def get_runtime_system_params(self):
+        """
+        Get the XBlockRuntimeSystem parameters appropriate for viewing and/or
+        editing XBlock content in Studio
+        """
+        return dict(
+            authored_data_store=BlockstoreFieldData(),
+            student_data_store=KvsFieldData(kvs=DictKeyValueStore()),
+        )
+
+    def get_site_root_url(self):
+        """
+        Get the absolute root URL to this site, e.g. 'https://studio.example.com'
+        Should not have any trailing slash.
+        """
+        scheme = "https" if settings.HTTPS == "on" else "http"
+        return scheme + '://' + settings.CMS_BASE
+        # or for the LMS version: configuration_helpers.get_value('LMS_ROOT_URL', settings.LMS_ROOT_URL)
+
+    def get_learning_context_params(self):
+        """
+        Get additional kwargs that are passed to learning context implementations
+        (LearningContext subclass constructors). For example, this can be used to
+        specify that the course learning context should load the course's list of
+        blocks from the _draft_ version of the course in studio, but from the
+        published version of the course in the LMS.
+        """
+        return {
+            "use_draft": self.BLOCKSTORE_DRAFT_NAME,
+        }
+
+
+def get_xblock_app_config():
+    """
+    Get whichever of the above AppConfig subclasses is active.
+    """
+    return apps.get_app_config(XBlockAppConfig.label)

openedx/core/djangoapps/xblock/learning_context/__init__.py

@@ -0,0 +1,5 @@
+"""
+A "Learning Context" is a course, a library, a program, or some other collection
+of content where learning happens.
+"""
+from .learning_context import LearningContext

openedx/core/djangoapps/xblock/learning_context/keys.py

@@ -0,0 +1,226 @@
+"""
+New key/locator types that work with Blockstore and the new "Learning Context"
+concept.
+
+We will probably move these key types into the "opaque-keys" repository once
+they are stable.
+"""
+# Disable warnings about _to_deprecated_string etc. which we don't want to implement.
+# And fix warnings about key fields, which pylint doesn't see as member variables.
+# pylint: disable=abstract-method, no-member
+from __future__ import absolute_import, division, print_function, unicode_literals
+import re
+from uuid import UUID
+import warnings
+
+from opaque_keys import InvalidKeyError, OpaqueKey
+from opaque_keys.edx.keys import DefinitionKey, UsageKey
+import six
+
+
+def check_key_string_field(value, regexp=r'^[\w\-.]+$'):
+    """
+    Helper method to verify that a key's string field(s) meet certain
+    requirements:
+        Are a non-empty string
+        Match the specified regular expression
+    """
+    if not isinstance(value, six.string_types):
+        raise TypeError("Expected a string")
+    if not value or not re.match(regexp, value):
+        raise ValueError("'{}' is not a valid field value for this key type.".format(value))
+
+
+def check_draft_name(value):
+    """
+    Check that the draft name is valid (unambiguously not a bundle version
+    nubmer).
+
+    Valid: studio_draft, foo-bar, import348975938
+    Invalid: 1, 15, 873452847357834
+    """
+    if not isinstance(value, six.string_types) or not value:
+        raise TypeError("Expected a non-empty string")
+    if value.isdigit():
+        raise ValueError("Cannot use an integer draft name as it conflicts with bundle version nubmers")
+
+
+class BundleDefinitionLocator(DefinitionKey):
+    """
+    Implementation of the DefinitionKey type, for XBlock content stored in
+    Blockstore bundles. This is a low-level identifier used within the Open edX
+    system for identifying and retrieving OLX.
+
+    A "Definition" is a specific OLX file in a specific BundleVersion
+    (or sometimes rather than a BundleVersion, it may point to a named draft.)
+    The OLX file, and thus the definition key, defines Scope.content fields as
+    well as defaults for Scope.settings and Scope.children fields. However the
+    definition has no parent and no position in any particular course or other
+    context - both of which require a *usage key* and not just a definition key.
+    The same block definition (.olx file) can be used in multiple places in a
+    course, each with a different usage key.
+
+    Example serialized definition keys follow.
+
+    The 'html' type OLX file "html/introduction/definition.xml" in bundle
+    11111111-1111-1111-1111-111111111111, bundle version 5:
+
+        bundle-olx:11111111-1111-1111-1111-111111111111:5:html:html/introduction/definition.xml
+
+    The 'problem' type OLX file "problem324234.xml" in bundle
+    22222222-2222-2222-2222-222222222222, draft 'studio-draft':
+
+        bundle-olx:22222222-2222-2222-2222-222222222222:studio-draft:problem:problem/324234.xml
+
+    (The serialized version is somewhat long and verbose because it should
+    rarely be used except for debugging - the in-memory python key instance will
+    be used most of the time, and users will rarely/never see definition keys.)
+
+    User state should never be stored using a BundleDefinitionLocator as the
+    key. State should always be stored against a usage locator, which refers to
+    a particular definition being used in a particular context.
+
+    Each BundleDefinitionLocator holds the following data
+        1. Bundle UUID and [bundle version OR draft name]
+        2. Block type (e.g. 'html', 'problem', etc.)
+        3. Path to OLX file
+
+    Note that since the data in an .olx file can only ever change in a bundle
+    draft (not in a specific bundle version), an XBlock that is actively making
+    changes to its Scope.content/Scope.settings field values must have a
+    BundleDefinitionLocator with a draft name (not a bundle version).
+    """
+    CANONICAL_NAMESPACE = 'bundle-olx'
+    KEY_FIELDS = ('bundle_uuid', 'block_type', 'olx_path', '_version_or_draft')
+    __slots__ = KEY_FIELDS
+    CHECKED_INIT = False
+
+    def __init__(self, bundle_uuid, block_type, olx_path, bundle_version=None, draft_name=None, _version_or_draft=None):
+        """
+        Instantiate a new BundleDefinitionLocator
+        """
+        if not isinstance(bundle_uuid, UUID):
+            bundle_uuid = UUID(bundle_uuid)
+        check_key_string_field(block_type)
+        check_key_string_field(olx_path, regexp=r'^[\w\-./]+$')
+        # For now the following is a convention; we could remove this restriction in the future given new use cases.
+        assert block_type + '/' in olx_path, 'path should contain block type, e.g. html/id/definition.xml for html'
+
+        if (bundle_version is not None) + (draft_name is not None) + (_version_or_draft is not None) != 1:
+            raise ValueError("Exactly one of [bundle_version, draft_name, _version_or_draft] must be specified")
+        if _version_or_draft is not None:
+            if isinstance(_version_or_draft, int):
+                pass  # This is a bundle version number.
+            else:
+                # This is a draft name, not a bundle version:
+                check_draft_name(_version_or_draft)
+        elif draft_name is not None:
+            check_draft_name(draft_name)
+            _version_or_draft = draft_name
+        else:
+            assert isinstance(bundle_version, int)
+            _version_or_draft = bundle_version
+
+        super(BundleDefinitionLocator, self).__init__(
+            bundle_uuid=bundle_uuid,
+            block_type=block_type,
+            olx_path=olx_path,
+            _version_or_draft=_version_or_draft,
+        )
+
+    @property
+    def bundle_version(self):
+        return self._version_or_draft if isinstance(self._version_or_draft, int) else None
+
+    @property
+    def draft_name(self):
+        return self._version_or_draft if not isinstance(self._version_or_draft, int) else None
+
+    def _to_string(self):
+        """
+        Return a string representing this BundleDefinitionLocator
+        """
+        return ":".join((
+            six.text_type(self.bundle_uuid), six.text_type(self._version_or_draft), self.block_type, self.olx_path,
+        ))
+
+    @classmethod
+    def _from_string(cls, serialized):
+        """
+        Return a BundleDefinitionLocator by parsing the given serialized string
+        """
+        try:
+            (bundle_uuid_str, _version_or_draft, block_type, olx_path) = serialized.split(':', 3)
+        except ValueError:
+            raise InvalidKeyError(cls, serialized)
+
+        bundle_uuid = UUID(bundle_uuid_str)
+        if not block_type or not olx_path:
+            raise InvalidKeyError(cls, serialized)
+
+        if _version_or_draft.isdigit():
+            _version_or_draft = int(_version_or_draft)
+
+        return cls(
+            bundle_uuid=bundle_uuid,
+            block_type=block_type,
+            olx_path=olx_path,
+            _version_or_draft=_version_or_draft,
+        )
+
+
+class LearningContextKey(OpaqueKey):
+    """
+    A key that idenitifies a course, a library, a program,
+    or some other collection of content where learning happens.
+    """
+    KEY_TYPE = 'context_key'
+    __slots__ = ()
+
+    def make_definition_usage(self, definition_key, usage_id=None):
+        """
+        Return a usage key, given the given the specified definition key and
+        usage_id.
+        """
+        raise NotImplementedError()
+
+
+class BlockUsageKeyV2(UsageKey):
+    """
+    Abstract base class that encodes an XBlock used in a specific learning
+    context (e.g. a course).
+
+    Definition + Learning Context = Usage
+    """
+    @property
+    def context_key(self):
+        raise NotImplementedError()
+
+    @property
+    def definition_key(self):
+        """
+        Returns the definition key for this usage.
+        """
+        # Because this key definition is likely going to be moved into the
+        # opaque-keys package, we cannot put the logic here for getting the
+        # definition.
+        raise NotImplementedError(
+            "To get the definition key, use: "
+            "get_learning_context_impl(usage_key).definition_for_usage(usage_key)"
+        )
+
+    @property
+    def course_key(self):
+        warnings.warn("Use .context_key instead of .course_key", DeprecationWarning, stacklevel=2)
+        return self.context_key
+
+    def html_id(self):
+        """
+        Return an id which can be used on an html page as an id attr of an html
+        element. This is only in here for backwards-compatibility with XModules;
+        don't use in new code.
+        """
+        warnings.warn(".html_id is deprecated", DeprecationWarning, stacklevel=2)
+        # HTML5 allows ID values to contain any characters at all other than spaces.
+        # These key types don't allow spaces either, so no transform is needed.
+        return six.text_type(self)

openedx/core/djangoapps/xblock/learning_context/learning_context.py

@@ -0,0 +1,98 @@
+"""
+A "Learning Context" is a course, a library, a program, or some other collection
+of content where learning happens.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+
+class LearningContext(object):
+    """
+    Abstract base class for learning context implementations.
+
+    A "Learning Context" is a course, a library, a program,
+    or some other collection of content where learning happens.
+
+    Additional learning contexts can be implemented as plugins, by subclassing
+    this class and registering in the 'openedx.learning_context' entry point.
+    """
+
+    def __init__(self, **kwargs):
+        """
+        Initialize this learning context.
+
+        Subclasses should pass **kwargs to this constructor to allow for future
+        parameters without changing the API.
+        """
+
+    def can_edit_block(self, user, usage_key):  # pylint: disable=unused-argument
+        """
+        Does the specified usage key exist in its context, and if so, does the
+        specified user have permission to edit it?
+
+        user: a Django User object (may be an AnonymousUser)
+
+        usage_key: the BlockUsageKeyV2 subclass used for this learning context
+
+        Must return a boolean.
+        """
+        return False
+
+    def can_view_block(self, user, usage_key):  # pylint: disable=unused-argument
+        """
+        Does the specified usage key exist in its context, and if so, does the
+        specified user have permission to view it and interact with it (call
+        handlers, save user state, etc.)?
+
+        user: a Django User object (may be an AnonymousUser)
+
+        usage_key: the BlockUsageKeyV2 subclass used for this learning context
+
+        Must return a boolean.
+        """
+        return False
+
+    def definition_for_usage(self, usage_key):
+        """
+        Given a usage key for an XBlock in this context, return the
+        BundleDefinitionLocator which specifies the actual XBlock definition
+        (as a path to an OLX in a specific blockstore bundle).
+
+        usage_key: the BlockUsageKeyV2 subclass used for this learning context
+
+        Must return a BundleDefinitionLocator if the XBlock exists in this
+        context, or None otherwise.
+        """
+        raise NotImplementedError
+
+    def usage_for_child_include(self, parent_usage, parent_definition, parsed_include):
+        """
+        Method that the runtime uses when loading a block's child, to get the
+        ID of the child. Must return a usage key.
+
+        The child is always from an <xblock-include /> element.
+
+        parent_usage: the BlockUsageKeyV2 subclass key of the parent
+
+        parent_definition: the BundleDefinitionLocator key of the parent (same
+            as returned by definition_for_usage(parent_usage) but included here
+            as an optimization since it's already known.)
+
+        parsed_include: the XBlockInclude tuple containing the data from the
+            parsed <xblock-include /> element. See xblock.runtime.olx_parsing.
+
+        Must return a BlockUsageKeyV2 subclass
+        """
+        raise NotImplementedError
+
+    # Future functionality:
+    # def get_field_overrides(self, user, usage_key):
+    #     """
+    #     Each learning context may have a way for authors to specify field
+    #     overrides that apply to XBlocks in the context.
+
+    #     For example, courses might allow an instructor to specify that all
+    #     'problem' blocks in her course have 'num_attempts' set to '5',
+    #     regardless of the 'num_attempts' value in the underlying problem XBlock
+    #     definitions.
+    #     """
+    #     raise NotImplementedError

openedx/core/djangoapps/xblock/learning_context/manager.py

@@ -0,0 +1,50 @@
+"""
+Helper methods for working with learning contexts
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from openedx.core.djangoapps.xblock.apps import get_xblock_app_config
+from openedx.core.lib.plugins import PluginManager
+from .keys import LearningContextKey, BlockUsageKeyV2
+
+
+class LearningContextPluginManager(PluginManager):
+    """
+    Plugin manager that uses stevedore extension points (entry points) to allow
+    learning contexts to register as plugins.
+
+    The key of the learning context must match the CANONICAL_NAMESPACE of its
+    LearningContextKey
+    """
+    NAMESPACE = 'openedx.learning_context'
+
+
+_learning_context_cache = {}
+
+
+def get_learning_context_impl(key):
+    """
+    Given an opaque key, get the implementation of its learning context.
+
+    Returns a subclass of LearningContext
+
+    Raises TypeError if the specified key isn't a type that has a learning
+    context.
+    Raises PluginError if there is some misconfiguration causing the context
+    implementation to not be installed.
+    """
+    if isinstance(key, LearningContextKey):
+        context_type = key.CANONICAL_NAMESPACE  # e.g. 'lib'
+    elif isinstance(key, BlockUsageKeyV2):
+        context_type = key.context_key.CANONICAL_NAMESPACE
+    else:
+        # Maybe this is an older modulestore key etc.
+        raise TypeError("Opaque key {} does not have a learning context.".format(key))
+
+    try:
+        return _learning_context_cache[context_type]
+    except KeyError:
+        # Load this learning context type.
+        params = get_xblock_app_config().get_learning_context_params()
+        _learning_context_cache[context_type] = LearningContextPluginManager.get_plugin(context_type)(**params)
+        return _learning_context_cache[context_type]

openedx/core/djangoapps/xblock/rest_api/urls.py

@@ -0,0 +1,29 @@
+"""
+URL configuration for the new XBlock API
+"""
+from django.conf.urls import include, url
+
+from . import views
+
+# Note that the exact same API URLs are used in Studio and the LMS, but the API
+# may act a bit differently in each (e.g. Studio stores user state ephemerally).
+# If necessary at some point in the future, these URLs could be duplicated into
+# urls_studio and urls_lms, and/or the views could be likewise duplicated.
+urlpatterns = [
+    url(r'^api/xblock/v2/', include([
+        url(r'^xblocks/(?P<usage_key_str>[^/]+)/', include([
+            # get metadata about an XBlock:
+            url(r'^$', views.block_metadata),
+            # render one of this XBlock's views (e.g. student_view)
+            url(r'^view/(?P<view_name>[\w\-]+)/$', views.render_block_view),
+            # get the URL needed to call this XBlock's handlers
+            url(r'^handler_url/(?P<handler_name>[\w\-]+)/$', views.get_handler_url),
+            # call one of this block's handlers
+            url(
+                r'^handler/(?P<user_id>\d+)-(?P<secure_token>\w+)/(?P<handler_name>[\w\-]+)/(?P<suffix>.+)?$',
+                views.xblock_handler,
+                name='xblock_handler',
+            ),
+        ])),
+    ])),
+]

openedx/core/djangoapps/xblock/rest_api/views.py

@@ -0,0 +1,102 @@
+"""
+Views that implement a RESTful API for interacting with XBlocks.
+
+Note that these views are only for interacting with existing blocks. Other
+Studio APIs cover use cases like adding/deleting/editing blocks.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from django.contrib.auth import get_user_model
+from rest_framework import permissions
+from rest_framework.decorators import api_view, permission_classes, authentication_classes
+from rest_framework.exceptions import PermissionDenied, AuthenticationFailed
+from rest_framework.response import Response
+from xblock.django.request import DjangoWebobRequest, webob_to_django_response
+
+from opaque_keys.edx.keys import UsageKey
+from openedx.core.lib.api.view_utils import view_auth_classes
+from ..api import (
+    get_block_metadata,
+    get_handler_url as _get_handler_url,
+    load_block,
+    render_block_view as _render_block_view,
+)
+from ..utils import validate_secure_token_for_xblock_handler
+
+User = get_user_model()
+
+
+@api_view(['GET'])
+@permission_classes((permissions.AllowAny, ))  # Permissions are handled at a lower level, by the learning context
+def block_metadata(request, usage_key_str):
+    """
+    Get metadata about the specified block.
+    """
+    usage_key = UsageKey.from_string(usage_key_str)
+    block = load_block(usage_key, request.user)
+    metadata_dict = get_block_metadata(block)
+    return Response(metadata_dict)
+
+
+@api_view(['GET'])
+@permission_classes((permissions.AllowAny, ))  # Permissions are handled at a lower level, by the learning context
+def render_block_view(request, usage_key_str, view_name):
+    """
+    Get the HTML, JS, and CSS needed to render the given XBlock.
+    """
+    usage_key = UsageKey.from_string(usage_key_str)
+    block = load_block(usage_key, request.user)
+    fragment = _render_block_view(block, view_name, request.user)
+    response_data = get_block_metadata(block)
+    response_data.update(fragment.to_dict())
+    return Response(response_data)
+
+
+@api_view(['GET'])
+@view_auth_classes(is_authenticated=True)
+def get_handler_url(request, usage_key_str, handler_name):
+    """
+    Get an absolute URL which can be used (without any authentication) to call
+    the given XBlock handler.
+
+    The URL will expire but is guaranteed to be valid for a minimum of 2 days.
+    """
+    usage_key = UsageKey.from_string(usage_key_str)
+    handler_url = _get_handler_url(usage_key, handler_name, request.user.id)
+    return Response({"handler_url": handler_url})
+
+
+@api_view(['GET', 'POST', 'PUT', 'PATCH', 'DELETE'])
+@authentication_classes([])  # Disable session authentication; we don't need it and don't want CSRF checks
+@permission_classes((permissions.AllowAny, ))
+def xblock_handler(request, user_id, secure_token, usage_key_str, handler_name, suffix):
+    """
+    Run an XBlock's handler and return the result
+    """
+    user_id = int(user_id)  # User ID comes from the URL, not session auth
+    usage_key = UsageKey.from_string(usage_key_str)
+
+    # To support sandboxed XBlocks, custom frontends, and other use cases, we
+    # authenticate requests using a secure token in the URL. see
+    # openedx.core.djangoapps.xblock.utils.get_secure_hash_for_xblock_handler
+    # for details and rationale.
+    if not validate_secure_token_for_xblock_handler(user_id, usage_key_str, secure_token):
+        raise PermissionDenied("Invalid/expired auth token.")
+    if request.user.is_authenticated:
+        # The user authenticated twice, e.g. with session auth and the token
+        # So just make sure the session auth matches the token
+        if request.user.id != user_id:
+            raise AuthenticationFailed("Authentication conflict.")
+        user = request.user
+    else:
+        user = User.objects.get(pk=user_id)
+
+    request_webob = DjangoWebobRequest(request)  # Convert from django request to the webob format that XBlocks expect
+    block = load_block(usage_key, user)
+    # Run the handler, and save any resulting XBlock field value changes:
+    response_webob = block.handle(handler_name, request_webob, suffix)
+    response = webob_to_django_response(response_webob)
+    # We need to set Access-Control-Allow-Origin: * to allow sandboxed XBlocks
+    # to call these handlers:
+    response['Access-Control-Allow-Origin'] = '*'
+    return response

openedx/core/djangoapps/xblock/runtime/blockstore_field_data.py

@@ -0,0 +1,249 @@
+"""
+Key-value store that holds XBlock field data read out of Blockstore
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+from collections import namedtuple
+from weakref import WeakKeyDictionary
+import logging
+
+from xblock.exceptions import InvalidScopeError, NoSuchDefinition
+from xblock.fields import Field, BlockScope, Scope, UserScope, Sentinel
+from xblock.field_data import FieldData
+
+from openedx.core.djangolib.blockstore_cache import (
+    get_bundle_version_files_cached,
+    get_bundle_draft_files_cached,
+)
+
+log = logging.getLogger(__name__)
+
+ActiveBlock = namedtuple('ActiveBlock', ['olx_hash', 'changed_fields'])
+
+DELETED = Sentinel('DELETED')  # Special value indicating a field was reset to its default value
+
+MAX_DEFINITIONS_LOADED = 100  # How many of the most recently used XBlocks' field data to keep in memory at max.
+
+
+class BlockInstanceUniqueKey(object):
+    """
+    An empty object used as a unique key for each XBlock instance, see
+    BlockstoreFieldData._get_active_block(). Every XBlock instance will get a
+    unique one of these keys, even if they are otherwise identical. Its purpose
+    is similar to `id(block)`.
+    """
+
+
+def get_olx_hash_for_definition_key(def_key):
+    """
+    Given a BundleDefinitionLocator, which identifies a specific version of an
+    OLX file, return the hash of the OLX file as given by the Blockstore API.
+    """
+    if def_key.bundle_version:
+        # This is referring to an immutable file (BundleVersions are immutable so this can be aggressively cached)
+        files_list = get_bundle_version_files_cached(def_key.bundle_uuid, def_key.bundle_version)
+    else:
+        # This is referring to a draft OLX file which may be recently updated:
+        files_list = get_bundle_draft_files_cached(def_key.bundle_uuid, def_key.draft_name)
+    for entry in files_list:
+        if entry.path == def_key.olx_path:
+            return entry.hash_digest
+    raise NoSuchDefinition("Could not load OLX file for key {}".format(def_key))
+
+
+class BlockstoreFieldData(FieldData):
+    """
+    An XBlock FieldData implementation that reads XBlock field data directly out
+    of Blockstore.
+
+    It requires that every XBlock have a BundleDefinitionLocator as its
+    "definition key", since the BundleDefinitionLocator is what specifies the
+    OLX file path and version to use.
+
+    Within Blockstore there is no mechanism for setting different field values
+    at the usage level compared to the definition level, so we treat
+    usage-scoped fields identically to definition-scoped fields.
+    """
+    def __init__(self):
+        """
+        Initialize this BlockstoreFieldData instance.
+        """
+        # loaded definitions: a dict where the key is the hash of the XBlock's
+        # olx file (as stated by the Blockstore API), and the values is the
+        # dict of field data as loaded from that OLX file. The field data dicts
+        # in this should be considered immutable, and never modified.
+        self.loaded_definitions = {}
+        # Active blocks: this holds the field data *changes* for all the XBlocks
+        # that are currently in memory being used for something. We only keep a
+        # weak reference so that the memory will be freed when the XBlock is no
+        # longer needed (e.g. at the end of a request)
+        # The key of this dictionary is on ID object owned by the XBlock itself
+        # (see _get_active_block()) and the value is an ActiveBlock object
+        # (which holds olx_hash and changed_fields)
+        self.active_blocks = WeakKeyDictionary()
+        super(BlockstoreFieldData, self).__init__()
+
+    def _getfield(self, block, name):
+        """
+        Return the field with the given `name` from `block`.
+        If the XBlock doesn't have such a field, raises a KeyError.
+        """
+        # First, get the field from the class, if defined
+        block_field = getattr(block.__class__, name, None)
+        if block_field is not None and isinstance(block_field, Field):
+            return block_field
+        # Not in the class, so name really doesn't name a field
+        raise KeyError(name)
+
+    def _check_field(self, block, name):
+        """
+        Given a block and the name of one of its fields, check that we will be
+        able to read/write it.
+        """
+        field = self._getfield(block, name)
+        if field.scope == Scope.children:
+            if name != 'children':
+                raise InvalidScopeError("Expect Scope.children only for field named 'children', not '{}'".format(name))
+        elif field.scope == Scope.parent:
+            # This field data store is focused on definition-level field data, and parent is mostly
+            # relevant at the usage level. Luckily this doesn't even seem to be used?
+            raise NotImplementedError("Setting Scope.parent is not supported by BlockstoreFieldData.")
+        else:
+            if field.scope.user != UserScope.NONE:
+                raise InvalidScopeError("BlockstoreFieldData only supports UserScope.NONE fields")
+            if field.scope.block not in (BlockScope.DEFINITION, BlockScope.USAGE):
+                raise InvalidScopeError(
+                    "BlockstoreFieldData does not support BlockScope.{} fields".format(field.scope.block)
+                )
+            # There is also BlockScope.TYPE but we don't need to support that;
+            # it's mostly relevant as Scope.preferences(UserScope.ONE, BlockScope.TYPE)
+            # Which would be handled by a user-aware FieldData implementation
+
+    def _get_active_block(self, block):
+        """
+        Get the ActiveBlock entry for the specified block, creating it if
+        necessary.
+        """
+        # We would like to make the XBlock instance 'block' itself the key of
+        # self.active_blocks, so that we have exactly one entry per XBlock
+        # instance in memory, and they'll each be automatically freed by the
+        # WeakKeyDictionary as needed. But because XModules implement
+        # __eq__() in a way that reads all field values, just attempting to use
+        # the block as a dict key here will trigger infinite recursion. So
+        # instead we key the dict on an arbitrary object,
+        # block key = BlockInstanceUniqueKey() which we create here. That way
+        # the weak reference will still cause the entry in self.active_blocks to
+        # be freed automatically when the block is no longer needed, and we
+        # still get one entry per XBlock instance.
+        if not hasattr(block, '_field_data_key_obj'):
+            block._field_data_key_obj = BlockInstanceUniqueKey()  # pylint: disable=protected-access
+        key = block._field_data_key_obj  # pylint: disable=protected-access
+        if key not in self.active_blocks:
+            self.active_blocks[key] = ActiveBlock(
+                olx_hash=get_olx_hash_for_definition_key(block.scope_ids.def_id),
+                changed_fields={},
+            )
+        return self.active_blocks[key]
+
+    def get(self, block, name):
+        """
+        Get the given field value from Blockstore
+
+        If the XBlock has been making changes to its fields, the value will be
+        in self._get_active_block(block).changed_fields[name]
+
+        Otherwise, the value comes from self.loaded_definitions which is a dict
+        of OLX file field data, keyed by the hash of the OLX file.
+        """
+        self._check_field(block, name)
+        entry = self._get_active_block(block)
+        if name in entry.changed_fields:
+            value = entry.changed_fields[name]
+            if value == DELETED:
+                raise KeyError  # KeyError means use the default value, since this field was deliberately set to default
+            return value
+        try:
+            saved_fields = self.loaded_definitions[entry.olx_hash]
+        except KeyError:
+            if name == 'children':
+                # Special case: parse_xml calls add_node_as_child which calls 'block.children.append()'
+                # BEFORE parse_xml is done, and .append() needs to read the value of children. So
+                return []  # start with an empty list, it will get filled in.
+            # Otherwise, this is an anomalous get() before the XML was fully loaded:
+            # This could happen if an XBlock's parse_xml() method tried to read a field before setting it,
+            # if an XBlock read field data in its constructor (forbidden), or if an XBlock was loaded via
+            # some means other than runtime.get_block()
+            log.exception(
+                "XBlock %s tried to read from field data (%s) that wasn't loaded from Blockstore!",
+                block.scope_ids.usage_id, name,
+            )
+            raise  # Just use the default value for now; any exception raised here is caught anyways
+        return saved_fields[name]
+        # If 'name' is not found, this will raise KeyError, which means to use the default value
+
+    def set(self, block, name, value):
+        """
+        Set the value of the field named `name`
+        """
+        entry = self._get_active_block(block)
+        entry.changed_fields[name] = value
+
+    def delete(self, block, name):
+        """
+        Reset the value of the field named `name` to the default
+        """
+        self.set(block, name, DELETED)
+
+    def default(self, block, name):
+        """
+        Get the default value for block's field 'name'.
+        The XBlock class will provide the default if KeyError is raised; this is
+        mostly for the purpose of context-specific overrides.
+        """
+        raise KeyError(name)
+
+    def cache_fields(self, block):
+        """
+        Cache field data:
+        This is called by the runtime after a block has parsed its OLX via its
+        parse_xml() methods and written all of its field values into this field
+        data store. The values will be stored in
+            self._get_active_block(block).changed_fields
+        so we know at this point that that isn't really "changed" field data,
+        it's the result of parsing the OLX. Save a copy into loaded_definitions.
+        """
+        entry = self._get_active_block(block)
+        self.loaded_definitions[entry.olx_hash] = entry.changed_fields.copy()
+        # Reset changed_fields to indicate this block hasn't actually made any field data changes, just loaded from XML:
+        entry.changed_fields.clear()
+
+        if len(self.loaded_definitions) > MAX_DEFINITIONS_LOADED:
+            self.free_unused_definitions()
+
+    def has_changes(self, block):
+        """
+        Does the specified block have any unsaved changes?
+        """
+        entry = self._get_active_block(block)
+        return bool(entry.changed_fields)
+
+    def has_cached_definition(self, definition_key):
+        """
+        Has the specified OLX file been loaded into memory?
+        """
+        olx_hash = get_olx_hash_for_definition_key(definition_key)
+        return olx_hash in self.loaded_definitions
+
+    def free_unused_definitions(self):
+        """
+        Free unused field data cache entries from self.loaded_definitions
+        as long as they're not in use.
+        """
+        olx_hashes = set(self.loaded_definitions.keys())
+        olx_hashes_needed = set(entry.olx_hash for entry in self.active_blocks.values())
+
+        olx_hashes_safe_to_delete = olx_hashes - olx_hashes_needed
+
+        # To avoid doing this too often, randomly cull unused entries until
+        # we have only half as many as MAX_DEFINITIONS_LOADED in memory, if possible.
+        while olx_hashes_safe_to_delete and (len(self.loaded_definitions) > MAX_DEFINITIONS_LOADED / 2):
+            del self.loaded_definitions[olx_hashes_safe_to_delete.pop()]

openedx/core/djangoapps/xblock/runtime/blockstore_runtime.py

@@ -0,0 +1,175 @@
+"""
+A runtime designed to work with Blockstore, reading and writing
+XBlock field data directly from Blockstore.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import logging
+
+from lxml import etree
+from xblock.exceptions import NoSuchDefinition, NoSuchUsage
+from xblock.fields import ScopeIds
+
+from openedx.core.djangoapps.xblock.learning_context.keys import BundleDefinitionLocator
+from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl
+from openedx.core.djangoapps.xblock.runtime.runtime import XBlockRuntime
+from openedx.core.djangoapps.xblock.runtime.olx_parsing import parse_xblock_include
+from openedx.core.djangoapps.xblock.runtime.serializer import serialize_xblock
+from openedx.core.djangolib.blockstore_cache import BundleCache, get_bundle_file_data_with_cache
+from openedx.core.lib import blockstore_api
+
+log = logging.getLogger(__name__)
+
+
+class BlockstoreXBlockRuntime(XBlockRuntime):
+    """
+    A runtime designed to work with Blockstore, reading and writing
+    XBlock field data directly from Blockstore.
+    """
+    def parse_xml_file(self, fileobj, id_generator=None):
+        raise NotImplementedError("Use parse_olx_file() instead")
+
+    def get_block(self, usage_id, for_parent=None):
+        """
+        Create an XBlock instance in this runtime.
+
+        The `usage_id` is used to find the XBlock class and data.
+        """
+        def_id = self.id_reader.get_definition_id(usage_id)
+        if def_id is None:
+            raise ValueError("Definition not found for usage {}".format(usage_id))
+        if not isinstance(def_id, BundleDefinitionLocator):
+            raise TypeError("This runtime can only load blocks stored in Blockstore bundles.")
+        try:
+            block_type = self.id_reader.get_block_type(def_id)
+        except NoSuchDefinition:
+            raise NoSuchUsage(repr(usage_id))
+        keys = ScopeIds(self.user_id, block_type, def_id, usage_id)
+
+        if self.system.authored_data_store.has_cached_definition(def_id):
+            return self.construct_xblock(block_type, keys, for_parent=for_parent)
+        else:
+            # We need to load this block's field data from its OLX file in blockstore:
+            xml_node = xml_for_definition(def_id)
+            if xml_node.get("url_name", None):
+                log.warning("XBlock at %s should not specify an old-style url_name attribute.", def_id.olx_path)
+            block_class = self.mixologist.mix(self.load_block_type(block_type))
+            if hasattr(block_class, 'parse_xml_new_runtime'):
+                # This is a (former) XModule with messy XML parsing code; let its parse_xml() method continue to work
+                # as it currently does in the old runtime, but let this parse_xml_new_runtime() method parse the XML in
+                # a simpler way that's free of tech debt, if defined.
+                # In particular, XmlParserMixin doesn't play well with this new runtime, so this is mostly about
+                # bypassing that mixin's code.
+                # When a former XModule no longer needs to support the old runtime, its parse_xml_new_runtime method
+                # should be removed and its parse_xml() method should be simplified to just call the super().parse_xml()
+                # plus some minor additional lines of code as needed.
+                block = block_class.parse_xml_new_runtime(xml_node, runtime=self, keys=keys)
+            else:
+                block = block_class.parse_xml(xml_node, runtime=self, keys=keys, id_generator=None)
+            # Update field data with parsed values. We can't call .save() because it will call save_block(), below.
+            block.force_save_fields(block._get_fields_to_save())  # pylint: disable=protected-access
+            self.system.authored_data_store.cache_fields(block)
+            # There is no way to set the parent via parse_xml, so do what
+            # HierarchyMixin would do:
+            if for_parent is not None:
+                block._parent_block = for_parent  # pylint: disable=protected-access
+                block._parent_block_id = for_parent.scope_ids.usage_id  # pylint: disable=protected-access
+            return block
+
+    def add_node_as_child(self, block, node, id_generator=None):
+        """
+        This runtime API should normally be used via
+        runtime.get_block() -> block.parse_xml() -> runtime.add_node_as_child
+        """
+        parent_usage = block.scope_ids.usage_id
+        parent_definition = block.scope_ids.def_id
+        learning_context = get_learning_context_impl(parent_usage)
+        parsed_include = parse_xblock_include(node)
+        usage_key = learning_context.usage_for_child_include(parent_usage, parent_definition, parsed_include)
+        block.children.append(usage_key)
+        if parent_definition.draft_name:
+            # Store the <xblock-include /> data which we'll need later if saving changes to this block
+            self.child_includes_of(block).append(parsed_include)
+
+    def add_child_include(self, block, parsed_include):
+        """
+        Given an XBlockInclude tuple that represents a new <xblock-include />
+        node, add it as a child of the specified XBlock. This is the only
+        supported API for adding a new child to an XBlock - one cannot just
+        modify block.children to append a usage ID, since that doesn't provide
+        enough information to serialize the block's <xblock-include /> elements.
+        """
+        learning_context = get_learning_context_impl(block.scope_ids.usage_id)
+        child_usage_key = learning_context.usage_for_child_include(
+            block.scope_ids.usage_id, block.scope_ids.def_id, parsed_include,
+        )
+        block.children.append(child_usage_key)
+        self.child_includes_of(block).append(parsed_include)
+
+    def child_includes_of(self, block):
+        """
+        Get the (mutable) list of <xblock-include /> directives that define the
+        children of this block's definition.
+        """
+        # A hack: when serializing an XBlock, we need to re-create the <xblock-include definition="..." usage="..." />
+        # elements that were in its original XML. But doing so requires the usage="..." hint attribute, which is
+        # technically part of the parent definition but which is not otherwise stored anywhere; we only have the derived
+        # usage_key, but asking the learning context to transform the usage_key back to the usage="..." hint attribute
+        # is non-trivial and could lead to bugs, because it could happen differently if the same parent definition is
+        # used in a library compared to a course (each would have different usage keys for the same usage hint).
+        # So, if this is a draft XBlock (we are editing it), we store the actual parsed <xblock-includes> so we can
+        # re-use them exactly when serializing this block back to XML.
+        # This implies that changes to an XBlock's children cannot be made by manipulating the .children field and
+        # then calling save().
+        assert block.scope_ids.def_id.draft_name, "Manipulating includes is only relevant for draft XBlocks."
+        attr_name = "children_includes_{}".format(id(block))  # Force use of this accessor method
+        if not hasattr(block, attr_name):
+            setattr(block, attr_name, [])
+        return getattr(block, attr_name)
+
+    def save_block(self, block):
+        """
+        Save any pending field data values to Blockstore.
+
+        This gets called by block.save() - do not call this directly.
+        """
+        if not self.system.authored_data_store.has_changes(block):
+            return  # No changes, so no action needed.
+        definition_key = block.scope_ids.def_id
+        if definition_key.draft_name is None:
+            raise RuntimeError(
+                "The Blockstore runtime does not support saving changes to blockstore without a draft. "
+                "Are you making changes to UserScope.NONE fields from the LMS rather than Studio?"
+            )
+        olx_str, static_files = serialize_xblock(block)
+        # Write the OLX file to the bundle:
+        draft_uuid = blockstore_api.get_or_create_bundle_draft(
+            definition_key.bundle_uuid, definition_key.draft_name
+        ).uuid
+        olx_path = definition_key.olx_path
+        blockstore_api.write_draft_file(draft_uuid, olx_path, olx_str)
+        # And the other files, if any:
+        for fh in static_files:
+            raise NotImplementedError(
+                "Need to write static file {} to blockstore but that's not yet implemented yet".format(fh.name)
+            )
+        # Now invalidate the blockstore data cache for the bundle:
+        BundleCache(definition_key.bundle_uuid, draft_name=definition_key.draft_name).clear()
+
+
+def xml_for_definition(definition_key):
+    """
+    Method for loading OLX from Blockstore and parsing it to an etree.
+    """
+    try:
+        xml_str = get_bundle_file_data_with_cache(
+            bundle_uuid=definition_key.bundle_uuid,
+            path=definition_key.olx_path,
+            bundle_version=definition_key.bundle_version,
+            draft_name=definition_key.draft_name,
+        )
+    except blockstore_api.BundleFileNotFound:
+        raise NoSuchDefinition("OLX file {} not found in bundle {}.".format(
+            definition_key.olx_path, definition_key.bundle_uuid,
+        ))
+    node = etree.fromstring(xml_str)
+    return node

openedx/core/djangoapps/xblock/runtime/id_managers.py

@@ -0,0 +1,89 @@
+"""
+Implementation of the APIs required for XBlock runtimes to work with
+our newer Open edX-specific opaque key formats.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from xblock.runtime import IdReader
+
+from openedx.core.djangoapps.xblock.learning_context.keys import BlockUsageKeyV2
+from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl
+
+
+class OpaqueKeyReader(IdReader):
+    """
+    IdReader for :class:`DefinitionKey` and :class:`UsageKey`s.
+    """
+    def get_definition_id(self, usage_id):
+        """Retrieve the definition that a usage is derived from.
+
+        Args:
+            usage_id: The id of the usage to query
+
+        Returns:
+            The `definition_id` the usage is derived from
+        """
+        if isinstance(usage_id, BlockUsageKeyV2):
+            return get_learning_context_impl(usage_id).definition_for_usage(usage_id)
+        raise TypeError("This version of get_definition_id doesn't support the given key type.")
+
+    def get_block_type(self, def_id):
+        """Retrieve the block_type of a particular definition
+
+        Args:
+            def_id: The id of the definition to query
+
+        Returns:
+            The `block_type` of the definition
+        """
+        return def_id.block_type
+
+    def get_usage_id_from_aside(self, aside_id):
+        """
+        Retrieve the XBlock `usage_id` associated with this aside usage id.
+
+        Args:
+            aside_id: The usage id of the XBlockAside.
+
+        Returns:
+            The `usage_id` of the usage the aside is commenting on.
+        """
+        return aside_id.usage_key
+
+    def get_definition_id_from_aside(self, aside_id):
+        """
+        Retrieve the XBlock `definition_id` associated with this aside definition id.
+
+        Args:
+            aside_id: The usage id of the XBlockAside.
+
+        Returns:
+            The `definition_id` of the usage the aside is commenting on.
+        """
+        return aside_id.definition_key
+
+    def get_aside_type_from_usage(self, aside_id):
+        """
+        Retrieve the XBlockAside `aside_type` associated with this aside
+        usage id.
+
+        Args:
+            aside_id: The usage id of the XBlockAside.
+
+        Returns:
+            The `aside_type` of the aside.
+        """
+        return aside_id.aside_type
+
+    def get_aside_type_from_definition(self, aside_id):
+        """
+        Retrieve the XBlockAside `aside_type` associated with this aside
+        definition id.
+
+        Args:
+            aside_id: The definition id of the XBlockAside.
+
+        Returns:
+            The `aside_type` of the aside.
+        """
+        return aside_id.aside_type

openedx/core/djangoapps/xblock/runtime/olx_parsing.py

@@ -0,0 +1,82 @@
+"""
+Helpful methods to use when parsing OLX (XBlock XML)
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+from collections import namedtuple
+
+from openedx.core.djangoapps.xblock.learning_context.keys import BundleDefinitionLocator
+from openedx.core.djangolib.blockstore_cache import get_bundle_direct_links_with_cache
+
+
+class BundleFormatException(Exception):
+    """
+    Raised when certain errors are found when parsing the OLX in a content
+    libary bundle.
+    """
+
+
+XBlockInclude = namedtuple('XBlockInclude', ['link_id', 'block_type', 'definition_id', 'usage_hint'])
+
+
+def parse_xblock_include(include_node):
+    """
+    Given an etree XML node that represents an <xblock-include /> element,
+    parse it and return the BundleDefinitionLocator that it points to.
+    """
+    # An XBlock include looks like:
+    # <xblock-include source="link_id" definition="block_type/definition_id" usage="alias" />
+    # Where "source" and "usage" are optional.
+    try:
+        definition_path = include_node.attrib['definition']
+    except KeyError:
+        raise BundleFormatException("<xblock-include> is missing the required definition=\"...\" attribute")
+    usage_hint = include_node.attrib.get("usage", None)
+    link_id = include_node.attrib.get("source", None)
+    # This is pointing to another definition in the same bundle. It looks like:
+    # <xblock-include definition="block_type/definition_id" />
+    try:
+        block_type, definition_id = definition_path.split("/")
+    except ValueError:
+        raise BundleFormatException("Invalid definition attribute: {}".format(definition_path))
+    return XBlockInclude(link_id=link_id, block_type=block_type, definition_id=definition_id, usage_hint=usage_hint)
+
+
+def definition_for_include(parsed_include, parent_definition_key):
+    """
+    Given a parsed <xblock-include /> element as a XBlockInclude tuple, get the
+    definition (OLX file) that it is pointing to.
+
+    Arguments:
+
+    parsed_include: An XBlockInclude tuple
+
+    parent_definition_key: The BundleDefinitionLocator for the XBlock whose OLX
+        contained the <xblock-include /> (i.e. the parent).
+
+    Returns: a BundleDefinitionLocator
+    """
+    if parsed_include.link_id:
+        links = get_bundle_direct_links_with_cache(
+            parent_definition_key.bundle_uuid,
+            # And one of the following will be set:
+            bundle_version=parent_definition_key.bundle_version,
+            draft_name=parent_definition_key.draft_name,
+        )
+        try:
+            link = links[parsed_include.link_id]
+        except KeyError:
+            raise BundleFormatException("Link not found: {}".format(parsed_include.link_id))
+        return BundleDefinitionLocator(
+            bundle_uuid=link.bundle_uuid,
+            block_type=parsed_include.block_type,
+            olx_path="{}/{}/definition.xml".format(parsed_include.block_type, parsed_include.definition_id),
+            bundle_version=link.version,
+        )
+    else:
+        return BundleDefinitionLocator(
+            bundle_uuid=parent_definition_key.bundle_uuid,
+            block_type=parsed_include.block_type,
+            olx_path="{}/{}/definition.xml".format(parsed_include.block_type, parsed_include.definition_id),
+            bundle_version=parent_definition_key.bundle_version,
+            draft_name=parent_definition_key.draft_name,
+        )

openedx/core/djangoapps/xblock/runtime/runtime.py

@@ -0,0 +1,226 @@
+"""
+Common base classes for all new XBlock runtimes.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import logging
+
+from django.utils.lru_cache import lru_cache
+from six.moves.urllib.parse import urljoin  # pylint: disable=import-error
+from xblock.exceptions import NoSuchServiceError
+from xblock.field_data import SplitFieldData
+from xblock.fields import Scope
+from xblock.runtime import Runtime, NullI18nService, MemoryIdManager
+from web_fragments.fragment import Fragment
+
+from openedx.core.djangoapps.xblock.apps import get_xblock_app_config
+from openedx.core.lib.xblock_utils import xblock_local_resource_url
+from xmodule.errortracker import make_error_tracker
+from .id_managers import OpaqueKeyReader
+from .shims import RuntimeShim, XBlockShim
+
+
+log = logging.getLogger(__name__)
+
+
+class XBlockRuntime(RuntimeShim, Runtime):
+    """
+    This class manages one or more instantiated XBlocks for a particular user,
+    providing those XBlocks with the standard XBlock runtime API (and some
+    Open edX-specific additions) so that it can interact with the platform,
+    and the platform can interact with it.
+
+    The main reason we cannot make the runtime a long-lived singleton is that
+    the XBlock runtime API requires 'user_id' to be a property of the runtime,
+    not an argument passed in when loading particular blocks.
+    """
+
+    # ** Do not add any XModule compatibility code to this class **
+    # Add it to RuntimeShim instead, to help keep legacy code isolated.
+
+    def __init__(self, system, user_id):
+        # type: (XBlockRuntimeSystem, int) -> None
+        super(XBlockRuntime, self).__init__(
+            id_reader=system.id_reader,
+            mixins=(
+                XBlockShim,
+            ),
+            services={
+                "i18n": NullI18nService(),
+            },
+            default_class=None,
+            select=None,
+            id_generator=system.id_generator,
+        )
+        self.system = system
+        self.user_id = user_id
+
+    def handler_url(self, block, handler_name, suffix='', query='', thirdparty=False):
+        """
+        Get the URL to a specific handler.
+        """
+        url = self.system.handler_url(
+            usage_key=block.scope_ids.usage_id,
+            handler_name=handler_name,
+            user_id=XBlockRuntimeSystem.ANONYMOUS_USER if thirdparty else self.user_id,
+        )
+        if suffix:
+            if not url.endswith('/'):
+                url += '/'
+            url += suffix
+        if query:
+            url += '&' if '?' in url else '?'
+            url += query
+        return url
+
+    def resource_url(self, resource):
+        raise NotImplementedError("resource_url is not supported by Open edX.")
+
+    def local_resource_url(self, block, uri):
+        """
+        Get the absolute URL to a resource file (like a CSS/JS file or an image)
+        that is part of an XBlock's python module.
+        """
+        relative_url = xblock_local_resource_url(block, uri)
+        site_root_url = get_xblock_app_config().get_site_root_url()
+        absolute_url = urljoin(site_root_url, relative_url)
+        return absolute_url
+
+    def publish(self, block, event_type, event_data):
+        # TODO: publish events properly
+        log.info("XBlock %s has published a '%s' event.", block.scope_ids.usage_id, event_type)
+
+    def applicable_aside_types(self, block):
+        """ Disable XBlock asides in this runtime """
+        return []
+
+    def parse_xml_file(self, fileobj, id_generator=None):
+        # Deny access to the inherited method
+        raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime")
+
+    def add_node_as_child(self, block, node, id_generator=None):
+        """
+        Called by XBlock.parse_xml to treat a child node as a child block.
+        """
+        # Deny access to the inherited method
+        raise NotImplementedError("XML Serialization is only supported with BlockstoreXBlockRuntime")
+
+    def service(self, block, service_name):
+        """
+        Return a service, or None.
+        Services are objects implementing arbitrary other interfaces.
+        """
+        # TODO: Do these declarations actually help with anything? Maybe this check should
+        # be removed from here and from XBlock.runtime
+        declaration = block.service_declaration(service_name)
+        if declaration is None:
+            raise NoSuchServiceError("Service {!r} was not requested.".format(service_name))
+        # Special case handling for some services:
+        service = self.system.get_service(block.scope_ids, service_name)
+        if service is None:
+            service = super(XBlockRuntime, self).service(block, service_name)
+        return service
+
+    def render(self, block, view_name, context=None):
+        """
+        Render a specific view of an XBlock.
+        """
+        # We only need to override this method because some XBlocks in the
+        # edx-platform codebase use methods like add_webpack_to_fragment()
+        # which create relative URLs (/static/studio/bundles/webpack-foo.js).
+        # We want all resource URLs to be absolute, such as is done when
+        # local_resource_url() is used.
+        fragment = super(XBlockRuntime, self).render(block, view_name, context)
+        needs_fix = False
+        for resource in fragment.resources:
+            if resource.kind == 'url' and resource.data.startswith('/'):
+                needs_fix = True
+                break
+        if needs_fix:
+            log.warning("XBlock %s returned relative resource URLs, which are deprecated", block.scope_ids.usage_id)
+            # The Fragment API is mostly immutable, so changing a resource requires this:
+            frag_data = fragment.to_dict()
+            for resource in frag_data['resources']:
+                if resource['kind'] == 'url' and resource['data'].startswith('/'):
+                    log.debug("-> Relative resource URL: %s", resource['data'])
+                    resource['data'] = get_xblock_app_config().get_site_root_url() + resource['data']
+            fragment = Fragment.from_dict(frag_data)
+        return fragment
+
+
+class XBlockRuntimeSystem(object):
+    """
+    This class is essentially a factory for XBlockRuntimes. This is a
+    long-lived object which provides the behavior specific to the application
+    that wants to use XBlocks. Unlike XBlockRuntime, a single instance of this
+    class can be used with many different XBlocks, whereas each XBlock gets its
+    own instance of XBlockRuntime.
+    """
+    ANONYMOUS_USER = 'anon'  # Special value passed to handler_url() methods
+
+    def __init__(
+        self,
+        handler_url,  # type: (Callable[[UsageKey, str, Union[int, ANONYMOUS_USER]], str]
+        authored_data_store,  # type: FieldData
+        student_data_store,  # type: FieldData
+        runtime_class,  # type: XBlockRuntime
+    ):
+        """
+        args:
+            handler_url: A method to get URLs to call XBlock handlers. It must
+                implement this signature:
+                handler_url(
+                    usage_key: UsageKey,
+                    handler_name: str,
+                    user_id: Union[int, ANONYMOUS_USER],
+                )
+                If user_id is ANONYMOUS_USER, the handler should execute without
+                any user-scoped fields.
+            authored_data_store: A FieldData instance used to retrieve/write
+                any fields with UserScope.NONE
+            student_data_store: A FieldData instance used to retrieve/write
+                any fields with UserScope.ONE or UserScope.ALL
+        """
+        self.handler_url = handler_url
+        self.id_reader = OpaqueKeyReader()
+        self.id_generator = MemoryIdManager()  # We don't really use id_generator until we need to support asides
+        self.runtime_class = runtime_class
+        self.authored_data_store = authored_data_store
+        self.field_data = SplitFieldData({
+            Scope.content: authored_data_store,
+            Scope.settings: authored_data_store,
+            Scope.parent: authored_data_store,
+            Scope.children: authored_data_store,
+            Scope.user_state_summary: student_data_store,
+            Scope.user_state: student_data_store,
+            Scope.user_info: student_data_store,
+            Scope.preferences: student_data_store,
+        })
+
+        self._error_trackers = {}
+
+    def get_runtime(self, user_id):
+        # type: (int) -> XBlockRuntime
+        return self.runtime_class(self, user_id)
+
+    def get_service(self, scope_ids, service_name):
+        """
+        Get a runtime service
+
+        Runtime services may come from this XBlockRuntimeSystem,
+        or if this method returns None, they may come from the
+        XBlockRuntime.
+        """
+        if service_name == "field-data":
+            return self.field_data
+        if service_name == 'error_tracker':
+            return self.get_error_tracker_for_context(scope_ids.usage_id.context_key)
+        return None  # None means see if XBlockRuntime offers this service
+
+    @lru_cache(maxsize=32)
+    def get_error_tracker_for_context(self, context_key):  # pylint: disable=unused-argument
+        """
+        Get an error tracker for the specified context.
+        lru_cache makes this error tracker long-lived, for
+        up to 32 contexts that have most recently been used.
+        """
+        return make_error_tracker()

openedx/core/djangoapps/xblock/runtime/serializer.py

@@ -0,0 +1,132 @@
+"""
+Code to serialize an XBlock to OLX
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+from collections import namedtuple
+from contextlib import contextmanager
+import logging
+import os
+
+from fs.memoryfs import MemoryFS
+from fs.wrapfs import WrapFS
+from lxml.etree import Element
+from lxml.etree import tostring as etree_tostring
+
+from xmodule.xml_module import XmlParserMixin
+
+log = logging.getLogger(__name__)
+
+# A static file required by an XBlock
+StaticFile = namedtuple('StaticFile', ['name', 'data'])
+
+
+def serialize_xblock(block):
+    """
+    Given an XBlock instance, serialize it to OLX
+
+    Returns
+        (olx_str, static_files)
+    where olx_str is the XML as a string, and static_files is a list of
+    StaticFile objects for any small data files that the XBlock may need for
+    complete serialization (e.g. video subtitle files or a .html data file for
+    an HTML block).
+    """
+    static_files = []
+    # Create an XML node to hold the exported data
+    olx_node = Element("root")  # The node name doesn't matter: add_xml_to_node will change it
+    # ^ Note: We could pass nsmap=xblock.core.XML_NAMESPACES here, but the
+    # resulting XML namespace attributes don't seem that useful?
+
+    with override_export_fs(block) as filesystem:  # Needed for XBlocks that inherit XModuleDescriptor
+        # Tell the block to serialize itself as XML/OLX:
+        if not block.has_children:
+            block.add_xml_to_node(olx_node)
+        else:
+            # We don't want the children serialized at this time, because
+            # otherwise we can't tell which files in 'filesystem' belong to
+            # this block and which belong to its children. So, temporarily
+            # disable any children:
+            children = block.children
+            block.children = []
+            block.add_xml_to_node(olx_node)
+            block.children = children
+
+        # Now the block/module may have exported addtional data as files in
+        # 'filesystem'. If so, store them:
+        for item in filesystem.walk():  # pylint: disable=not-callable
+            for unit_file in item.files:
+                file_path = os.path.join(item.path, unit_file.name)
+                with filesystem.open(file_path, 'rb') as fh:
+                    data = fh.read()
+                static_files.append(StaticFile(name=unit_file.name, data=data))
+
+    # Remove 'url_name' - we store the definition key in the folder name
+    # that holds the OLX and the usage key elsewhere, so specifying it
+    # within the OLX file is redundant and can lead to issues if the file is
+    # copied and pasted elsewhere in the bundle with a new definition key.
+    olx_node.attrib.pop('url_name', None)
+
+    # Add  <xblock-include /> tags for each child:
+    if block.has_children and block.children:
+        try:
+            child_includes = block.runtime.child_includes_of(block)
+        except AttributeError:
+            raise RuntimeError("Cannot get child includes of block. Make sure it's using BlockstoreXBlockRuntime")
+        if len(child_includes) != len(block.children):
+            raise RuntimeError(
+                "Mistmatch between block.children and runtime.child_includes_of()."
+                "Make sure the block was loaded via runtime.get_block() and that "
+                "the block.children field was not modified directly (use "
+                "block.runtime.add_child_include() instead)."
+            )
+        for include_data in child_includes:
+            definition_str = include_data.block_type + "/" + include_data.definition_id
+            attrs = {"definition": definition_str}
+            if include_data.usage_hint:
+                attrs["usage"] = include_data.usage_hint
+            if include_data.link_id:
+                attrs["source"] = include_data.link_id
+            olx_node.append(olx_node.makeelement("xblock-include", attrs))
+
+    # Serialize the resulting XML to a string:
+    olx_str = etree_tostring(olx_node, encoding="utf-8", pretty_print=True)
+    return (olx_str, static_files)
+
+
+@contextmanager
+def override_export_fs(block):
+    """
+    Hack required for some legacy XBlocks which inherit
+        XModuleDescriptor.add_xml_to_node()
+    instead of the usual
+        XmlSerializationMixin.add_xml_to_node()
+    method.
+
+    This method temporarily replaces a block's runtime's 'export_fs' system with
+    an in-memory filesystem. This method also abuses the
+        XmlParserMixin.export_to_file()
+    API to prevent the XModule export code from exporting each block as two
+    files (one .olx pointing to one .xml file). The export_to_file was meant to
+    be used only by the customtag XModule but it makes our lives here much
+    easier.
+    """
+    fs = WrapFS(MemoryFS())
+    fs.makedir('course')
+    fs.makedir('course/static')  # Video XBlock requires this directory to exists, to put srt files etc.
+
+    old_export_fs = block.runtime.export_fs
+    block.runtime.export_fs = fs
+    if hasattr(block, 'export_to_file'):
+        old_export_to_file = block.export_to_file
+        block.export_to_file = lambda: False
+    old_global_export_to_file = XmlParserMixin.export_to_file
+    XmlParserMixin.export_to_file = lambda _: False  # So this applies to child blocks that get loaded during export
+    try:
+        yield fs
+    except:
+        raise
+    finally:
+        block.runtime.export_fs = old_export_fs
+        if hasattr(block, 'export_to_file'):
+            block.export_to_file = old_export_to_file
+        XmlParserMixin.export_to_file = old_global_export_to_file

openedx/core/djangoapps/xblock/runtime/shims.py

@@ -0,0 +1,395 @@
+"""
+Code to implement backwards compatibility
+"""
+# pylint: disable=no-member
+from __future__ import absolute_import, division, print_function, unicode_literals
+import hashlib
+import warnings
+
+from django.conf import settings
+from django.core.cache import cache
+from django.template import TemplateDoesNotExist
+from django.utils.functional import cached_property
+from fs.memoryfs import MemoryFS
+
+from edxmako.shortcuts import render_to_string
+import six
+
+
+class RuntimeShim(object):
+    """
+    All the old/deprecated APIs that our XBlock runtime(s) need to
+    support are captured in this mixin.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(RuntimeShim, self).__init__(*args, **kwargs)
+        self._active_block = None
+
+    def render(self, block, view_name, context=None):
+        """
+        Render a block by invoking its view.
+        """
+        # The XBlock runtime code assumes a runtime is long-lived and serves
+        # multiple blocks. But the previous edX runtime had a separate runtime
+        # instance for each XBlock instance. So in order to implement its API,
+        # we need this hacky way to get the "current" XBlock. As XBlocks are
+        # modified to not use these deprecated APIs, this should be used less
+        # and less
+        old_active_block = self._active_block
+        self._active_block = block
+        try:
+            return super(RuntimeShim, self).render(block, view_name, context)
+        finally:
+            # Reset the active view to what it was before entering this method
+            self._active_block = old_active_block
+
+    def handle(self, block, handler_name, request, suffix=''):
+        """
+        Render a block by invoking its view.
+        """
+        # See comment in render() above
+        old_active_block = self._active_block
+        self._active_block = block
+        try:
+            return super(RuntimeShim, self).handle(block, handler_name, request, suffix)
+        finally:
+            # Reset the active view to what it was before entering this method
+            self._active_block = old_active_block
+
+    @property
+    def anonymous_student_id(self):
+        """
+        Get an anonymized identifier for this user.
+        """
+        # TODO: Change this to a runtime service or method so that we can have
+        # access to the context_key without relying on self._active_block.
+        if self.user_id is None:
+            raise NotImplementedError("TODO: anonymous ID for anonymous users.")
+        #### TEMPORARY IMPLEMENTATION:
+        # TODO: Update student.models.AnonymousUserId to have a 'context_key'
+        # column instead of 'course_key' (no DB migration needed). Then change
+        # this to point to the existing student.models.anonymous_id_for_user
+        # code. (Or do we want a separate new table for the new runtime?)
+        # The reason we can't do this now is that ContextKey doesn't yet exist
+        # in the opaque-keys repo, so there is no working 'ContextKeyField'
+        # class in opaque-keys that accepts either old CourseKeys or new
+        # LearningContextKeys.
+        hasher = hashlib.md5()
+        hasher.update(settings.SECRET_KEY)
+        hasher.update(six.text_type(self.user_id))
+        digest = hasher.hexdigest()
+        return digest
+
+    @property
+    def cache(self):
+        """
+        Access to a cache.
+
+        Seems only to be used by capa. Remove this if capa can be refactored.
+        """
+        # TODO: Refactor capa to access this directly, don't bother the runtime. Then remove it from here.
+        return cache
+
+    @property
+    def can_execute_unsafe_code(self):
+        """
+        Determine if capa problems in this context/course are allowed to run
+        unsafe code. See common/lib/xmodule/xmodule/util/sandboxing.py
+
+        Seems only to be used by capa.
+        """
+        # TODO: Refactor capa to access this directly, don't bother the runtime. Then remove it from here.
+        return False  # Change this if/when we need to support unsafe courses in the new runtime.
+
+    @property
+    def DEBUG(self):
+        """
+        Should DEBUG mode (?) be used? This flag is only read by capa.
+        """
+        # TODO: Refactor capa to access this directly, don't bother the runtime. Then remove it from here.
+        return False
+
+    def get_python_lib_zip(self):
+        """
+        A function returning a bytestring or None. The bytestring is the
+        contents of a zip file that should be importable by other Python code
+        running in the module.
+
+        Only used for capa problems.
+        """
+        # TODO: load the python code from Blockstore. Ensure it's not publicly accessible.
+        return None
+
+    @property
+    def error_tracker(self):
+        """
+        Accessor for the course's error tracker
+        """
+        warnings.warn(
+            "Use of system.error_tracker is deprecated; use self.runtime.service(self, 'error_tracker') instead",
+            DeprecationWarning, stacklevel=2,
+        )
+        return self.service(self._active_block, 'error_tracker')
+
+    def get_policy(self, _usage_id):
+        """
+        A function that takes a usage id and returns a dict of policy to apply.
+        """
+        # TODO: implement?
+        return {}
+
+    @property
+    def filestore(self):
+        """
+        Alternate name for 'resources_fs'. Not sure if either name is deprecated
+        but we should deprecate one :)
+        """
+        return self.resources_fs
+
+    @property
+    def node_path(self):
+        """
+        Get the path to Node.js
+
+        Seems only to be used by capa. Remove this if capa can be refactored.
+        """
+        # TODO: Refactor capa to access this directly, don't bother the runtime. Then remove it from here.
+        return getattr(settings, 'NODE_PATH', None)  # Only defined in the LMS
+
+    def render_template(self, template_name, dictionary, namespace='main'):
+        """
+        Render a mako template
+        """
+        warnings.warn(
+            "Use of runtime.render_template is deprecated. "
+            "Use xblockutils.resources.ResourceLoader.render_mako_template or a JavaScript-based template instead.",
+            DeprecationWarning, stacklevel=2,
+        )
+        try:
+            return render_to_string(template_name, dictionary, namespace=namespace)
+        except TemplateDoesNotExist:
+            # From Studio, some templates might be in the LMS namespace only
+            return render_to_string(template_name, dictionary, namespace="lms." + namespace)
+
+    def process_xml(self, xml):
+        """
+        Code to handle parsing of child XML for old blocks that use XmlParserMixin.
+        """
+        # We can't parse XML in a vacuum - we need to know the parent block and/or the
+        # OLX file that holds this XML in order to generate useful definition keys etc.
+        # The older ImportSystem runtime could do this because it stored the course_id
+        # as part of the runtime.
+        raise NotImplementedError("This newer runtime does not support process_xml()")
+
+    def replace_urls(self, html_str):
+        """
+        Given an HTML string, replace any static file URLs like
+            /static/foo.png
+        with working URLs like
+            https://s3.example.com/course/this/assets/foo.png
+        See common/djangoapps/static_replace/__init__.py
+        """
+        # TODO: implement or deprecate.
+        # Can we replace this with a filesystem service that has a .get_url
+        # method on all files? See comments in the 'resources_fs' property.
+        # See also the version in openedx/core/lib/xblock_utils/__init__.py
+        return html_str  # For now just return without changes.
+
+    def replace_course_urls(self, html_str):
+        """
+        Given an HTML string, replace any course-relative URLs like
+            /course/blah
+        with working URLs like
+            /course/:course_id/blah
+        See common/djangoapps/static_replace/__init__.py
+        """
+        # TODO: implement or deprecate.
+        # See also the version in openedx/core/lib/xblock_utils/__init__.py
+        return html_str
+
+    def replace_jump_to_id_urls(self, html_str):
+        """
+        Replace /jump_to_id/ URLs in the HTML with expanded versions.
+        See common/djangoapps/static_replace/__init__.py
+        """
+        # TODO: implement or deprecate.
+        # See also the version in openedx/core/lib/xblock_utils/__init__.py
+        return html_str
+
+    @property
+    def resources_fs(self):
+        """
+        A filesystem that XBlocks can use to read large binary assets.
+        """
+        # TODO: implement this to serve any static assets that
+        # self._active_block has in its blockstore "folder". But this API should
+        # be deprecated and we should instead get compatible XBlocks to use a
+        # runtime filesystem service. Some initial exploration of that (as well
+        # as of the 'FileField' concept) has been done and is included in the
+        # XBlock repo at xblock.reference.plugins.FSService and is available in
+        # the old runtime as the 'fs' service.
+        warnings.warn(
+            "Use of legacy runtime.resources_fs or .filestore won't be able to find resources.",
+            stacklevel=3,
+        )
+        fake_fs = MemoryFS()
+        fake_fs.root_path = 'mem://'  # Required for the video XBlock's use of edxval create_transcript_objects
+        return fake_fs
+
+    export_fs = object()  # Same as above, see resources_fs ^
+
+    @property
+    def seed(self):
+        """
+        A number to seed the random number generator. Used by capa and the
+        randomize module.
+
+        Should be based on the user ID, per the existing implementation.
+        """
+        # TODO: Refactor capa to use the user ID or anonymous ID as the seed, don't bother the runtime.
+        # Then remove it from here.
+        return self.user_id if self.user_id is not None else 0
+
+    @property
+    def STATIC_URL(self):
+        """
+        Get the django STATIC_URL path.
+
+        Seems only to be used by capa. Remove this if capa can be refactored.
+        """
+        # TODO: Refactor capa to access this directly, don't bother the runtime. Then remove it from here.
+        return settings.STATIC_URL
+
+    @cached_property
+    def user_is_staff(self):
+        """
+        Is the current user a global staff user?
+        """
+        warnings.warn(
+            "runtime.user_is_staff is deprecated. Use the user service instead:\n"
+            "    user = self.runtime.service(self, 'user').get_current_user()\n"
+            "    is_staff = user.opt_attrs.get('edx-platform.user_is_staff')",
+            DeprecationWarning, stacklevel=2,
+        )
+        if self.user_id:
+            from django.contrib.auth import get_user_model
+            try:
+                user = get_user_model().objects.get(id=self.user_id)
+            except get_user_model().DoesNotExist:
+                return False
+            return user.is_staff
+        return False
+
+    @cached_property
+    def xqueue(self):
+        """
+        An accessor for XQueue, the platform's interface for external grader
+        services.
+
+        Seems only to be used by capa. Remove this if capa can be refactored.
+        """
+        # TODO: Refactor capa to access this directly, don't bother the runtime. Then remove it from here.
+        return {
+            'interface': None,
+            'construct_callback': None,
+            'default_queuename': None,
+            'waittime': 5,  # seconds; should come from settings.XQUEUE_WAITTIME_BETWEEN_REQUESTS
+        }
+
+    def get_field_provenance(self, xblock, field):
+        """
+        A Studio-specific method that was implemented on DescriptorSystem.
+        Used by the problem block.
+
+        For the given xblock, return a dict for the field's current state:
+        {
+            'default_value': what json'd value will take effect if field is unset: either the field default or
+            inherited value,
+            'explicitly_set': boolean for whether the current value is set v default/inherited,
+        }
+        """
+        result = {}
+        result['explicitly_set'] = xblock._field_data.has(xblock, field.name)  # pylint: disable=protected-access
+        try:
+            result['default_value'] = xblock._field_data.default(xblock, field.name)  # pylint: disable=protected-access
+        except KeyError:
+            result['default_value'] = field.to_json(field.default)
+        return result
+
+    @property
+    def user_location(self):
+        """
+        Old API to get the user's country code (or None)
+        Used by the Video XBlock to select a CDN for user.
+        """
+        # Studio always returned None so we just return None for now.
+        # TBD: support this API or deprecate+remove it.
+        return None
+
+    @property
+    def course_id(self):
+        """
+        Old API to get the course ID.
+        """
+        warnings.warn(
+            "runtime.course_id is deprecated. Use context_key instead:\n"
+            "    block.scope_ids.usage_id.context_key\n",
+            DeprecationWarning, stacklevel=2,
+        )
+        return self._active_block.scope_ids.usage_id.context_key
+
+    def _css_classes_for(self, block, view):
+        """
+        Get the list of CSS classes that the wrapping <div> should have for the
+        specified xblock or aside's view.
+        """
+        css_classes = super(RuntimeShim, self)._css_classes_for(block, view)
+        # Many CSS styles for former XModules use
+        # .xmodule_display.xmodule_VideoBlock
+        # as their selector, so add those classes:
+        if view == 'student_view':
+            css_classes.append('xmodule_display')
+        elif view == 'studio_view':
+            css_classes.append('xmodule_edit')
+        css_classes.append('xmodule_{}'.format(block.unmixed_class.__name__))
+        return css_classes
+
+
+class XBlockShim(object):
+    """
+    Mixin added to XBlock classes in this runtime, to support
+    older/XModule APIs
+    """
+    @property
+    def location(self):
+        """
+        Accessor for the usage ID
+        """
+        warnings.warn(
+            "Use of block.location should be replaced with block.scope_ids.usage_id",
+            DeprecationWarning, stacklevel=2,
+        )
+        return self.scope_ids.usage_id
+
+    @property
+    def system(self):
+        """
+        Accessor for the XModule runtime
+        """
+        warnings.warn(
+            "Use of block.system should be replaced with block.runtime",
+            DeprecationWarning, stacklevel=2,
+        )
+        return self.runtime
+
+    @property
+    def graded(self):
+        """
+        Not sure what this is or how it's supposed to be set. Capa seems to
+        expect a 'graded' attribute to be present on itself. Possibly through
+        contentstore's update_section_grader_type() ?
+        """
+        if self.scope_ids.block_type != 'problem':
+            raise AttributeError(".graded shim is only for capa")
+        return False

openedx/core/djangoapps/xblock/utils.py

@@ -0,0 +1,69 @@
+"""
+Useful helper methods related to the XBlock runtime
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import hashlib
+import hmac
+import math
+import time
+
+from django.conf import settings
+from six import text_type
+
+
+def get_secure_token_for_xblock_handler(user_id, block_key_str, time_idx=0):
+    """
+    Get a secure token (one-way hash) used to authenticate XBlock handler
+    requests. This token replaces both the session ID cookie (or OAuth
+    bearer token) and the CSRF token for such requests.
+
+    The token is specific to one user and one XBlock usage ID, though may
+    be used for any handler. It expires and is only valid for 2-4 days (our
+    current best guess at a reasonable trade off between "what if the user left
+    their browser open overnight and tried to continue the next day" which
+    should work vs. "for security, tokens should not last too long")
+
+    We use this token because XBlocks may sometimes be sandboxed (run in a
+    client-side JavaScript environment with no access to cookies) and
+    because the XBlock python and JavaScript handler_url APIs do not provide
+    any way of authenticating the handler requests, other than assuming
+    cookies are present or including this sort of secure token in the
+    handler URL.
+
+    For security, we need these tokens to have an expiration date. So: the
+    hash incorporates the current time, rounded to the lowest TOKEN_PERIOD
+    value. When checking this, you should check both time_idx=0 and
+    time_idx=-1 in case we just recently moved from one time period to
+    another (i.e. at the stroke of midnight UTC or similar). The effect of
+    this is that each token is valid for 2-4 days.
+
+    (Alternatively, we could make each token expire after exactly X days, but
+    that requires storing the expiration date of each token on the server side,
+    making the implementation needlessly complex. The "time window" approach we
+    are using here also has the advantage that throughout a typical day, the
+    token each user gets for a given XBlock doesn't change, which makes
+    debugging and reasoning about the system simpler.)
+    """
+    TOKEN_PERIOD = 24 * 60 * 60 * 2  # These URLs are valid for 2-4 days
+    time_token = math.floor(time.time() / TOKEN_PERIOD)
+    time_token += TOKEN_PERIOD * time_idx
+    check_string = text_type(time_token) + ':' + text_type(user_id) + ':' + block_key_str
+    secure_key = hmac.new(settings.SECRET_KEY.encode('utf-8'), check_string.encode('utf-8'), hashlib.sha256).hexdigest()
+    return secure_key[:20]
+
+
+def validate_secure_token_for_xblock_handler(user_id, block_key_str, token):
+    """
+    Returns True if the specified handler authentication token is valid for the
+    given XBlock ID and user ID. Otherwise returns false.
+
+    See get_secure_token_for_xblock_handler
+    """
+    token = token.encode('utf-8')  # This line isn't needed after python 3, nor the .encode('utf-8') below
+    token_expected = get_secure_token_for_xblock_handler(user_id, block_key_str).encode('utf-8')
+    prev_token_expected = get_secure_token_for_xblock_handler(user_id, block_key_str, -1).encode('utf-8')
+    result1 = hmac.compare_digest(token, token_expected)
+    result2 = hmac.compare_digest(token, prev_token_expected)
+    # All computations happen above this line so this function always takes a
+    # constant time to produce its answer (security best practice).
+    return bool(result1 or result2)

openedx/core/djangolib/blockstore_cache.py

@@ -0,0 +1,251 @@
+"""
+An API for caching data related to Blockstore bundles
+
+The whole point of this is to make the hard problem of cache invalidation
+somewhat less hard.
+
+This cache prefixes all keys with the bundle/draft version number, so that when
+any change is made to the bundle/draft, we will look up entries using a new key
+and won't find the now-invalid cached data.
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+from datetime import datetime
+from uuid import UUID
+
+from django.core.cache import caches, InvalidCacheBackendError
+from django.utils.lru_cache import lru_cache
+from pytz import UTC
+import requests
+
+from openedx.core.lib import blockstore_api
+
+try:
+    # Use a dedicated cache for blockstore, if available:
+    cache = caches['blockstore']
+except InvalidCacheBackendError:
+    cache = caches['default']
+
+# MAX_BLOCKSTORE_CACHE_DELAY:
+# The per-bundle/draft caches are automatically invalidated when a newer version
+# of the bundle/draft is available, but that automatic check for the current
+# version is cached for this many seconds. So in the absence of explicit calls
+# to invalidate the cache, data may be out of date by up to this many seconds.
+# (Note that we do usually explicitly invalidate this cache during write
+# operations though, so this setting mostly affects actions by external systems
+# on Blockstore or bugs where we left out the cache invalidation step.)
+MAX_BLOCKSTORE_CACHE_DELAY = 60
+
+
+class BundleCache(object):
+    """
+    Data cache that ties every key-value to a particular version of a blockstore
+    bundle/draft, so that if/when the bundle/draft is updated, the cache is
+    automatically invalidated.
+
+    The automatic invalidation may take up to MAX_BLOCKSTORE_CACHE_DELAY
+    seconds, although the cache can also be manually invalidated for any
+    particular bundle versoin/draft by calling .clear()
+    """
+
+    def __init__(self, bundle_uuid, draft_name=None):
+        """
+        Instantiate this wrapper for the bundle with the specified UUID, and
+        optionally the specified draft name.
+        """
+        self.bundle_uuid = bundle_uuid
+        self.draft_name = draft_name
+
+    def get(self, key_parts, default=None):
+        """
+        Get a cached value related to this Blockstore bundle/draft.
+
+        key_parts: an arbitrary list of strings to identify the cached value.
+            For example, if caching the XBlock type of an OLX file, one could
+            request:
+                get(bundle_uuid, ["olx_type", "/path/to/file"])
+        default: default value if the key is not set in the cache
+        draft_name: read keys related to the specified draft
+        """
+        assert isinstance(key_parts, (list, tuple))
+        full_key = _get_versioned_cache_key(self.bundle_uuid, self.draft_name, key_parts)
+        return cache.get(full_key, default)
+
+    def set(self, key_parts, value):
+        """
+        Set a cached value related to this Blockstore bundle/draft.
+
+        key_parts: an arbitrary list of strings to identify the cached value.
+            For example, if caching the XBlock type of an OLX file, one could
+            request:
+                set(bundle_uuid, ["olx_type", "/path/to/file"], "html")
+        value: value to set in the cache
+        """
+        assert isinstance(key_parts, (list, tuple))
+        full_key = _get_versioned_cache_key(self.bundle_uuid, self.draft_name, key_parts)
+        return cache.set(full_key, value)
+
+    def clear(self):
+        """
+        Clear the cache for the specified bundle or draft.
+
+        This doesn't actually delete keys from the cache, but if the bundle or
+        draft has been modified, this will ensure we use the latest version
+        number, which will change the key prefix used by this cache, causing the
+        old version's keys to become unaddressable and eventually expire.
+        """
+        # Note: if we switch from memcached to redis at some point, this can be
+        # improved because Redis makes it easy to delete all keys with a
+        # specific prefix (e.g. a specific bundle UUID), which memcached cannot.
+        # With memcached, we just have to leave the invalid keys in the cache
+        # (unused) until they expire.
+        cache_key = 'bundle_version:{}:{}'.format(self.bundle_uuid, self.draft_name or '')
+        cache.delete(cache_key)
+
+
+def _get_versioned_cache_key(bundle_uuid, draft_name, key_parts):
+    """
+    Generate a cache key string that can be used to store data about the current
+    version/draft of the given bundle. The key incorporates the bundle/draft's
+    current version number such that if the bundle/draft is updated, a new key
+    will be used and the old key will no longer be valid and will expire.
+
+    Pass draft_name=None if you want to use the published version of the bundle.
+    """
+    assert isinstance(bundle_uuid, UUID)
+    version_num = get_bundle_version_number(bundle_uuid, draft_name)
+    return str(bundle_uuid) + ":" + str(version_num) + ":" + ":".join(key_parts)
+
+
+def get_bundle_version_number(bundle_uuid, draft_name=None):
+    """
+    Get the current version number of the specified bundle/draft. If a draft is
+    specified, the update timestamp is used in lieu of a version number.
+    """
+    cache_key = 'bundle_version:{}:{}'.format(bundle_uuid, draft_name or '')
+    version = cache.get(cache_key)
+    if version is not None:
+        return version
+    else:
+        version = 0  # Default to 0 in case bundle/draft is empty or doesn't exist
+
+    bundle_metadata = blockstore_api.get_bundle(bundle_uuid)
+    if draft_name:
+        draft_uuid = bundle_metadata.drafts.get(draft_name)  # pylint: disable=no-member
+        if draft_uuid:
+            draft_metadata = blockstore_api.get_draft(draft_uuid)
+            # Convert the 'updated_at' datetime info an integer value with microsecond accuracy.
+            updated_at_timestamp = (draft_metadata.updated_at - datetime(1970, 1, 1, tzinfo=UTC)).total_seconds()
+            version = int(updated_at_timestamp * 1e6)
+    # If we're not using a draft or the draft does not exist [anymore], fall
+    # back to the bundle version, if any versions have been published:
+    if version == 0 and bundle_metadata.latest_version:
+        version = bundle_metadata.latest_version
+    cache.set(cache_key, version, timeout=MAX_BLOCKSTORE_CACHE_DELAY)
+    return version
+
+
+@lru_cache(maxsize=200)
+def get_bundle_version_files_cached(bundle_uuid, bundle_version):
+    """
+    Get the files in the specified BundleVersion. Since BundleVersions are
+    immutable, this should be cached as aggressively as possible (ideally
+    it would be infinitely but we don't want to use up all available memory).
+    So this method uses lru_cache() to cache results in process-local memory.
+
+    (Note: This can't use BundleCache because BundleCache only associates data
+    with the most recent bundleversion, not a specified bundleversion)
+    """
+    return blockstore_api.get_bundle_version_files(bundle_uuid, bundle_version)
+
+
+def get_bundle_draft_files_cached(bundle_uuid, draft_name):
+    """
+    Get the files in the specified bundle draft. Cached using BundleCache so we
+    get automatic cache invalidation when the draft is updated.
+    """
+    bundle_cache = BundleCache(bundle_uuid, draft_name)
+    cache_key = ('bundle_draft_files', )
+    result = bundle_cache.get(cache_key)
+    if result is None:
+        result = blockstore_api.get_bundle_files(bundle_uuid, use_draft=draft_name)
+        bundle_cache.set(cache_key, result)
+    return result
+
+
+def get_bundle_files_cached(bundle_uuid, bundle_version=None, draft_name=None):
+    """
+    Get the list of files in the bundle, optionally with a version and/or draft
+    specified.
+    """
+    if draft_name:
+        return get_bundle_draft_files_cached(bundle_uuid, draft_name)
+    else:
+        if bundle_version is None:
+            bundle_version = get_bundle_version_number(bundle_uuid)
+        return get_bundle_version_files_cached(bundle_uuid, bundle_version)
+
+
+def get_bundle_file_metadata_with_cache(bundle_uuid, path, bundle_version=None, draft_name=None):
+    """
+    Get metadata about a file in a Blockstore Bundle[Version] or Draft, using the
+    cached list of files in each bundle if available.
+    """
+    for file_info in get_bundle_files_cached(bundle_uuid, bundle_version, draft_name):
+        if file_info.path == path:
+            return file_info
+    raise blockstore_api.BundleFileNotFound("Could not load {} from bundle {}".format(path, bundle_uuid))
+
+
+def get_bundle_file_data_with_cache(bundle_uuid, path, bundle_version=None, draft_name=None):
+    """
+    Method to read a file out of a Blockstore Bundle[Version] or Draft, using the
+    cached list of files in each bundle if available.
+    """
+    file_info = get_bundle_file_metadata_with_cache(bundle_uuid, path, bundle_version, draft_name)
+    with requests.get(file_info.url, stream=True) as r:
+        return r.content
+
+
+@lru_cache(maxsize=200)
+def get_bundle_version_direct_links_cached(bundle_uuid, bundle_version):
+    """
+    Get the direct links in the specified BundleVersion. Since BundleVersions
+    are immutable, this should be cached as aggressively as possible (ideally
+    it would be infinitely but we don't want to use up all available memory).
+    So this method uses lru_cache() to cache results in process-local memory.
+
+    (Note: This can't use BundleCache because BundleCache only associates data
+    with the most recent bundleversion, not a specified bundleversion)
+    """
+    return {
+        link.name: link.direct for link in blockstore_api.get_bundle_version_links(bundle_uuid, bundle_version).values()
+    }
+
+
+def get_bundle_draft_direct_links_cached(bundle_uuid, draft_name):
+    """
+    Get the direct links in the specified bundle draft. Cached using BundleCache
+    so we get automatic cache invalidation when the draft is updated.
+    """
+    bundle_cache = BundleCache(bundle_uuid, draft_name)
+    cache_key = ('bundle_draft_direct_links', )
+    result = bundle_cache.get(cache_key)
+    if result is None:
+        links = blockstore_api.get_bundle_links(bundle_uuid, use_draft=draft_name).values()
+        result = {link.name: link.direct for link in links}
+        bundle_cache.set(cache_key, result)
+    return result
+
+
+def get_bundle_direct_links_with_cache(bundle_uuid, bundle_version=None, draft_name=None):
+    """
+    Get a dictionary of the direct links of the specified bundle, from cache if
+    possible.
+    """
+    if draft_name:
+        links = get_bundle_draft_direct_links_cached(bundle_uuid, draft_name)
+    else:
+        if bundle_version is None:
+            bundle_version = get_bundle_version_number(bundle_uuid)
+        links = get_bundle_version_direct_links_cached(bundle_uuid, bundle_version)
+    return links

openedx/core/djangolib/tests/test_blockstore_cache.py

@@ -0,0 +1,114 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for BundleCache
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import unittest
+
+from django.conf import settings
+from mock import patch
+from openedx.core.djangolib.blockstore_cache import BundleCache
+from openedx.core.lib import blockstore_api as api
+
+
+class TestWithBundleMixin(object):
+    """
+    Mixin that gives every test method access to a bundle + draft
+    """
+
+    @classmethod
+    def setUpClass(cls):
+        super(TestWithBundleMixin, cls).setUpClass()
+        cls.collection = api.create_collection(title="Collection")
+        cls.bundle = api.create_bundle(cls.collection.uuid, title="Test Bundle", slug="test")
+        cls.draft = api.get_or_create_bundle_draft(cls.bundle.uuid, draft_name="test-draft")
+
+
+@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server")
+@patch('openedx.core.djangolib.blockstore_cache.MAX_BLOCKSTORE_CACHE_DELAY', 0)
+class BundleCacheTest(TestWithBundleMixin, unittest.TestCase):
+    """
+    Tests for BundleCache
+    """
+
+    def test_bundle_cache(self):
+        """
+        Test caching data related to a bundle (no draft)
+        """
+        cache = BundleCache(self.bundle.uuid)
+
+        key1 = ("some", "key", "1")
+        key2 = ("key2", )
+
+        value1 = "value1"
+        cache.set(key1, value1)
+        value2 = {"this is": "a dict", "for": "key2"}
+        cache.set(key2, value2)
+        self.assertEqual(cache.get(key1), value1)
+        self.assertEqual(cache.get(key2), value2)
+
+        # Now publish a new version of the bundle:
+        api.write_draft_file(self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version")
+        api.commit_draft(self.draft.uuid)
+
+        # Now the cache should be invalidated
+        # (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0)
+        self.assertEqual(cache.get(key1), None)
+        self.assertEqual(cache.get(key2), None)
+
+    def test_bundle_draft_cache(self):
+        """
+        Test caching data related to a bundle draft
+        """
+        cache = BundleCache(self.bundle.uuid, draft_name=self.draft.name)
+
+        key1 = ("some", "key", "1")
+        key2 = ("key2", )
+
+        value1 = "value1"
+        cache.set(key1, value1)
+        value2 = {"this is": "a dict", "for": "key2"}
+        cache.set(key2, value2)
+        self.assertEqual(cache.get(key1), value1)
+        self.assertEqual(cache.get(key2), value2)
+
+        # Now make a change to the draft (doesn't matter if we commit it or not)
+        api.write_draft_file(self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version")
+
+        # Now the cache should be invalidated
+        # (immediately since we set MAX_BLOCKSTORE_CACHE_DELAY to 0)
+        self.assertEqual(cache.get(key1), None)
+        self.assertEqual(cache.get(key2), None)
+
+
+@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server")
+class BundleCacheClearTest(TestWithBundleMixin, unittest.TestCase):
+    """
+    Tests for BundleCache's clear() method.
+    Requires MAX_BLOCKSTORE_CACHE_DELAY to be non-zero. This clear() method does
+    not actually clear the cache but rather just means "a new bundle/draft
+    version has been created, so immediately start reading/writing cache keys
+    using the new version number.
+    """
+
+    def test_bundle_cache_clear(self):
+        """
+        Test the cache clear() method
+        """
+        cache = BundleCache(self.bundle.uuid)
+        key1 = ("some", "key", "1")
+        value1 = "value1"
+        cache.set(key1, value1)
+        self.assertEqual(cache.get(key1), value1)
+
+        # Now publish a new version of the bundle:
+        api.write_draft_file(self.draft.uuid, "test.txt", "we need a changed file in order to publish a new version")
+        api.commit_draft(self.draft.uuid)
+
+        # Now the cache will not be immediately invalidated; it takes up to MAX_BLOCKSTORE_CACHE_DELAY seconds.
+        # Since this is a new bundle and we _just_ accessed the cache for the first time, we can be confident
+        # it won't yet be automatically invalidated.
+        self.assertEqual(cache.get(key1), value1)
+        # Now "clear" the cache, forcing the check of the new version:
+        cache.clear()
+        self.assertEqual(cache.get(key1), None)

openedx/core/lib/blockstore_api/__init__.py

@@ -0,0 +1,52 @@
+"""
+API Client for Blockstore
+
+This API does not do any caching; consider using BundleCache or (in
+openedx.core.djangolib.blockstore_cache) together with these API methods for
+improved performance.
+"""
+from .models import (
+    Collection,
+    Bundle,
+    Draft,
+    BundleFile,
+    DraftFile,
+    LinkReference,
+    LinkDetails,
+    DraftLinkDetails,
+)
+from .methods import (
+    # Collections:
+    get_collection,
+    create_collection,
+    update_collection,
+    delete_collection,
+    # Bundles:
+    get_bundle,
+    create_bundle,
+    update_bundle,
+    delete_bundle,
+    # Drafts:
+    get_draft,
+    get_or_create_bundle_draft,
+    write_draft_file,
+    set_draft_link,
+    commit_draft,
+    delete_draft,
+    # Bundles or drafts:
+    get_bundle_files,
+    get_bundle_files_dict,
+    get_bundle_file_metadata,
+    get_bundle_file_data,
+    get_bundle_version_files,
+    # Links:
+    get_bundle_links,
+    get_bundle_version_links,
+)
+from .exceptions import (
+    BlockstoreException,
+    CollectionNotFound,
+    BundleNotFound,
+    DraftNotFound,
+    BundleFileNotFound,
+)

openedx/core/lib/blockstore_api/exceptions.py

@@ -0,0 +1,28 @@
+"""
+Exceptions that may be raised by the Blockstore API
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+
+class BlockstoreException(Exception):
+    pass
+
+
+class NotFound(BlockstoreException):
+    pass
+
+
+class CollectionNotFound(NotFound):
+    pass
+
+
+class BundleNotFound(NotFound):
+    pass
+
+
+class DraftNotFound(NotFound):
+    pass
+
+
+class BundleFileNotFound(NotFound):
+    pass

openedx/core/lib/blockstore_api/methods.py

@@ -0,0 +1,392 @@
+"""
+API Client methods for working with Blockstore bundles and drafts
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import base64
+from uuid import UUID
+
+import dateutil.parser
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+import requests
+import six
+
+from .models import (
+    Bundle,
+    Collection,
+    Draft,
+    BundleFile,
+    DraftFile,
+    LinkDetails,
+    LinkReference,
+    DraftLinkDetails,
+)
+from .exceptions import (
+    NotFound,
+    CollectionNotFound,
+    BundleNotFound,
+    DraftNotFound,
+    BundleFileNotFound,
+)
+
+
+def api_url(*path_parts):
+    if not settings.BLOCKSTORE_API_URL or not settings.BLOCKSTORE_API_URL.endswith('/api/v1/'):
+        raise ImproperlyConfigured('BLOCKSTORE_API_URL must be set and should end with /api/v1/')
+    return settings.BLOCKSTORE_API_URL + '/'.join(path_parts)
+
+
+def api_request(method, url, **kwargs):
+    """
+    Helper method for making a request to the Blockstore REST API
+    """
+    if not settings.BLOCKSTORE_API_AUTH_TOKEN:
+        raise ImproperlyConfigured("Cannot use Blockstore unless BLOCKSTORE_API_AUTH_TOKEN is set.")
+    kwargs.setdefault('headers', {})['Authorization'] = "Token {}".format(settings.BLOCKSTORE_API_AUTH_TOKEN)
+    response = requests.request(method, url, **kwargs)
+    if response.status_code == 404:
+        raise NotFound
+    response.raise_for_status()
+    if response.status_code == 204:
+        return None  # No content
+    return response.json()
+
+
+def _collection_from_response(data):
+    """
+    Given data about a Collection returned by any blockstore REST API, convert it to
+    a Collection instance.
+    """
+    return Collection(uuid=UUID(data['uuid']), title=data['title'])
+
+
+def _bundle_from_response(data):
+    """
+    Given data about a Bundle returned by any blockstore REST API, convert it to
+    a Bundle instance.
+    """
+    return Bundle(
+        uuid=UUID(data['uuid']),
+        title=data['title'],
+        description=data['description'],
+        slug=data['slug'],
+        # drafts: Convert from a dict of URLs to a dict of UUIDs:
+        drafts={draft_name: UUID(url.split('/')[-1]) for (draft_name, url) in data['drafts'].items()},
+        # versions field: take the last one and convert it from URL to an int
+        # i.e.: [..., 'https://blockstore/api/v1/bundle_versions/bundle_uuid,15'] -> 15
+        latest_version=int(data['versions'][-1].split(',')[-1]) if data['versions'] else 0,
+    )
+
+
+def _draft_from_response(data):
+    """
+    Given data about a Draft returned by any blockstore REST API, convert it to
+    a Draft instance.
+    """
+    return Draft(
+        uuid=UUID(data['uuid']),
+        bundle_uuid=UUID(data['bundle_uuid']),
+        name=data['name'],
+        updated_at=dateutil.parser.parse(data['staged_draft']['updated_at']),
+        files={
+            path: DraftFile(path=path, **file)
+            for path, file in data['staged_draft']['files'].items()
+        },
+        links={
+            name: DraftLinkDetails(
+                name=name,
+                direct=LinkReference(**link["direct"]),
+                indirect=[LinkReference(**ind) for ind in link["indirect"]],
+                modified=link["modified"],
+            )
+            for name, link in data['staged_draft']['links'].items()
+        }
+    )
+
+
+def get_collection(collection_uuid):
+    """
+    Retrieve metadata about the specified collection
+
+    Raises CollectionNotFound if the collection does not exist
+    """
+    assert isinstance(collection_uuid, UUID)
+    try:
+        data = api_request('get', api_url('collections', str(collection_uuid)))
+    except NotFound:
+        raise CollectionNotFound("Collection {} does not exist.".format(collection_uuid))
+    return _collection_from_response(data)
+
+
+def create_collection(title):
+    """
+    Create a new collection.
+    """
+    result = api_request('post', api_url('collections'), json={"title": title})
+    return _collection_from_response(result)
+
+
+def update_collection(collection_uuid, title):
+    """
+    Update a collection's title
+    """
+    assert isinstance(collection_uuid, UUID)
+    data = {"title": title}
+    result = api_request('patch', api_url('collections', str(collection_uuid)), json=data)
+    return _collection_from_response(result)
+
+
+def delete_collection(collection_uuid):
+    """
+    Delete a collection
+    """
+    assert isinstance(collection_uuid, UUID)
+    api_request('delete', api_url('collections', str(collection_uuid)))
+
+
+def get_bundle(bundle_uuid):
+    """
+    Retrieve metadata about the specified bundle
+
+    Raises BundleNotFound if the bundle does not exist
+    """
+    assert isinstance(bundle_uuid, UUID)
+    try:
+        data = api_request('get', api_url('bundles', str(bundle_uuid)))
+    except NotFound:
+        raise BundleNotFound("Bundle {} does not exist.".format(bundle_uuid))
+    return _bundle_from_response(data)
+
+
+def create_bundle(collection_uuid, slug, title="New Bundle", description=""):
+    """
+    Create a new bundle.
+
+    Note that description is currently required.
+    """
+    result = api_request('post', api_url('bundles'), json={
+        "collection_uuid": str(collection_uuid),
+        "slug": slug,
+        "title": title,
+        "description": description,
+    })
+    return _bundle_from_response(result)
+
+
+def update_bundle(bundle_uuid, **fields):
+    """
+    Update a bundle's title, description, slug, or collection.
+    """
+    assert isinstance(bundle_uuid, UUID)
+    data = {}
+    # Most validation will be done by Blockstore, so we don't worry too much about data validation
+    for str_field in ("title", "description", "slug"):
+        if str_field in fields:
+            data[str_field] = fields.pop(str_field)
+    if "collection_uuid" in fields:
+        data["collection_uuid"] = str(fields.pop("collection_uuid"))
+    if fields:
+        raise ValueError("Unexpected extra fields passed to update_bundle: {}".format(fields.keys()))
+    result = api_request('patch', api_url('bundles', str(bundle_uuid)), json=data)
+    return _bundle_from_response(result)
+
+
+def delete_bundle(bundle_uuid):
+    """
+    Delete a bundle
+    """
+    assert isinstance(bundle_uuid, UUID)
+    api_request('delete', api_url('bundles', str(bundle_uuid)))
+
+
+def get_draft(draft_uuid):
+    """
+    Retrieve metadata about the specified draft.
+    If you don't know the draft's UUID, look it up using get_bundle()
+    """
+    assert isinstance(draft_uuid, UUID)
+    try:
+        data = api_request('get', api_url('drafts', str(draft_uuid)))
+    except NotFound:
+        raise DraftNotFound("Draft does not exist: {}".format(draft_uuid))
+    return _draft_from_response(data)
+
+
+def get_or_create_bundle_draft(bundle_uuid, draft_name):
+    """
+    Retrieve metadata about the specified draft.
+    """
+    bundle = get_bundle(bundle_uuid)
+    try:
+        return get_draft(bundle.drafts[draft_name])  # pylint: disable=unsubscriptable-object
+    except KeyError:
+        # The draft doesn't exist yet, so create it:
+        response = api_request('post', api_url('drafts'), json={
+            "bundle_uuid": str(bundle_uuid),
+            "name": draft_name,
+        })
+        # The result of creating a draft doesn't include all the fields we want, so retrieve it now:
+        return get_draft(UUID(response["uuid"]))
+
+
+def commit_draft(draft_uuid):
+    """
+    Commit all of the pending changes in the draft, creating a new version of
+    the associated bundle.
+
+    Does not return any value.
+    """
+    api_request('post', api_url('drafts', str(draft_uuid), 'commit'))
+
+
+def delete_draft(draft_uuid):
+    """
+    Delete the specified draft, removing any staged changes/files/deletes.
+
+    Does not return any value.
+    """
+    api_request('delete', api_url('drafts', str(draft_uuid)))
+
+
+def get_bundle_version_files(bundle_uuid, version_number):
+    """
+    Get a list of the files in the specified bundle version
+    """
+    if version_number == 0:
+        return []
+    version_url = api_url('bundle_versions', str(bundle_uuid) + ',' + str(version_number))
+    version_info = api_request('get', version_url)
+    return [BundleFile(path=path, **file_metadata) for path, file_metadata in version_info["snapshot"]["files"].items()]
+
+
+def get_bundle_version_links(bundle_uuid, version_number):
+    """
+    Get a dictionary of the links in the specified bundle version
+    """
+    if version_number == 0:
+        return []
+    version_url = api_url('bundle_versions', str(bundle_uuid) + ',' + str(version_number))
+    version_info = api_request('get', version_url)
+    return {
+        name: LinkDetails(
+            name=name,
+            direct=LinkReference(**link["direct"]),
+            indirect=[LinkReference(**ind) for ind in link["indirect"]],
+        )
+        for name, link in version_info['snapshot']['links'].items()
+    }
+
+
+def get_bundle_files_dict(bundle_uuid, use_draft=None):
+    """
+    Get a dict of all the files in the specified bundle.
+
+    Returns a dict where the keys are the paths (strings) and the values are
+    BundleFile or DraftFile tuples.
+    """
+    bundle = get_bundle(bundle_uuid)
+    if use_draft and use_draft in bundle.drafts:  # pylint: disable=unsupported-membership-test
+        draft_uuid = bundle.drafts[use_draft]  # pylint: disable=unsubscriptable-object
+        return get_draft(draft_uuid).files
+    elif not bundle.latest_version:
+        # This bundle has no versions so definitely does not contain any files
+        return {}
+    else:
+        return {file_meta.path: file_meta for file_meta in get_bundle_version_files(bundle_uuid, bundle.latest_version)}
+
+
+def get_bundle_files(bundle_uuid, use_draft=None):
+    """
+    Get a flat list of all the files in the specified bundle or draft.
+    """
+    return get_bundle_files_dict(bundle_uuid, use_draft).values()
+
+
+def get_bundle_links(bundle_uuid, use_draft=None):
+    """
+    Get a dict of all the links in the specified bundle.
+
+    Returns a dict where the keys are the link names (strings) and the values
+    are LinkDetails or DraftLinkDetails tuples.
+    """
+    bundle = get_bundle(bundle_uuid)
+    if use_draft and use_draft in bundle.drafts:  # pylint: disable=unsupported-membership-test
+        draft_uuid = bundle.drafts[use_draft]  # pylint: disable=unsubscriptable-object
+        return get_draft(draft_uuid).links
+    elif not bundle.latest_version:
+        # This bundle has no versions so definitely does not contain any links
+        return {}
+    else:
+        return get_bundle_version_links(bundle_uuid, bundle.latest_version)
+
+
+def get_bundle_file_metadata(bundle_uuid, path, use_draft=None):
+    """
+    Get the metadata of the specified file.
+    """
+    assert isinstance(bundle_uuid, UUID)
+    files_dict = get_bundle_files_dict(bundle_uuid, use_draft=use_draft)
+    try:
+        return files_dict[path]
+    except KeyError:
+        raise BundleFileNotFound(
+            "Bundle {} (draft: {}) does not contain a file {}".format(bundle_uuid, use_draft, path)
+        )
+
+
+def get_bundle_file_data(bundle_uuid, path, use_draft=None):
+    """
+    Read all the data in the given bundle file and return it as a
+    binary string.
+
+    Do not use this for large files!
+    """
+    metadata = get_bundle_file_metadata(bundle_uuid, path, use_draft)
+    with requests.get(metadata.url, stream=True) as r:
+        return r.content
+
+
+def write_draft_file(draft_uuid, path, contents):
+    """
+    Create or overwrite the file at 'path' in the specified draft with the given
+    contents. To delete a file, pass contents=None.
+
+    If you don't know the draft's UUID, look it up using
+    get_or_create_bundle_draft()
+
+    Does not return anything.
+    """
+    api_request('patch', api_url('drafts', str(draft_uuid)), json={
+        'files': {
+            path: encode_str_for_draft(contents) if contents is not None else None,
+        },
+    })
+
+
+def set_draft_link(draft_uuid, link_name, bundle_uuid, version):
+    """
+    Create or replace the link with the given name in the specified draft so
+    that it points to the specified bundle version. To delete a link, pass
+    bundle_uuid=None, version=None.
+
+    If you don't know the draft's UUID, look it up using
+    get_or_create_bundle_draft()
+
+    Does not return anything.
+    """
+    api_request('patch', api_url('drafts', str(draft_uuid)), json={
+        'links': {
+            link_name: {"bundle_uuid": str(bundle_uuid), "version": version} if bundle_uuid is not None else None,
+        },
+    })
+
+
+def encode_str_for_draft(input_str):
+    """
+    Given a string, return UTF-8 representation that is then base64 encoded.
+    """
+    if isinstance(input_str, six.text_type):
+        binary = input_str.encode('utf8')
+    else:
+        binary = input_str
+    return base64.b64encode(binary)

openedx/core/lib/blockstore_api/models.py

@@ -0,0 +1,98 @@
+"""
+Data models used for Blockstore API Client
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+from datetime import datetime
+from uuid import UUID
+
+import attr
+import six
+
+
+def _convert_to_uuid(value):
+    if not isinstance(value, UUID):
+        return UUID(value)
+    return value
+
+
+@attr.s(frozen=True)
+class Collection(object):
+    """
+    Metadata about a blockstore collection
+    """
+    uuid = attr.ib(type=UUID, converter=_convert_to_uuid)
+    title = attr.ib(type=six.text_type)
+
+
+@attr.s(frozen=True)
+class Bundle(object):
+    """
+    Metadata about a blockstore bundle
+    """
+    uuid = attr.ib(type=UUID, converter=_convert_to_uuid)
+    title = attr.ib(type=six.text_type)
+    description = attr.ib(type=six.text_type)
+    slug = attr.ib(type=six.text_type)
+    drafts = attr.ib(type=dict)  # Dict of drafts, where keys are the draft names and values are draft UUIDs
+    # Note that if latest_version is 0, it means that no versions yet exist
+    latest_version = attr.ib(type=int, validator=attr.validators.instance_of(int))
+
+
+@attr.s(frozen=True)
+class Draft(object):
+    """
+    Metadata about a blockstore draft
+    """
+    uuid = attr.ib(type=UUID, converter=_convert_to_uuid)
+    bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid)
+    name = attr.ib(type=six.text_type)
+    updated_at = attr.ib(type=datetime, validator=attr.validators.instance_of(datetime))
+    files = attr.ib(type=dict)
+    links = attr.ib(type=dict)
+
+
+@attr.s(frozen=True)
+class BundleFile(object):
+    """
+    Metadata about a file in a blockstore bundle or draft.
+    """
+    path = attr.ib(type=six.text_type)
+    size = attr.ib(type=int)
+    url = attr.ib(type=six.text_type)
+    hash_digest = attr.ib(type=six.text_type)
+
+
+@attr.s(frozen=True)
+class DraftFile(BundleFile):
+    """
+    Metadata about a file in a blockstore draft.
+    """
+    modified = attr.ib(type=bool)  # Was this file modified in the draft?
+
+
+@attr.s(frozen=True)
+class LinkReference(object):
+    """
+    A pointer to a specific BundleVersion
+    """
+    bundle_uuid = attr.ib(type=UUID, converter=_convert_to_uuid)
+    version = attr.ib(type=int)
+    snapshot_digest = attr.ib(type=six.text_type)
+
+
+@attr.s(frozen=True)
+class LinkDetails(object):
+    """
+    Details about a specific link in a BundleVersion or Draft
+    """
+    name = attr.ib(type=str)
+    direct = attr.ib(type=LinkReference)
+    indirect = attr.ib(type=list)  # List of LinkReference objects
+
+
+@attr.s(frozen=True)
+class DraftLinkDetails(LinkDetails):
+    """
+    Details about a specific link in a Draft
+    """
+    modified = attr.ib(type=bool)

openedx/core/lib/blockstore_api/tests/test_blockstore_api.py

@@ -0,0 +1,195 @@
+# -*- coding: utf-8 -*-
+"""
+Tests for xblock_utils.py
+"""
+from __future__ import absolute_import, division, print_function, unicode_literals
+import unittest
+from uuid import UUID
+
+from django.conf import settings
+from openedx.core.lib import blockstore_api as api
+
+# A fake UUID that won't represent any real bundle/draft/collection:
+BAD_UUID = UUID('12345678-0000-0000-0000-000000000000')
+
+
+@unittest.skipUnless(settings.RUN_BLOCKSTORE_TESTS, "Requires a running Blockstore server")
+class BlockstoreApiClientTest(unittest.TestCase):
+    """
+    Test for the Blockstore API Client.
+
+    The goal of these tests is not to test that Blockstore works correctly, but
+    that the API client can interact with it and all the API client methods
+    work.
+    """
+
+    # Collections
+
+    def test_nonexistent_collection(self):
+        """ Request a collection that doesn't exist -> CollectionNotFound """
+        with self.assertRaises(api.CollectionNotFound):
+            api.get_collection(BAD_UUID)
+
+    def test_collection_crud(self):
+        """ Create, Fetch, Update, and Delete a Collection """
+        title = "Fire 🔥 Collection"
+        # Create:
+        coll = api.create_collection(title)
+        self.assertEqual(coll.title, title)
+        self.assertIsInstance(coll.uuid, UUID)
+        # Fetch:
+        coll2 = api.get_collection(coll.uuid)
+        self.assertEqual(coll, coll2)
+        # Update:
+        new_title = "Air 🌀 Collection"
+        coll3 = api.update_collection(coll.uuid, title=new_title)
+        self.assertEqual(coll3.title, new_title)
+        coll4 = api.get_collection(coll.uuid)
+        self.assertEqual(coll4.title, new_title)
+        # Delete:
+        api.delete_collection(coll.uuid)
+        with self.assertRaises(api.CollectionNotFound):
+            api.get_collection(coll.uuid)
+
+    # Bundles
+
+    def test_nonexistent_bundle(self):
+        """ Request a bundle that doesn't exist -> BundleNotFound """
+        with self.assertRaises(api.BundleNotFound):
+            api.get_bundle(BAD_UUID)
+
+    def test_bundle_crud(self):
+        """ Create, Fetch, Update, and Delete a Bundle """
+        coll = api.create_collection("Test Collection")
+        args = {
+            "title": "Water 💧 Bundle",
+            "slug": "h2o",
+            "description": "Sploosh",
+        }
+        # Create:
+        bundle = api.create_bundle(coll.uuid, **args)
+        for attr, value in args.items():
+            self.assertEqual(getattr(bundle, attr), value)
+        self.assertIsInstance(bundle.uuid, UUID)
+        # Fetch:
+        bundle2 = api.get_bundle(bundle.uuid)
+        self.assertEqual(bundle, bundle2)
+        # Update:
+        new_description = "Water Nation Bending Lessons"
+        bundle3 = api.update_bundle(bundle.uuid, description=new_description)
+        self.assertEqual(bundle3.description, new_description)
+        bundle4 = api.get_bundle(bundle.uuid)
+        self.assertEqual(bundle4.description, new_description)
+        # Delete:
+        api.delete_bundle(bundle.uuid)
+        with self.assertRaises(api.BundleNotFound):
+            api.get_bundle(bundle.uuid)
+
+    # Drafts, files, and reading/writing file contents:
+
+    def test_nonexistent_draft(self):
+        """ Request a draft that doesn't exist -> DraftNotFound """
+        with self.assertRaises(api.DraftNotFound):
+            api.get_draft(BAD_UUID)
+
+    def test_drafts_and_files(self):
+        """
+        Test creating, reading, writing, committing, and reverting drafts and
+        files.
+        """
+        coll = api.create_collection("Test Collection")
+        bundle = api.create_bundle(coll.uuid, title="Earth 🗿 Bundle", slug="earth", description="another test bundle")
+        # Create a draft
+        draft = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft")
+        self.assertEqual(draft.bundle_uuid, bundle.uuid)
+        self.assertEqual(draft.name, "test-draft")
+        self.assertGreaterEqual(draft.updated_at.year, 2019)
+        # And retrieve it again:
+        draft2 = api.get_or_create_bundle_draft(bundle.uuid, draft_name="test-draft")
+        self.assertEqual(draft, draft2)
+        # Also test retrieving using get_draft
+        draft3 = api.get_draft(draft.uuid)
+        self.assertEqual(draft, draft3)
+
+        # Write a file into the bundle:
+        api.write_draft_file(draft.uuid, "test.txt", "initial version")
+        # Now the file should be visible in the draft:
+        draft_contents = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name)
+        self.assertEqual(draft_contents, "initial version")
+        api.commit_draft(draft.uuid)
+
+        # Write a new version into the draft:
+        api.write_draft_file(draft.uuid, "test.txt", "modified version")
+        published_contents = api.get_bundle_file_data(bundle.uuid, "test.txt")
+        self.assertEqual(published_contents, "initial version")
+        draft_contents2 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name)
+        self.assertEqual(draft_contents2, "modified version")
+        # Now delete the draft:
+        api.delete_draft(draft.uuid)
+        draft_contents3 = api.get_bundle_file_data(bundle.uuid, "test.txt", use_draft=draft.name)
+        # Confirm the file is now reset:
+        self.assertEqual(draft_contents3, "initial version")
+
+        # Finaly, test the get_bundle_file* methods:
+        file_info1 = api.get_bundle_file_metadata(bundle.uuid, "test.txt")
+        self.assertEqual(file_info1.path, "test.txt")
+        self.assertEqual(file_info1.size, len("initial version"))
+        self.assertEqual(file_info1.hash_digest, "a45a5c6716276a66c4005534a51453ab16ea63c4")
+
+        self.assertEqual(api.get_bundle_files(bundle.uuid), [file_info1])
+        self.assertEqual(api.get_bundle_files_dict(bundle.uuid), {
+            "test.txt": file_info1,
+        })
+
+    # Links
+
+    def test_links(self):
+        """
+        Test operations involving bundle links.
+        """
+        coll = api.create_collection("Test Collection")
+        # Create two library bundles and a course bundle:
+        lib1_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib1")
+        lib1_draft = api.get_or_create_bundle_draft(lib1_bundle.uuid, draft_name="test-draft")
+        lib2_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="lib2")
+        lib2_draft = api.get_or_create_bundle_draft(lib2_bundle.uuid, draft_name="other-draft")
+        course_bundle = api.create_bundle(coll.uuid, title="Library 1", slug="course")
+        course_draft = api.get_or_create_bundle_draft(course_bundle.uuid, draft_name="test-draft")
+
+        # To create links, we need valid BundleVersions, which requires having committed at least one change:
+        api.write_draft_file(lib1_draft.uuid, "lib1-data.txt", "hello world")
+        api.commit_draft(lib1_draft.uuid)  # Creates version 1
+        api.write_draft_file(lib2_draft.uuid, "lib2-data.txt", "hello world")
+        api.commit_draft(lib2_draft.uuid)  # Creates version 1
+
+        # Lib2 has no links:
+        self.assertFalse(api.get_bundle_links(lib2_bundle.uuid))
+
+        # Create a link from lib2 to lib1
+        link1_name = "lib2_to_lib1"
+        api.set_draft_link(lib2_draft.uuid, link1_name, lib1_bundle.uuid, version=1)
+        # Now confirm the link exists in the draft:
+        lib2_draft_links = api.get_bundle_links(lib2_bundle.uuid, use_draft=lib2_draft.name)
+        self.assertIn(link1_name, lib2_draft_links)
+        self.assertEqual(lib2_draft_links[link1_name].direct.bundle_uuid, lib1_bundle.uuid)
+        self.assertEqual(lib2_draft_links[link1_name].direct.version, 1)
+        # Now commit the change to lib2:
+        api.commit_draft(lib2_draft.uuid)  # Creates version 2
+
+        # Now create a link from course to lib2
+        link2_name = "course_to_lib2"
+        api.set_draft_link(course_draft.uuid, link2_name, lib2_bundle.uuid, version=2)
+        api.commit_draft(course_draft.uuid)
+
+        # And confirm the link exists in the resulting bundle version:
+        course_links = api.get_bundle_links(course_bundle.uuid)
+        self.assertIn(link2_name, course_links)
+        self.assertEqual(course_links[link2_name].direct.bundle_uuid, lib2_bundle.uuid)
+        self.assertEqual(course_links[link2_name].direct.version, 2)
+        # And since the links go course->lib2->lib1, course has an indirect link to lib1:
+        self.assertEqual(course_links[link2_name].indirect[0].bundle_uuid, lib1_bundle.uuid)
+        self.assertEqual(course_links[link2_name].indirect[0].version, 1)
+
+        # Finally, test deleting a link from course's draft:
+        api.set_draft_link(course_draft.uuid, link2_name, None, None)
+        self.assertFalse(api.get_bundle_links(course_bundle.uuid, use_draft=course_draft.name))

openedx/core/lib/xblock_builtin/xblock_discussion/xblock_discussion/__init__.py

@@ -263,6 +263,8 @@ class DiscussionXBlock(XBlock, StudioEditableXBlockMixin, XmlParserMixin):
         """
         Attempt to load definition XML from "discussion" folder in OLX, than parse it and update block fields
         """
+        if node.get('url_name') is None:
+            return  # Newer/XBlock XML format - no need to load an additional file.
         try:
             definition_xml, _ = cls.load_definition_xml(node, runtime, block.scope_ids.def_id)
         except Exception as err:  # pylint: disable=broad-except

setup.py

@@ -72,6 +72,7 @@ setup(
             "announcements = openedx.features.announcements.apps:AnnouncementsConfig",
             "ace_common = openedx.core.djangoapps.ace_common.apps:AceCommonConfig",
             "credentials = openedx.core.djangoapps.credentials.apps:CredentialsConfig",
+            "content_libraries = openedx.core.djangoapps.content_libraries.apps:ContentLibrariesConfig",
             "discussion = lms.djangoapps.discussion.apps:DiscussionConfig",
             "grades = lms.djangoapps.grades.apps:GradesConfig",
             "plugins = openedx.core.djangoapps.plugins.apps:PluginsConfig",
@@ -87,6 +88,7 @@ setup(
         "cms.djangoapp": [
             "announcements = openedx.features.announcements.apps:AnnouncementsConfig",
             "ace_common = openedx.core.djangoapps.ace_common.apps:AceCommonConfig",
+            "content_libraries = openedx.core.djangoapps.content_libraries.apps:ContentLibrariesConfig",
             # Importing an LMS app into the Studio process is not a good
             # practice. We're ignoring this for Discussions here because its
             # placement in LMS is a historical artifact. The eventual goal is to
@@ -102,5 +104,17 @@ setup(
             "user_authn = openedx.core.djangoapps.user_authn.apps:UserAuthnConfig",
             "instructor = lms.djangoapps.instructor.apps:InstructorConfig",
         ],
+        'definition_key': [
+            'bundle-olx = openedx.core.djangoapps.xblock.learning_context.keys:BundleDefinitionLocator',
+        ],
+        'context_key': [
+            'lib = openedx.core.djangoapps.content_libraries.keys:LibraryLocatorV2',
+        ],
+        'usage_key': [
+            'lb = openedx.core.djangoapps.content_libraries.keys:LibraryUsageLocatorV2',
+        ],
+        'openedx.learning_context': [
+            'lib = openedx.core.djangoapps.content_libraries.library_context:LibraryContextImpl',
+        ],
     }
 )