WIP: xblock pipeline work (#10176)

[PERF-303] Integer XBlocks/XModules into the static asset pipeline.

This PR, based on hackathon work from Christina/Andy, implements a way to discover all installed XBlocks and XModules and to enumerate their public assets, then pulling them in during the collectstatic phase and hashing them.  In turn, the methods for generating URLs to resources will then returned the hashed name for assets, allowing them to be served from nginx/CDNs, and cached heavily.

.jshintrc

@@ -148,6 +148,7 @@
         "JSON",
 
         // edX globals
-        "edx"
+        "edx",
+        "XBlock"
     ]
 }

cms/djangoapps/contentstore/views/preview.py

@@ -9,8 +9,9 @@ from django.http import Http404, HttpResponseBadRequest
 from django.contrib.auth.decorators import login_required
 from edxmako.shortcuts import render_to_string
 
-from openedx.core.lib.xblock_utils import replace_static_urls, wrap_xblock, wrap_fragment, wrap_xblock_aside,\
-    request_token
+from openedx.core.lib.xblock_utils import (
+    replace_static_urls, wrap_xblock, wrap_fragment, wrap_xblock_aside, request_token, xblock_local_resource_url,
+)
 from xmodule.x_module import PREVIEW_VIEWS, STUDENT_VIEW, AUTHOR_VIEW
 from xmodule.contentstore.django import contentstore
 from xmodule.error_module import ErrorDescriptor
@@ -31,7 +32,6 @@ from xblock_django.user_service import DjangoXBlockUserService
 
 from lms.djangoapps.lms_xblock.field_data import LmsFieldData
 from cms.lib.xblock.field_data import CmsFieldData
-from cms.lib.xblock.runtime import local_resource_url
 
 from util.sandboxing import can_execute_unsafe_code, get_python_lib_zip
 
@@ -115,7 +115,7 @@ class PreviewModuleSystem(ModuleSystem):  # pylint: disable=abstract-method
         }) + '?' + query
 
     def local_resource_url(self, block, uri):
-        return local_resource_url(block, uri)
+        return xblock_local_resource_url(block, uri)
 
     def applicable_aside_types(self, block):
         """

cms/envs/common.py

@@ -524,6 +524,7 @@ STATICFILES_FINDERS = [
     'openedx.core.djangoapps.theming.finders.ComprehensiveThemeFinder',
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
     'pipeline.finders.PipelineFinder',
 ]
 

cms/envs/devstack_optimized.py

@@ -32,6 +32,9 @@ DEBUG = True
 # Set REQUIRE_DEBUG to false so that it behaves like production
 REQUIRE_DEBUG = False
 
+# Fetch static files out of the pipeline's static root
+STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
+
 #  Serve static files at /static directly from the staticfiles directory under test root.
 # Note: optimized files for testing are generated with settings from test_static_optimized
 STATIC_URL = "/static/"

cms/envs/test_static_optimized.py

@@ -31,6 +31,7 @@ STATICFILES_STORAGE = 'openedx.core.lib.django_require.staticstorage.OptimizedCa
 STATICFILES_FINDERS = [
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
 ]
 
 # Redirect to the test_root folder within the repo

cms/lib/xblock/runtime.py

@@ -23,13 +23,3 @@ def handler_url(block, handler_name, suffix='', query='', thirdparty=False):
         url += '?' + query
 
     return url
-
-
-def local_resource_url(block, uri):
-    """
-    local_resource_url for Studio
-    """
-    return reverse('xblock_resource_url', kwargs={
-        'block_type': block.scope_ids.block_type,
-        'uri': uri,
-    })

cms/startup.py

@@ -13,6 +13,7 @@ from monkey_patch import (
     third_party_auth,
     django_db_models_options
 )
+from openedx.core.lib.xblock_utils import xblock_local_resource_url
 
 import xmodule.x_module
 import cms.lib.xblock.runtime
@@ -46,7 +47,7 @@ def run():
     # TODO: Remove this code when Runtimes are no longer created by modulestores
     # https://openedx.atlassian.net/wiki/display/PLAT/Convert+from+Storage-centric+runtimes+to+Application-centric+runtimes
     xmodule.x_module.descriptor_global_handler_url = cms.lib.xblock.runtime.handler_url
-    xmodule.x_module.descriptor_global_local_resource_url = cms.lib.xblock.runtime.local_resource_url
+    xmodule.x_module.descriptor_global_local_resource_url = xblock_local_resource_url
 
 
 def add_mimetypes():

common/lib/xmodule/xmodule/annotatable_module.py

@@ -162,3 +162,4 @@ class AnnotatableModule(AnnotatableFields, XModule):
 class AnnotatableDescriptor(AnnotatableFields, RawDescriptor):
     module_class = AnnotatableModule
     mako_template = "widgets/raw-edit.html"
+    resources_dir = None

common/lib/xmodule/xmodule/assets/split_test/public/js/split_test_author_view.js

@@ -1,5 +1,6 @@
 /* JavaScript for editing operations that can be done on the split test author view. */
 window.SplitTestAuthorView = function (runtime, element) {
+    "use strict";
     var $element = $(element);
     var splitTestLocator = $element.closest('.studio-xblock-wrapper').data('locator');
 

common/lib/xmodule/xmodule/assets/split_test/public/js/split_test_staff.js

@@ -1,11 +1,7 @@
-
-/**
- * Creates a new selector for managing toggling which child to show
- * @constructor
- */
-
-function ABTestSelector(runtime, elem) {
-    var _this = this;
+/* Creates a new selector for managing toggling which child to show. */
+window.ABTestSelector = function (runtime, elem) {
+    "use strict";
+    var _this = {};
     _this.elem = $(elem);
     _this.children = _this.elem.find('.split-test-child');
     _this.content_container = _this.elem.find('.split-test-child-container');
@@ -23,17 +19,13 @@ function ABTestSelector(runtime, elem) {
         });
     }
 
-    select = _this.elem.find('.split-test-select');
-    cur_group_id = select.val();
+    var select = _this.elem.find('.split-test-select');
+    var cur_group_id = select.val();
     select_child(cur_group_id);
 
     // bind the change event to the dropdown
     select.change(function() {
-        group_id = $(this).val()
+        var group_id = $(this).val();
         select_child(group_id);
     });
-
-}
-
-
-
+};

common/lib/xmodule/xmodule/assets/split_test/public/js/split_test_student.js

@@ -1,5 +1,6 @@
 /* Javascript for the Split Test XBlock. */
-function SplitTestStudentView(runtime, element) {
+window.SplitTestStudentView = function (runtime, element) {
+    "use strict";
     $.post(runtime.handlerUrl(element, 'log_child_render'));
     return {};
-}
+};

common/lib/xmodule/xmodule/assets/vertical/public/js/vertical_student_view.js

@@ -1,7 +1,6 @@
 /* JavaScript for Vertical Student View. */
 window.VerticalStudentView = function (runtime, element) {
-
-    'use strict';
+    "use strict";
     RequireJS.require(['js/bookmarks/views/bookmark_button'], function (BookmarkButton) {
         var $element = $(element);
         var $bookmarkButtonElement = $element.find('.bookmark-button');

common/lib/xmodule/xmodule/backcompat_module.py

@@ -60,6 +60,8 @@ def process_includes(fn):
 
 
 class SemanticSectionDescriptor(XModuleDescriptor):
+    resources_dir = None
+
     @classmethod
     @process_includes
     def from_xml(cls, xml_data, system, id_generator):
@@ -82,6 +84,8 @@ class SemanticSectionDescriptor(XModuleDescriptor):
 
 
 class TranslateCustomTagDescriptor(XModuleDescriptor):
+    resources_dir = None
+
     @classmethod
     def from_xml(cls, xml_data, system, id_generator):
         """

common/lib/xmodule/xmodule/capa_module.py

@@ -131,6 +131,7 @@ class CapaDescriptor(CapaFields, RawDescriptor):
     INDEX_CONTENT_TYPE = 'CAPA'
 
     module_class = CapaModule
+    resources_dir = None
 
     has_score = True
     show_in_read_only_mode = True

common/lib/xmodule/xmodule/conditional_module.py

@@ -184,6 +184,8 @@ class ConditionalDescriptor(ConditionalFields, SequenceDescriptor):
 
     module_class = ConditionalModule
 
+    resources_dir = None
+
     filename_extension = "xml"
 
     has_score = False

common/lib/xmodule/xmodule/course_module.py

@@ -789,6 +789,8 @@ class CourseDescriptor(CourseFields, SequenceDescriptor, LicenseMixin):
     """
     module_class = CourseModule
 
+    resources_dir = None
+
     def __init__(self, *args, **kwargs):
         """
         Expects the same arguments as XModuleDescriptor.__init__

common/lib/xmodule/xmodule/discussion_module.py

@@ -108,8 +108,9 @@ class DiscussionModule(DiscussionFields, XModule):
 
 
 class DiscussionDescriptor(DiscussionFields, MetadataOnlyEditingDescriptor, RawDescriptor):
-
     module_class = DiscussionModule
+    resources_dir = None
+
     # The discussion XML format uses `id` and `for` attributes,
     # but these would overload other module attributes, so we prefix them
     # for actual use in the code

common/lib/xmodule/xmodule/editing_module.py

@@ -20,6 +20,8 @@ class EditingDescriptor(EditingFields, MakoModuleDescriptor):
 
     This class is intended to be used as a mixin.
     """
+    resources_dir = None
+
     mako_template = "widgets/raw-edit.html"
 
     @property

common/lib/xmodule/xmodule/error_module.py

@@ -75,6 +75,7 @@ class ErrorDescriptor(ErrorFields, XModuleDescriptor):
     Module that provides a raw editing view of broken xml.
     """
     module_class = ErrorModule
+    resources_dir = None
 
     def get_html(self):
         return u''

common/lib/xmodule/xmodule/hidden_module.py

@@ -15,3 +15,4 @@ class HiddenModule(XModule):
 
 class HiddenDescriptor(RawDescriptor):
     module_class = HiddenModule
+    resources_dir = None

common/lib/xmodule/xmodule/html_module.py

@@ -118,6 +118,7 @@ class HtmlDescriptor(HtmlBlock, XmlDescriptor, EditingDescriptor):  # pylint: di
     """
     mako_template = "widgets/html-edit.html"
     module_class = HtmlModule
+    resources_dir = None
     filename_extension = "xml"
     template_dir_name = "html"
     show_in_read_only_mode = True

common/lib/xmodule/xmodule/imageannotation_module.py

@@ -151,6 +151,7 @@ class ImageAnnotationModule(AnnotatableFields, XModule):
 class ImageAnnotationDescriptor(AnnotatableFields, RawDescriptor):
     ''' Image annotation descriptor '''
     module_class = ImageAnnotationModule
+    resources_dir = None
     mako_template = "widgets/raw-edit.html"
 
     @property

common/lib/xmodule/xmodule/js/public

@@ -1 +0,0 @@
-../public/

common/lib/xmodule/xmodule/js/public/js/library_content_edit.js

@@ -0,0 +1 @@
+../../../assets/library_content/public/js/library_content_edit.js

common/lib/xmodule/xmodule/js/public/js/split_test_author_view.js

@@ -0,0 +1 @@
+../../../assets/split_test/public/js/split_test_author_view.js

common/lib/xmodule/xmodule/js/public/js/split_test_staff.js

@@ -0,0 +1 @@
+../../../assets/split_test/public/js/split_test_staff.js

common/lib/xmodule/xmodule/js/public/js/split_test_student.js

@@ -0,0 +1 @@
+../../../assets/split_test/public/js/split_test_student.js

common/lib/xmodule/xmodule/js/public/js/vertical_student_view.js

@@ -0,0 +1 @@
+../../../assets/vertical/public/js/vertical_student_view.js

common/lib/xmodule/xmodule/library_content_module.py

@@ -367,6 +367,9 @@ class LibraryContentDescriptor(LibraryContentFields, MakoModuleDescriptor, XmlDe
     """
     Descriptor class for LibraryContentModule XBlock.
     """
+
+    resources_dir = 'assets/library_content'
+
     module_class = LibraryContentModule
     mako_template = 'widgets/metadata-edit.html'
     js = {'coffee': [resource_string(__name__, 'js/src/vertical/edit.coffee')]}

common/lib/xmodule/xmodule/library_root_xblock.py

@@ -22,6 +22,8 @@ class LibraryRoot(XBlock):
     the library are its children. It contains metadata such as the library's
     display_name.
     """
+    resources_dir = None
+
     display_name = String(
         help=_("Enter the name of the library as it should appear in Studio."),
         default="Library",

common/lib/xmodule/xmodule/lti_module.py

@@ -899,6 +899,7 @@ class LTIDescriptor(LTIFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescri
     Descriptor for LTI Xmodule.
     """
     module_class = LTIModule
+    resources_dir = None
     grade_handler = module_attr('grade_handler')
     preview_handler = module_attr('preview_handler')
     lti_2_0_result_rest_handler = module_attr('lti_2_0_result_rest_handler')

common/lib/xmodule/xmodule/mako_module.py

@@ -59,5 +59,7 @@ class MakoModuleDescriptor(MakoTemplateBlockBase, XModuleDescriptor):  # pylint:
     """
     Mixin to use for XModule descriptors.
     """
+    resources_dir = None
+
     def get_html(self):
         return self.studio_view(None).content

common/lib/xmodule/xmodule/poll_module.py

@@ -145,6 +145,7 @@ class PollDescriptor(PollFields, MakoModuleDescriptor, XmlDescriptor):
     _child_tag_name = 'answer'
 
     module_class = PollModule
+    resources_dir = None
 
     @classmethod
     def definition_from_xml(cls, xml_object, system):

common/lib/xmodule/xmodule/randomize_module.py

@@ -98,6 +98,7 @@ class RandomizeModule(RandomizeFields, XModule):
 class RandomizeDescriptor(RandomizeFields, SequenceDescriptor):
     # the editing interface can be the same as for sequences -- just a container
     module_class = RandomizeModule
+    resources_dir = None
 
     filename_extension = "xml"
 

common/lib/xmodule/xmodule/raw_module.py

@@ -13,6 +13,8 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
     Module that provides a raw editing view of its data and children.  It
     requires that the definition xml is valid.
     """
+    resources_dir = None
+
     data = String(help="XML data for the module", default="", scope=Scope.content)
 
     @classmethod
@@ -42,6 +44,8 @@ class EmptyDataRawDescriptor(XmlDescriptor, XMLEditingDescriptor):
     Version of RawDescriptor for modules which may have no XML data,
     but use XMLEditingDescriptor for import/export handling.
     """
+    resources_dir = None
+
     data = String(default='', scope=Scope.content)
 
     @classmethod

common/lib/xmodule/xmodule/seq_module.py

@@ -402,6 +402,7 @@ class SequenceDescriptor(SequenceFields, ProctoringFields, MakoModuleDescriptor,
     """
     mako_template = 'widgets/sequence-edit.html'
     module_class = SequenceModule
+    resources_dir = None
 
     show_in_read_only_mode = True
 

common/lib/xmodule/xmodule/split_test_module.py

@@ -376,6 +376,8 @@ class SplitTestDescriptor(SplitTestFields, SequenceDescriptor, StudioEditableDes
     # the editing interface can be the same as for sequences -- just a container
     module_class = SplitTestModule
 
+    resources_dir = 'assets/split_test'
+
     filename_extension = "xml"
 
     mako_template = "widgets/metadata-only-edit.html"

common/lib/xmodule/xmodule/template_module.py

@@ -37,6 +37,7 @@ class CustomTagModule(XModule):
 class CustomTagDescriptor(RawDescriptor):
     """ Descriptor for custom tags.  Loads the template when created."""
     module_class = CustomTagModule
+    resources_dir = None
     template_dir_name = 'customtag'
 
     def render_template(self, system, xml_data):

common/lib/xmodule/xmodule/textannotation_module.py

@@ -147,6 +147,7 @@ class TextAnnotationModule(AnnotatableFields, XModule):
 class TextAnnotationDescriptor(AnnotatableFields, RawDescriptor):
     ''' Text Annotation Descriptor '''
     module_class = TextAnnotationModule
+    resources_dir = None
     mako_template = "widgets/raw-edit.html"
 
     @property

common/lib/xmodule/xmodule/vertical_block.py

@@ -24,6 +24,9 @@ class VerticalBlock(SequenceFields, XModuleFields, StudioEditableBlock, XmlParse
     """
     Layout XBlock for rendering subblocks vertically.
     """
+
+    resources_dir = 'assets/vertical'
+
     mako_template = 'widgets/sequence-edit.html'
     js_module_name = "VerticalBlock"
 

common/lib/xmodule/xmodule/videoannotation_module.py

@@ -154,6 +154,7 @@ class VideoAnnotationModule(AnnotatableFields, XModule):
 class VideoAnnotationDescriptor(AnnotatableFields, RawDescriptor):
     ''' Video annotation descriptor '''
     module_class = VideoAnnotationModule
+    resources_dir = None
     mako_template = "widgets/raw-edit.html"
 
     @property

common/lib/xmodule/xmodule/word_cloud_module.py

@@ -248,4 +248,5 @@ class WordCloudModule(WordCloudFields, XModule):
 class WordCloudDescriptor(WordCloudFields, MetadataOnlyEditingDescriptor, EmptyDataRawDescriptor):
     """Descriptor for WordCloud Xmodule."""
     module_class = WordCloudModule
+    resources_dir = None
     template_dir_name = 'word_cloud'

common/lib/xmodule/xmodule/xml_module.py

@@ -503,6 +503,8 @@ class XmlDescriptor(XmlParserMixin, XModuleDescriptor):  # pylint: disable=abstr
     """
     Mixin class for standardized parsing of XModule xml.
     """
+    resources_dir = None
+
     @classmethod
     def from_xml(cls, xml_data, system, id_generator):
         """

lms/djangoapps/lms_xblock/runtime.py

@@ -3,12 +3,13 @@ Module implementing `xblock.runtime.Runtime` functionality for the LMS
 """
 import re
 
-from django.core.urlresolvers import reverse
 from django.conf import settings
+from django.core.urlresolvers import reverse
 
 from badges.service import BadgingService
 from badges.utils import badges_enabled
 from openedx.core.djangoapps.user_api.course_tag import api as user_course_tag_api
+from openedx.core.lib.xblock_utils import xblock_local_resource_url
 from request_cache.middleware import RequestCache
 import xblock.reference.plugins
 from xmodule.library_tools import LibraryToolsService
@@ -126,11 +127,7 @@ def local_resource_url(block, uri):
     """
     local_resource_url for Studio
     """
-    path = reverse('xblock_resource_url', kwargs={
-        'block_type': block.scope_ids.block_type,
-        'uri': uri,
-    })
-    return '//{}{}'.format(settings.SITE_NAME, path)
+    return xblock_local_resource_url(block, uri)
 
 
 class LmsPartitionService(PartitionService):

lms/envs/common.py

@@ -1185,6 +1185,7 @@ STATICFILES_FINDERS = [
     'openedx.core.djangoapps.theming.finders.ComprehensiveThemeFinder',
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
     'pipeline.finders.PipelineFinder',
 ]
 

lms/envs/devstack_optimized.py

@@ -32,6 +32,9 @@ DEBUG = True
 # Set REQUIRE_DEBUG to false so that it behaves like production
 REQUIRE_DEBUG = False
 
+# Fetch static files out of the pipeline's static root
+STATICFILES_STORAGE = 'pipeline.storage.PipelineCachedStorage'
+
 #  Serve static files at /static directly from the staticfiles directory under test root.
 # Note: optimized files for testing are generated with settings from test_static_optimized
 STATIC_URL = "/static/"

lms/envs/test_static_optimized.py

@@ -44,6 +44,7 @@ STATICFILES_STORAGE = 'openedx.core.lib.django_require.staticstorage.OptimizedCa
 STATICFILES_FINDERS = [
     'django.contrib.staticfiles.finders.FileSystemFinder',
     'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+    'openedx.core.lib.xblock_pipeline.finder.XBlockPipelineFinder',
 ]
 
 # Redirect to the test_root folder within the repo

openedx/core/djangoapps/common_views/xblock.py

@@ -21,14 +21,10 @@ def xblock_resource(request, block_type, uri):  # pylint: disable=unused-argumen
     Return a package resource for the specified XBlock.
     """
     try:
+        # Figure out what the XBlock class is from the block type, and
+        # then open whatever resource has been requested.
         xblock_class = XBlock.load_class(block_type, select=settings.XBLOCK_SELECT_FUNCTION)
-        # Note: in debug mode, return any file rather than going through the XBlock which
-        # will only return public files. This allows unbundled files to be served up
-        # during development.
-        if settings.DEBUG:
-            content = pkg_resources.resource_stream(xblock_class.__module__, uri)
-        else:
-            content = xblock_class.open_local_resource(uri)
+        content = xblock_class.open_local_resource(uri)
     except IOError:
         log.info('Failed to load xblock resource', exc_info=True)
         raise Http404

openedx/core/lib/django_require/staticstorage.py

@@ -4,9 +4,10 @@
 
 from pipeline.storage import PipelineCachedStorage
 from require.storage import OptimizedFilesMixin
+from django_pipeline_forgiving.storages import PipelineForgivingStorage
 
 
-class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineCachedStorage):
+class OptimizedCachedRequireJsStorage(OptimizedFilesMixin, PipelineForgivingStorage):
     """
     Custom storage backend that is used by Django-require.
     """

openedx/core/lib/xblock_pipeline/finder.py

@@ -0,0 +1,142 @@
+"""
+Django pipeline finder for handling static assets required by XBlocks.
+"""
+from datetime import datetime
+import os
+from pkg_resources import resource_exists, resource_listdir, resource_isdir, resource_filename
+
+from xblock.core import XBlock
+
+from django.contrib.staticfiles import utils
+from django.contrib.staticfiles.finders import BaseFinder
+from django.contrib.staticfiles.storage import FileSystemStorage
+from django.core.files.storage import Storage
+
+
+class XBlockPackageStorage(Storage):
+    """
+    Storage implementation for accessing XBlock package resources.
+    """
+
+    RESOURCE_PREFIX = 'xblock/resources/'
+
+    def __init__(self, module, base_dir, *args, **kwargs):
+        """
+        Returns a static file storage if available in the given app.
+        """
+        super(XBlockPackageStorage, self).__init__(*args, **kwargs)
+        self.module = module
+        self.base_dir = base_dir
+
+        # Register a prefix that collectstatic will add to each path
+        self.prefix = os.path.join(self.RESOURCE_PREFIX, module)
+
+    def path(self, name):
+        """
+        Returns a file system filename for the specified file name.
+        """
+        return resource_filename(self.module, os.path.join(self.base_dir, name))
+
+    def exists(self, path):
+        """
+        Returns True if the specified path exists.
+        """
+        if self.base_dir is None:
+            return False
+
+        return resource_exists(self.module, os.path.join(self.base_dir, path))
+
+    def listdir(self, path):
+        """
+        Lists the directories beneath the specified path.
+        """
+        directories = []
+        files = []
+        for item in resource_listdir(self.module, os.path.join(self.base_dir, path)):
+            __, file_extension = os.path.splitext(item)
+            if file_extension not in [".py", ".pyc", ".scss"]:
+                if resource_isdir(self.module, os.path.join(self.base_dir, path, item)):
+                    directories.append(item)
+                else:
+                    files.append(item)
+        return directories, files
+
+    def open(self, name, mode='rb'):
+        """
+        Retrieves the specified file from storage.
+        """
+        path = self.path(name)
+        return FileSystemStorage(path).open(path, mode)
+
+    def size(self, name):
+        """
+        Returns the size of the package resource.
+        """
+        return os.path.getsize(self.path(name))
+
+    def accessed_time(self, name):
+        """
+        Returns a URL to the package resource.
+        """
+        return datetime.fromtimestamp(os.path.getatime(self.path(name)))
+
+    def created_time(self, name):
+        """
+        Returns the created time of the package resource.
+        """
+        return datetime.fromtimestamp(os.path.getctime(self.path(name)))
+
+    def modified_time(self, name):
+        """
+        Returns the modified time of the resource.
+        """
+        return datetime.fromtimestamp(os.path.getmtime(self.path(name)))
+
+    def url(self, name):
+        """
+        Note: package resources do not support URLs
+        """
+        raise NotImplementedError("Package resources do not support URLs")
+
+    def delete(self, name):
+        """
+        Note: deleting files from a package is not supported.
+        """
+        raise NotImplementedError("Deleting files from a package is not supported")
+
+
+class XBlockPipelineFinder(BaseFinder):
+    """
+    A static files finder that gets static assets from xblocks.
+    """
+    def __init__(self, *args, **kwargs):
+        super(XBlockPipelineFinder, self).__init__(*args, **kwargs)
+        xblock_classes = set()
+        for __, xblock_class in XBlock.load_classes():
+            xblock_classes.add(xblock_class)
+        self.package_storages = [
+            XBlockPackageStorage(xblock_class.__module__, xblock_class.get_resources_dir())
+            for xblock_class in xblock_classes
+        ]
+
+    def list(self, ignore_patterns):
+        """
+        List all static files in all xblock packages.
+        """
+        for storage in self.package_storages:
+            if storage.exists(''):  # check if storage location exists
+                for path in utils.get_files(storage, ignore_patterns):
+                    yield path, storage
+
+    def find(self, path, all=False):  # pylint: disable=redefined-builtin
+        """
+        Looks for files in the xblock package directories.
+        """
+        matches = []
+        for storage in self.package_storages:
+            if storage.exists(path):
+                match = storage.path(path)
+                if not all:
+                    return match
+                matches.append(match)
+        return matches

openedx/core/lib/xblock_utils.py

@@ -13,6 +13,8 @@ from lxml import html, etree
 from contracts import contract
 
 from django.conf import settings
+from django.contrib.staticfiles.storage import staticfiles_storage
+from django.core.urlresolvers import reverse
 from django.utils.timezone import UTC
 from django.utils.html import escape
 from django.contrib.auth.models import User
@@ -445,3 +447,23 @@ def get_course_update_items(course_updates, provided_index=0):
                         return payload
 
     return course_update_items
+
+
+def xblock_local_resource_url(block, uri):
+    """
+    Returns the URL for an XBlock's local resource.
+
+    Note: when running with the full Django pipeline, the file will be accessed
+    as a static asset which will use a CDN in production.
+    """
+    xblock_class = getattr(block.__class__, 'unmixed_class', block.__class__)
+    if settings.PIPELINE_ENABLED or not settings.REQUIRE_DEBUG:
+        return staticfiles_storage.url('xblock/resources/{package_name}/{path}'.format(
+            package_name=xblock_class.__module__,
+            path=uri
+        ))
+    else:
+        return reverse('xblock_resource_url', kwargs={
+            'block_type': block.scope_ids.block_type,
+            'uri': uri,
+        })

openedx/core/storage.py

@@ -1,6 +1,7 @@
 """
 Django storage backends for Open edX.
 """
+from django_pipeline_forgiving.storages import PipelineForgivingStorage
 from django.contrib.staticfiles.storage import StaticFilesStorage, CachedFilesMixin
 from pipeline.storage import PipelineMixin, NonPackagingMixin
 from require.storage import OptimizedFilesMixin
@@ -8,6 +9,7 @@ from openedx.core.djangoapps.theming.storage import ComprehensiveThemingAwareMix
 
 
 class ProductionStorage(
+        PipelineForgivingStorage,
         ComprehensiveThemingAwareMixin,
         OptimizedFilesMixin,
         PipelineMixin,

requirements/edx/base.txt

@@ -24,6 +24,7 @@ django-model-utils==2.3.1
 django-mptt==0.7.4
 django-oauth-plus==2.2.8
 django-oauth-toolkit==0.10.0
+django-pipeline-forgiving==1.0.0
 django-sekizai==0.8.2
 django-ses==0.7.0
 django-simple-history==1.6.3

requirements/edx/github.txt

@@ -70,7 +70,7 @@ git+https://github.com/edx/lettuce.git@0.2.20.002#egg=lettuce==0.2.20.002
 git+https://github.com/edx/pa11ycrawler.git@0.0.1#egg=pa11ycrawler
 
 # Our libraries:
-git+https://github.com/edx/XBlock.git@xblock-0.4.8#egg=XBlock==0.4.8
+git+https://github.com/edx/XBlock.git@xblock-0.4.10#egg=XBlock==0.4.10
 -e git+https://github.com/edx/codejail.git@6b17c33a89bef0ac510926b1d7fea2748b73aadd#egg=codejail
 -e git+https://github.com/edx/event-tracking.git@0.2.1#egg=event-tracking==0.2.1
 -e git+https://github.com/edx/django-splash.git@v0.2#egg=django-splash==0.2