Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #12573 from edx/PERF-325

Browse Source 
[PERF-325] Add versioned course asset URLs when canonicalizing asset paths.
This commit is contained in:
Toby Lawrence
2016-06-15 09:05:13 -04:00
committed by GitHub
parent acc3a9bf76 849ebc5f22
commit 9abce102ff
9 changed files with 406 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

95
common/lib/xmodule/xmodule/contentstore/content.py
View File

@@ -5,16 +5,16 @@ from xmodule.assetstore.assetmgr import AssetManager
XASSET_LOCATION_TAG = 'c4x'
XASSET_SRCREF_PREFIX = 'xasset:'
XASSET_THUMBNAIL_TAIL_NAME = '.jpg'
STREAM_DATA_CHUNK_SIZE = 1024
VERSIONED_ASSETS_PREFIX = '/assets/courseware'
VERSIONED_ASSETS_PATTERN = r'/assets/courseware/([a-f0-9]{32})'
import os
import logging
import StringIO
from urlparse import urlparse, urlunparse, parse_qsl
from urllib import urlencode
from urllib import urlencode, quote_plus
from opaque_keys.edx.locator import AssetLocator
from opaque_keys.edx.keys import CourseKey, AssetKey
@@ -26,7 +26,7 @@ from PIL import Image
class StaticContent(object):
def __init__(self, loc, name, content_type, data, last_modified_at=None, thumbnail_location=None, import_path=None,
length=None, locked=False):
length=None, locked=False, content_digest=None):
self.location = loc
self.name = name # a display string which can be edited, and thus not part of the location which needs to be fixed
self.content_type = content_type
@@ -38,6 +38,7 @@ class StaticContent(object):
# cycles
self.import_path = import_path
self.locked = locked
self.content_digest = content_digest
@property
def is_thumbnail(self):
@@ -145,6 +146,40 @@ class StaticContent(object):
# try stripping off the leading slash and try again
return AssetKey.from_string(path[1:])
@staticmethod
def is_versioned_asset_path(path):
"""Determines whether the given asset path is versioned."""
return path.startswith(VERSIONED_ASSETS_PREFIX)
@staticmethod
def parse_versioned_asset_path(path):
"""
Examines an asset path and breaks it apart if it is versioned,
returning both the asset digest and the unversioned asset path,
which will normally be an AssetKey.
"""
asset_digest = None
asset_path = path
if StaticContent.is_versioned_asset_path(asset_path):
result = re.match(VERSIONED_ASSETS_PATTERN, asset_path)
if result is not None:
asset_digest = result.groups()[0]
asset_path = re.sub(VERSIONED_ASSETS_PATTERN, '', asset_path)
return (asset_digest, asset_path)
@staticmethod
def add_version_to_asset_path(path, version):
"""
Adds a prefix to an asset path indicating the asset's version.
"""
# Don't version an already-versioned path.
if StaticContent.is_versioned_asset_path(path):
return path
return VERSIONED_ASSETS_PREFIX + '/' + version + path
@staticmethod
def get_asset_key_from_path(course_key, path):
"""
@@ -172,7 +207,16 @@ class StaticContent(object):
return StaticContent.compute_location(course_key, path)
@staticmethod
def get_canonicalized_asset_path(course_key, path, base_url, excluded_exts):
def is_excluded_asset_type(path, excluded_exts):
"""
Check if this is an allowed file extension to serve.
Some files aren't served through the CDN in order to avoid same-origin policy/CORS-related issues.
"""
return any(path.lower().endswith(excluded_ext.lower()) for excluded_ext in excluded_exts)
@staticmethod
def get_canonicalized_asset_path(course_key, path, base_url, excluded_exts, encode=True):
"""
Returns a fully-qualified path to a piece of static content.
@@ -188,25 +232,27 @@ class StaticContent(object):
"""
# Break down the input path.
_, _, relative_path, params, query_string, fragment = urlparse(path)
_, _, relative_path, params, query_string, _ = urlparse(path)
# Convert our path to an asset key if it isn't one already.
asset_key = StaticContent.get_asset_key_from_path(course_key, relative_path)
# Check the status of the asset to see if this can be served via CDN aka publicly.
serve_from_cdn = False
content_digest = None
try:
content = AssetManager.find(asset_key, as_stream=True)
is_locked = getattr(content, "locked", True)
serve_from_cdn = not is_locked
serve_from_cdn = not getattr(content, "locked", True)
content_digest = content.content_digest
except (ItemNotFoundError, NotFoundError):
# If we can't find the item, just treat it as if it's locked.
serve_from_cdn = False
# See if this is an allowed file extension to serve. Some files aren't served through the
# CDN in order to avoid same-origin policy/CORS-related issues.
if any(relative_path.lower().endswith(excluded_ext.lower()) for excluded_ext in excluded_exts):
# Do a generic check to see if anything about this asset disqualifies it from being CDN'd.
is_excluded = False
if StaticContent.is_excluded_asset_type(relative_path, excluded_exts):
serve_from_cdn = False
is_excluded = True
# Update any query parameter values that have asset paths in them. This is for assets that
# require their own after-the-fact values, like a Flash file that needs the path of a config
@@ -215,15 +261,29 @@ class StaticContent(object):
updated_query_params = []
for query_name, query_val in query_params:
if query_val.startswith("/static/"):
new_val = StaticContent.get_canonicalized_asset_path(course_key, query_val, base_url, excluded_exts)
new_val = StaticContent.get_canonicalized_asset_path(
course_key, query_val, base_url, excluded_exts, encode=False)
updated_query_params.append((query_name, new_val))
else:
updated_query_params.append((query_name, query_val))
# Make sure we're encoding Unicode strings down to their byte string
# representation so that `urlencode` can handle it.
updated_query_params.append((query_name, query_val.encode('utf-8')))
serialized_asset_key = StaticContent.serialize_asset_key_with_slash(asset_key)
base_url = base_url if serve_from_cdn else ''
asset_path = serialized_asset_key
return urlunparse((None, base_url, serialized_asset_key, params, urlencode(updated_query_params), fragment))
# If the content has a digest (i.e. md5sum) value specified, create a versioned path to the asset using it.
if not is_excluded and content_digest:
asset_path = StaticContent.add_version_to_asset_path(serialized_asset_key, content_digest)
# Only encode this if told to. Important so that we don't double encode
# when working with paths that are in query parameters.
asset_path = asset_path.encode('utf-8')
if encode:
asset_path = quote_plus(asset_path, '/:+@')
return urlunparse((None, base_url.encode('utf-8'), asset_path, params, urlencode(updated_query_params), None))
def stream_data(self):
yield self._data
@@ -242,10 +302,10 @@ class StaticContent(object):
class StaticContentStream(StaticContent):
def __init__(self, loc, name, content_type, stream, last_modified_at=None, thumbnail_location=None, import_path=None,
length=None, locked=False):
length=None, locked=False, content_digest=None):
super(StaticContentStream, self).__init__(loc, name, content_type, None, last_modified_at=last_modified_at,
thumbnail_location=thumbnail_location, import_path=import_path,
length=length, locked=locked)
length=length, locked=locked, content_digest=content_digest)
self._stream = stream
def stream_data(self):
@@ -277,7 +337,8 @@ class StaticContentStream(StaticContent):
self._stream.seek(0)
content = StaticContent(self.location, self.name, self.content_type, self._stream.read(),
last_modified_at=self.last_modified_at, thumbnail_location=self.thumbnail_location,
import_path=self.import_path, length=self.length, locked=self.locked)
import_path=self.import_path, length=self.length, locked=self.locked,
content_digest=self.content_digest)
return content

6
common/lib/xmodule/xmodule/contentstore/mongo.py
View File

@@ -128,7 +128,8 @@ class MongoContentStore(ContentStore):
location, fp.displayname, fp.content_type, fp, last_modified_at=fp.uploadDate,
thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False)
length=fp.length, locked=getattr(fp, 'locked', False),
content_digest=getattr(fp, 'md5', None),
)
else:
with self.fs.get(content_id) as fp:
@@ -142,7 +143,8 @@ class MongoContentStore(ContentStore):
location, fp.displayname, fp.content_type, fp.read(), last_modified_at=fp.uploadDate,
thumbnail_location=thumbnail_location,
import_path=getattr(fp, 'import_path', None),
length=fp.length, locked=getattr(fp, 'locked', False)
length=fp.length, locked=getattr(fp, 'locked', False),
content_digest=getattr(fp, 'md5', None),
)
except NoFile:
if throw_on_not_found: