Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #11025 from edx/PERF-224

Browse Source 
[PERF-224] Serve course assets from a CDN
This commit is contained in:
Toby Lawrence
2016-01-05 13:40:54 -05:00
parent 54d4ea8a80 77343df0d9
commit 56fe511a81
7 changed files with 428 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
common/lib/xmodule/xmodule/contentstore/content.py
View File

@@ -1,5 +1,8 @@
import re
import uuid
from xmodule.assetstore.assetmgr import AssetManager
XASSET_LOCATION_TAG = 'c4x'
XASSET_SRCREF_PREFIX = 'xasset:'
@@ -16,6 +19,8 @@ from urllib import urlencode
from opaque_keys.edx.locator import AssetLocator
from opaque_keys.edx.keys import CourseKey, AssetKey
from opaque_keys import InvalidKeyError
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.exceptions import NotFoundError
from PIL import Image
@@ -137,32 +142,79 @@ class StaticContent(object):
return AssetKey.from_string(path[1:])
@staticmethod
def convert_legacy_static_url_with_course_id(path, course_id):
def get_asset_key_from_path(course_key, path):
"""
Returns a path to a piece of static content when we are provided with a filepath and
a course_id
"""
# Generate url of urlparse.path component
scheme, netloc, orig_path, params, query, fragment = urlparse(path)
loc = StaticContent.compute_location(course_id, orig_path)
loc_url = StaticContent.serialize_asset_key_with_slash(loc)
Parses a path, extracting an asset key or creating one.
# parse the query params for "^/static/" and replace with the location url
orig_query = parse_qsl(query)
new_query_list = []
for query_name, query_value in orig_query:
Args:
course_key: key to the course which owns this asset
path: the path to said content
Returns:
AssetKey: the asset key that represents the path
"""
# Clean up the path, removing any static prefix and any leading slash.
if path.startswith('/static/'):
path = path[len('/static/'):]
path = path.lstrip('/')
try:
return AssetKey.from_string(path)
except InvalidKeyError:
# If we couldn't parse the path, just let compute_location figure it out.
# It's most likely a path like /image.png or something.
return StaticContent.compute_location(course_key, path)
@staticmethod
def get_canonicalized_asset_path(course_key, path, base_url):
"""
Returns a fully-qualified path to a piece of static content.
If a static asset CDN is configured, this path will include it.
Otherwise, the path will simply be relative.
Args:
course_key: key to the course which owns this asset
path: the path to said content
Returns:
string: fully-qualified path to asset
"""
# Break down the input path.
_, _, relative_path, params, query_string, fragment = urlparse(path)
# Convert our path to an asset key if it isn't one already.
asset_key = StaticContent.get_asset_key_from_path(course_key, relative_path)
# Check the status of the asset to see if this can be served via CDN aka publicly.
serve_from_cdn = False
try:
content = AssetManager.find(asset_key, as_stream=True)
is_locked = getattr(content, "locked", True)
serve_from_cdn = not is_locked
except (ItemNotFoundError, NotFoundError):
# If we can't find the item, just treat it as if it's locked.
serve_from_cdn = False
# Update any query parameter values that have asset paths in them. This is for assets that
# require their own after-the-fact values, like a Flash file that needs the path of a config
# file passed to it e.g. /static/visualization.swf?configFile=/static/visualization.xml
query_params = parse_qsl(query_string)
updated_query_params = []
for query_name, query_value in query_params:
if query_value.startswith("/static/"):
new_query = StaticContent.compute_location(
course_id,
query_value[len('/static/'):],
)
new_query_url = StaticContent.serialize_asset_key_with_slash(new_query)
new_query_list.append((query_name, new_query_url))
new_query_value = StaticContent.get_canonicalized_asset_path(course_key, query_value, base_url)
updated_query_params.append((query_name, new_query_value))
else:
new_query_list.append((query_name, query_value))
updated_query_params.append((query_name, query_value))
# Reconstruct with new path
return urlunparse((scheme, netloc, loc_url, params, urlencode(new_query_list), fragment))
serialized_asset_key = StaticContent.serialize_asset_key_with_slash(asset_key)
base_url = base_url if serve_from_cdn else ''
return urlunparse((None, base_url, serialized_asset_key, params, urlencode(updated_query_params), fragment))
def stream_data(self):
yield self._data

10
common/lib/xmodule/xmodule/tests/test_content.py
View File

@@ -4,6 +4,7 @@ import os
import unittest
import ddt
from path import Path as path
from xmodule.contentstore.content import StaticContent, StaticContentStream
from xmodule.contentstore.content import ContentStore
from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation
@@ -94,11 +95,6 @@ class ContentTest(unittest.TestCase):
content = StaticContent('loc', 'name', 'content_type', 'data')
self.assertIsNone(content.thumbnail_location)
def test_static_url_generation_from_courseid(self):
course_key = SlashSeparatedCourseKey('foo', 'bar', 'bz')
url = StaticContent.convert_legacy_static_url_with_course_id('images_course_image.jpg', course_key)
self.assertEqual(url, '/c4x/foo/bar/asset/images_course_image.jpg')
@ddt.data(
(u"monsters__.jpg", u"monsters__.jpg"),
(u"monsters__.png", u"monsters__-png.jpg"),
@@ -122,9 +118,9 @@ class ContentTest(unittest.TestCase):
self.assertEqual(AssetLocation(u'mitX', u'400', u'ignore', u'asset', u'subs__1eo_jXvZnE_.srt.sjson', None), asset_location)
def test_get_location_from_path(self):
asset_location = StaticContent.get_location_from_path(u'/c4x/foo/bar/asset/images_course_image.jpg')
asset_location = StaticContent.get_location_from_path(u'/c4x/a/b/asset/images_course_image.jpg')
self.assertEqual(
AssetLocation(u'foo', u'bar', None, u'asset', u'images_course_image.jpg', None),
AssetLocation(u'a', u'b', None, u'asset', u'images_course_image.jpg', None),
asset_location
)