Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Move contentserver to openedx/core

Browse Source
This commit is contained in:
Andy Armstrong
2016-10-07 09:07:37 -04:00
parent 9fa126dfac
commit 684aec8740
16 changed files with 39 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
common/djangoapps/contentserver/__init__.py
View File

@@ -1,3 +0,0 @@
"""
Serves course assets to end users.
"""

49
common/djangoapps/contentserver/admin.py
View File

@@ -1,49 +0,0 @@
"""
Django admin page for CourseAssetCacheTtlConfig, which allows you to configure the TTL
that gets used when sending cachability headers back with request course assets.
"""
from django.contrib import admin
from config_models.admin import ConfigurationModelAdmin
from .models import CourseAssetCacheTtlConfig, CdnUserAgentsConfig
class CourseAssetCacheTtlConfigAdmin(ConfigurationModelAdmin):
"""
Basic configuration for cache TTL.
"""
list_display = [
'cache_ttl'
]
def get_list_display(self, request):
"""
Restore default list_display behavior.
ConfigurationModelAdmin overrides this, but in a way that doesn't
respect the ordering. This lets us customize it the usual Django admin
way.
"""
return self.list_display
class CdnUserAgentsConfigAdmin(ConfigurationModelAdmin):
"""
Basic configuration for CDN user agent whitelist.
"""
list_display = [
'cdn_user_agents'
]
def get_list_display(self, request):
"""
Restore default list_display behavior.
ConfigurationModelAdmin overrides this, but in a way that doesn't
respect the ordering. This lets us customize it the usual Django admin
way.
"""
return self.list_display
admin.site.register(CourseAssetCacheTtlConfig, CourseAssetCacheTtlConfigAdmin)
admin.site.register(CdnUserAgentsConfig, CdnUserAgentsConfigAdmin)

49
common/djangoapps/contentserver/caching.py
View File

@@ -1,49 +0,0 @@
"""
Helper functions for caching course assets.
"""
from django.core.cache import caches
from django.core.cache.backends.base import InvalidCacheBackendError
from opaque_keys import InvalidKeyError
from xmodule.contentstore.content import STATIC_CONTENT_VERSION
# See if there's a "course_assets" cache configured, and if not, fallback to the default cache.
CONTENT_CACHE = caches['default']
try:
CONTENT_CACHE = caches['course_assets']
except InvalidCacheBackendError:
pass
def set_cached_content(content):
"""
Stores the given piece of content in the cache, using its location as the key.
"""
CONTENT_CACHE.set(unicode(content.location).encode("utf-8"), content, version=STATIC_CONTENT_VERSION)
def get_cached_content(location):
"""
Retrieves the given piece of content by its location if cached.
"""
return CONTENT_CACHE.get(unicode(location).encode("utf-8"), version=STATIC_CONTENT_VERSION)
def del_cached_content(location):
"""
Delete content for the given location, as well versions of the content without a run.
It's possible that the content could have been cached without knowing the course_key,
and so without having the run.
"""
def location_str(loc):
"""Force the location to a Unicode string."""
return unicode(loc).encode("utf-8")
locations = [location_str(location)]
try:
locations.append(location_str(location.replace(run=None)))
except InvalidKeyError:
# although deprecated keys allowed run=None, new keys don't if there is no version.
pass
CONTENT_CACHE.delete_many(locations, version=STATIC_CONTENT_VERSION)

323
common/djangoapps/contentserver/middleware.py
View File

@@ -1,323 +0,0 @@
"""
Middleware to serve assets.
"""
import logging
import datetime
import newrelic.agent
from django.http import (
HttpResponse, HttpResponseNotModified, HttpResponseForbidden,
HttpResponseBadRequest, HttpResponseNotFound, HttpResponsePermanentRedirect)
from student.models import CourseEnrollment
from contentserver.models import CourseAssetCacheTtlConfig, CdnUserAgentsConfig
from header_control import force_header_for_response
from xmodule.assetstore.assetmgr import AssetManager
from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
from xmodule.modulestore import InvalidLocationError
from opaque_keys import InvalidKeyError
from opaque_keys.edx.locator import AssetLocator
from .caching import get_cached_content, set_cached_content
from xmodule.modulestore.exceptions import ItemNotFoundError
from xmodule.exceptions import NotFoundError
# TODO: Soon as we have a reasonable way to serialize/deserialize AssetKeys, we need
# to change this file so instead of using course_id_partial, we're just using asset keys
log = logging.getLogger(__name__)
HTTP_DATE_FORMAT = "%a, %d %b %Y %H:%M:%S GMT"
class StaticContentServer(object):
"""
Serves course assets to end users. Colloquially referred to as "contentserver."
"""
def is_asset_request(self, request):
"""Determines whether the given request is an asset request"""
return (
request.path.startswith('/' + XASSET_LOCATION_TAG + '/')
or
request.path.startswith('/' + AssetLocator.CANONICAL_NAMESPACE)
or
StaticContent.is_versioned_asset_path(request.path)
)
def process_request(self, request):
"""Process the given request"""
asset_path = request.path
if self.is_asset_request(request):
# Make sure we can convert this request into a location.
if AssetLocator.CANONICAL_NAMESPACE in asset_path:
asset_path = asset_path.replace('block/', 'block@', 1)
# If this is a versioned request, pull out the digest and chop off the prefix.
requested_digest = None
if StaticContent.is_versioned_asset_path(asset_path):
requested_digest, asset_path = StaticContent.parse_versioned_asset_path(asset_path)
# Make sure we have a valid location value for this asset.
try:
loc = StaticContent.get_location_from_path(asset_path)
except (InvalidLocationError, InvalidKeyError):
return HttpResponseBadRequest()
# Attempt to load the asset to make sure it exists, and grab the asset digest
# if we're able to load it.
actual_digest = None
try:
content = self.load_asset_from_location(loc)
actual_digest = getattr(content, "content_digest", None)
except (ItemNotFoundError, NotFoundError):
return HttpResponseNotFound()
# If this was a versioned asset, and the digest doesn't match, redirect
# them to the actual version.
if requested_digest is not None and actual_digest is not None and (actual_digest != requested_digest):
actual_asset_path = StaticContent.add_version_to_asset_path(asset_path, actual_digest)
return HttpResponsePermanentRedirect(actual_asset_path)
# Set the basics for this request. Make sure that the course key for this
# asset has a run, which old-style courses do not. Otherwise, this will
# explode when the key is serialized to be sent to NR.
safe_course_key = loc.course_key
if safe_course_key.run is None:
safe_course_key = safe_course_key.replace(run='only')
newrelic.agent.add_custom_parameter('course_id', safe_course_key)
newrelic.agent.add_custom_parameter('org', loc.org)
newrelic.agent.add_custom_parameter('contentserver.path', loc.path)
# Figure out if this is a CDN using us as the origin.
is_from_cdn = StaticContentServer.is_cdn_request(request)
newrelic.agent.add_custom_parameter('contentserver.from_cdn', is_from_cdn)
# Check if this content is locked or not.
locked = self.is_content_locked(content)
newrelic.agent.add_custom_parameter('contentserver.locked', locked)
# Check that user has access to the content.
if not self.is_user_authorized(request, content, loc):
return HttpResponseForbidden('Unauthorized')
# Figure out if the client sent us a conditional request, and let them know
# if this asset has changed since then.
last_modified_at_str = content.last_modified_at.strftime(HTTP_DATE_FORMAT)
if 'HTTP_IF_MODIFIED_SINCE' in request.META:
if_modified_since = request.META['HTTP_IF_MODIFIED_SINCE']
if if_modified_since == last_modified_at_str:
return HttpResponseNotModified()
# *** File streaming within a byte range ***
# If a Range is provided, parse Range attribute of the request
# Add Content-Range in the response if Range is structurally correct
# Request -> Range attribute structure: "Range: bytes=first-[last]"
# Response -> Content-Range attribute structure: "Content-Range: bytes first-last/totalLength"
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
response = None
if request.META.get('HTTP_RANGE'):
# If we have a StaticContent, get a StaticContentStream. Can't manipulate the bytes otherwise.
if type(content) == StaticContent:
content = AssetManager.find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:
unit, ranges = parse_range_header(header_value, content.length)
except ValueError as exception:
# If the header field is syntactically invalid it should be ignored.
log.exception(
u"%s in Range header: %s for content: %s", exception.message, header_value, unicode(loc)
)
else:
if unit != 'bytes':
# Only accept ranges in bytes
log.warning(u"Unknown unit in Range header: %s for content: %s", header_value, unicode(loc))
elif len(ranges) > 1:
# According to Http/1.1 spec content for multiple ranges should be sent as a multipart message.
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.16
# But we send back the full content.
log.warning(
u"More than 1 ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
else:
first, last = ranges[0]
if 0 <= first <= last < content.length:
# If the byte range is satisfiable
response = HttpResponse(content.stream_data_in_range(first, last))
response['Content-Range'] = 'bytes {first}-{last}/{length}'.format(
first=first, last=last, length=content.length
)
response['Content-Length'] = str(last - first + 1)
response.status_code = 206 # Partial Content
newrelic.agent.add_custom_parameter('contentserver.ranged', True)
else:
log.warning(
u"Cannot satisfy ranges in Range header: %s for content: %s", header_value, unicode(loc)
)
return HttpResponse(status=416) # Requested Range Not Satisfiable
# If Range header is absent or syntactically invalid return a full content response.
if response is None:
response = HttpResponse(content.stream_data())
response['Content-Length'] = content.length
newrelic.agent.add_custom_parameter('contentserver.content_len', content.length)
newrelic.agent.add_custom_parameter('contentserver.content_type', content.content_type)
# "Accept-Ranges: bytes" tells the user that only "bytes" ranges are allowed
response['Accept-Ranges'] = 'bytes'
response['Content-Type'] = content.content_type
# Set any caching headers, and do any response cleanup needed. Based on how much
# middleware we have in place, there's no easy way to use the built-in Django
# utilities and properly sanitize and modify a response to ensure that it is as
# cacheable as possible, which is why we do it ourselves.
self.set_caching_headers(content, response)
return response
def set_caching_headers(self, content, response):
"""
Sets caching headers based on whether or not the asset is locked.
"""
is_locked = getattr(content, "locked", False)
# We want to signal to the end user's browser, and to any intermediate proxies/caches,
# whether or not this asset is cacheable. If we have a TTL configured, we inform the
# caller, for unlocked assets, how long they are allowed to cache it. Since locked
# assets should be restricted to enrolled students, we simply send headers that
# indicate there should be no caching whatsoever.
cache_ttl = CourseAssetCacheTtlConfig.get_cache_ttl()
if cache_ttl > 0 and not is_locked:
newrelic.agent.add_custom_parameter('contentserver.cacheable', True)
response['Expires'] = StaticContentServer.get_expiration_value(datetime.datetime.utcnow(), cache_ttl)
response['Cache-Control'] = "public, max-age={ttl}, s-maxage={ttl}".format(ttl=cache_ttl)
elif is_locked:
newrelic.agent.add_custom_parameter('contentserver.cacheable', False)
response['Cache-Control'] = "private, no-cache, no-store"
response['Last-Modified'] = content.last_modified_at.strftime(HTTP_DATE_FORMAT)
# Force the Vary header to only vary responses on Origin, so that XHR and browser requests get cached
# separately and don't screw over one another. i.e. a browser request that doesn't send Origin, and
# caches a version of the response without CORS headers, in turn breaking XHR requests.
force_header_for_response(response, 'Vary', 'Origin')
@staticmethod
def is_cdn_request(request):
"""
Attempts to determine whether or not the given request is coming from a CDN.
Currently, this is a static check because edx.org only uses CloudFront, but may
be expanded in the future.
"""
cdn_user_agents = CdnUserAgentsConfig.get_cdn_user_agents()
user_agent = request.META.get('HTTP_USER_AGENT', '')
if user_agent in cdn_user_agents:
# This is a CDN request.
return True
return False
@staticmethod
def get_expiration_value(now, cache_ttl):
"""Generates an RFC1123 datetime string based on a future offset."""
expire_dt = now + datetime.timedelta(seconds=cache_ttl)
return expire_dt.strftime(HTTP_DATE_FORMAT)
def is_content_locked(self, content):
"""
Determines whether or not the given content is locked.
"""
return bool(getattr(content, "locked", False))
def is_user_authorized(self, request, content, location):
"""
Determines whether or not the user for this request is authorized to view the given asset.
"""
if not self.is_content_locked(content):
return True
if not hasattr(request, "user") or not request.user.is_authenticated():
return False
if not request.user.is_staff:
deprecated = getattr(location, 'deprecated', False)
if deprecated and not CourseEnrollment.is_enrolled_by_partial(request.user, location.course_key):
return False
if not deprecated and not CourseEnrollment.is_enrolled(request.user, location.course_key):
return False
return True
def load_asset_from_location(self, location):
"""
Loads an asset based on its location, either retrieving it from a cache
or loading it directly from the contentstore.
"""
# See if we can load this item from cache.
content = get_cached_content(location)
if content is None:
# Not in cache, so just try and load it from the asset manager.
try:
content = AssetManager.find(location, as_stream=True)
except (ItemNotFoundError, NotFoundError):
raise
# Now that we fetched it, let's go ahead and try to cache it. We cap this at 1MB
# because it's the default for memcached and also we don't want to do too much
# buffering in memory when we're serving an actual request.
if content.length is not None and content.length < 1048576:
content = content.copy_to_in_mem()
set_cached_content(content)
return content
def parse_range_header(header_value, content_length):
"""
Returns the unit and a list of (start, end) tuples of ranges.
Raises ValueError if header is syntactically invalid or does not contain a range.
See spec for details: http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
"""
unit = None
ranges = []
if '=' in header_value:
unit, byte_ranges_string = header_value.split('=')
# Parse the byte ranges.
for byte_range_string in byte_ranges_string.split(','):
byte_range_string = byte_range_string.strip()
# Case 0:
if '-' not in byte_range_string: # Invalid syntax of header value.
raise ValueError('Invalid syntax.')
# Case 1: -500
elif byte_range_string.startswith('-'):
first = max(0, (content_length + int(byte_range_string)))
last = content_length - 1
# Case 2: 500-
elif byte_range_string.endswith('-'):
first = int(byte_range_string[0:-1])
last = content_length - 1
# Case 3: 500-999
else:
first, last = byte_range_string.split('-')
first = int(first)
last = min(int(last), content_length - 1)
ranges.append((first, last))
if len(ranges) == 0:
raise ValueError('Invalid syntax')
return unit, ranges

27
common/djangoapps/contentserver/migrations/0001_initial.py
View File

@@ -1,27 +0,0 @@
# -*- coding: utf-8 -*-
#pylint: skip-file
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
from django.conf import settings
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
]
operations = [
migrations.CreateModel(
name='CourseAssetCacheTtlConfig',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')),
('enabled', models.BooleanField(default=False, verbose_name='Enabled')),
('cache_ttl', models.PositiveIntegerField(default=0, help_text=b'The time, in seconds, to report that a course asset is allowed to be cached for.')),
('changed_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, editable=False, to=settings.AUTH_USER_MODEL, null=True, verbose_name='Changed by')),
],
),
]

27
common/djangoapps/contentserver/migrations/0002_cdnuseragentsconfig.py
View File

@@ -1,27 +0,0 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
from django.conf import settings
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('contentserver', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='CdnUserAgentsConfig',
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('change_date', models.DateTimeField(auto_now_add=True, verbose_name='Change date')),
('enabled', models.BooleanField(default=False, verbose_name='Enabled')),
('cdn_user_agents', models.TextField(default=b'Amazon CloudFront', help_text=b'A newline-separated list of user agents that should be considered CDNs.')),
('changed_by', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, editable=False, to=settings.AUTH_USER_MODEL, null=True, verbose_name='Changed by')),
],
),
]

0
common/djangoapps/contentserver/migrations/__init__.py
View File

52
common/djangoapps/contentserver/models.py
View File

@@ -1,52 +0,0 @@
"""
Models for contentserver
"""
from django.db.models.fields import PositiveIntegerField, TextField
from config_models.models import ConfigurationModel
class CourseAssetCacheTtlConfig(ConfigurationModel):
"""Configuration for the TTL of course assets."""
class Meta(object):
app_label = 'contentserver'
cache_ttl = PositiveIntegerField(
default=0,
help_text="The time, in seconds, to report that a course asset is allowed to be cached for."
)
@classmethod
def get_cache_ttl(cls):
"""Gets the cache TTL for course assets, if present"""
return cls.current().cache_ttl
def __repr__(self):
return '<CourseAssetCacheTtlConfig(cache_ttl={})>'.format(self.get_cache_ttl())
def __unicode__(self):
return unicode(repr(self))
class CdnUserAgentsConfig(ConfigurationModel):
"""Configuration for the user agents we expect to see from CDNs."""
class Meta(object):
app_label = 'contentserver'
cdn_user_agents = TextField(
default='Amazon CloudFront',
help_text="A newline-separated list of user agents that should be considered CDNs."
)
@classmethod
def get_cdn_user_agents(cls):
"""Gets the list of CDN user agents, if present"""
return cls.current().cdn_user_agents
def __repr__(self):
return '<WhitelistedCdnConfig(cdn_user_agents={})>'.format(self.get_cdn_user_agents())
def __unicode__(self):
return unicode(repr(self))

0
common/djangoapps/contentserver/test/__init__.py
View File

448
common/djangoapps/contentserver/test/test_contentserver.py
View File

@@ -1,448 +0,0 @@
"""
Tests for StaticContentServer
"""
import copy
import datetime
import ddt
import logging
import unittest
from uuid import uuid4
from django.conf import settings
from django.test import RequestFactory
from django.test.client import Client
from django.test.utils import override_settings
from mock import patch
from xmodule.contentstore.django import contentstore
from xmodule.contentstore.content import StaticContent, VERSIONED_ASSETS_PREFIX
from xmodule.modulestore.django import modulestore
from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase
from xmodule.modulestore.xml_importer import import_course_from_xml
from xmodule.assetstore.assetmgr import AssetManager
from opaque_keys import InvalidKeyError
from xmodule.modulestore.exceptions import ItemNotFoundError
from contentserver.middleware import parse_range_header, HTTP_DATE_FORMAT, StaticContentServer
from student.models import CourseEnrollment
from student.tests.factories import UserFactory, AdminFactory
log = logging.getLogger(__name__)
TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE)
TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex
TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT
FAKE_MD5_HASH = 'ffffffffffffffffffffffffffffffff'
def get_versioned_asset_url(asset_path):
"""
Creates a versioned asset URL.
"""
try:
locator = StaticContent.get_location_from_path(asset_path)
content = AssetManager.find(locator, as_stream=True)
return StaticContent.add_version_to_asset_path(asset_path, content.content_digest)
except (InvalidKeyError, ItemNotFoundError):
pass
return asset_path
def get_old_style_versioned_asset_url(asset_path):
"""
Creates an old-style versioned asset URL.
"""
try:
locator = StaticContent.get_location_from_path(asset_path)
content = AssetManager.find(locator, as_stream=True)
return u'{}/{}{}'.format(VERSIONED_ASSETS_PREFIX, content.content_digest, asset_path)
except (InvalidKeyError, ItemNotFoundError):
pass
return asset_path
@ddt.ddt
@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE)
class ContentStoreToyCourseTest(SharedModuleStoreTestCase):
"""
Tests that use the toy course.
"""
@classmethod
def setUpClass(cls):
super(ContentStoreToyCourseTest, cls).setUpClass()
cls.contentstore = contentstore()
cls.modulestore = modulestore()
cls.course_key = cls.modulestore.make_course_key('edX', 'toy', '2012_Fall')
import_course_from_xml(
cls.modulestore, 1, TEST_DATA_DIR, ['toy'],
static_content_store=cls.contentstore, verbose=True
)
# A locked asset
cls.locked_asset = cls.course_key.make_asset_key('asset', 'sample_static.html')
cls.url_locked = unicode(cls.locked_asset)
cls.url_locked_versioned = get_versioned_asset_url(cls.url_locked)
cls.url_locked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_locked)
cls.contentstore.set_attr(cls.locked_asset, 'locked', True)
# An unlocked asset
cls.unlocked_asset = cls.course_key.make_asset_key('asset', 'another_static.txt')
cls.url_unlocked = unicode(cls.unlocked_asset)
cls.url_unlocked_versioned = get_versioned_asset_url(cls.url_unlocked)
cls.url_unlocked_versioned_old_style = get_old_style_versioned_asset_url(cls.url_unlocked)
cls.length_unlocked = cls.contentstore.get_attr(cls.unlocked_asset, 'length')
def setUp(self):
"""
Create user and login.
"""
super(ContentStoreToyCourseTest, self).setUp()
self.staff_usr = AdminFactory.create()
self.non_staff_usr = UserFactory.create()
self.client = Client()
def test_unlocked_asset(self):
"""
Test that unlocked assets are being served.
"""
self.client.logout()
resp = self.client.get(self.url_unlocked)
self.assertEqual(resp.status_code, 200)
def test_unlocked_versioned_asset(self):
"""
Test that unlocked assets that are versioned are being served.
"""
self.client.logout()
resp = self.client.get(self.url_unlocked_versioned)
self.assertEqual(resp.status_code, 200)
def test_unlocked_versioned_asset_old_style(self):
"""
Test that unlocked assets that are versioned (old-style) are being served.
"""
self.client.logout()
resp = self.client.get(self.url_unlocked_versioned_old_style)
self.assertEqual(resp.status_code, 200)
def test_unlocked_versioned_asset_with_nonexistent_version(self):
"""
Test that unlocked assets that are versioned, but have a nonexistent version,
are sent back as a 301 redirect which tells the caller the correct URL.
"""
url_unlocked_versioned_old = StaticContent.add_version_to_asset_path(self.url_unlocked, FAKE_MD5_HASH)
self.client.logout()
resp = self.client.get(url_unlocked_versioned_old)
self.assertEqual(resp.status_code, 301)
self.assertTrue(resp.url.endswith(self.url_unlocked_versioned)) # pylint: disable=no-member
def test_locked_versioned_asset(self):
"""
Test that locked assets that are versioned are being served.
"""
CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
self.client.login(username=self.non_staff_usr, password='test')
resp = self.client.get(self.url_locked_versioned)
self.assertEqual(resp.status_code, 200)
def test_locked_versioned_old_styleasset(self):
"""
Test that locked assets that are versioned (old-style) are being served.
"""
CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
self.client.login(username=self.non_staff_usr, password='test')
resp = self.client.get(self.url_locked_versioned_old_style)
self.assertEqual(resp.status_code, 200)
def test_locked_asset_not_logged_in(self):
"""
Test that locked assets behave appropriately in case the user is not
logged in.
"""
self.client.logout()
resp = self.client.get(self.url_locked)
self.assertEqual(resp.status_code, 403)
def test_locked_asset_not_registered(self):
"""
Test that locked assets behave appropriately in case user is logged in
in but not registered for the course.
"""
self.client.login(username=self.non_staff_usr, password='test')
resp = self.client.get(self.url_locked)
self.assertEqual(resp.status_code, 403)
def test_locked_asset_registered(self):
"""
Test that locked assets behave appropriately in case user is logged in
and registered for the course.
"""
CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
self.client.login(username=self.non_staff_usr, password='test')
resp = self.client.get(self.url_locked)
self.assertEqual(resp.status_code, 200)
def test_locked_asset_staff(self):
"""
Test that locked assets behave appropriately in case user is staff.
"""
self.client.login(username=self.staff_usr, password='test')
resp = self.client.get(self.url_locked)
self.assertEqual(resp.status_code, 200)
def test_range_request_full_file(self):
"""
Test that a range request from byte 0 to last,
outputs partial content status code and valid Content-Range and Content-Length.
"""
resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes=0-')
self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT
self.assertEqual(
resp['Content-Range'],
'bytes {first}-{last}/{length}'.format(
first=0, last=self.length_unlocked - 1,
length=self.length_unlocked
)
)
self.assertEqual(resp['Content-Length'], str(self.length_unlocked))
def test_range_request_partial_file(self):
"""
Test that a range request for a partial file,
outputs partial content status code and valid Content-Range and Content-Length.
first_byte and last_byte are chosen to be simple but non trivial values.
"""
first_byte = self.length_unlocked / 4
last_byte = self.length_unlocked / 2
resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format(
first=first_byte, last=last_byte))
self.assertEqual(resp.status_code, 206) # HTTP_206_PARTIAL_CONTENT
self.assertEqual(resp['Content-Range'], 'bytes {first}-{last}/{length}'.format(
first=first_byte, last=last_byte, length=self.length_unlocked))
self.assertEqual(resp['Content-Length'], str(last_byte - first_byte + 1))
def test_range_request_multiple_ranges(self):
"""
Test that multiple ranges in request outputs the full content.
"""
first_byte = self.length_unlocked / 4
last_byte = self.length_unlocked / 2
resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}, -100'.format(
first=first_byte, last=last_byte))
self.assertEqual(resp.status_code, 200)
self.assertNotIn('Content-Range', resp)
self.assertEqual(resp['Content-Length'], str(self.length_unlocked))
@ddt.data(
'bytes 0-',
'bits=0-',
'bytes=0',
'bytes=one-',
)
def test_syntax_errors_in_range(self, header_value):
"""
Test that syntactically invalid Range values result in a 200 OK full content response.
"""
resp = self.client.get(self.url_unlocked, HTTP_RANGE=header_value)
self.assertEqual(resp.status_code, 200)
self.assertNotIn('Content-Range', resp)
def test_range_request_malformed_invalid_range(self):
"""
Test that a range request with malformed Range (first_byte > last_byte) outputs
416 Requested Range Not Satisfiable.
"""
resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format(
first=(self.length_unlocked / 2), last=(self.length_unlocked / 4)))
self.assertEqual(resp.status_code, 416)
def test_range_request_malformed_out_of_bounds(self):
"""
Test that a range request with malformed Range (first_byte, last_byte == totalLength, offset by 1 error)
outputs 416 Requested Range Not Satisfiable.
"""
resp = self.client.get(self.url_unlocked, HTTP_RANGE='bytes={first}-{last}'.format(
first=(self.length_unlocked), last=(self.length_unlocked)))
self.assertEqual(resp.status_code, 416)
def test_vary_header_sent(self):
"""
Tests that we're properly setting the Vary header to ensure browser requests don't get
cached in a way that breaks XHR requests to the same asset.
"""
resp = self.client.get(self.url_unlocked)
self.assertEqual(resp.status_code, 200)
self.assertEquals('Origin', resp['Vary'])
@patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
def test_cache_headers_with_ttl_unlocked(self, mock_get_cache_ttl):
"""
Tests that when a cache TTL is set, an unlocked asset will be sent back with
the correct cache control/expires headers.
"""
mock_get_cache_ttl.return_value = 10
resp = self.client.get(self.url_unlocked)
self.assertEqual(resp.status_code, 200)
self.assertIn('Expires', resp)
self.assertEquals('public, max-age=10, s-maxage=10', resp['Cache-Control'])
@patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
def test_cache_headers_with_ttl_locked(self, mock_get_cache_ttl):
"""
Tests that when a cache TTL is set, a locked asset will be sent back without
any cache control/expires headers.
"""
mock_get_cache_ttl.return_value = 10
CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
self.client.login(username=self.non_staff_usr, password='test')
resp = self.client.get(self.url_locked)
self.assertEqual(resp.status_code, 200)
self.assertNotIn('Expires', resp)
self.assertEquals('private, no-cache, no-store', resp['Cache-Control'])
@patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
def test_cache_headers_without_ttl_unlocked(self, mock_get_cache_ttl):
"""
Tests that when a cache TTL is not set, an unlocked asset will be sent back without
any cache control/expires headers.
"""
mock_get_cache_ttl.return_value = 0
resp = self.client.get(self.url_unlocked)
self.assertEqual(resp.status_code, 200)
self.assertNotIn('Expires', resp)
self.assertNotIn('Cache-Control', resp)
@patch('contentserver.models.CourseAssetCacheTtlConfig.get_cache_ttl')
def test_cache_headers_without_ttl_locked(self, mock_get_cache_ttl):
"""
Tests that when a cache TTL is not set, a locked asset will be sent back with a
cache-control header that indicates this asset should not be cached.
"""
mock_get_cache_ttl.return_value = 0
CourseEnrollment.enroll(self.non_staff_usr, self.course_key)
self.assertTrue(CourseEnrollment.is_enrolled(self.non_staff_usr, self.course_key))
self.client.login(username=self.non_staff_usr, password='test')
resp = self.client.get(self.url_locked)
self.assertEqual(resp.status_code, 200)
self.assertNotIn('Expires', resp)
self.assertEquals('private, no-cache, no-store', resp['Cache-Control'])
def test_get_expiration_value(self):
start_dt = datetime.datetime.strptime("Thu, 01 Dec 1983 20:00:00 GMT", HTTP_DATE_FORMAT)
near_expire_dt = StaticContentServer.get_expiration_value(start_dt, 55)
self.assertEqual("Thu, 01 Dec 1983 20:00:55 GMT", near_expire_dt)
@patch('contentserver.models.CdnUserAgentsConfig.get_cdn_user_agents')
def test_cache_is_cdn_with_normal_request(self, mock_get_cdn_user_agents):
"""
Tests that when a normal request is made -- i.e. from an end user with their
browser -- that we don't classify the request as coming from a CDN.
"""
mock_get_cdn_user_agents.return_value = 'Amazon CloudFront'
request_factory = RequestFactory()
browser_request = request_factory.get('/fake', HTTP_USER_AGENT='Chrome 1234')
is_from_cdn = StaticContentServer.is_cdn_request(browser_request)
self.assertEqual(is_from_cdn, False)
@patch('contentserver.models.CdnUserAgentsConfig.get_cdn_user_agents')
def test_cache_is_cdn_with_cdn_request(self, mock_get_cdn_user_agents):
"""
Tests that when a CDN request is made -- i.e. from an edge node back to the
origin -- that we classify the request as coming from a CDN.
"""
mock_get_cdn_user_agents.return_value = 'Amazon CloudFront'
request_factory = RequestFactory()
browser_request = request_factory.get('/fake', HTTP_USER_AGENT='Amazon CloudFront')
is_from_cdn = StaticContentServer.is_cdn_request(browser_request)
self.assertEqual(is_from_cdn, True)
@patch('contentserver.models.CdnUserAgentsConfig.get_cdn_user_agents')
def test_cache_is_cdn_with_cdn_request_multiple_user_agents(self, mock_get_cdn_user_agents):
"""
Tests that when a CDN request is made -- i.e. from an edge node back to the
origin -- that we classify the request as coming from a CDN when multiple UAs
are configured.
"""
mock_get_cdn_user_agents.return_value = 'Amazon CloudFront\nAkamai GHost'
request_factory = RequestFactory()
browser_request = request_factory.get('/fake', HTTP_USER_AGENT='Amazon CloudFront')
is_from_cdn = StaticContentServer.is_cdn_request(browser_request)
self.assertEqual(is_from_cdn, True)
@ddt.ddt
class ParseRangeHeaderTestCase(unittest.TestCase):
"""
Tests for the parse_range_header function.
"""
def setUp(self):
super(ParseRangeHeaderTestCase, self).setUp()
self.content_length = 10000
def test_bytes_unit(self):
unit, __ = parse_range_header('bytes=100-', self.content_length)
self.assertEqual(unit, 'bytes')
@ddt.data(
('bytes=100-', 1, [(100, 9999)]),
('bytes=1000-', 1, [(1000, 9999)]),
('bytes=100-199, 200-', 2, [(100, 199), (200, 9999)]),
('bytes=100-199, 200-499', 2, [(100, 199), (200, 499)]),
('bytes=-100', 1, [(9900, 9999)]),
('bytes=-100, -200', 2, [(9900, 9999), (9800, 9999)])
)
@ddt.unpack
def test_valid_syntax(self, header_value, excepted_ranges_length, expected_ranges):
__, ranges = parse_range_header(header_value, self.content_length)
self.assertEqual(len(ranges), excepted_ranges_length)
self.assertEqual(ranges, expected_ranges)
@ddt.data(
('bytes=one-20', ValueError, 'invalid literal for int()'),
('bytes=-one', ValueError, 'invalid literal for int()'),
('bytes=-', ValueError, 'invalid literal for int()'),
('bytes=--', ValueError, 'invalid literal for int()'),
('bytes', ValueError, 'Invalid syntax'),
('bytes=', ValueError, 'Invalid syntax'),
('bytes=0', ValueError, 'Invalid syntax'),
('bytes=0-10,0', ValueError, 'Invalid syntax'),
('bytes=0=', ValueError, 'too many values to unpack'),
)
@ddt.unpack
def test_invalid_syntax(self, header_value, exception_class, exception_message_regex):
self.assertRaisesRegexp(
exception_class, exception_message_regex, parse_range_header, header_value, self.content_length
)