Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Wrap all asset read operations.

Browse Source 
First step in moving assets out of the contentstore (optionally).
https://openedx.atlassian.net/browse/PLAT-77
This commit is contained in:
John Eskew
2014-10-27 12:33:07 -04:00
parent 5a1df635d0
commit 786a32e7cf
6 changed files with 135 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
cms/djangoapps/contentstore/views/tests/test_assets.py
View File

@@ -13,6 +13,8 @@ import json
from contentstore.tests.utils import CourseTestCase
from contentstore.views import assets
from contentstore.utils import reverse_course_url
from xmodule.assetstore.assetmgr import UnknownAssetType, AssetMetadataFoundTemporary
from xmodule.assetstore import AssetMetadata
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
from xmodule.modulestore.django import modulestore
@@ -134,6 +136,55 @@ class UploadTestCase(AssetsTestCase):
self.assertEquals(resp.status_code, 400)
class DownloadTestCase(AssetsTestCase):
"""
Unit tests for downloading a file.
"""
def setUp(self):
super(DownloadTestCase, self).setUp()
self.url = reverse_course_url('assets_handler', self.course.id)
# First, upload something.
self.asset_name = 'download_test'
resp = self.upload_asset(self.asset_name)
self.assertEquals(resp.status_code, 200)
self.uploaded_url = json.loads(resp.content)['asset']['url']
def test_download(self):
# Now, download it.
resp = self.client.get(self.uploaded_url, HTTP_ACCEPT='text/html')
self.assertEquals(resp.status_code, 200)
self.assertEquals(resp.content, self.asset_name)
def test_download_not_found_throw(self):
url = self.uploaded_url.replace(self.asset_name, 'not_the_asset_name')
resp = self.client.get(url, HTTP_ACCEPT='text/html')
self.assertEquals(resp.status_code, 404)
def test_download_unknown_asset_type(self):
# Change the asset type to something unknown.
url = self.uploaded_url.replace('/asset/', '/unknown_type/')
with self.assertRaises((UnknownAssetType, NameError)):
self.client.get(url, HTTP_ACCEPT='text/html')
def test_metadata_found_in_modulestore(self):
# Insert asset metadata into the modulestore (with no accompanying asset).
asset_key = self.course.id.make_asset_key(AssetMetadata.ASSET_TYPE, 'pic1.jpg')
asset_md = AssetMetadata(asset_key, {
'internal_name': 'EKMND332DDBK',
'basename': 'pix/archive',
'locked': False,
'curr_version': '14',
'prev_version': '13'
})
modulestore().save_asset_metadata(self.course.id, asset_md, 15)
# Get the asset metadata and have it be found in the modulestore.
# Currently, no asset metadata should be found in the modulestore. The code is not yet storing it there.
# If asset metadata *is* found there, an exception is raised. This test ensures the exception is indeed raised.
# THIS IS TEMPORARY. Soon, asset metadata *will* be stored in the modulestore.
with self.assertRaises((AssetMetadataFoundTemporary, NameError)):
self.client.get(unicode(asset_key), HTTP_ACCEPT='text/html')
class AssetToJsonTestCase(AssetsTestCase):
"""
Unit test for transforming asset information into something

6
common/djangoapps/contentserver/middleware.py
View File

@@ -9,7 +9,7 @@ from django.http import (
)
from student.models import CourseEnrollment
from xmodule.contentstore.django import contentstore
from xmodule.assetstore.assetmgr import AssetManager
from xmodule.contentstore.content import StaticContent, XASSET_LOCATION_TAG
from xmodule.modulestore import InvalidLocationError
from opaque_keys import InvalidKeyError
@@ -42,7 +42,7 @@ class StaticContentServer(object):
if content is None:
# nope, not in cache, let's fetch from DB
try:
content = contentstore().find(loc, as_stream=True)
content = AssetManager.find(loc, as_stream=True)
except NotFoundError:
response = HttpResponse()
response.status_code = 404
@@ -94,7 +94,7 @@ class StaticContentServer(object):
if request.META.get('HTTP_RANGE'):
# Data from cache (StaticContent) has no easy byte management, so we use the DB instead (StaticContentStream)
if type(content) == StaticContent:
content = contentstore().find(loc, as_stream=True)
content = AssetManager.find(loc, as_stream=True)
header_value = request.META['HTTP_RANGE']
try:

78
common/lib/xmodule/xmodule/assetstore/assetmgr.py Normal file
View File

@@ -0,0 +1,78 @@
"""
Asset Manager
Interface allowing course asset saving/retrieving.
Handles:
- saving asset in the BlobStore -and- saving asset metadata in course modulestore.
- retrieving asset metadata from course modulestore -and- returning URL to asset -or- asset bytes.
Phase 1: Checks to see if an asset's metadata can be found in the course's modulestore.
If not found, fails over to access the asset from the contentstore.
At first, the asset metadata will never be found, since saving isn't implemented yet.
"""
from contracts import contract, new_contract
from opaque_keys.edx.keys import AssetKey
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
new_contract('AssetKey', AssetKey)
class AssetException(Exception):
"""
Base exception class for all exceptions related to assets.
"""
pass
class AssetMetadataNotFound(AssetException):
"""
Thrown when no asset metadata is present in the course modulestore for the particular asset requested.
"""
pass
class UnknownAssetType(AssetException):
"""
Thrown when the asset type is not recognized.
"""
pass
class AssetMetadataFoundTemporary(AssetException):
"""
TEMPORARY: Thrown if asset metadata is actually found in the course modulestore.
"""
pass
class AssetManager(object):
"""
Manager for saving/loading course assets.
"""
@staticmethod
@contract(asset_key='AssetKey', throw_on_not_found='bool', as_stream='bool')
def find(asset_key, throw_on_not_found=True, as_stream=False):
"""
Finds a course asset either in the assetstore -or- in the deprecated contentstore.
"""
store = modulestore()
content_md = None
asset_type = asset_key.asset_type
if asset_type == AssetThumbnailMetadata.ASSET_TYPE:
content_md = store.find_asset_thumbnail_metadata(asset_key)
elif asset_type == AssetMetadata.ASSET_TYPE:
content_md = store.find_asset_metadata(asset_key)
else:
raise UnknownAssetType()
# If found, raise an exception.
if content_md:
# For now, no asset metadata should be found in the modulestore.
raise AssetMetadataFoundTemporary()
else:
# If not found, load the asset via the contentstore.
return contentstore().find(asset_key, throw_on_not_found, as_stream)

2
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -892,6 +892,8 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite):
Asset info for the course, index of asset/thumbnail in list (None if asset/thumbnail does not exist)
"""
course_assets = self._find_course_assets(course_key)
if course_assets is None:
return None, None
if get_thumbnail:
all_assets = course_assets['thumbnails']

1
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -1462,6 +1462,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# Using the course_key, find or insert the course asset metadata document.
# A single document exists per course to store the course asset metadata.
course_key = self.fill_in_run(course_key)
course_assets = self.asset_collection.find_one(
{'course_id': unicode(course_key)},
fields=('course_id', 'storage', 'assets', 'thumbnails')

1
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
View File

@@ -105,7 +105,6 @@ class TestMongoModuleStoreBase(unittest.TestCase):
'port': PORT,
'db': DB,
'collection': COLLECTION,
#'asset_collection': ASSET_COLLECTION,
}
cls.add_asset_collection(doc_store_config)