Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #5847 from edx/dhm/generalize_assets

Browse Source 
Generalize assets to any block_type but using
This commit is contained in:
Don Mitchell
2014-11-05 17:32:56 -05:00
parent 6d5bb9b195 fc83c299ca
commit 5491418cd5
14 changed files with 275 additions and 621 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
cms/djangoapps/contentstore/views/tests/test_assets.py
View File

@@ -1,11 +1,6 @@
"""
Unit tests for the asset upload endpoint.
"""
# pylint: disable=C0111
# pylint: disable=W0621
# pylint: disable=W0212
from datetime import datetime
from io import BytesIO
from pytz import UTC
@@ -13,7 +8,7 @@ import json
from contentstore.tests.utils import CourseTestCase
from contentstore.views import assets
from contentstore.utils import reverse_course_url
from xmodule.assetstore.assetmgr import UnknownAssetType, AssetMetadataFoundTemporary
from xmodule.assetstore.assetmgr import AssetMetadataFoundTemporary
from xmodule.assetstore import AssetMetadata
from xmodule.contentstore.content import StaticContent
from xmodule.contentstore.django import contentstore
@@ -35,12 +30,18 @@ class AssetsTestCase(CourseTestCase):
self.url = reverse_course_url('assets_handler', self.course.id)
def upload_asset(self, name="asset-1"):
"""
Post to the asset upload url
"""
f = BytesIO(name)
f.name = name + ".txt"
return self.client.post(self.url, {"name": name, "file": f})
class BasicAssetsTestCase(AssetsTestCase):
"""
Test getting assets via html w/o additional args
"""
def test_basic(self):
resp = self.client.get(self.url, HTTP_ACCEPT='text/html')
self.assertEquals(resp.status_code, 200)
@@ -81,6 +82,9 @@ class PaginationTestCase(AssetsTestCase):
Tests the pagination of assets returned from the REST API.
"""
def test_json_responses(self):
"""
Test the ajax asset interfaces
"""
self.upload_asset("asset-1")
self.upload_asset("asset-2")
self.upload_asset("asset-3")
@@ -100,20 +104,26 @@ class PaginationTestCase(AssetsTestCase):
self.assert_correct_asset_response(self.url + "?page_size=3&page=1", 0, 3, 3)
def assert_correct_asset_response(self, url, expected_start, expected_length, expected_total):
"""
Get from the url and ensure it contains the expected number of responses
"""
resp = self.client.get(url, HTTP_ACCEPT='application/json')
json_response = json.loads(resp.content)
assets = json_response['assets']
assets_response = json_response['assets']
self.assertEquals(json_response['start'], expected_start)
self.assertEquals(len(assets), expected_length)
self.assertEquals(len(assets_response), expected_length)
self.assertEquals(json_response['totalCount'], expected_total)
def assert_correct_sort_response(self, url, sort, direction):
"""
Get from the url w/ a sort option and ensure items honor that sort
"""
resp = self.client.get(url + '?sort=' + sort + '&direction=' + direction, HTTP_ACCEPT='application/json')
json_response = json.loads(resp.content)
assets = json_response['assets']
name1 = assets[0][sort]
name2 = assets[1][sort]
name3 = assets[2][sort]
assets_response = json_response['assets']
name1 = assets_response[0][sort]
name2 = assets_response[1][sort]
name3 = assets_response[2][sort]
if direction == 'asc':
self.assertLessEqual(name1, name2)
self.assertLessEqual(name2, name3)
@@ -163,12 +173,6 @@ class DownloadTestCase(AssetsTestCase):
resp = self.client.get(url, HTTP_ACCEPT='text/html')
self.assertEquals(resp.status_code, 404)
def test_download_unknown_asset_type(self):
# Change the asset type to something unknown.
url = self.uploaded_url.replace('/asset/', '/unknown_type/')
with self.assertRaises((UnknownAssetType, NameError)):
self.client.get(url, HTTP_ACCEPT='text/html')
def test_metadata_found_in_modulestore(self):
# Insert asset metadata into the modulestore (with no accompanying asset).
asset_key = self.course.id.make_asset_key(AssetMetadata.ASSET_TYPE, 'pic1.jpg')
@@ -179,7 +183,7 @@ class DownloadTestCase(AssetsTestCase):
'curr_version': '14',
'prev_version': '13'
})
modulestore().save_asset_metadata(self.course.id, asset_md, 15)
modulestore().save_asset_metadata(asset_md, 15)
# Get the asset metadata and have it be found in the modulestore.
# Currently, no asset metadata should be found in the modulestore. The code is not yet storing it there.
# If asset metadata *is* found there, an exception is raised. This test ensures the exception is indeed raised.
@@ -201,6 +205,7 @@ class AssetToJsonTestCase(AssetsTestCase):
location = course_key.make_asset_key('asset', 'my_file_name.jpg')
thumbnail_location = course_key.make_asset_key('thumbnail', 'my_file_name_thumb.jpg')
# pylint: disable=protected-access
output = assets._get_asset_json("my_file", upload_date, location, thumbnail_location, True)
self.assertEquals(output["display_name"], "my_file")
@@ -239,6 +244,7 @@ class LockAssetTestCase(AssetsTestCase):
resp = self.client.post(
url,
# pylint: disable=protected-access
json.dumps(assets._get_asset_json("sample_static.txt", upload_date, asset_location, None, lock)),
"application/json"
)

92
common/lib/xmodule/xmodule/assetstore/__init__.py
View File

@@ -1,5 +1,5 @@
"""
Classes representing asset & asset thumbnail metadata.
Classes representing asset metadata.
"""
from datetime import datetime
@@ -13,74 +13,70 @@ new_contract('datetime', datetime)
new_contract('basestring', basestring)
class IncorrectAssetIdType(Exception):
"""
Raised when the asset ID passed-in to create an AssetMetadata or
AssetThumbnailMetadata is of the wrong type.
"""
pass
class AssetMetadata(object):
"""
Stores the metadata associated with a particular course asset. The asset metadata gets stored
in the modulestore.
"""
TOP_LEVEL_ATTRS = ['basename', 'internal_name', 'locked', 'contenttype', 'md5']
TOP_LEVEL_ATTRS = ['basename', 'internal_name', 'locked', 'contenttype', 'thumbnail', 'fields']
EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_on']
ALLOWED_ATTRS = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS
# All AssetMetadata objects should have AssetLocators with this type.
# Default type for AssetMetadata objects. A constant for convenience.
ASSET_TYPE = 'asset'
@contract(asset_id='AssetKey', basename='basestring|None', internal_name='basestring|None', locked='bool|None', contenttype='basestring|None',
md5='basestring|None', curr_version='basestring|None', prev_version='basestring|None', edited_by='int|None', edited_on='datetime|None')
fields='dict | None', curr_version='basestring|None', prev_version='basestring|None', edited_by='int|None', edited_on='datetime|None')
def __init__(self, asset_id,
basename=None, internal_name=None,
locked=None, contenttype=None, md5=None,
locked=None, contenttype=None, thumbnail=None, fields=None,
curr_version=None, prev_version=None,
edited_by=None, edited_on=None, field_decorator=None):
edited_by=None, edited_on=None,
field_decorator=None,):
"""
Construct a AssetMetadata object.
Arguments:
asset_id (AssetKey): Key identifying this particular asset.
basename (str): Original path to file at asset upload time.
internal_name (str): Name under which the file is stored internally.
internal_name (str): Name, url, or handle for the storage system to access the file.
locked (bool): If True, only course participants can access the asset.
contenttype (str): MIME type of the asset.
thumbnail (str): the internal_name for the thumbnail if one exists
fields (dict): fields to save w/ the metadata
curr_version (str): Current version of the asset.
prev_version (str): Previous version of the asset.
edited_by (str): Username of last user to upload this asset.
edited_on (datetime): Datetime of last upload of this asset.
field_decorator (function): used by strip_key to convert OpaqueKeys to the app's understanding
field_decorator (function): used by strip_key to convert OpaqueKeys to the app's understanding.
Not saved.
"""
if asset_id.asset_type != self.ASSET_TYPE:
raise IncorrectAssetIdType()
self.asset_id = asset_id if field_decorator is None else field_decorator(asset_id)
self.basename = basename # Path w/o filename.
self.internal_name = internal_name
self.locked = locked
self.contenttype = contenttype
self.md5 = md5
self.thumbnail = thumbnail
self.curr_version = curr_version
self.prev_version = prev_version
self.edited_by = edited_by
self.edited_on = edited_on or datetime.now(pytz.utc)
self.fields = fields or {}
def __repr__(self):
return """AssetMetadata{!r}""".format((
self.asset_id,
self.basename, self.internal_name,
self.locked, self.contenttype, self.md5,
self.locked, self.contenttype, self.fields,
self.curr_version, self.prev_version,
self.edited_by, self.edited_on
))
def update(self, attr_dict):
"""
Set the attributes on the metadata. Ignore all those outside the known fields.
Set the attributes on the metadata. Any which are not in ALLOWED_ATTRS get put into
fields.
Arguments:
attr_dict: Prop, val dictionary of all attributes to set.
@@ -88,6 +84,8 @@ class AssetMetadata(object):
for attr, val in attr_dict.iteritems():
if attr in self.ALLOWED_ATTRS:
setattr(self, attr, val)
else:
self.fields[attr] = val
def to_mongo(self):
"""
@@ -99,7 +97,8 @@ class AssetMetadata(object):
'internal_name': self.internal_name,
'locked': self.locked,
'contenttype': self.contenttype,
'md5': self.md5,
'thumbnail': self.thumbnail,
'fields': self.fields,
'curr_version': self.curr_version,
'prev_version': self.prev_version,
'edited_by': self.edited_by,
@@ -119,54 +118,9 @@ class AssetMetadata(object):
self.internal_name = asset_doc['internal_name']
self.locked = asset_doc['locked']
self.contenttype = asset_doc['contenttype']
self.md5 = asset_doc['md5']
self.thumbnail = asset_doc['thumbnail']
self.fields = asset_doc['fields']
self.curr_version = asset_doc['curr_version']
self.prev_version = asset_doc['prev_version']
self.edited_by = asset_doc['edited_by']
self.edited_on = asset_doc['edited_on']
class AssetThumbnailMetadata(object):
"""
Stores the metadata associated with the thumbnail of a course asset.
"""
# All AssetThumbnailMetadata objects should have AssetLocators with this type.
ASSET_TYPE = 'thumbnail'
@contract(asset_id='AssetKey', internal_name='basestring|None')
def __init__(self, asset_id, internal_name=None, field_decorator=None):
"""
Construct a AssetThumbnailMetadata object.
Arguments:
asset_id (AssetKey): Key identifying this particular asset.
internal_name (str): Name under which the file is stored internally.
"""
if asset_id.asset_type != self.ASSET_TYPE:
raise IncorrectAssetIdType()
self.asset_id = asset_id if field_decorator is None else field_decorator(asset_id)
self.internal_name = internal_name
def __repr__(self):
return """AssetMetadata{!r}""".format((self.asset_id, self.internal_name))
def to_mongo(self):
"""
Converts metadata properties into a MongoDB-storable dict.
"""
return {
'filename': self.asset_id.path,
'internal_name': self.internal_name
}
@contract(thumbnail_doc='dict|None')
def from_mongo(self, thumbnail_doc):
"""
Fill in all metadata fields from a MongoDB document.
The asset_id prop is initialized upon construction only.
"""
if thumbnail_doc is None:
return
self.internal_name = thumbnail_doc['internal_name']

18
common/lib/xmodule/xmodule/assetstore/assetmgr.py
View File

@@ -15,7 +15,6 @@ from contracts import contract, new_contract
from opaque_keys.edx.keys import AssetKey
from xmodule.modulestore.django import modulestore
from xmodule.contentstore.django import contentstore
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
new_contract('AssetKey', AssetKey)
@@ -35,13 +34,6 @@ class AssetMetadataNotFound(AssetException):
pass
class UnknownAssetType(AssetException):
"""
Thrown when the asset type is not recognized.
"""
pass
class AssetMetadataFoundTemporary(AssetException):
"""
TEMPORARY: Thrown if asset metadata is actually found in the course modulestore.
@@ -59,15 +51,7 @@ class AssetManager(object):
"""
Finds a course asset either in the assetstore -or- in the deprecated contentstore.
"""
store = modulestore()
content_md = None
asset_type = asset_key.asset_type
if asset_type == AssetThumbnailMetadata.ASSET_TYPE:
content_md = store.find_asset_thumbnail_metadata(asset_key)
elif asset_type == AssetMetadata.ASSET_TYPE:
content_md = store.find_asset_metadata(asset_key)
else:
raise UnknownAssetType()
content_md = modulestore().find_asset_metadata(asset_key)
# If found, raise an exception.
if content_md:

216
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -23,7 +23,7 @@ from xblock.plugin import default_select
from .exceptions import InvalidLocationError, InsufficientSpecificationError
from xmodule.errortracker import make_error_tracker
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from xmodule.assetstore import AssetMetadata
from opaque_keys.edx.keys import CourseKey, UsageKey, AssetKey
from opaque_keys.edx.locations import Location # For import backwards compatibility
from opaque_keys import InvalidKeyError
@@ -36,7 +36,6 @@ log = logging.getLogger('edx.modulestore')
new_contract('CourseKey', CourseKey)
new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata)
class ModuleStoreEnum(object):
@@ -281,70 +280,44 @@ class ModuleStoreAssetInterface(object):
"""
def _find_course_assets(self, course_key):
"""
Base method to override.
Finds the persisted repr of the asset metadata not converted to AssetMetadata yet.
Returns the container holding a dict indexed by asset block_type whose values are a list
of raw metadata documents
"""
raise NotImplementedError()
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
def _find_course_asset(self, asset_key):
"""
Internal; finds or creates course asset info -and- finds existing asset (or thumbnail) metadata.
Returns same as _find_course_assets plus the index to the given asset or None. Does not convert
to AssetMetadata; thus, is internal.
Arguments:
course_key (CourseKey): course identifier
filename (str): filename of the asset or thumbnail
get_thumbnail (bool): True gets thumbnail data, False gets asset data
asset_key (AssetKey): what to look for
Returns:
Asset info for the course, index of asset/thumbnail in list (None if asset/thumbnail does not exist)
AssetMetadata[] for all assets of the given asset_key's type, & the index of asset in list
(None if asset does not exist)
"""
course_assets = self._find_course_assets(course_key)
course_assets = self._find_course_assets(asset_key.course_key)
if course_assets is None:
return None, None
info = 'thumbnails' if get_thumbnail else 'assets'
all_assets = SortedListWithKey([], key=itemgetter('filename'))
# Assets should be pre-sorted, so add them efficiently without sorting.
# extend() will raise a ValueError if the passed-in list is not sorted.
all_assets.extend(course_assets.get(info, []))
all_assets.extend(course_assets.setdefault(asset_key.block_type, []))
# See if this asset already exists by checking the external_filename.
# Studio doesn't currently support using multiple course assets with the same filename.
# So use the filename as the unique identifier.
idx = None
idx_left = all_assets.bisect_left({'filename': filename})
idx_right = all_assets.bisect_right({'filename': filename})
idx_left = all_assets.bisect_left({'filename': asset_key.block_id})
idx_right = all_assets.bisect_right({'filename': asset_key.block_id})
if idx_left != idx_right:
# Asset was found in the list.
idx = idx_left
return course_assets, idx
@contract(asset_key='AssetKey')
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
"""
Find the info for a particular course asset/thumbnail.
Arguments:
asset_key (AssetKey): key containing original asset filename
thumbnail (bool): True if finding thumbnail, False if finding asset metadata
Returns:
asset/thumbnail metadata (AssetMetadata/AssetThumbnailMetadata) -or- None if not found
"""
course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, thumbnail)
if asset_idx is None:
return None
if thumbnail:
info = 'thumbnails'
mdata = AssetThumbnailMetadata(asset_key, asset_key.path, **kwargs)
else:
info = 'assets'
mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[info]
mdata.from_mongo(all_assets[asset_idx])
return mdata
@contract(asset_key='AssetKey')
def find_asset_metadata(self, asset_key, **kwargs):
"""
@@ -356,38 +329,33 @@ class ModuleStoreAssetInterface(object):
Returns:
asset metadata (AssetMetadata) -or- None if not found
"""
return self._find_asset_info(asset_key, thumbnail=False, **kwargs)
course_assets, asset_idx = self._find_course_asset(asset_key)
if asset_idx is None:
return None
@contract(asset_key='AssetKey')
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
info = asset_key.block_type
mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[info]
mdata.from_mongo(all_assets[asset_idx])
return mdata
@contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='tuple(str,(int,>=1,<=2))|None',)
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
"""
Find the metadata for a particular course asset.
Returns a list of asset metadata for all assets of the given asset_type in the course.
Arguments:
asset_key (AssetKey): key containing original asset filename
Returns:
asset metadata (AssetMetadata) -or- None if not found
"""
return self._find_asset_info(asset_key, thumbnail=True, **kwargs)
@contract(course_key='CourseKey', start='int|None', maxresults='int|None',
sort='tuple(str,(int,>=1,<=2))|None', get_thumbnails='bool')
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending),
get_thumbnails=False, **kwargs):
"""
Args:
course_key (CourseKey): course identifier
asset_type (str): the block_type of the assets to return
start (int): optional - start at this asset number. Zero-based!
maxresults (int): optional - return at most this many, -1 means no limit
sort (array): optional - None means no sort
(sort_by (str), sort_order (str))
sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of SortOrder.ascending or SortOrder.descending
get_thumbnails (bool): True if getting thumbnail metadata, else getting asset metadata
Returns:
List of AssetMetadata or AssetThumbnailMetadata objects.
List of AssetMetadata objects.
"""
course_assets = self._find_course_assets(course_key)
if course_assets is None:
@@ -399,13 +367,12 @@ class ModuleStoreAssetInterface(object):
sort_field = 'filename'
sort_order = ModuleStoreEnum.SortOrder.ascending
if sort:
if sort[0] == 'uploadDate' and not get_thumbnails:
if sort[0] == 'uploadDate':
sort_field = 'edited_on'
if sort[1] == ModuleStoreEnum.SortOrder.descending:
sort_order = ModuleStoreEnum.SortOrder.descending
info = 'thumbnails' if get_thumbnails else 'assets'
all_assets = SortedListWithKey(course_assets.get(info, []), key=itemgetter(sort_field))
all_assets = SortedListWithKey(course_assets.get(asset_type, []), key=itemgetter(sort_field))
num_assets = len(all_assets)
start_idx = start
@@ -423,102 +390,30 @@ class ModuleStoreAssetInterface(object):
ret_assets = []
for idx in xrange(start_idx, end_idx, step_incr):
asset = all_assets[idx]
if get_thumbnails:
thumb = AssetThumbnailMetadata(
course_key.make_asset_key('thumbnail', asset['filename']),
internal_name=asset['filename'],
**kwargs
)
ret_assets.append(thumb)
else:
new_asset = AssetMetadata(
course_key.make_asset_key('asset', asset['filename']),
basename=asset['filename'],
internal_name=asset['internal_name'],
locked=asset['locked'],
contenttype=asset['contenttype'],
md5=asset['md5'],
curr_version=asset['curr_version'],
prev_version=asset['prev_version'],
edited_on=asset['edited_on'],
edited_by=asset['edited_by'],
**kwargs
)
ret_assets.append(new_asset)
raw_asset = all_assets[idx]
new_asset = AssetMetadata(course_key.make_asset_key(asset_type, raw_asset['filename']))
new_asset.from_mongo(raw_asset)
ret_assets.append(new_asset)
return ret_assets
@contract(course_key='CourseKey', start='int|None', maxresults='int|None', sort='tuple(str,int)|None')
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
"""
Returns a list of static assets for a course.
By default all assets are returned, but start and maxresults can be provided to limit the query.
Args:
course_key (CourseKey): course identifier
start (int): optional - start at this asset number
maxresults (int): optional - return at most this many, -1 means no limit
sort (array): optional - None means no sort
(sort_by (str), sort_order (str))
sort_by - one of 'uploadDate' or 'displayname'
sort_order - one of SortOrder.ascending or SortOrder.descending
Returns:
List of AssetMetadata objects.
"""
return self._get_all_asset_metadata(course_key, start, maxresults, sort, get_thumbnails=False, **kwargs)
@contract(course_key='CourseKey')
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
"""
Returns a list of thumbnails for all course assets.
Args:
course_key (CourseKey): course identifier
Returns:
List of AssetThumbnailMetadata objects.
"""
return self._get_all_asset_metadata(course_key, get_thumbnails=True, **kwargs)
class ModuleStoreAssetWriteInterface(ModuleStoreAssetInterface):
"""
The write operations for assets and asset metadata
"""
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
"""
Base method to over-ride in modulestore.
"""
raise NotImplementedError()
@contract(course_key='CourseKey', asset_metadata='AssetMetadata')
def save_asset_metadata(self, course_key, asset_metadata, user_id):
@contract(asset_metadata='AssetMetadata')
def save_asset_metadata(self, asset_metadata, user_id):
"""
Saves the asset metadata for a particular course's asset.
Arguments:
course_key (CourseKey): course identifier
asset_metadata (AssetMetadata): data about the course asset data
asset_metadata (AssetMetadata): data about the course asset data (must have asset_id
set)
Returns:
True if metadata save was successful, else False
"""
return self._save_asset_info(course_key, asset_metadata, user_id, thumbnail=False)
@contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata')
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata, user_id):
"""
Saves the asset thumbnail metadata for a particular course asset's thumbnail.
Arguments:
course_key (CourseKey): course identifier
asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail
Returns:
True if thumbnail metadata save was successful, else False
"""
return self._save_asset_info(course_key, asset_thumbnail_metadata, user_id, thumbnail=True)
raise NotImplementedError()
def set_asset_metadata_attrs(self, asset_key, attrs, user_id):
"""
@@ -526,7 +421,7 @@ class ModuleStoreAssetWriteInterface(ModuleStoreAssetInterface):
"""
raise NotImplementedError()
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
def delete_asset_metadata(self, asset_key, user_id):
"""
Base method to over-ride in modulestore.
"""
@@ -548,36 +443,13 @@ class ModuleStoreAssetWriteInterface(ModuleStoreAssetInterface):
"""
return self.set_asset_metadata_attrs(asset_key, {attr: value}, user_id)
@contract(asset_key='AssetKey')
def delete_asset_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): locator containing original asset filename
Returns:
Number of asset metadata entries deleted (0 or 1)
"""
return self._delete_asset_data(asset_key, user_id, thumbnail=False)
@contract(asset_key='AssetKey')
def delete_asset_thumbnail_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): locator containing original asset filename
Returns:
Number of asset metadata entries deleted (0 or 1)
"""
return self._delete_asset_data(asset_key, user_id, thumbnail=True)
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copy all the course assets from source_course_key to dest_course_key.
NOTE: unlike get_all_asset_metadata, this does not take an asset type because
this function is intended for things like cloning or exporting courses not for
clients to list assets.
Arguments:
source_course_key (CourseKey): identifier of course to copy from

82
common/lib/xmodule/xmodule/modulestore/mixed.py
View File

@@ -14,7 +14,7 @@ from contracts import contract, new_contract
from opaque_keys import InvalidKeyError
from opaque_keys.edx.keys import CourseKey, AssetKey
from opaque_keys.edx.locations import SlashSeparatedCourseKey
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from xmodule.assetstore import AssetMetadata
from . import ModuleStoreWriteBase
from . import ModuleStoreEnum
@@ -25,7 +25,6 @@ from .split_migrator import SplitMigrator
new_contract('CourseKey', CourseKey)
new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata)
log = logging.getLogger(__name__)
@@ -315,8 +314,8 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
store = self._get_modulestore_for_courseid(course_key)
return store.delete_course(course_key, user_id)
@contract(course_key='CourseKey', asset_metadata='AssetMetadata')
def save_asset_metadata(self, course_key, asset_metadata, user_id):
@contract(asset_metadata='AssetMetadata')
def save_asset_metadata(self, asset_metadata, user_id):
"""
Saves the asset metadata for a particular course's asset.
@@ -324,20 +323,8 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
course_key (CourseKey): course identifier
asset_metadata (AssetMetadata): data about the course asset data
"""
store = self._get_modulestore_for_courseid(course_key)
return store.save_asset_metadata(course_key, asset_metadata, user_id)
@contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata')
def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata, user_id):
"""
Saves the asset thumbnail metadata for a particular course asset's thumbnail.
Arguments:
course_key (CourseKey): course identifier
asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail
"""
store = self._get_modulestore_for_courseid(course_key)
return store.save_asset_thumbnail_metadata(course_key, asset_thumbnail_metadata, user_id)
store = self._get_modulestore_for_courseid(asset_metadata.asset_id.course_key)
return store.save_asset_metadata(asset_metadata, user_id)
@strip_key
@contract(asset_key='AssetKey')
@@ -354,24 +341,9 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.find_asset_metadata(asset_key, **kwargs)
@strip_key
@contract(asset_key='AssetKey')
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
"""
Find the metadata for a particular course asset.
Arguments:
asset_key (AssetKey): key containing original asset filename
Returns:
asset metadata (AssetMetadata) -or- None if not found
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.find_asset_thumbnail_metadata(asset_key, **kwargs)
@strip_key
@contract(course_key='CourseKey', start=int, maxresults=int, sort='tuple|None')
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
"""
Returns a list of static assets for a course.
By default all assets are returned, but start and maxresults can be provided to limit the query.
@@ -394,22 +366,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
md5: An md5 hash of the asset content
"""
store = self._get_modulestore_for_courseid(course_key)
return store.get_all_asset_metadata(course_key, start, maxresults, sort, **kwargs)
@strip_key
@contract(course_key='CourseKey')
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
"""
Returns a list of thumbnails for all course assets.
Args:
course_key (CourseKey): course identifier
Returns:
List of AssetThumbnailMetadata objects.
"""
store = self._get_modulestore_for_courseid(course_key)
return store.get_all_asset_thumbnail_metadata(course_key, **kwargs)
return store.get_all_asset_metadata(course_key, asset_type, start, maxresults, sort, **kwargs)
@contract(asset_key='AssetKey')
def delete_asset_metadata(self, asset_key, user_id):
@@ -425,31 +382,6 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.delete_asset_metadata(asset_key, user_id)
@contract(asset_key='AssetKey')
def delete_asset_thumbnail_metadata(self, asset_key, user_id):
"""
Deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): locator containing original asset filename
Returns:
Number of asset metadata entries deleted (0 or 1)
"""
store = self._get_modulestore_for_courseid(asset_key.course_key)
return store.delete_asset_thumbnail_metadata(asset_key, user_id)
@contract(course_key='CourseKey')
def delete_all_asset_metadata(self, course_key, user_id):
"""
Delete all of the assets which use this course_key as an identifier.
Arguments:
course_key (CourseKey): course_identifier
"""
store = self._get_modulestore_for_courseid(course_key)
return store.delete_all_asset_metadata(course_key, user_id)
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""

73
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -47,14 +47,13 @@ from opaque_keys.edx.locator import CourseLocator
from opaque_keys.edx.keys import UsageKey, CourseKey, AssetKey
from xmodule.exceptions import HeartbeatFailure
from xmodule.modulestore.edit_info import EditInfoRuntimeMixin
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from xmodule.assetstore import AssetMetadata
log = logging.getLogger(__name__)
new_contract('CourseKey', CourseKey)
new_contract('AssetKey', AssetKey)
new_contract('AssetMetadata', AssetMetadata)
new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata)
# sort order that returns DRAFT items first
SORT_REVISION_FAVOR_DRAFT = ('_id.revision', pymongo.DESCENDING)
@@ -1467,25 +1466,22 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
course_key = self.fill_in_run(course_key)
course_assets = self.asset_collection.find_one(
{'course_id': unicode(course_key)},
fields=('course_id', 'storage', 'assets', 'thumbnails')
)
if course_assets is None:
# Not found, so create.
course_assets = {'course_id': unicode(course_key), 'storage': 'FILLMEIN-TMP', 'assets': [], 'thumbnails': []}
course_assets = {'course_id': unicode(course_key), 'storage': 'FILLMEIN-TMP', 'assets': []}
course_assets['_id'] = self.asset_collection.insert(course_assets)
return course_assets
@contract(course_key='CourseKey', asset_metadata='AssetMetadata | AssetThumbnailMetadata')
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
@contract(asset_metadata='AssetMetadata')
def save_asset_metadata(self, asset_metadata, user_id):
"""
Saves the info for a particular course's asset/thumbnail.
Saves the info for a particular course's asset.
Arguments:
course_key (CourseKey): course identifier
asset_metadata (AssetMetadata/AssetThumbnailMetadata): data about the course asset/thumbnail
thumbnail (bool): True if saving thumbnail metadata, False if saving asset metadata
asset_metadata (AssetMetadata): data about the course asset
Returns:
True if info save was successful, else False
@@ -1493,16 +1489,12 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if self.asset_collection is None:
return False
course_assets, asset_idx = self._find_course_asset(course_key, asset_metadata.asset_id.path, thumbnail)
info = 'thumbnails' if thumbnail else 'assets'
course_assets, asset_idx = self._find_course_asset(asset_metadata.asset_id)
all_assets = SortedListWithKey([], key=itemgetter('filename'))
# Assets should be pre-sorted, so add them efficiently without sorting.
# extend() will raise a ValueError if the passed-in list is not sorted.
all_assets.extend(course_assets[info])
# Set the edited information for assets only - not thumbnails.
if not thumbnail:
asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
all_assets.extend(course_assets[asset_metadata.asset_id.block_type])
asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
# Translate metadata to Mongo format.
metadata_to_insert = asset_metadata.to_mongo()
@@ -1514,30 +1506,31 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
all_assets[asset_idx] = metadata_to_insert
# Update the document.
self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets.as_list()}})
self.asset_collection.update(
{'_id': course_assets['_id']},
{'$set': {asset_metadata.asset_id.block_type: all_assets.as_list()}}
)
return True
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copy all the course assets from source_course_key to dest_course_key.
If dest_course already has assets, this removes the previous value.
It doesn't combine the assets in dest.
Arguments:
source_course_key (CourseKey): identifier of course to copy from
dest_course_key (CourseKey): identifier of course to copy to
"""
source_assets = self._find_course_assets(source_course_key)
dest_assets = self._find_course_assets(dest_course_key)
dest_assets['assets'] = source_assets.get('assets', [])
dest_assets['thumbnails'] = source_assets.get('thumbnails', [])
dest_assets = source_assets.copy()
dest_assets['course_id'] = unicode(dest_course_key)
del dest_assets['_id']
self.asset_collection.remove({'course_id': unicode(dest_course_key)})
# Update the document.
self.asset_collection.update(
{'_id': dest_assets['_id']},
{'$set': {'assets': dest_assets['assets'],
'thumbnails': dest_assets['thumbnails']}
}
)
self.asset_collection.insert(dest_assets)
@contract(asset_key='AssetKey', attr_dict=dict)
def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id):
@@ -1555,12 +1548,12 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
if self.asset_collection is None:
return
course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path)
course_assets, asset_idx = self._find_course_asset(asset_key)
if asset_idx is None:
raise ItemNotFoundError(asset_key)
# Form an AssetMetadata.
all_assets = course_assets['assets']
all_assets = course_assets[asset_key.block_type]
md = AssetMetadata(asset_key, asset_key.path)
md.from_mongo(all_assets[asset_idx])
md.update(attr_dict)
@@ -1568,34 +1561,34 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# Generate a Mongo doc from the metadata and update the course asset info.
all_assets[asset_idx] = md.to_mongo()
self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {'assets': all_assets}})
self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {asset_key.block_type: all_assets}})
@contract(asset_key='AssetKey')
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
def delete_asset_metadata(self, asset_key, user_id):
"""
Internal; deletes a single asset's metadata -or- thumbnail.
Internal; deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): key containing original asset/thumbnail filename
thumbnail: True if thumbnail deletion, False if asset metadata deletion
asset_key (AssetKey): key containing original asset filename
Returns:
Number of asset metadata/thumbnail entries deleted (0 or 1)
Number of asset metadata entries deleted (0 or 1)
"""
if self.asset_collection is None:
return 0
course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, get_thumbnail=thumbnail)
course_assets, asset_idx = self._find_course_asset(asset_key)
if asset_idx is None:
return 0
info = 'thumbnails' if thumbnail else 'assets'
all_asset_info = course_assets[info]
all_asset_info = course_assets[asset_key.block_type]
all_asset_info.pop(asset_idx)
# Update the document.
self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_asset_info}})
self.asset_collection.update(
{'_id': course_assets['_id']},
{'$set': {asset_key.block_type: all_asset_info}}
)
return 1
# pylint: disable=unused-argument

1
common/lib/xmodule/xmodule/modulestore/mongo/draft.py
View File

@@ -155,6 +155,7 @@ class DraftModuleStore(MongoModuleStore):
# delete all of the db records for the course
course_query = self._course_key_to_son(course_key)
self.collection.remove(course_query, multi=True)
self.delete_all_asset_metadata(course_key, user_id)
def clone_course(self, source_course_id, dest_course_id, user_id, fields=None, **kwargs):
"""

69
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -2125,26 +2125,30 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
"""
Split specific lookup
"""
return self._lookup_course(course_key).structure
return self._lookup_course(course_key).structure.get('assets', {})
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
structure = self._lookup_course(course_key).structure
return structure, self._lookup_course_asset(structure, filename, get_thumbnail)
def _find_course_asset(self, asset_key):
"""
Return the raw dict of assets type as well as the index to the one being sought from w/in
it's subvalue (or None)
"""
assets = self._lookup_course(asset_key.course_key).structure.get('assets', {})
return assets, self._lookup_course_asset(assets, asset_key)
def _lookup_course_asset(self, structure, filename, get_thumbnail=False):
def _lookup_course_asset(self, structure, asset_key):
"""
Find the course asset in the structure or return None if it does not exist
"""
# See if this asset already exists by checking the external_filename.
# Studio doesn't currently support using multiple course assets with the same filename.
# So use the filename as the unique identifier.
accessor = 'thumbnails' if get_thumbnail else 'assets'
for idx, asset in enumerate(structure.get(accessor, [])):
if asset['filename'] == filename:
accessor = asset_key.block_type
for idx, asset in enumerate(structure.setdefault(accessor, [])):
if asset['filename'] == asset_key.block_id:
return idx
return None
def _update_course_assets(self, user_id, asset_key, update_function, get_thumbnail=False):
def _update_course_assets(self, user_id, asset_key, update_function):
"""
A wrapper for functions wanting to manipulate assets. Gets and versions the structure,
passes the mutable array for either 'assets' or 'thumbnails' as well as the idx to the function for it to
@@ -2158,10 +2162,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
index_entry = self._get_index_if_valid(asset_key.course_key)
new_structure = self.version_structure(asset_key.course_key, original_structure, user_id)
accessor = 'thumbnails' if get_thumbnail else 'assets'
asset_idx = self._lookup_course_asset(new_structure, asset_key.path, get_thumbnail)
asset_idx = self._lookup_course_asset(new_structure.setdefault('assets', {}), asset_key)
new_structure[accessor] = update_function(new_structure.get(accessor, []), asset_idx)
new_structure['assets'][asset_key.block_type] = update_function(
new_structure['assets'][asset_key.block_type], asset_idx
)
# update index if appropriate and structures
self.update_structure(asset_key.course_key, new_structure)
@@ -2170,7 +2175,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# update the index entry if appropriate
self._update_head(asset_key.course_key, index_entry, asset_key.branch, new_structure['_id'])
def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False):
def save_asset_metadata(self, asset_metadata, user_id):
"""
The guts of saving a new or updated asset
"""
@@ -2186,7 +2191,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
all_assets[asset_idx] = metadata_to_insert
return all_assets
return self._update_course_assets(user_id, asset_metadata.asset_id, _internal_method, thumbnail)
return self._update_course_assets(user_id, asset_metadata.asset_id, _internal_method)
@contract(asset_key='AssetKey', attr_dict=dict)
def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id):
@@ -2217,19 +2222,18 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
all_assets[asset_idx] = mdata.to_mongo()
return all_assets
self._update_course_assets(user_id, asset_key, _internal_method, False)
self._update_course_assets(user_id, asset_key, _internal_method)
@contract(asset_key='AssetKey')
def _delete_asset_data(self, asset_key, user_id, thumbnail=False):
def delete_asset_metadata(self, asset_key, user_id):
"""
Internal; deletes a single asset's metadata -or- thumbnail.
Internal; deletes a single asset's metadata.
Arguments:
asset_key (AssetKey): key containing original asset/thumbnail filename
thumbnail: True if thumbnail deletion, False if asset metadata deletion
asset_key (AssetKey): key containing original asset filename
Returns:
Number of asset metadata/thumbnail entries deleted (0 or 1)
Number of asset metadata entries deleted (0 or 1)
"""
def _internal_method(all_asset_info, asset_idx):
"""
@@ -2242,34 +2246,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
return all_asset_info
try:
self._update_course_assets(user_id, asset_key, _internal_method, thumbnail)
self._update_course_assets(user_id, asset_key, _internal_method)
return 1
except ItemNotFoundError:
return 0
@contract(course_key='CourseKey')
def delete_all_asset_metadata(self, course_key, user_id):
"""
Delete all of the assets which use this course_key as an identifier.
Arguments:
course_key (CourseKey): course_identifier
"""
with self.bulk_operations(course_key):
original_structure = self._lookup_course(course_key).structure
index_entry = self._get_index_if_valid(course_key)
new_structure = self.version_structure(course_key, original_structure, user_id)
new_structure['assets'] = []
new_structure['thumbnails'] = []
# update index if appropriate and structures
self.update_structure(course_key, new_structure)
if index_entry is not None:
# update the index entry if appropriate
self._update_head(course_key, index_entry, course_key.branch, new_structure['_id'])
@contract(source_course_key='CourseKey', dest_course_key='CourseKey')
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""

35
common/lib/xmodule/xmodule/modulestore/split_mongo/split_draft.py
View File

@@ -11,6 +11,7 @@ from xmodule.modulestore.draft_and_published import (
)
from opaque_keys.edx.locator import CourseLocator
from xmodule.modulestore.split_mongo import BlockKey
from contracts import contract
class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPublished):
@@ -446,33 +447,34 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
setattr(xblock, '_published_by', published_block['edit_info']['edited_by'])
setattr(xblock, '_published_on', published_block['edit_info']['edited_on'])
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
return super(DraftVersioningModuleStore, self)._find_asset_info(
self._map_revision_to_branch(asset_key), thumbnail, **kwargs
@contract(asset_key='AssetKey')
def find_asset_metadata(self, asset_key, **kwargs):
return super(DraftVersioningModuleStore, self).find_asset_metadata(
self._map_revision_to_branch(asset_key), **kwargs
)
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
return super(DraftVersioningModuleStore, self)._get_all_asset_metadata(
self._map_revision_to_branch(course_key), start, maxresults, sort, get_thumbnails, **kwargs
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
return super(DraftVersioningModuleStore, self).get_all_asset_metadata(
self._map_revision_to_branch(course_key), asset_type, start, maxresults, sort, **kwargs
)
def _update_course_assets(self, user_id, asset_key, update_function, get_thumbnail=False):
def _update_course_assets(self, user_id, asset_key, update_function):
"""
Updates both the published and draft branches
"""
# if one call gets an exception, don't do the other call but pass on the exception
super(DraftVersioningModuleStore, self)._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.published_only),
update_function, get_thumbnail
update_function
)
super(DraftVersioningModuleStore, self)._update_course_assets(
user_id, self._map_revision_to_branch(asset_key, ModuleStoreEnum.RevisionOption.draft_only),
update_function, get_thumbnail
update_function
)
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
def _find_course_asset(self, asset_key):
return super(DraftVersioningModuleStore, self)._find_course_asset(
self._map_revision_to_branch(course_key), filename, get_thumbnail=get_thumbnail
self._map_revision_to_branch(asset_key)
)
def _find_course_assets(self, course_key):
@@ -483,17 +485,6 @@ class DraftVersioningModuleStore(SplitMongoModuleStore, ModuleStoreDraftAndPubli
self._map_revision_to_branch(course_key)
)
def delete_all_asset_metadata(self, course_key, user_id):
"""
Deletes from both branches
"""
super(DraftVersioningModuleStore, self).delete_all_asset_metadata(
self._map_revision_to_branch(course_key, ModuleStoreEnum.RevisionOption.published_only), user_id
)
super(DraftVersioningModuleStore, self).delete_all_asset_metadata(
self._map_revision_to_branch(course_key, ModuleStoreEnum.RevisionOption.draft_only), user_id
)
def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id):
"""
Copies to and from both branches

225
common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py
View File

@@ -7,7 +7,7 @@ import pytz
import unittest
import ddt
from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata
from xmodule.assetstore import AssetMetadata
from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.factories import CourseFactory
@@ -46,17 +46,20 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
"""
Make a single test asset metadata.
"""
return AssetMetadata(asset_loc, internal_name='EKMND332DDBK',
basename='pictures/historical', contenttype='image/jpeg',
locked=False, md5='77631ca4f0e08419b70726a447333ab6',
edited_by=ModuleStoreEnum.UserID.test, edited_on=datetime.now(pytz.utc),
curr_version='v1.0', prev_version='v0.95')
return AssetMetadata(
asset_loc, internal_name='EKMND332DDBK',
basename='pictures/historical', contenttype='image/jpeg',
locked=False, fields={'md5': '77631ca4f0e08419b70726a447333ab6'},
edited_by=ModuleStoreEnum.UserID.test, edited_on=datetime.now(pytz.utc),
curr_version='v1.0', prev_version='v0.95'
)
def _make_asset_thumbnail_metadata(self, asset_key):
def _make_asset_thumbnail_metadata(self, asset_md):
"""
Make a single test asset thumbnail metadata.
Add thumbnail to the asset_md
"""
return AssetThumbnailMetadata(asset_key, internal_name='ABC39XJUDN2')
asset_md.thumbnail = 'ABC39XJUDN2'
return asset_md
def setup_assets(self, course1_key, course2_key, store=None):
"""
@@ -81,41 +84,13 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
asset_key = course1_key.make_asset_key('asset', asset[0])
asset_md = AssetMetadata(asset_key, **asset_dict)
if store is not None:
store.save_asset_metadata(course1_key, asset_md, asset[4])
store.save_asset_metadata(asset_md, asset[4])
elif course2_key:
asset_key = course2_key.make_asset_key('asset', asset[0])
asset_md = AssetMetadata(asset_key, **asset_dict)
# Don't save assets 5 and 6.
if store is not None and i not in (4, 5):
store.save_asset_metadata(course2_key, asset_md, asset[4])
def setup_thumbnails(self, course1_key, course2_key, store=None):
"""
Setup thumbs. Save in store if given
"""
thumbnail_fields = ('filename', 'internal_name')
all_thumbnail_data = (
('cat_thumb.jpg', 'XYXYXYXYXYXY'),
('kitten_thumb.jpg', '123ABC123ABC'),
('puppy_thumb.jpg', 'ADAM12ADAM12'),
('meerkat_thumb.jpg', 'CHIPSPONCH14'),
('corgi_thumb.jpg', 'RON8LDXFFFF10'),
)
for i, thumb in enumerate(all_thumbnail_data):
thumb_dict = dict(zip(thumbnail_fields[1:], thumb[1:]))
if i in (0, 1) and course1_key:
thumb_key = course1_key.make_asset_key('thumbnail', thumb[0])
thumb_md = AssetThumbnailMetadata(thumb_key, **thumb_dict)
if store is not None:
store.save_asset_thumbnail_metadata(course1_key, thumb_md, ModuleStoreEnum.UserID.test)
elif course2_key:
thumb_key = course2_key.make_asset_key('thumbnail', thumb[0])
thumb_md = AssetThumbnailMetadata(thumb_key, **thumb_dict)
# Don't save assets 5 and 6.
if store is not None and i not in (4, 5):
store.save_asset_thumbnail_metadata(course2_key, thumb_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(asset_md, asset[4])
@ddt.data(*MODULESTORE_SETUPS)
def test_save_one_and_confirm(self, storebuilder):
@@ -132,19 +107,12 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
self.assertIsNone(store.find_asset_metadata(new_asset_loc))
# Save the asset's metadata.
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
# Find the asset's metadata and confirm it's the same.
found_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(found_asset_md)
self.assertEquals(new_asset_md, found_asset_md)
# Confirm that only two setup plus one asset's metadata exists.
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 1)
# Delete all metadata and confirm it's gone.
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
# Now delete the non-existent metadata and ensure it doesn't choke
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS)
def test_delete(self, storebuilder):
@@ -157,12 +125,12 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
# Attempt to delete an asset that doesn't exist.
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_find_non_existing_assets(self, storebuilder):
@@ -188,14 +156,12 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
# Add asset metadata.
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 1)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
# Add *the same* asset metadata.
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
# Still one here?
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 1)
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id)), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS)
def test_lock_unlock_assets(self, storebuilder):
@@ -207,7 +173,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
locked_state = new_asset_md.locked
# Flip the course asset's locked status.
@@ -227,7 +193,8 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
('internal_name', 'new_filename.txt'),
('locked', True),
('contenttype', 'image/png'),
('md5', '5346682d948cc3f683635b6918f9b3d0'),
('thumbnail', 'new_filename_thumb.jpg'),
('fields', {'md5': '5346682d948cc3f683635b6918f9b3d0'}),
('curr_version', 'v1.01'),
('prev_version', 'v1.0'),
('edited_by', 'Mork'),
@@ -253,7 +220,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
for attr, value in self.ALLOWED_ATTRS:
# Set the course asset's attr.
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test)
@@ -273,7 +240,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
for attr, value in self.DISALLOWED_ATTRS:
original_attr_val = getattr(new_asset_md, attr)
# Set the course asset's attr.
@@ -295,7 +262,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(course.id, new_asset_md, ModuleStoreEnum.UserID.test)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
for attr, value in self.UNKNOWN_ATTRS:
# Set the course asset's attr.
store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test)
@@ -307,56 +274,55 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
self.assertEquals(getattr(updated_asset_md, attr), value)
@ddt.data(*MODULESTORE_SETUPS)
def test_save_one_thumbnail_and_delete_one_thumbnail(self, storebuilder):
def test_save_one_different_asset(self, storebuilder):
"""
saving and deleting thumbnails
saving and deleting things which are not 'asset'
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
thumbnail_filename = 'burn_thumb.jpg'
asset_key = course.id.make_asset_key('thumbnail', thumbnail_filename)
new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key)
store.save_asset_thumbnail_metadata(course.id, new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 1)
self.assertEquals(store.delete_asset_thumbnail_metadata(asset_key, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_find_thumbnail(self, storebuilder):
"""
finding thumbnails
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
thumbnail_filename = 'burn_thumb.jpg'
asset_key = course.id.make_asset_key('thumbnail', thumbnail_filename)
new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key)
store.save_asset_thumbnail_metadata(course.id, new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertIsNotNone(store.find_asset_thumbnail_metadata(asset_key))
unknown_asset_key = course.id.make_asset_key('thumbnail', 'nosuchfile.jpg')
self.assertIsNone(store.find_asset_thumbnail_metadata(unknown_asset_key))
@ddt.data(*MODULESTORE_SETUPS)
def test_delete_all_thumbnails(self, storebuilder):
"""
deleting all thumbnails
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
thumbnail_filename = 'burn_thumb.jpg'
asset_key = course.id.make_asset_key('thumbnail', thumbnail_filename)
new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key)
store.save_asset_thumbnail_metadata(
course.id, new_asset_thumbnail, ModuleStoreEnum.UserID.test
asset_key = course.id.make_asset_key('different', 'burn.jpg')
new_asset_thumbnail = self._make_asset_thumbnail_metadata(
self._make_asset_metadata(asset_key)
)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 1)
store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_find_different(self, storebuilder):
"""
finding things which are of type other than 'asset'
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
asset_key = course.id.make_asset_key('different', 'burn.jpg')
new_asset_thumbnail = self._make_asset_thumbnail_metadata(
self._make_asset_metadata(asset_key)
)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertIsNotNone(store.find_asset_metadata(asset_key))
unknown_asset_key = course.id.make_asset_key('different', 'nosuchfile.jpg')
self.assertIsNone(store.find_asset_metadata(unknown_asset_key))
@ddt.data(*MODULESTORE_SETUPS)
def test_delete_all_different_type(self, storebuilder):
"""
deleting all assets of a given but not 'asset' type
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
course = CourseFactory.create(modulestore=store)
asset_key = course.id.make_asset_key('different', 'burn_thumb.jpg')
new_asset_thumbnail = self._make_asset_thumbnail_metadata(
self._make_asset_metadata(asset_key)
)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
@ddt.data(*MODULESTORE_SETUPS)
def test_get_all_assets_with_paging(self, storebuilder):
@@ -397,14 +363,18 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
# First, with paging across all sorts.
for sort_test in expected_sorts_by_2:
for i in xrange(3):
asset_page = store.get_all_asset_metadata(course2.id, start=2 * i, maxresults=2, sort=sort_test[0])
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=2 * i, maxresults=2, sort=sort_test[0]
)
self.assertEquals(len(asset_page), sort_test[2][i])
self.assertEquals(asset_page[0].asset_id.path, sort_test[1][2 * i])
if sort_test[2][i] == 2:
self.assertEquals(asset_page[1].asset_id.path, sort_test[1][(2 * i) + 1])
# Now fetch everything.
asset_page = store.get_all_asset_metadata(course2.id, start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending))
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(asset_page), 5)
self.assertEquals(asset_page[0].asset_id.path, 'code.tgz')
self.assertEquals(asset_page[1].asset_id.path, 'demo.swf')
@@ -413,11 +383,19 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp')
# Some odd conditions.
asset_page = store.get_all_asset_metadata(course2.id, start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending))
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(asset_page), 0)
asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending))
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=3, maxresults=0,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(asset_page), 0)
asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=-12345, sort=('displayname', ModuleStoreEnum.SortOrder.descending))
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=3, maxresults=-12345,
sort=('displayname', ModuleStoreEnum.SortOrder.descending)
)
self.assertEquals(len(asset_page), 2)
@ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())]))
@@ -428,15 +406,14 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
with storebuilder.build(None) as store:
course_key = store.make_course_key("org", "course", "run")
asset_key = course_key.make_asset_key('asset', 'foo.jpg')
for method in ['_find_asset_info', 'find_asset_metadata', 'find_asset_thumbnail_metadata']:
for method in ['find_asset_metadata']:
with self.assertRaises(NotImplementedError):
getattr(store, method)(asset_key)
with self.assertRaises(NotImplementedError):
# pylint: disable=protected-access
store._find_course_asset(course_key, asset_key.block_id)
for method in ['_get_all_asset_metadata', 'get_all_asset_metadata', 'get_all_asset_thumbnail_metadata']:
with self.assertRaises(NotImplementedError):
getattr(store, method)(course_key)
store._find_course_asset(asset_key)
with self.assertRaises(NotImplementedError):
store.get_all_asset_metadata(course_key, 'asset')
@ddt.data(*MODULESTORE_SETUPS)
def test_copy_all_assets(self, storebuilder):
@@ -448,19 +425,13 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
course1 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store)
self.setup_assets(course1.id, None, store)
self.setup_thumbnails(course1.id, None, store)
self.assertEquals(len(store.get_all_asset_metadata(course1.id)), 2)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course1.id)), 2)
self.assertEquals(len(store.get_all_asset_metadata(course2.id)), 0)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course2.id)), 0)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
self.assertEquals(len(store.get_all_asset_metadata(course1.id)), 2)
self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course1.id)), 2)
all_assets = store.get_all_asset_metadata(course2.id, sort=('displayname', ModuleStoreEnum.SortOrder.ascending))
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
all_assets = store.get_all_asset_metadata(
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(all_assets), 2)
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
all_thumbnails = store.get_all_asset_thumbnail_metadata(course2.id, sort=('uploadDate', ModuleStoreEnum.SortOrder.descending))
self.assertEquals(len(all_thumbnails), 2)
self.assertEquals(all_thumbnails[0].asset_id.path, 'kitten_thumb.jpg')
self.assertEquals(all_thumbnails[1].asset_id.path, 'cat_thumb.jpg')

2
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
View File

@@ -40,7 +40,7 @@ COMMON_DOCSTORE_CONFIG = {
'host': MONGO_HOST,
'port': MONGO_PORT_NUM,
}
DATA_DIR = path(__file__).dirname().parent.parent.parent.parent.parent / "test" / "data"
DATA_DIR = path(__file__).dirname().parent.parent / "tests" / "data" / "xml-course-root"
XBLOCK_MIXINS = (InheritanceMixin, XModuleMixin)

6
common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py
View File

@@ -686,11 +686,7 @@ class TestMongoModuleStoreWithNoAssetCollection(TestMongoModuleStore):
courses = self.draft_store.get_courses()
course = courses[0]
# Confirm that no asset collection means no asset metadata.
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None)
# Now delete the non-existent asset metadata.
self.draft_store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test)
# Should still be nothing.
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None)
self.assertEquals(self.draft_store.get_all_asset_metadata(course.id, 'asset'), None)
class TestMongoKeyValueStore(object):

32
common/lib/xmodule/xmodule/modulestore/xml.py
View File

@@ -847,14 +847,7 @@ class XMLModuleStore(ModuleStoreReadBase):
raise ValueError(u"Cannot set branch setting to {} on a ReadOnly store".format(branch_setting))
yield
def _find_course_asset(self, course_key, filename, get_thumbnail=False):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def _find_asset_info(self, asset_key, thumbnail=False, **kwargs):
def _find_course_asset(self, asset_key):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
@@ -868,28 +861,7 @@ class XMLModuleStore(ModuleStoreReadBase):
"""
raise NotImplementedError()
def find_asset_thumbnail_metadata(self, asset_key, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.
"""
raise NotImplementedError()
def get_all_asset_thumbnail_metadata(self, course_key, **kwargs):
def get_all_asset_metadata(self, course_key, asset_type, start=0, maxresults=-1, sort=None, **kwargs):
"""
For now this is not implemented, but others should feel free to implement using the asset.json
which export produces.

1
common/lib/xmodule/xmodule/tests/data/xml-course-root Symbolic link
View File

@@ -0,0 +1 @@
../../../../../test/data/