From b857a0ed7b56852822c141af6baf1175874a1dbe Mon Sep 17 00:00:00 2001 From: John Eskew Date: Fri, 15 Aug 2014 11:37:02 -0400 Subject: [PATCH] Phase 1 of adding asset metadata saving to old Mongo --- .../xmodule/xmodule/assetstore/__init__.py | 168 ++++++++ .../xmodule/xmodule/modulestore/__init__.py | 279 +++++++++++- .../lib/xmodule/xmodule/modulestore/mixed.py | 211 ++++++++- .../xmodule/xmodule/modulestore/mongo/base.py | 158 ++++++- .../split_mongo/mongo_connection.py | 6 +- .../tests/test_mixed_modulestore.py | 2 + .../xmodule/modulestore/tests/test_mongo.py | 406 +++++++++++++++++- lms/envs/common.py | 1 + 8 files changed, 1215 insertions(+), 16 deletions(-) create mode 100644 common/lib/xmodule/xmodule/assetstore/__init__.py diff --git a/common/lib/xmodule/xmodule/assetstore/__init__.py b/common/lib/xmodule/xmodule/assetstore/__init__.py new file mode 100644 index 0000000000..622b42f264 --- /dev/null +++ b/common/lib/xmodule/xmodule/assetstore/__init__.py @@ -0,0 +1,168 @@ +""" +Classes representing asset & asset thumbnail metadata. +""" + +from datetime import datetime +from contracts import contract, new_contract +from opaque_keys.edx.keys import CourseKey, AssetKey + +new_contract('AssetKey', AssetKey) +new_contract('datetime', datetime) + + +class IncorrectAssetIdType(Exception): + """ + Raised when the asset ID passed-in to create an AssetMetadata or + AssetThumbnailMetadata is of the wrong type. + """ + pass + + +class AssetMetadata(object): + """ + Stores the metadata associated with a particular course asset. The asset metadata gets stored + in the modulestore. + """ + + TOP_LEVEL_ATTRS = ['basename', 'internal_name', 'locked', 'contenttype', 'md5'] + EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_on'] + ALLOWED_ATTRS = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS + + # All AssetMetadata objects should have AssetLocators with this type. + ASSET_TYPE = 'asset' + + @contract(asset_id='AssetKey', basename='str | unicode | None', internal_name='str | None', locked='bool | None', + contenttype='str | unicode | None', md5='str | None', curr_version='str | None', prev_version='str | None') + def __init__(self, asset_id, + basename=None, internal_name=None, + locked=None, contenttype=None, md5=None, + curr_version=None, prev_version=None): + """ + Construct a AssetMetadata object. + + Arguments: + asset_id (AssetKey): Key identifying this particular asset. + basename (str): Original path to file at asset upload time. + internal_name (str): Name under which the file is stored internally. + locked (bool): If True, only course participants can access the asset. + contenttype (str): MIME type of the asset. + curr_version (str): Current version of the asset. + prev_version (str): Previous version of the asset. + """ + if asset_id.asset_type != self.ASSET_TYPE: + raise IncorrectAssetIdType() + self.asset_id = asset_id + self.basename = basename # Path w/o filename. + self.internal_name = internal_name + self.locked = locked + self.contenttype = contenttype + self.md5 = md5 + self.curr_version = curr_version + self.prev_version = prev_version + self.edited_by = None + self.edited_on = None + + def __repr__(self): + return """AssetMetadata{!r}""".format(( + self.asset_id, + self.basename, self.internal_name, + self.locked, self.contenttype, self.md5, + self.curr_version, self.prev_version, + self.edited_by, self.edited_on + )) + + def update(self, attr_dict): + """ + Set the attributes on the metadata. Ignore all those outside the known fields. + + Arguments: + attr_dict: Prop, val dictionary of all attributes to set. + """ + for attr, val in attr_dict.iteritems(): + if attr in self.ALLOWED_ATTRS: + setattr(self, attr, val) + + def to_mongo(self): + """ + Converts metadata properties into a MongoDB-storable dict. + """ + return { + 'filename': self.asset_id.path, + 'basename': self.basename, + 'internal_name': self.internal_name, + 'locked': self.locked, + 'contenttype': self.contenttype, + 'md5': self.md5, + 'edit_info': { + 'curr_version': self.curr_version, + 'prev_version': self.prev_version, + 'edited_by': self.edited_by, + 'edited_on': self.edited_on + } + } + + @contract(asset_doc='dict | None') + def from_mongo(self, asset_doc): + """ + Fill in all metadata fields from a MongoDB document. + + The asset_id prop is initialized upon construction only. + """ + if asset_doc is None: + return + self.basename = asset_doc['basename'] + self.internal_name = asset_doc['internal_name'] + self.locked = asset_doc['locked'] + self.contenttype = asset_doc['contenttype'] + self.md5 = asset_doc['md5'] + edit_info = asset_doc['edit_info'] + self.curr_version = edit_info['curr_version'] + self.prev_version = edit_info['prev_version'] + self.edited_by = edit_info['edited_by'] + self.edited_on = edit_info['edited_on'] + + +class AssetThumbnailMetadata(object): + """ + Stores the metadata associated with the thumbnail of a course asset. + """ + + # All AssetThumbnailMetadata objects should have AssetLocators with this type. + ASSET_TYPE = 'thumbnail' + + @contract(asset_id='AssetKey', internal_name='str | unicode | None') + def __init__(self, asset_id, internal_name=None): + """ + Construct a AssetThumbnailMetadata object. + + Arguments: + asset_id (AssetKey): Key identifying this particular asset. + internal_name (str): Name under which the file is stored internally. + """ + if asset_id.asset_type != self.ASSET_TYPE: + raise IncorrectAssetIdType() + self.asset_id = asset_id + self.internal_name = internal_name + + def __repr__(self): + return """AssetMetadata{!r}""".format((self.asset_id, self.internal_name)) + + def to_mongo(self): + """ + Converts metadata properties into a MongoDB-storable dict. + """ + return { + 'filename': self.asset_id.path, + 'internal_name': self.internal_name + } + + @contract(thumbnail_doc='dict | None') + def from_mongo(self, thumbnail_doc): + """ + Fill in all metadata fields from a MongoDB document. + + The asset_id prop is initialized upon construction only. + """ + if thumbnail_doc is None: + return + self.internal_name = thumbnail_doc['internal_name'] diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 92f78b94fd..04684c3bba 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -12,23 +12,30 @@ from uuid import uuid4 from collections import namedtuple, defaultdict import collections from contextlib import contextmanager +import functools +import threading from abc import ABCMeta, abstractmethod +from contracts import contract, new_contract from xblock.plugin import default_select from .exceptions import InvalidLocationError, InsufficientSpecificationError from xmodule.errortracker import make_error_tracker -from opaque_keys.edx.keys import CourseKey, UsageKey +from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata +from opaque_keys.edx.keys import CourseKey, UsageKey, AssetKey from opaque_keys.edx.locations import Location # For import backwards compatibility from opaque_keys import InvalidKeyError from opaque_keys.edx.locations import SlashSeparatedCourseKey from xblock.runtime import Mixologist from xblock.core import XBlock -import functools -import threading log = logging.getLogger('edx.modulestore') +new_contract('CourseKey', CourseKey) +new_contract('AssetKey', AssetKey) +new_contract('AssetMetadata', AssetMetadata) +new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata) + class ModuleStoreEnum(object): """ @@ -740,6 +747,9 @@ class ModuleStoreReadBase(BulkOperationsMixin, ModuleStoreRead): """ @functools.wraps(func) def wrapper(self, *args, **kwargs): + """ + Wraps a method to memoize results. + """ if self.request_cache: cache_key = '&'.join([hashvalue(arg) for arg in args]) if cache_key in self.request_cache.data.setdefault(func.__name__, {}): @@ -863,6 +873,269 @@ class ModuleStoreWriteBase(ModuleStoreReadBase, ModuleStoreWrite): parent.children.append(item.location) self.update_item(parent, user_id) + def _find_course_assets(self, course_key): + """ + Base method to override. + """ + raise NotImplementedError() + + def _find_course_asset(self, course_key, filename, get_thumbnail=False): + """ + Internal; finds or creates course asset info -and- finds existing asset (or thumbnail) metadata. + + Arguments: + course_key (CourseKey): course identifier + filename (str): filename of the asset or thumbnail + get_thumbnail (bool): True gets thumbnail data, False gets asset data + + Returns: + Asset info for the course, index of asset/thumbnail in list (None if asset/thumbnail does not exist) + """ + course_assets = self._find_course_assets(course_key) + + if get_thumbnail: + all_assets = course_assets['thumbnails'] + else: + all_assets = course_assets['assets'] + + # See if this asset already exists by checking the external_filename. + # Studio doesn't currently support using multiple course assets with the same filename. + # So use the filename as the unique identifier. + for idx, asset in enumerate(all_assets): + if asset['filename'] == filename: + return course_assets, idx + + return course_assets, None + + def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False): + """ + Base method to over-ride in modulestore. + """ + raise NotImplementedError() + + @contract(course_key='CourseKey', asset_metadata='AssetMetadata', user_id='str | unicode') + def save_asset_metadata(self, course_key, asset_metadata, user_id): + """ + Saves the asset metadata for a particular course's asset. + + Arguments: + course_key (CourseKey): course identifier + asset_metadata (AssetMetadata): data about the course asset data + + Returns: + True if metadata save was successful, else False + """ + return self._save_asset_info(course_key, asset_metadata, user_id, thumbnail=False) + + @contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata') + def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata): + """ + Saves the asset thumbnail metadata for a particular course asset's thumbnail. + + Arguments: + course_key (CourseKey): course identifier + asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail + + Returns: + True if thumbnail metadata save was successful, else False + """ + return self._save_asset_info(course_key, asset_thumbnail_metadata, '', thumbnail=True) + + @contract(asset_key='AssetKey') + def _find_asset_info(self, asset_key, thumbnail=False): + """ + Find the info for a particular course asset/thumbnail. + + Arguments: + asset_key (AssetKey): key containing original asset filename + thumbnail (bool): True if finding thumbnail, False if finding asset metadata + + Returns: + asset/thumbnail metadata (AssetMetadata/AssetThumbnailMetadata) -or- None if not found + """ + course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, thumbnail) + if asset_idx is None: + return None + + if thumbnail: + info = 'thumbnails' + mdata = AssetThumbnailMetadata(asset_key, asset_key.path) + else: + info = 'assets' + mdata = AssetMetadata(asset_key, asset_key.path) + all_assets = course_assets[info] + mdata.from_mongo(all_assets[asset_idx]) + return mdata + + @contract(asset_key='AssetKey') + def find_asset_metadata(self, asset_key): + """ + Find the metadata for a particular course asset. + + Arguments: + asset_key (AssetKey): key containing original asset filename + + Returns: + asset metadata (AssetMetadata) -or- None if not found + """ + return self._find_asset_info(asset_key, thumbnail=False) + + @contract(asset_key='AssetKey') + def find_asset_thumbnail_metadata(self, asset_key): + """ + Find the metadata for a particular course asset. + + Arguments: + asset_key (AssetKey): key containing original asset filename + + Returns: + asset metadata (AssetMetadata) -or- None if not found + """ + return self._find_asset_info(asset_key, thumbnail=True) + + @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None', get_thumbnails='bool') + def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False): + """ + Returns a list of static asset (or thumbnail) metadata for a course. + + Args: + course_key (CourseKey): course identifier + start (int): optional - start at this asset number + maxresults (int): optional - return at most this many, -1 means no limit + sort (array): optional - None means no sort + (sort_by (str), sort_order (str)) + sort_by - one of 'uploadDate' or 'displayname' + sort_order - one of 'ascending' or 'descending' + get_thumbnails (bool): True if getting thumbnail metadata, else getting asset metadata + + Returns: + List of AssetMetadata or AssetThumbnailMetadata objects. + """ + course_assets = self._find_course_assets(course_key) + if course_assets is None: + # If no course assets are found, return None instead of empty list + # to distinguish zero assets from "not able to retrieve assets". + return None + + if get_thumbnails: + all_assets = course_assets['thumbnails'] + else: + all_assets = course_assets['assets'] + + # DO_NEXT: Add start/maxresults/sort functionality as part of https://openedx.atlassian.net/browse/PLAT-74 + if start and maxresults and sort: + pass + + ret_assets = [] + for asset in all_assets: + if get_thumbnails: + thumb = AssetThumbnailMetadata(course_key.make_asset_key('thumbnail', asset['filename']), + internal_name=asset['filename']) + ret_assets.append(thumb) + else: + one_asset = AssetMetadata(course_key.make_asset_key('asset', asset['filename'])) + one_asset.from_mongo(asset) + ret_assets.append(one_asset) + return ret_assets + + @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None') + def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None): + """ + Returns a list of static assets for a course. + By default all assets are returned, but start and maxresults can be provided to limit the query. + + Args: + course_key (CourseKey): course identifier + start (int): optional - start at this asset number + maxresults (int): optional - return at most this many, -1 means no limit + sort (array): optional - None means no sort + (sort_by (str), sort_order (str)) + sort_by - one of 'uploadDate' or 'displayname' + sort_order - one of 'ascending' or 'descending' + + Returns: + List of AssetMetadata objects. + """ + return self._get_all_asset_metadata(course_key, start, maxresults, sort, get_thumbnails=False) + + @contract(course_key='CourseKey') + def get_all_asset_thumbnail_metadata(self, course_key): + """ + Returns a list of thumbnails for all course assets. + + Args: + course_key (CourseKey): course identifier + + Returns: + List of AssetThumbnailMetadata objects. + """ + return self._get_all_asset_metadata(course_key, get_thumbnails=True) + + def set_asset_metadata_attrs(self, asset_key, attrs, user_id): + """ + Base method to over-ride in modulestore. + """ + raise NotImplementedError() + + def _delete_asset_data(self, asset_key, thumbnail=False): + """ + Base method to over-ride in modulestore. + """ + raise NotImplementedError() + + @contract(asset_key='AssetKey', attr=str) + def set_asset_metadata_attr(self, asset_key, attr, value, user_id): + """ + Add/set the given attr on the asset at the given location. Value can be any type which pymongo accepts. + + Arguments: + asset_key (AssetKey): asset identifier + attr (str): which attribute to set + value: the value to set it to (any type pymongo accepts such as datetime, number, string) + + Raises: + ItemNotFoundError if no such item exists + AttributeError is attr is one of the build in attrs. + """ + return self.set_asset_metadata_attrs(asset_key, {attr: value}, user_id) + + @contract(asset_key='AssetKey') + def delete_asset_metadata(self, asset_key): + """ + Deletes a single asset's metadata. + + Arguments: + asset_key (AssetKey): locator containing original asset filename + + Returns: + Number of asset metadata entries deleted (0 or 1) + """ + return self._delete_asset_data(asset_key, thumbnail=False) + + @contract(asset_key='AssetKey') + def delete_asset_thumbnail_metadata(self, asset_key): + """ + Deletes a single asset's metadata. + + Arguments: + asset_key (AssetKey): locator containing original asset filename + + Returns: + Number of asset metadata entries deleted (0 or 1) + """ + return self._delete_asset_data(asset_key, thumbnail=True) + + @contract(source_course_key='CourseKey', dest_course_key='CourseKey') + def copy_all_asset_metadata(self, source_course_key, dest_course_key): + """ + Copy all the course assets from source_course_key to dest_course_key. + + Arguments: + source_course_key (CourseKey): identifier of course to copy from + dest_course_key (CourseKey): identifier of course to copy to + """ + pass + def only_xmodules(identifier, entry_points): """Only use entry_points that are supplied by the xmodule package""" diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 99c01438a4..5ab6a5c69b 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -9,10 +9,12 @@ import logging from contextlib import contextmanager import itertools import functools +from contracts import contract, new_contract from opaque_keys import InvalidKeyError -from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.keys import CourseKey, AssetKey from opaque_keys.edx.locations import SlashSeparatedCourseKey +from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata from . import ModuleStoreWriteBase from . import ModuleStoreEnum @@ -20,6 +22,10 @@ from .exceptions import ItemNotFoundError, DuplicateCourseError from .draft_and_published import ModuleStoreDraftAndPublished from .split_migrator import SplitMigrator +new_contract('CourseKey', CourseKey) +new_contract('AssetKey', AssetKey) +new_contract('AssetMetadata', AssetMetadata) +new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata) log = logging.getLogger(__name__) @@ -309,6 +315,209 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): store = self._get_modulestore_for_courseid(course_key) return store.delete_course(course_key, user_id) + def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False): + """ + Base method to over-ride in modulestore. + """ + raise NotImplementedError() + + def _delete_asset_data(self, asset_key, thumbnail=False): + """ + Base method to over-ride in modulestore. + """ + raise NotImplementedError() + + def _find_course_assets(self, course_key): + """ + Base method to override. + """ + raise NotImplementedError() + + @contract(course_key='CourseKey', asset_metadata='AssetMetadata') + def save_asset_metadata(self, course_key, asset_metadata, user_id): + """ + Saves the asset metadata for a particular course's asset. + + Args: + course_key (CourseKey): course identifier + asset_metadata (AssetMetadata): data about the course asset data + + Returns: + bool: True if metadata save was successful, else False + """ + store = self._get_modulestore_for_courseid(course_key) + return store.save_asset_metadata(course_key, asset_metadata, user_id) + + @contract(course_key='CourseKey', asset_thumbnail_metadata='AssetThumbnailMetadata') + def save_asset_thumbnail_metadata(self, course_key, asset_thumbnail_metadata): + """ + Saves the asset thumbnail metadata for a particular course asset's thumbnail. + + Arguments: + course_key (CourseKey): course identifier + asset_thumbnail_metadata (AssetThumbnailMetadata): data about the course asset thumbnail + + Returns: + True if thumbnail metadata save was successful, else False + """ + store = self._get_modulestore_for_courseid(course_key) + return store.save_asset_metadata(course_key, asset_thumbnail_metadata) + + @contract(asset_key='AssetKey') + def find_asset_metadata(self, asset_key): + """ + Find the metadata for a particular course asset. + + Args: + asset_key (AssetKey): locator containing original asset filename + + Returns: + asset metadata (AssetMetadata) -or- None if not found + """ + store = self._get_modulestore_for_courseid(asset_key.course_key) + return store.find_asset_metadata(asset_key) + + @contract(asset_key='AssetKey') + def find_asset_thumbnail_metadata(self, asset_key): + """ + Find the metadata for a particular course asset. + + Arguments: + asset_key (AssetKey): key containing original asset filename + + Returns: + asset metadata (AssetMetadata) -or- None if not found + """ + store = self._get_modulestore_for_courseid(asset_key.course_key) + return store.find_asset_thumbnail_metadata(asset_key) + + @contract(course_key='CourseKey', start=int, maxresults=int, sort='list | None') + def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None): + """ + Returns a list of static assets for a course. + By default all assets are returned, but start and maxresults can be provided to limit the query. + + Args: + course_key (CourseKey): course identifier + start (int): optional - start at this asset number + maxresults (int): optional - return at most this many, -1 means no limit + sort (array): optional - None means no sort + (sort_by (str), sort_order (str)) + sort_by - one of 'uploadDate' or 'displayname' + sort_order - one of 'ascending' or 'descending' + + Returns: + List of asset data dictionaries, which have the following keys: + asset_key (AssetKey): asset identifier + displayname: The human-readable name of the asset + uploadDate (datetime.datetime): The date and time that the file was uploaded + contentType: The mimetype string of the asset + md5: An md5 hash of the asset content + """ + store = self._get_modulestore_for_courseid(course_key) + return store.get_all_asset_metadata(course_key, start, maxresults, sort) + + @contract(course_key='CourseKey') + def get_all_asset_thumbnail_metadata(self, course_key): + """ + Returns a list of thumbnails for all course assets. + + Args: + course_key (CourseKey): course identifier + + Returns: + List of AssetThumbnailMetadata objects. + """ + store = self._get_modulestore_for_courseid(course_key) + return store.get_all_asset_thumbnail_metadata(course_key) + + @contract(asset_key='AssetKey') + def delete_asset_metadata(self, asset_key): + """ + Deletes a single asset's metadata. + + Arguments: + asset_id (AssetKey): locator containing original asset filename + + Returns: + Number of asset metadata entries deleted (0 or 1) + """ + store = self._get_modulestore_for_courseid(asset_key.course_key) + return store.delete_asset_metadata(asset_key) + + @contract(asset_key='AssetKey') + def delete_asset_thumbnail_metadata(self, asset_key): + """ + Deletes a single asset's metadata. + + Arguments: + asset_key (AssetKey): locator containing original asset filename + + Returns: + Number of asset metadata entries deleted (0 or 1) + """ + store = self._get_modulestore_for_courseid(asset_key.course_key) + return store.delete_asset_metadata(asset_key) + + @contract(course_key='CourseKey') + def delete_all_asset_metadata(self, course_key): + """ + Delete all of the assets which use this course_key as an identifier. + + Arguments: + course_key (CourseKey): course_identifier + """ + store = self._get_modulestore_for_courseid(course_key) + return store.delete_all_asset_metadata(course_key) + + @contract(source_course_key='CourseKey', dest_course_key='CourseKey') + def copy_all_asset_metadata(self, source_course_key, dest_course_key): + """ + Copy all the course assets from source_course_key to dest_course_key. + + Arguments: + source_course_key (CourseKey): identifier of course to copy from + dest_course_key (CourseKey): identifier of course to copy to + """ + # When implementing this in https://openedx.atlassian.net/browse/PLAT-78 , consider this: + # Check the modulestores of both the source and dest course_keys. If in different modulestores, + # export all asset data from one modulestore and import it into the dest one. + store = self._get_modulestore_for_courseid(source_course_key) + return store.copy_all_asset_metadata(source_course_key, dest_course_key) + + @contract(asset_key='AssetKey', attr=str) + def set_asset_metadata_attr(self, asset_key, attr, value, user_id): + """ + Add/set the given attr on the asset at the given location. Value can be any type which pymongo accepts. + + Arguments: + asset_key (AssetKey): asset identifier + attr (str): which attribute to set + value: the value to set it to (any type pymongo accepts such as datetime, number, string) + + Raises: + NotFoundError if no such item exists + AttributeError is attr is one of the build in attrs. + """ + store = self._get_modulestore_for_courseid(asset_key.course_key) + return store.set_asset_metadata_attrs(asset_key, attr, value, user_id) + + @contract(asset_key='AssetKey', attr_dict=dict) + def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id): + """ + Add/set the given dict of attrs on the asset at the given location. Value can be any type which pymongo accepts. + + Arguments: + asset_key (AssetKey): asset identifier + attr_dict (dict): attribute/value pairs to set + + Raises: + NotFoundError if no such item exists + AttributeError is attr is one of the build in attrs. + """ + store = self._get_modulestore_for_courseid(asset_key.course_key) + return store.set_asset_metadata_attrs(asset_key, attr_dict, user_id) + @strip_key def get_parent_location(self, location, **kwargs): """ diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 1f3e67b03b..4fcb3f0497 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -24,6 +24,7 @@ from fs.osfs import OSFS from path import path from datetime import datetime from pytz import UTC +from contracts import contract, new_contract from importlib import import_module from xmodule.errortracker import null_error_tracker, exc_info_to_str @@ -41,12 +42,18 @@ from xmodule.modulestore.inheritance import InheritanceMixin, inherit_metadata, from xblock.core import XBlock from opaque_keys.edx.locations import SlashSeparatedCourseKey from opaque_keys.edx.locator import CourseLocator -from opaque_keys.edx.keys import UsageKey, CourseKey +from opaque_keys.edx.keys import UsageKey, CourseKey, AssetKey from xmodule.exceptions import HeartbeatFailure from xmodule.modulestore.edit_info import EditInfoRuntimeMixin +from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata log = logging.getLogger(__name__) +new_contract('CourseKey', CourseKey) +new_contract('AssetKey', AssetKey) +new_contract('AssetMetadata', AssetMetadata) +new_contract('AssetThumbnailMetadata', AssetThumbnailMetadata) + # sort order that returns DRAFT items first SORT_REVISION_FAVOR_DRAFT = ('_id.revision', pymongo.DESCENDING) @@ -195,7 +202,6 @@ class CachingDescriptorSystem(MakoDescriptorSystem, EditInfoRuntimeMixin): category = json_data['location']['category'] class_ = self.load_block_type(category) - definition = json_data.get('definition', {}) metadata = json_data.get('metadata', {}) for old_name, new_name in getattr(class_, 'metadata_translations', {}).items(): @@ -443,7 +449,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo super(MongoModuleStore, self).__init__(contentstore=contentstore, **kwargs) def do_connection( - db, collection, host, port=27017, tz_aware=True, user=None, password=None, **kwargs + db, collection, host, port=27017, tz_aware=True, user=None, password=None, asset_collection=None, **kwargs ): """ Create & open the connection, authenticate, and provide pointers to the collection @@ -460,6 +466,11 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo ) self.collection = self.database[collection] + # Collection which stores asset metadata. + self.asset_collection = None + if asset_collection is not None: + self.asset_collection = self.database[asset_collection] + if user is not None and password is not None: self.database.authenticate(user, password) @@ -1436,6 +1447,147 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo field_data = KvsFieldData(kvs) return field_data + def _find_course_assets(self, course_key): + """ + Internal; finds (or creates) course asset info about all assets for a particular course + + Arguments: + course_key (CourseKey): course identifier + + Returns: + Asset info for the course + """ + if self.asset_collection is None: + return None + + # Using the course_key, find or insert the course asset metadata document. + # A single document exists per course to store the course asset metadata. + course_assets = self.asset_collection.find_one( + {'course_id': unicode(course_key)}, + fields=('course_id', 'storage', 'assets', 'thumbnails') + ) + + if course_assets is None: + # Not found, so create. + course_assets = {'course_id': unicode(course_key), 'storage': 'FILLMEIN-TMP', 'assets': [], 'thumbnails': []} + course_assets['_id'] = self.asset_collection.insert(course_assets) + + return course_assets + + @contract(course_key='CourseKey', asset_metadata='AssetMetadata | AssetThumbnailMetadata', user_id='str | unicode') + def _save_asset_info(self, course_key, asset_metadata, user_id, thumbnail=False): + """ + Saves the info for a particular course's asset/thumbnail. + + Arguments: + course_key (CourseKey): course identifier + asset_metadata (AssetMetadata/AssetThumbnailMetadata): data about the course asset/thumbnail + thumbnail (bool): True if saving thumbnail metadata, False if saving asset metadata + + Returns: + True if info save was successful, else False + """ + if self.asset_collection is None: + return False + + course_assets, asset_idx = self._find_course_asset(course_key, asset_metadata.asset_id.path, thumbnail) + info = 'thumbnails' if thumbnail else 'assets' + all_assets = course_assets[info] + + # Set the edited information for assets only - not thumbnails. + if not thumbnail: + asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)}) + + # Translate metadata to Mongo format. + metadata_to_insert = asset_metadata.to_mongo() + if asset_idx is None: + # Append new metadata. + # Future optimization: Insert in order & binary search to retrieve. + all_assets.append(metadata_to_insert) + else: + # Replace existing metadata. + all_assets[asset_idx] = metadata_to_insert + + # Update the document. + self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets}}) + return True + + @contract(asset_key='AssetKey', attr_dict=dict) + def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id): + """ + Add/set the given dict of attrs on the asset at the given location. Value can be any type which pymongo accepts. + + Arguments: + asset_key (AssetKey): asset identifier + attr_dict (dict): attribute: value pairs to set + + Raises: + ItemNotFoundError if no such item exists + AttributeError is attr is one of the build in attrs. + """ + if self.asset_collection is None: + return + + course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path) + if asset_idx is None: + raise ItemNotFoundError(asset_key) + + # Form an AssetMetadata. + all_assets = course_assets['assets'] + md = AssetMetadata(asset_key, asset_key.path) + md.from_mongo(all_assets[asset_idx]) + md.update(attr_dict) + md.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)}) + + # Generate a Mongo doc from the metadata and update the course asset info. + all_assets[asset_idx] = md.to_mongo() + + self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {'assets': all_assets}}) + + @contract(asset_key='AssetKey') + def _delete_asset_data(self, asset_key, thumbnail=False): + """ + Internal; deletes a single asset's metadata -or- thumbnail. + + Arguments: + asset_key (AssetKey): key containing original asset/thumbnail filename + thumbnail: True if thumbnail deletion, False if asset metadata deletion + + Returns: + Number of asset metadata/thumbnail entries deleted (0 or 1) + """ + if self.asset_collection is None: + return 0 + + course_assets, asset_idx = self._find_course_asset(asset_key.course_key, asset_key.path, get_thumbnail=thumbnail) + if asset_idx is None: + return 0 + + info = 'thumbnails' if thumbnail else 'assets' + + all_asset_info = course_assets[info] + all_asset_info.pop(asset_idx) + + # Update the document. + self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_asset_info}}) + return 1 + + @contract(course_key='CourseKey') + def delete_all_asset_metadata(self, course_key): + """ + Delete all of the assets which use this course_key as an identifier. + + Arguments: + course_key (CourseKey): course_identifier + """ + if self.asset_collection is None: + return + + # Using the course_id, find the course asset metadata document. + # A single document exists per course to store the course asset metadata. + course_assets = self._find_course_assets(course_key) + self.asset_collection.remove(course_assets['_id']) + def heartbeat(self): """ Check that the db is reachable. diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py index f5e8bd0105..b28e3f77a6 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/mongo_connection.py @@ -99,7 +99,7 @@ class MongoConnection(object): Segregation of pymongo functions from the data modeling mechanisms for split modulestore. """ def __init__( - self, db, collection, host, port=27017, tz_aware=True, user=None, password=None, **kwargs + self, db, collection, host, port=27017, tz_aware=True, user=None, password=None, asset_collection=None, **kwargs ): """ Create & open the connection, authenticate, and provide pointers to the collections @@ -114,6 +114,10 @@ class MongoConnection(object): db ) + # Remove when adding official Split support for asset metadata storage. + if asset_collection: + pass + if user is not None and password is not None: self.database.authenticate(user, password) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py index 0027e55b4c..fc17af92b9 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -47,6 +47,7 @@ class TestMixedModuleStore(CourseComparisonTest): PORT = MONGO_PORT_NUM DB = 'test_mongo_%s' % uuid4().hex[:5] COLLECTION = 'modulestore' + ASSET_COLLECTION = 'assetstore' FS_ROOT = DATA_DIR DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor' RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': '' @@ -67,6 +68,7 @@ class TestMixedModuleStore(CourseComparisonTest): 'port': PORT, 'db': DB, 'collection': COLLECTION, + 'asset_collection': ASSET_COLLECTION, } OPTIONS = { 'mappings': { diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py index 45e9537d7c..6d7b101b2b 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py @@ -14,6 +14,7 @@ from datetime import datetime from pytz import UTC import unittest from xblock.core import XBlock +from ddt import ddt, data from xblock.fields import Scope, Reference, ReferenceList, ReferenceValueDict from xblock.runtime import KeyValueStore @@ -30,6 +31,7 @@ from opaque_keys.edx.keys import UsageKey from xmodule.modulestore.xml_exporter import export_to_xml from xmodule.modulestore.xml_importer import import_from_xml, perform_xlint from xmodule.contentstore.mongo import MongoContentStore +from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata from nose.tools import assert_in from xmodule.exceptions import NotFoundError @@ -45,6 +47,7 @@ HOST = MONGO_HOST PORT = MONGO_PORT_NUM DB = 'test_mongo_%s' % uuid4().hex[:5] COLLECTION = 'modulestore' +ASSET_COLLECTION = 'assetstore' FS_ROOT = DATA_DIR # TODO (vshnayder): will need a real fs_root for testing load_item DEFAULT_CLASS = 'xmodule.raw_module.RawDescriptor' RENDER_TEMPLATE = lambda t_n, d, ctx = None, nsp = 'main': '' @@ -60,8 +63,10 @@ class ReferenceTestXBlock(XBlock, XModuleMixin): reference_dict = ReferenceValueDict(scope=Scope.settings) -class TestMongoModuleStore(unittest.TestCase): - '''Tests!''' +class TestMongoModuleStoreBase(unittest.TestCase): + ''' + Basic setup for all tests + ''' # Explicitly list the courses to load (don't want the big one) courses = ['toy', 'simple', 'simple_with_draft', 'test_unicode'] @@ -87,6 +92,13 @@ class TestMongoModuleStore(unittest.TestCase): cls.connection.drop_database(DB) cls.connection.close() + @classmethod + def add_asset_collection(cls, doc_store_config): + """ + No asset collection. + """ + pass + @classmethod def initdb(cls): # connect to the db @@ -95,7 +107,10 @@ class TestMongoModuleStore(unittest.TestCase): 'port': PORT, 'db': DB, 'collection': COLLECTION, + #'asset_collection': ASSET_COLLECTION, } + cls.add_asset_collection(doc_store_config) + # since MongoModuleStore and MongoContentStore are basically assumed to be together, create this class # as well content_store = MongoContentStore(HOST, DB, port=PORT) @@ -136,14 +151,33 @@ class TestMongoModuleStore(unittest.TestCase): # Destroy the test db. connection.drop_database(DB) - def setUp(self): - # make a copy for convenience - self.connection = TestMongoModuleStore.connection - self.dummy_user = ModuleStoreEnum.UserID.test + @classmethod + def setUp(cls): + cls.dummy_user = ModuleStoreEnum.UserID.test - def tearDown(self): + @classmethod + def tearDown(cls): pass + +class TestMongoModuleStore(TestMongoModuleStoreBase): + '''Module store tests''' + + @classmethod + def add_asset_collection(cls, doc_store_config): + """ + No asset collection - it's not used in the tests below. + """ + pass + + @classmethod + def setupClass(cls): + super(TestMongoModuleStore, cls).setupClass() + + @classmethod + def teardownClass(cls): + super(TestMongoModuleStore, cls).teardownClass() + def test_init(self): '''Make sure the db loads''' ids = list(self.connection[DB][COLLECTION].find({}, {'_id': True})) @@ -233,7 +267,6 @@ class TestMongoModuleStore(unittest.TestCase): self.draft_store.get_item(Location('edX', 'test_unicode', '2012_Fall', 'chapter', 'Overview')), ) - def test_find_one(self): assert_not_none( self.draft_store._find_one(Location('edX', 'toy', '2012_Fall', 'course', '2012_Fall')), @@ -632,6 +665,363 @@ class TestMongoModuleStore(unittest.TestCase): shutil.rmtree(root_dir) +@ddt +class TestMongoAssetMetadataStorage(TestMongoModuleStore): + """ + Tests for storing/querying course asset metadata from Mongo storage. + """ + def _make_asset_metadata(self, asset_loc): + """ + Make a single test asset metadata. + """ + return AssetMetadata(asset_loc, internal_name='EKMND332DDBK', + basename='pictures/historical', contenttype='image/jpeg', + locked=False, md5='77631ca4f0e08419b70726a447333ab6', + curr_version='v1.0', prev_version='v0.95') + + def _make_asset_thumbnail_metadata(self, asset_key): + """ + Make a single test asset thumbnail metadata. + """ + return AssetThumbnailMetadata(asset_key, internal_name='ABC39XJUDN2') + + @classmethod + def add_asset_collection(cls, doc_store_config): + """ + Valid asset collection. + """ + doc_store_config['asset_collection'] = ASSET_COLLECTION + + @classmethod + def setupClass(cls): + super(TestMongoAssetMetadataStorage, cls).setupClass() + + @classmethod + def teardownClass(cls): + super(TestMongoAssetMetadataStorage, cls).teardownClass() + + def setup_assets(self): + """ + Setup assets. + """ + asset_fields = ('filename', 'internal_name', 'basename', 'locked', 'curr_version', 'prev_version') + asset1_vals = ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, '14', '13') + asset2_vals = ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, '1', None) + asset3_vals = ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, 'AB', 'AA') + asset4_vals = ('dog.png', 'PUPY4242X', 'pictures/animals', True, '5', '4') + asset5_vals = ('not_here.txt', 'JJJCCC747', '/dev/null', False, '50', '49') + + asset1 = dict(zip(asset_fields[1:], asset1_vals[1:])) + asset2 = dict(zip(asset_fields[1:], asset2_vals[1:])) + asset3 = dict(zip(asset_fields[1:], asset3_vals[1:])) + asset4 = dict(zip(asset_fields[1:], asset4_vals[1:])) + non_existent_asset = dict(zip(asset_fields[1:], asset5_vals[1:])) + + # Asset6 and thumbnail6 have equivalent information on purpose. + asset6_vals = ('asset.txt', 'JJJCCC747858', '/dev/null', False, '50', '49') + asset6 = dict(zip(asset_fields[1:], asset6_vals[1:])) + + asset1_key = self.course1.id.make_asset_key('asset', asset1_vals[0]) + asset2_key = self.course1.id.make_asset_key('asset', asset2_vals[0]) + asset3_key = self.course2.id.make_asset_key('asset', asset3_vals[0]) + asset4_key = self.course2.id.make_asset_key('asset', asset4_vals[0]) + asset5_key = self.course2.id.make_asset_key('asset', asset5_vals[0]) + asset6_key = self.course2.id.make_asset_key('asset', asset6_vals[0]) + + asset1_md = AssetMetadata(asset1_key, **asset1) + asset2_md = AssetMetadata(asset2_key, **asset2) + asset3_md = AssetMetadata(asset3_key, **asset3) + asset4_md = AssetMetadata(asset4_key, **asset4) + asset5_md = AssetMetadata(asset5_key, **non_existent_asset) + asset6_md = AssetMetadata(asset6_key, **asset6) + + editing_user = 'Oliver Twist' + self.assertTrue(self.draft_store.save_asset_metadata(self.course1.id, asset1_md, editing_user)) + self.assertTrue(self.draft_store.save_asset_metadata(self.course1.id, asset2_md, editing_user)) + self.assertTrue(self.draft_store.save_asset_metadata(self.course2.id, asset3_md, editing_user)) + self.assertTrue(self.draft_store.save_asset_metadata(self.course2.id, asset4_md, editing_user)) + # asset5 and asset6 are not saved on purpose! + + return (asset1_md, asset2_md, asset3_md, asset4_md, asset5_md, asset6_md) + + def setup_thumbnails(self): + """ + Setup thumbs. + """ + thumbnail_fields = ('filename', 'internal_name') + thumbnail1_vals = ('cat_thumb.jpg', 'XYXYXYXYXYXY') + thumbnail2_vals = ('kitten_thumb.jpg', '123ABC123ABC') + thumbnail3_vals = ('puppy_thumb.jpg', 'ADAM12ADAM12') + thumbnail4_vals = ('meerkat_thumb.jpg', 'CHIPSPONCH14') + thumbnail5_vals = ('corgi_thumb.jpg', 'RON8LDXFFFF10') + + thumbnail1 = dict(zip(thumbnail_fields[1:], thumbnail1_vals[1:])) + thumbnail2 = dict(zip(thumbnail_fields[1:], thumbnail2_vals[1:])) + thumbnail3 = dict(zip(thumbnail_fields[1:], thumbnail3_vals[1:])) + thumbnail4 = dict(zip(thumbnail_fields[1:], thumbnail4_vals[1:])) + non_existent_thumbnail = dict(zip(thumbnail_fields[1:], thumbnail5_vals[1:])) + + # Asset6 and thumbnail6 have equivalent information on purpose. + thumbnail6_vals = ('asset.txt', 'JJJCCC747858') + thumbnail6 = dict(zip(thumbnail_fields[1:], thumbnail6_vals[1:])) + + thumb1_key = self.course1.id.make_asset_key('thumbnail', thumbnail1_vals[0]) + thumb2_key = self.course1.id.make_asset_key('thumbnail', thumbnail2_vals[0]) + thumb3_key = self.course2.id.make_asset_key('thumbnail', thumbnail3_vals[0]) + thumb4_key = self.course2.id.make_asset_key('thumbnail', thumbnail4_vals[0]) + thumb5_key = self.course2.id.make_asset_key('thumbnail', thumbnail5_vals[0]) + thumb6_key = self.course2.id.make_asset_key('thumbnail', thumbnail6_vals[0]) + + thumb1_md = AssetThumbnailMetadata(thumb1_key, **thumbnail1) + thumb2_md = AssetThumbnailMetadata(thumb2_key, **thumbnail2) + thumb3_md = AssetThumbnailMetadata(thumb3_key, **thumbnail3) + thumb4_md = AssetThumbnailMetadata(thumb4_key, **thumbnail4) + thumb5_md = AssetThumbnailMetadata(thumb5_key, **non_existent_thumbnail) + thumb6_md = AssetThumbnailMetadata(thumb6_key, **thumbnail6) + + self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course1.id, thumb1_md)) + self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course1.id, thumb2_md)) + self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course2.id, thumb3_md)) + self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course2.id, thumb4_md)) + # thumb5 and thumb6 are not saved on purpose! + + return (thumb1_md, thumb2_md, thumb3_md, thumb4_md, thumb5_md, thumb6_md) + + def setUp(self): + """ + Set up a quantity of test asset metadata for testing purposes. + """ + super(TestMongoAssetMetadataStorage, self).setUp() + + courses = self.draft_store.get_courses() + self.course1 = courses[0] + self.course2 = courses[1] + + (self.asset1_md, self.asset2_md, self.asset3_md, self.asset4_md, self.asset5_md, self.asset6_md) = self.setup_assets() + (self.thumb1_md, self.thumb2_md, self.thumb3_md, self.thumb4_md, self.thumb5_md, self.thumb6_md) = self.setup_thumbnails() + + def tearDown(self): + self.draft_store.delete_all_asset_metadata(self.course1.id) + self.draft_store.delete_all_asset_metadata(self.course2.id) + + def test_save_one_and_confirm(self): + courses = self.draft_store.get_courses() + course = courses[0] + asset_filename = 'burnside.jpg' + new_asset_loc = course.id.make_asset_key('asset', asset_filename) + # Confirm that the asset's metadata is not present. + self.assertIsNone(self.draft_store.find_asset_metadata(new_asset_loc)) + # Save the asset's metadata. + new_asset_md = self._make_asset_metadata(new_asset_loc) + self.assertTrue(self.draft_store.save_asset_metadata(course.id, new_asset_md, 'John Doe')) + # Find the asset's metadata and confirm it's the same. + found_asset_md = self.draft_store.find_asset_metadata(new_asset_loc) + self.assertIsNotNone(found_asset_md) + self.assertEquals(new_asset_md.asset_id, found_asset_md.asset_id) + # Confirm that only two setup plus one asset's metadata exists. + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 3) + # Delete all metadata and confirm it's gone. + self.draft_store.delete_all_asset_metadata(course.id) + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0) + + def test_delete_all_without_creation(self): + courses = self.draft_store.get_courses() + course = courses[0] + # Confirm that only setup asset metadata exists. + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 2) + # Now delete the metadata. + self.draft_store.delete_all_asset_metadata(course.id) + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0) + # Now delete the non-existent metadata. + self.draft_store.delete_all_asset_metadata(course.id) + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0) + + def test_save_many_and_delete_one(self): + # Make sure there's two assets. + self.assertEquals(len(self.draft_store.get_all_asset_metadata(self.course1.id)), 2) + # Delete one of the assets. + self.assertEquals(self.draft_store.delete_asset_metadata(self.asset1_md.asset_id), 1) + self.assertEquals(len(self.draft_store.get_all_asset_metadata(self.course1.id)), 1) + # Attempt to delete an asset that doesn't exist. + self.assertEquals(self.draft_store.delete_asset_metadata(self.asset5_md.asset_id), 0) + self.assertEquals(len(self.draft_store.get_all_asset_metadata(self.course1.id)), 1) + + def test_find_existing_and_non_existing_assets(self): + # Find existing asset metadata. + asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(asset_md) + # Find non-existent asset metadata. + asset_md = self.draft_store.find_asset_metadata(self.asset5_md.asset_id) + self.assertIsNone(asset_md) + + def test_add_same_asset_twice(self): + courses = self.draft_store.get_courses() + course = courses[0] + asset_filename = 'burnside.jpg' + new_asset_loc = course.id.make_asset_key('asset', asset_filename) + new_asset_md = self._make_asset_metadata(new_asset_loc) + # Only the setup stuff here? + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 2) + # Add asset metadata. + self.assertTrue(self.draft_store.save_asset_metadata(course.id, new_asset_md, "John Do")) + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 3) + # Add *the same* asset metadata. + self.assertTrue(self.draft_store.save_asset_metadata(course.id, new_asset_md, "John Dont")) + # Still one here? + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 3) + self.draft_store.delete_all_asset_metadata(course.id) + self.assertEquals(len(self.draft_store.get_all_asset_metadata(course.id)), 0) + + def test_lock_unlock_assets(self): + # Find a course asset and check its locked status. + asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(asset_md) + locked_state = asset_md.locked + # Flip the course asset's locked status. + self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, "locked", not locked_state, 'John Doe') + # Find the same course and check its locked status. + updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(updated_asset_md) + self.assertEquals(updated_asset_md.locked, not locked_state) + # Now flip it back. + self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, "locked", locked_state, 'John Doe') + reupdated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(reupdated_asset_md) + self.assertEquals(reupdated_asset_md.locked, locked_state) + + ALLOWED_ATTRS = ( + ('basename', '/new/path'), + ('internal_name', 'new_filename.txt'), + ('locked', True), + ('contenttype', 'image/png'), + ('md5', '5346682d948cc3f683635b6918f9b3d0'), + ('curr_version', 'v1.01'), + ('prev_version', 'v1.0'), + ('edited_by', 'Mork'), + ('edited_on', datetime(1969, 1, 1, tzinfo=UTC)), + ) + + DISALLOWED_ATTRS = ( + ('asset_id', 'IAmBogus'), + ) + + UNKNOWN_ATTRS = ( + ('lunch_order', 'burger_and_fries'), + ('villain', 'Khan') + ) + + @data(*ALLOWED_ATTRS) + def test_set_all_attrs(self, attr_pair): + # Find a course asset. + asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(asset_md) + # Set the course asset's attr. + editing_user = 'user_who_edited' + self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, *attr_pair, user_id=editing_user) + # Find the same course asset and check its changed attr. + updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(updated_asset_md) + self.assertIsNotNone(getattr(updated_asset_md, attr_pair[0], None)) + if attr_pair[0] == 'edited_by': + # No matter what the edited_by attr_pair is, it gets over-ridden by the passed-in user_id. + self.assertEquals(getattr(updated_asset_md, attr_pair[0], None), editing_user) + elif attr_pair[0] == 'edited_on': + # edited_on is also over-ridden to be the time of update. + pass + else: + self.assertEquals(getattr(updated_asset_md, attr_pair[0], None), attr_pair[1]) + + @data(*DISALLOWED_ATTRS) + def test_set_disallowed_attrs(self, attr_pair): + # Find a course asset. + asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(asset_md) + original_attr_val = getattr(asset_md, attr_pair[0]) + # Set the course asset's attr. + self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, *attr_pair, user_id='John Doe') + # Find the same course and check its changed attr. + updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(updated_asset_md) + self.assertIsNotNone(getattr(updated_asset_md, attr_pair[0], None)) + # Make sure that the attr is unchanged from its original value. + self.assertEquals(getattr(updated_asset_md, attr_pair[0], None), original_attr_val) + + @data(*UNKNOWN_ATTRS) + def test_set_unknown_attrs(self, attr_pair): + # Find a course asset. + asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(asset_md) + # Set the course asset's attr. + self.draft_store.set_asset_metadata_attr(self.asset1_md.asset_id, *attr_pair, user_id='John Smith') + # Find the same course and check its changed attr. + updated_asset_md = self.draft_store.find_asset_metadata(self.asset1_md.asset_id) + self.assertIsNotNone(updated_asset_md) + # Make sure the unknown field was *not* added. + with self.assertRaises(AttributeError): + self.assertEquals(getattr(updated_asset_md, attr_pair[0]), attr_pair[1]) + + def test_save_one_thumbnail_and_delete_one_thumbnail(self): + thumbnail_filename = 'burn_thumb.jpg' + asset_key = self.course1.id.make_asset_key('thumbnail', thumbnail_filename) + new_asset_thumbnail = self._make_asset_thumbnail_metadata(asset_key) + self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 2) + self.assertTrue(self.draft_store.save_asset_thumbnail_metadata(self.course1.id, new_asset_thumbnail)) + self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 3) + self.assertEquals(self.draft_store.delete_asset_thumbnail_metadata(asset_key), 1) + self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 2) + + def test_find_thumbnail(self): + self.assertIsNotNone(self.draft_store.find_asset_thumbnail_metadata(self.thumb1_md.asset_id)) + self.assertIsNone(self.draft_store.find_asset_thumbnail_metadata(self.thumb5_md.asset_id)) + + def test_delete_all_thumbnails(self): + self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 2) + self.draft_store.delete_all_asset_metadata(self.course1.id) + self.assertEquals(len(self.draft_store.get_all_asset_thumbnail_metadata(self.course1.id)), 0) + + def test_asset_object_equivalence(self): + # Assets are only equivalent to themselves. + self.assertTrue(self.asset6_md != self.thumb6_md) + self.assertEquals(self.asset1_md, self.asset1_md) + + def test_get_all_assets_with_paging(self): + pass + + def test_copy_all_assets(self): + pass + + +class TestMongoModuleStoreWithNoAssetCollection(TestMongoModuleStore): + ''' + Tests a situation where no asset_collection is specified. + ''' + + @classmethod + def add_asset_collection(cls, doc_store_config): + """ + No asset collection. + """ + pass + + @classmethod + def setupClass(cls): + super(TestMongoModuleStoreWithNoAssetCollection, cls).setupClass() + + @classmethod + def teardownClass(cls): + super(TestMongoModuleStoreWithNoAssetCollection, cls).teardownClass() + + def test_no_asset_collection(self): + courses = self.draft_store.get_courses() + course = courses[0] + # Confirm that no asset collection means no asset metadata. + self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None) + # Now delete the non-existent asset metadata. + self.draft_store.delete_all_asset_metadata(course.id) + # Should still be nothing. + self.assertEquals(self.draft_store.get_all_asset_metadata(course.id), None) + + class TestMongoKeyValueStore(object): """ Tests for MongoKeyValueStore. diff --git a/lms/envs/common.py b/lms/envs/common.py index 5569e6223f..1ad593393d 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -566,6 +566,7 @@ DOC_STORE_CONFIG = { 'host': 'localhost', 'db': 'xmodule', 'collection': 'modulestore', + 'asset_collection': 'assetstore', } MODULESTORE = { 'default': {