From 062e0b217483eed861eeada788a3021590bc6aa4 Mon Sep 17 00:00:00 2001 From: John Eskew Date: Fri, 31 Oct 2014 14:57:25 -0400 Subject: [PATCH 1/3] Implement course-wide asset paging methods. Add SortedCollection class to help sort lists efficiently. https://openedx.atlassian.net/browse/PLAT-74 --- .../xmodule/xmodule/assetstore/__init__.py | 24 +++-- .../xmodule/xmodule/modulestore/__init__.py | 79 +++++++++++----- .../lib/xmodule/xmodule/modulestore/mixed.py | 2 +- .../xmodule/xmodule/modulestore/mongo/base.py | 17 ++-- .../modulestore/tests/test_assetstore.py | 94 ++++++++++++++++++- docs/shared/requirements.txt | 1 + 6 files changed, 168 insertions(+), 49 deletions(-) diff --git a/common/lib/xmodule/xmodule/assetstore/__init__.py b/common/lib/xmodule/xmodule/assetstore/__init__.py index 63b98c7df3..48e3ff935e 100644 --- a/common/lib/xmodule/xmodule/assetstore/__init__.py +++ b/common/lib/xmodule/xmodule/assetstore/__init__.py @@ -5,6 +5,7 @@ Classes representing asset & asset thumbnail metadata. from datetime import datetime import pytz from contracts import contract, new_contract +from bisect import bisect_left, bisect_right from opaque_keys.edx.keys import CourseKey, AssetKey new_contract('AssetKey', AssetKey) @@ -33,8 +34,8 @@ class AssetMetadata(object): # All AssetMetadata objects should have AssetLocators with this type. ASSET_TYPE = 'asset' - @contract(asset_id='AssetKey', basename='basestring | None', internal_name='str | None', locked='bool | None', contenttype='basestring | None', - md5='str | None', curr_version='str | None', prev_version='str | None', edited_by='int | None', edited_on='datetime | None') + @contract(asset_id='AssetKey', basename='basestring|None', internal_name='basestring|None', locked='bool|None', contenttype='basestring|None', + md5='basestring|None', curr_version='basestring|None', prev_version='basestring|None', edited_by='int|None', edited_on='datetime|None') def __init__(self, asset_id, basename=None, internal_name=None, locked=None, contenttype=None, md5=None, @@ -99,12 +100,10 @@ class AssetMetadata(object): 'locked': self.locked, 'contenttype': self.contenttype, 'md5': self.md5, - 'edit_info': { - 'curr_version': self.curr_version, - 'prev_version': self.prev_version, - 'edited_by': self.edited_by, - 'edited_on': self.edited_on - } + 'curr_version': self.curr_version, + 'prev_version': self.prev_version, + 'edited_by': self.edited_by, + 'edited_on': self.edited_on } @contract(asset_doc='dict | None') @@ -121,11 +120,10 @@ class AssetMetadata(object): self.locked = asset_doc['locked'] self.contenttype = asset_doc['contenttype'] self.md5 = asset_doc['md5'] - edit_info = asset_doc['edit_info'] - self.curr_version = edit_info['curr_version'] - self.prev_version = edit_info['prev_version'] - self.edited_by = edit_info['edited_by'] - self.edited_on = edit_info['edited_on'] + self.curr_version = asset_doc['curr_version'] + self.prev_version = asset_doc['prev_version'] + self.edited_by = asset_doc['edited_by'] + self.edited_on = asset_doc['edited_on'] class AssetThumbnailMetadata(object): diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index a436bd34c0..64d5c10c01 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -14,6 +14,8 @@ import collections from contextlib import contextmanager import functools import threading +from operator import itemgetter +from sortedcontainers import SortedListWithKey from abc import ABCMeta, abstractmethod from contracts import contract, new_contract @@ -292,19 +294,23 @@ class ModuleStoreAssetInterface(object): if course_assets is None: return None, None - if get_thumbnail: - all_assets = course_assets['thumbnails'] - else: - all_assets = course_assets['assets'] + info = 'thumbnails' if get_thumbnail else 'assets' + all_assets = SortedListWithKey([], key=itemgetter('filename')) + # Assets should be pre-sorted, so add them efficiently without sorting. + # extend() will raise a ValueError if the passed-in list is not sorted. + all_assets.extend(course_assets.get(info, [])) # See if this asset already exists by checking the external_filename. # Studio doesn't currently support using multiple course assets with the same filename. # So use the filename as the unique identifier. - for idx, asset in enumerate(all_assets): - if asset['filename'] == filename: - return course_assets, idx + idx = None + idx_left = all_assets.bisect_left({'filename': filename}) + idx_right = all_assets.bisect_right({'filename': filename}) + if idx_left != idx_right: + # Asset was found in the list. + idx = idx_left - return course_assets, None + return course_assets, idx @contract(asset_key='AssetKey') def _find_asset_info(self, asset_key, thumbnail=False, **kwargs): @@ -358,14 +364,14 @@ class ModuleStoreAssetInterface(object): """ return self._find_asset_info(asset_key, thumbnail=True, **kwargs) - @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None', get_thumbnails='bool') + @contract(course_key='CourseKey', start='int|None', maxresults='int|None', sort='tuple(str,str)|None', get_thumbnails='bool') def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs): """ Returns a list of static asset (or thumbnail) metadata for a course. Args: course_key (CourseKey): course identifier - start (int): optional - start at this asset number + start (int): optional - start at this asset number. Zero-based! maxresults (int): optional - return at most this many, -1 means no limit sort (array): optional - None means no sort (sort_by (str), sort_order (str)) @@ -382,35 +388,60 @@ class ModuleStoreAssetInterface(object): # to distinguish zero assets from "not able to retrieve assets". return None - if get_thumbnails: - all_assets = course_assets.get('thumbnails', []) - else: - all_assets = course_assets.get('assets', []) + # Determine the proper sort - with defaults of ('displayname', 'ascending'). + sort_field = 'filename' + sort_order = 'ascending' + if sort: + if sort[0] == 'uploadDate': + sort_field = 'edited_on' + if sort[1] == 'descending': + sort_order = 'descending' - # DO_NEXT: Add start/maxresults/sort functionality as part of https://openedx.atlassian.net/browse/PLAT-74 - if start and maxresults and sort: - pass + info = 'thumbnails' if get_thumbnails else 'assets' + all_assets = SortedListWithKey(course_assets.get(info, []), key=itemgetter(sort_field)) + num_assets = len(all_assets) + + start_idx = start + end_idx = min(num_assets, start + maxresults) + if maxresults < 0: + # No limit on the results. + end_idx = num_assets + + step_incr = 1 + if sort_order == 'descending': + # Flip the indices and iterate backwards. + step_incr = -1 + start_idx = (num_assets - 1) - start_idx + end_idx = (num_assets - 1) - end_idx ret_assets = [] - for asset in all_assets: + for idx in range(start_idx, end_idx, step_incr): + asset = all_assets[idx] if get_thumbnails: thumb = AssetThumbnailMetadata( course_key.make_asset_key('thumbnail', asset['filename']), - internal_name=asset['filename'], **kwargs + internal_name=asset['filename'], + **kwargs ) ret_assets.append(thumb) else: - asset = AssetMetadata( + new_asset = AssetMetadata( course_key.make_asset_key('asset', asset['filename']), basename=asset['filename'], - edited_on=asset['edit_info']['edited_on'], + internal_name=asset['internal_name'], + locked=asset['locked'], contenttype=asset['contenttype'], - md5=str(asset['md5']), **kwargs + md5=asset['md5'], + curr_version=asset['curr_version'], + prev_version=asset['prev_version'], + edited_on=asset['edited_on'], + edited_by=asset['edited_by'], + **kwargs ) - ret_assets.append(asset) + ret_assets.append(new_asset) return ret_assets - @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='list | None') + @contract(course_key='CourseKey', start='int|None', maxresults='int|None', sort='tuple(str,str)|None') def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs): """ Returns a list of static assets for a course. diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 652d09a52c..c7c71fc8fb 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -370,7 +370,7 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): return store.find_asset_thumbnail_metadata(asset_key, **kwargs) @strip_key - @contract(course_key='CourseKey', start=int, maxresults=int, sort='list | None') + @contract(course_key='CourseKey', start=int, maxresults=int, sort='tuple|None') def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs): """ Returns a list of static assets for a course. diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 6adf74b150..02b33e3fd3 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -25,6 +25,8 @@ from path import path from datetime import datetime from pytz import UTC from contracts import contract, new_contract +from operator import itemgetter +from sortedcontainers import SortedListWithKey from importlib import import_module from xmodule.errortracker import null_error_tracker, exc_info_to_str @@ -1493,7 +1495,10 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo course_assets, asset_idx = self._find_course_asset(course_key, asset_metadata.asset_id.path, thumbnail) info = 'thumbnails' if thumbnail else 'assets' - all_assets = course_assets[info] + all_assets = SortedListWithKey([], key=itemgetter('filename')) + # Assets should be pre-sorted, so add them efficiently without sorting. + # extend() will raise a ValueError if the passed-in list is not sorted. + all_assets.extend(course_assets[info]) # Set the edited information for assets only - not thumbnails. if not thumbnail: @@ -1502,15 +1507,15 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo # Translate metadata to Mongo format. metadata_to_insert = asset_metadata.to_mongo() if asset_idx is None: - # Append new metadata. - # Future optimization: Insert in order & binary search to retrieve. - all_assets.append(metadata_to_insert) + # Add new metadata sorted into the list. + all_assets.add(metadata_to_insert) else: # Replace existing metadata. - all_assets[asset_idx] = metadata_to_insert + all_assets.pop(asset_idx) + all_assets.insert(asset_idx, metadata_to_insert) # Update the document. - self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets}}) + self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets.as_list()}}) return True @contract(asset_key='AssetKey', attr_dict=dict) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py index 1b0d83c63a..40498490e2 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py @@ -6,6 +6,7 @@ from datetime import datetime, timedelta import pytz import unittest import ddt +from time import sleep from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata from xmodule.modulestore import ModuleStoreEnum @@ -79,12 +80,23 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): asset6_vals = ('asset.txt', 'JJJCCC747858', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49') asset6 = dict(zip(asset_fields[1:], asset6_vals[1:])) + # More assets. + asset7_vals = ('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, ModuleStoreEnum.UserID.test * 7, datetime.now(pytz.utc), '1.1', '1.01') + asset8_vals = ('weather_patterns.bmp', '928SJXX2EB', 'science', False, ModuleStoreEnum.UserID.test * 8, datetime.now(pytz.utc), '52', '51') + asset9_vals = ('demo.swf', 'DFDFGGGG14', 'demos/easy', False, ModuleStoreEnum.UserID.test * 9, datetime.now(pytz.utc), '5', '4') + asset7 = dict(zip(asset_fields[1:], asset7_vals[1:])) + asset8 = dict(zip(asset_fields[1:], asset8_vals[1:])) + asset9 = dict(zip(asset_fields[1:], asset9_vals[1:])) + asset1_key = course1_key.make_asset_key('asset', asset1_vals[0]) asset2_key = course1_key.make_asset_key('asset', asset2_vals[0]) asset3_key = course2_key.make_asset_key('asset', asset3_vals[0]) asset4_key = course2_key.make_asset_key('asset', asset4_vals[0]) asset5_key = course2_key.make_asset_key('asset', asset5_vals[0]) asset6_key = course2_key.make_asset_key('asset', asset6_vals[0]) + asset7_key = course2_key.make_asset_key('asset', asset7_vals[0]) + asset8_key = course2_key.make_asset_key('asset', asset8_vals[0]) + asset9_key = course2_key.make_asset_key('asset', asset9_vals[0]) asset1_md = AssetMetadata(asset1_key, **asset1) asset2_md = AssetMetadata(asset2_key, **asset2) @@ -92,13 +104,26 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): asset4_md = AssetMetadata(asset4_key, **asset4) asset5_md = AssetMetadata(asset5_key, **non_existent_asset) asset6_md = AssetMetadata(asset6_key, **asset6) + asset7_md = AssetMetadata(asset7_key, **asset7) + asset8_md = AssetMetadata(asset8_key, **asset8) + asset9_md = AssetMetadata(asset9_key, **asset9) if store is not None: + # Sleeps are to ensure that edited_on order is correct. store.save_asset_metadata(course1_key, asset1_md, ModuleStoreEnum.UserID.test) - store.save_asset_metadata(course1_key, asset2_md, ModuleStoreEnum.UserID.test) - store.save_asset_metadata(course2_key, asset3_md, ModuleStoreEnum.UserID.test) - store.save_asset_metadata(course2_key, asset4_md, ModuleStoreEnum.UserID.test) + sleep(0.0001) + store.save_asset_metadata(course1_key, asset2_md, ModuleStoreEnum.UserID.test * 2) + sleep(0.0001) + store.save_asset_metadata(course2_key, asset3_md, ModuleStoreEnum.UserID.test * 3) + sleep(0.0001) + store.save_asset_metadata(course2_key, asset4_md, ModuleStoreEnum.UserID.test * 4) + sleep(0.0001) # 5 & 6 are not saved on purpose! + store.save_asset_metadata(course2_key, asset7_md, ModuleStoreEnum.UserID.test * 7) + sleep(0.0001) + store.save_asset_metadata(course2_key, asset8_md, ModuleStoreEnum.UserID.test * 8) + sleep(0.0001) + store.save_asset_metadata(course2_key, asset9_md, ModuleStoreEnum.UserID.test * 9) return (asset1_md, asset2_md, asset3_md, asset4_md, asset5_md, asset6_md) @@ -387,8 +412,67 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): store.delete_all_asset_metadata(course.id, ModuleStoreEnum.UserID.test) self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course.id)), 0) - def test_get_all_assets_with_paging(self): - pass + @ddt.data(*MODULESTORE_SETUPS) + def test_get_all_assets_with_paging(self, storebuilder): + """ + Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all. + """ + # Temporarily only perform this test for Old Mongo - not Split. + if not isinstance(storebuilder, MongoModulestoreBuilder): + raise unittest.SkipTest + with MongoContentstoreBuilder().build() as contentstore: + with storebuilder.build(contentstore) as store: + course1 = CourseFactory.create(modulestore=store) + course2 = CourseFactory.create(modulestore=store) + self.setup_assets(course1.id, course2.id, store) + + expected_sorts_by_2 = ( + ( + ('displayname', 'ascending'), + ('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'), + (2, 2, 1) + ), + ( + ('displayname', 'descending'), + ('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'), + (2, 2, 1) + ), + ( + ('uploadDate', 'ascending'), + ('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'), + (2, 2, 1) + ), + ( + ('uploadDate', 'descending'), + ('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'), + (2, 2, 1) + ), + ) + # First, with paging across all sorts. + for sort_test in expected_sorts_by_2: + for i in xrange(3): + asset_page = store.get_all_asset_metadata(course2.id, start=2 * i, maxresults=2, sort=sort_test[0]) + self.assertEquals(len(asset_page), sort_test[2][i]) + self.assertEquals(asset_page[0].asset_id.path, sort_test[1][2 * i]) + if sort_test[2][i] == 2: + self.assertEquals(asset_page[1].asset_id.path, sort_test[1][(2 * i) + 1]) + + # Now fetch everything. + asset_page = store.get_all_asset_metadata(course2.id, start=0, sort=('displayname', 'ascending')) + self.assertEquals(len(asset_page), 5) + self.assertEquals(asset_page[0].asset_id.path, 'code.tgz') + self.assertEquals(asset_page[1].asset_id.path, 'demo.swf') + self.assertEquals(asset_page[2].asset_id.path, 'dog.png') + self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf') + self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp') + + # Some odd conditions. + asset_page = store.get_all_asset_metadata(course2.id, start=100, sort=('displayname', 'ascending')) + self.assertEquals(len(asset_page), 0) + asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=0, sort=('displayname', 'ascending')) + self.assertEquals(len(asset_page), 0) + asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=-12345, sort=('displayname', 'descending')) + self.assertEquals(len(asset_page), 2) def test_copy_all_assets(self): pass diff --git a/docs/shared/requirements.txt b/docs/shared/requirements.txt index 860e55356d..330a7a1a21 100644 --- a/docs/shared/requirements.txt +++ b/docs/shared/requirements.txt @@ -56,6 +56,7 @@ PyYAML==3.10 requests==2.3.0 Shapely==1.2.16 sorl-thumbnail==11.12 +sortedcontainers==0.9.2 South==0.7.6 sympy==0.7.1 xmltodict==0.4.1 From 098bba16e16ec7a5fac6248e2af56d69e611ad3d Mon Sep 17 00:00:00 2001 From: John Eskew Date: Sat, 1 Nov 2014 12:29:57 -0400 Subject: [PATCH 2/3] Implement copy_all_asset_metadata for Old Mongo->Old Mongo. Add tests for copying all asset metadata. https://openedx.atlassian.net/browse/PLAT-74 --- .../xmodule/xmodule/assetstore/__init__.py | 6 +- .../xmodule/xmodule/modulestore/__init__.py | 2 +- .../xmodule/xmodule/modulestore/mongo/base.py | 22 +++ .../modulestore/tests/test_assetstore.py | 144 +++++++++++------- 4 files changed, 111 insertions(+), 63 deletions(-) diff --git a/common/lib/xmodule/xmodule/assetstore/__init__.py b/common/lib/xmodule/xmodule/assetstore/__init__.py index 48e3ff935e..a84558c401 100644 --- a/common/lib/xmodule/xmodule/assetstore/__init__.py +++ b/common/lib/xmodule/xmodule/assetstore/__init__.py @@ -106,7 +106,7 @@ class AssetMetadata(object): 'edited_on': self.edited_on } - @contract(asset_doc='dict | None') + @contract(asset_doc='dict|None') def from_mongo(self, asset_doc): """ Fill in all metadata fields from a MongoDB document. @@ -134,7 +134,7 @@ class AssetThumbnailMetadata(object): # All AssetThumbnailMetadata objects should have AssetLocators with this type. ASSET_TYPE = 'thumbnail' - @contract(asset_id='AssetKey', internal_name='str | unicode | None') + @contract(asset_id='AssetKey', internal_name='basestring|None') def __init__(self, asset_id, internal_name=None, field_decorator=None): """ Construct a AssetThumbnailMetadata object. @@ -160,7 +160,7 @@ class AssetThumbnailMetadata(object): 'internal_name': self.internal_name } - @contract(thumbnail_doc='dict | None') + @contract(thumbnail_doc='dict|None') def from_mongo(self, thumbnail_doc): """ Fill in all metadata fields from a MongoDB document. diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 64d5c10c01..204f09e4d6 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -392,7 +392,7 @@ class ModuleStoreAssetInterface(object): sort_field = 'filename' sort_order = 'ascending' if sort: - if sort[0] == 'uploadDate': + if sort[0] == 'uploadDate' and not get_thumbnails: sort_field = 'edited_on' if sort[1] == 'descending': sort_order = 'descending' diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 02b33e3fd3..5241466c07 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -1518,6 +1518,28 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets.as_list()}}) return True + @contract(source_course_key='CourseKey', dest_course_key='CourseKey') + def copy_all_asset_metadata(self, source_course_key, dest_course_key, user_id): + """ + Copy all the course assets from source_course_key to dest_course_key. + + Arguments: + source_course_key (CourseKey): identifier of course to copy from + dest_course_key (CourseKey): identifier of course to copy to + """ + source_assets = self._find_course_assets(source_course_key) + dest_assets = self._find_course_assets(dest_course_key) + dest_assets['assets'] = source_assets.get('assets', []) + dest_assets['thumbnails'] = source_assets.get('thumbnails', []) + + # Update the document. + self.asset_collection.update( + {'_id': dest_assets['_id']}, + {'$set': {'assets': dest_assets['assets'], + 'thumbnails': dest_assets['thumbnails']} + } + ) + @contract(asset_key='AssetKey', attr_dict=dict) def set_asset_metadata_attrs(self, asset_key, attr_dict, user_id): """ diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py index 40498490e2..33757bea10 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py @@ -88,44 +88,45 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): asset8 = dict(zip(asset_fields[1:], asset8_vals[1:])) asset9 = dict(zip(asset_fields[1:], asset9_vals[1:])) - asset1_key = course1_key.make_asset_key('asset', asset1_vals[0]) - asset2_key = course1_key.make_asset_key('asset', asset2_vals[0]) - asset3_key = course2_key.make_asset_key('asset', asset3_vals[0]) - asset4_key = course2_key.make_asset_key('asset', asset4_vals[0]) - asset5_key = course2_key.make_asset_key('asset', asset5_vals[0]) - asset6_key = course2_key.make_asset_key('asset', asset6_vals[0]) - asset7_key = course2_key.make_asset_key('asset', asset7_vals[0]) - asset8_key = course2_key.make_asset_key('asset', asset8_vals[0]) - asset9_key = course2_key.make_asset_key('asset', asset9_vals[0]) - - asset1_md = AssetMetadata(asset1_key, **asset1) - asset2_md = AssetMetadata(asset2_key, **asset2) - asset3_md = AssetMetadata(asset3_key, **asset3) - asset4_md = AssetMetadata(asset4_key, **asset4) - asset5_md = AssetMetadata(asset5_key, **non_existent_asset) - asset6_md = AssetMetadata(asset6_key, **asset6) - asset7_md = AssetMetadata(asset7_key, **asset7) - asset8_md = AssetMetadata(asset8_key, **asset8) - asset9_md = AssetMetadata(asset9_key, **asset9) + if course1_key: + asset1_key = course1_key.make_asset_key('asset', asset1_vals[0]) + asset2_key = course1_key.make_asset_key('asset', asset2_vals[0]) + asset1_md = AssetMetadata(asset1_key, **asset1) + asset2_md = AssetMetadata(asset2_key, **asset2) + if course2_key: + asset3_key = course2_key.make_asset_key('asset', asset3_vals[0]) + asset4_key = course2_key.make_asset_key('asset', asset4_vals[0]) + asset5_key = course2_key.make_asset_key('asset', asset5_vals[0]) + asset6_key = course2_key.make_asset_key('asset', asset6_vals[0]) + asset7_key = course2_key.make_asset_key('asset', asset7_vals[0]) + asset8_key = course2_key.make_asset_key('asset', asset8_vals[0]) + asset9_key = course2_key.make_asset_key('asset', asset9_vals[0]) + asset3_md = AssetMetadata(asset3_key, **asset3) + asset4_md = AssetMetadata(asset4_key, **asset4) + asset5_md = AssetMetadata(asset5_key, **non_existent_asset) # pylint: disable=W0612 + asset6_md = AssetMetadata(asset6_key, **asset6) # pylint: disable=W0612 + asset7_md = AssetMetadata(asset7_key, **asset7) + asset8_md = AssetMetadata(asset8_key, **asset8) + asset9_md = AssetMetadata(asset9_key, **asset9) if store is not None: # Sleeps are to ensure that edited_on order is correct. - store.save_asset_metadata(course1_key, asset1_md, ModuleStoreEnum.UserID.test) - sleep(0.0001) - store.save_asset_metadata(course1_key, asset2_md, ModuleStoreEnum.UserID.test * 2) - sleep(0.0001) - store.save_asset_metadata(course2_key, asset3_md, ModuleStoreEnum.UserID.test * 3) - sleep(0.0001) - store.save_asset_metadata(course2_key, asset4_md, ModuleStoreEnum.UserID.test * 4) - sleep(0.0001) - # 5 & 6 are not saved on purpose! - store.save_asset_metadata(course2_key, asset7_md, ModuleStoreEnum.UserID.test * 7) - sleep(0.0001) - store.save_asset_metadata(course2_key, asset8_md, ModuleStoreEnum.UserID.test * 8) - sleep(0.0001) - store.save_asset_metadata(course2_key, asset9_md, ModuleStoreEnum.UserID.test * 9) - - return (asset1_md, asset2_md, asset3_md, asset4_md, asset5_md, asset6_md) + if course1_key: + store.save_asset_metadata(course1_key, asset1_md, ModuleStoreEnum.UserID.test) + sleep(0.0001) + store.save_asset_metadata(course1_key, asset2_md, ModuleStoreEnum.UserID.test * 2) + sleep(0.0001) + if course2_key: + store.save_asset_metadata(course2_key, asset3_md, ModuleStoreEnum.UserID.test * 3) + sleep(0.0001) + store.save_asset_metadata(course2_key, asset4_md, ModuleStoreEnum.UserID.test * 4) + sleep(0.0001) + # 5 & 6 are not saved on purpose! + store.save_asset_metadata(course2_key, asset7_md, ModuleStoreEnum.UserID.test * 7) + sleep(0.0001) + store.save_asset_metadata(course2_key, asset8_md, ModuleStoreEnum.UserID.test * 8) + sleep(0.0001) + store.save_asset_metadata(course2_key, asset9_md, ModuleStoreEnum.UserID.test * 9) def setup_thumbnails(self, course1_key, course2_key, store=None): """ @@ -148,28 +149,29 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): thumbnail6_vals = ('asset.txt', 'JJJCCC747858') thumbnail6 = dict(zip(thumbnail_fields[1:], thumbnail6_vals[1:])) - thumb1_key = course1_key.make_asset_key('thumbnail', thumbnail1_vals[0]) - thumb2_key = course1_key.make_asset_key('thumbnail', thumbnail2_vals[0]) - thumb3_key = course2_key.make_asset_key('thumbnail', thumbnail3_vals[0]) - thumb4_key = course2_key.make_asset_key('thumbnail', thumbnail4_vals[0]) - thumb5_key = course2_key.make_asset_key('thumbnail', thumbnail5_vals[0]) - thumb6_key = course2_key.make_asset_key('thumbnail', thumbnail6_vals[0]) - - thumb1_md = AssetThumbnailMetadata(thumb1_key, **thumbnail1) - thumb2_md = AssetThumbnailMetadata(thumb2_key, **thumbnail2) - thumb3_md = AssetThumbnailMetadata(thumb3_key, **thumbnail3) - thumb4_md = AssetThumbnailMetadata(thumb4_key, **thumbnail4) - thumb5_md = AssetThumbnailMetadata(thumb5_key, **non_existent_thumbnail) - thumb6_md = AssetThumbnailMetadata(thumb6_key, **thumbnail6) + if course1_key: + thumb1_key = course1_key.make_asset_key('thumbnail', thumbnail1_vals[0]) + thumb2_key = course1_key.make_asset_key('thumbnail', thumbnail2_vals[0]) + thumb1_md = AssetThumbnailMetadata(thumb1_key, **thumbnail1) + thumb2_md = AssetThumbnailMetadata(thumb2_key, **thumbnail2) + if course2_key: + thumb3_key = course2_key.make_asset_key('thumbnail', thumbnail3_vals[0]) + thumb4_key = course2_key.make_asset_key('thumbnail', thumbnail4_vals[0]) + thumb5_key = course2_key.make_asset_key('thumbnail', thumbnail5_vals[0]) + thumb6_key = course2_key.make_asset_key('thumbnail', thumbnail6_vals[0]) + thumb3_md = AssetThumbnailMetadata(thumb3_key, **thumbnail3) + thumb4_md = AssetThumbnailMetadata(thumb4_key, **thumbnail4) + thumb5_md = AssetThumbnailMetadata(thumb5_key, **non_existent_thumbnail) # pylint: disable=W0612 + thumb6_md = AssetThumbnailMetadata(thumb6_key, **thumbnail6) # pylint: disable=W0612 if store is not None: - store.save_asset_thumbnail_metadata(course1_key, thumb1_md, ModuleStoreEnum.UserID.test) - store.save_asset_thumbnail_metadata(course1_key, thumb2_md, ModuleStoreEnum.UserID.test) - store.save_asset_thumbnail_metadata(course2_key, thumb3_md, ModuleStoreEnum.UserID.test) - store.save_asset_thumbnail_metadata(course2_key, thumb4_md, ModuleStoreEnum.UserID.test) - # thumb5 and thumb6 are not saved on purpose! - - return (thumb1_md, thumb2_md, thumb3_md, thumb4_md, thumb5_md, thumb6_md) + if course1_key: + store.save_asset_thumbnail_metadata(course1_key, thumb1_md, ModuleStoreEnum.UserID.test) + store.save_asset_thumbnail_metadata(course1_key, thumb2_md, ModuleStoreEnum.UserID.test) + if course2_key: + store.save_asset_thumbnail_metadata(course2_key, thumb3_md, ModuleStoreEnum.UserID.test) + store.save_asset_thumbnail_metadata(course2_key, thumb4_md, ModuleStoreEnum.UserID.test) + # thumb5 and thumb6 are not saved on purpose! @ddt.data(*MODULESTORE_SETUPS) def test_save_one_and_confirm(self, storebuilder): @@ -467,16 +469,13 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp') # Some odd conditions. - asset_page = store.get_all_asset_metadata(course2.id, start=100, sort=('displayname', 'ascending')) + asset_page = store.get_all_asset_metadata(course2.id, start=100, sort=('uploadDate', 'ascending')) self.assertEquals(len(asset_page), 0) asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=0, sort=('displayname', 'ascending')) self.assertEquals(len(asset_page), 0) asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=-12345, sort=('displayname', 'descending')) self.assertEquals(len(asset_page), 2) - def test_copy_all_assets(self): - pass - @ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())])) def test_xml_not_yet_implemented(self, storebuilder): """ @@ -494,3 +493,30 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): for method in ['_get_all_asset_metadata', 'get_all_asset_metadata', 'get_all_asset_thumbnail_metadata']: with self.assertRaises(NotImplementedError): getattr(store, method)(course_key) + + @ddt.data(*MODULESTORE_SETUPS) + def test_copy_all_assets(self, storebuilder): + """ + Create a course with assets and such, copy it all to another course, and check on it. + """ + with MongoContentstoreBuilder().build() as contentstore: + with storebuilder.build(contentstore) as store: + course1 = CourseFactory.create(modulestore=store) + course2 = CourseFactory.create(modulestore=store) + self.setup_assets(course1.id, None, store) + self.setup_thumbnails(course1.id, None, store) + self.assertEquals(len(store.get_all_asset_metadata(course1.id)), 2) + self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course1.id)), 2) + self.assertEquals(len(store.get_all_asset_metadata(course2.id)), 0) + self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course2.id)), 0) + store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) + self.assertEquals(len(store.get_all_asset_metadata(course1.id)), 2) + self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course1.id)), 2) + all_assets = store.get_all_asset_metadata(course2.id, sort=('displayname', 'ascending')) + self.assertEquals(len(all_assets), 2) + self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') + self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') + all_thumbnails = store.get_all_asset_thumbnail_metadata(course2.id, sort=('uploadDate', 'descending')) + self.assertEquals(len(all_thumbnails), 2) + self.assertEquals(all_thumbnails[0].asset_id.path, 'kitten_thumb.jpg') + self.assertEquals(all_thumbnails[1].asset_id.path, 'cat_thumb.jpg') From 3534ba70cada19babdfcdeaed3e5b0774e51d81e Mon Sep 17 00:00:00 2001 From: John Eskew Date: Mon, 3 Nov 2014 21:52:28 -0500 Subject: [PATCH 3/3] Changes for PR review: More compact tests, enumeration for metadata sort order, etc. https://openedx.atlassian.net/browse/PLAT-74 --- .../xmodule/xmodule/modulestore/__init__.py | 35 ++-- .../xmodule/xmodule/modulestore/mongo/base.py | 3 +- .../modulestore/tests/test_assetstore.py | 166 ++++++------------ docs/shared/requirements.txt | 1 - requirements/edx/base.txt | 1 + 5 files changed, 78 insertions(+), 128 deletions(-) diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 204f09e4d6..97d10b6560 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -100,6 +100,13 @@ class ModuleStoreEnum(object): # user ID to use for tests that do not have a django user available test = -3 + class SortOrder(object): + """ + Values for sorting asset metadata. + """ + ascending = 1 + descending = 2 + class BulkOpsRecord(object): """ @@ -364,19 +371,19 @@ class ModuleStoreAssetInterface(object): """ return self._find_asset_info(asset_key, thumbnail=True, **kwargs) - @contract(course_key='CourseKey', start='int|None', maxresults='int|None', sort='tuple(str,str)|None', get_thumbnails='bool') - def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, get_thumbnails=False, **kwargs): + @contract(course_key='CourseKey', start='int|None', maxresults='int|None', + sort='tuple(str,(int,>=1,<=2))|None', get_thumbnails='bool') + def _get_all_asset_metadata(self, course_key, start=0, maxresults=-1, + sort=('displayname', ModuleStoreEnum.SortOrder.ascending), + get_thumbnails=False, **kwargs): """ - Returns a list of static asset (or thumbnail) metadata for a course. - - Args: course_key (CourseKey): course identifier start (int): optional - start at this asset number. Zero-based! maxresults (int): optional - return at most this many, -1 means no limit sort (array): optional - None means no sort (sort_by (str), sort_order (str)) sort_by - one of 'uploadDate' or 'displayname' - sort_order - one of 'ascending' or 'descending' + sort_order - one of SortOrder.ascending or SortOrder.descending get_thumbnails (bool): True if getting thumbnail metadata, else getting asset metadata Returns: @@ -388,14 +395,14 @@ class ModuleStoreAssetInterface(object): # to distinguish zero assets from "not able to retrieve assets". return None - # Determine the proper sort - with defaults of ('displayname', 'ascending'). + # Determine the proper sort - with defaults of ('displayname', SortOrder.ascending). sort_field = 'filename' - sort_order = 'ascending' + sort_order = ModuleStoreEnum.SortOrder.ascending if sort: if sort[0] == 'uploadDate' and not get_thumbnails: sort_field = 'edited_on' - if sort[1] == 'descending': - sort_order = 'descending' + if sort[1] == ModuleStoreEnum.SortOrder.descending: + sort_order = ModuleStoreEnum.SortOrder.descending info = 'thumbnails' if get_thumbnails else 'assets' all_assets = SortedListWithKey(course_assets.get(info, []), key=itemgetter(sort_field)) @@ -408,14 +415,14 @@ class ModuleStoreAssetInterface(object): end_idx = num_assets step_incr = 1 - if sort_order == 'descending': + if sort_order == ModuleStoreEnum.SortOrder.descending: # Flip the indices and iterate backwards. step_incr = -1 start_idx = (num_assets - 1) - start_idx end_idx = (num_assets - 1) - end_idx ret_assets = [] - for idx in range(start_idx, end_idx, step_incr): + for idx in xrange(start_idx, end_idx, step_incr): asset = all_assets[idx] if get_thumbnails: thumb = AssetThumbnailMetadata( @@ -441,7 +448,7 @@ class ModuleStoreAssetInterface(object): ret_assets.append(new_asset) return ret_assets - @contract(course_key='CourseKey', start='int|None', maxresults='int|None', sort='tuple(str,str)|None') + @contract(course_key='CourseKey', start='int|None', maxresults='int|None', sort='tuple(str,int)|None') def get_all_asset_metadata(self, course_key, start=0, maxresults=-1, sort=None, **kwargs): """ Returns a list of static assets for a course. @@ -454,7 +461,7 @@ class ModuleStoreAssetInterface(object): sort (array): optional - None means no sort (sort_by (str), sort_order (str)) sort_by - one of 'uploadDate' or 'displayname' - sort_order - one of 'ascending' or 'descending' + sort_order - one of SortOrder.ascending or SortOrder.descending Returns: List of AssetMetadata objects. diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 5241466c07..a37e931376 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -1511,8 +1511,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo all_assets.add(metadata_to_insert) else: # Replace existing metadata. - all_assets.pop(asset_idx) - all_assets.insert(asset_idx, metadata_to_insert) + all_assets[asset_idx] = metadata_to_insert # Update the document. self.asset_collection.update({'_id': course_assets['_id']}, {'$set': {info: all_assets.as_list()}}) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py index 33757bea10..95538451ee 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py @@ -6,14 +6,13 @@ from datetime import datetime, timedelta import pytz import unittest import ddt -from time import sleep from xmodule.assetstore import AssetMetadata, AssetThumbnailMetadata from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.tests.factories import CourseFactory from xmodule.modulestore.tests.test_cross_modulestore_import_export import ( - MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder + MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder ) @@ -64,114 +63,59 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): Setup assets. Save in store if given """ asset_fields = ('filename', 'internal_name', 'basename', 'locked', 'edited_by', 'edited_on', 'curr_version', 'prev_version') - asset1_vals = ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '14', '13') - asset2_vals = ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '1', None) - asset3_vals = ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, ModuleStoreEnum.UserID.test * 2, datetime.now(pytz.utc), 'AB', 'AA') - asset4_vals = ('dog.png', 'PUPY4242X', 'pictures/animals', True, ModuleStoreEnum.UserID.test * 3, datetime.now(pytz.utc), '5', '4') - asset5_vals = ('not_here.txt', 'JJJCCC747', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49') + all_asset_data = ( + ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '14', '13'), + ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, ModuleStoreEnum.UserID.test, datetime.now(pytz.utc), '1', None), + ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, ModuleStoreEnum.UserID.test * 2, datetime.now(pytz.utc), 'AB', 'AA'), + ('dog.png', 'PUPY4242X', 'pictures/animals', True, ModuleStoreEnum.UserID.test * 3, datetime.now(pytz.utc), '5', '4'), + ('not_here.txt', 'JJJCCC747', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49'), + ('asset.txt', 'JJJCCC747858', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49'), + ('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, ModuleStoreEnum.UserID.test * 7, datetime.now(pytz.utc), '1.1', '1.01'), + ('weather_patterns.bmp', '928SJXX2EB', 'science', False, ModuleStoreEnum.UserID.test * 8, datetime.now(pytz.utc), '52', '51'), + ('demo.swf', 'DFDFGGGG14', 'demos/easy', False, ModuleStoreEnum.UserID.test * 9, datetime.now(pytz.utc), '5', '4'), + ) - asset1 = dict(zip(asset_fields[1:], asset1_vals[1:])) - asset2 = dict(zip(asset_fields[1:], asset2_vals[1:])) - asset3 = dict(zip(asset_fields[1:], asset3_vals[1:])) - asset4 = dict(zip(asset_fields[1:], asset4_vals[1:])) - non_existent_asset = dict(zip(asset_fields[1:], asset5_vals[1:])) - - # Asset6 and thumbnail6 have equivalent information on purpose. - asset6_vals = ('asset.txt', 'JJJCCC747858', '/dev/null', False, ModuleStoreEnum.UserID.test * 4, datetime.now(pytz.utc), '50', '49') - asset6 = dict(zip(asset_fields[1:], asset6_vals[1:])) - - # More assets. - asset7_vals = ('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, ModuleStoreEnum.UserID.test * 7, datetime.now(pytz.utc), '1.1', '1.01') - asset8_vals = ('weather_patterns.bmp', '928SJXX2EB', 'science', False, ModuleStoreEnum.UserID.test * 8, datetime.now(pytz.utc), '52', '51') - asset9_vals = ('demo.swf', 'DFDFGGGG14', 'demos/easy', False, ModuleStoreEnum.UserID.test * 9, datetime.now(pytz.utc), '5', '4') - asset7 = dict(zip(asset_fields[1:], asset7_vals[1:])) - asset8 = dict(zip(asset_fields[1:], asset8_vals[1:])) - asset9 = dict(zip(asset_fields[1:], asset9_vals[1:])) - - if course1_key: - asset1_key = course1_key.make_asset_key('asset', asset1_vals[0]) - asset2_key = course1_key.make_asset_key('asset', asset2_vals[0]) - asset1_md = AssetMetadata(asset1_key, **asset1) - asset2_md = AssetMetadata(asset2_key, **asset2) - if course2_key: - asset3_key = course2_key.make_asset_key('asset', asset3_vals[0]) - asset4_key = course2_key.make_asset_key('asset', asset4_vals[0]) - asset5_key = course2_key.make_asset_key('asset', asset5_vals[0]) - asset6_key = course2_key.make_asset_key('asset', asset6_vals[0]) - asset7_key = course2_key.make_asset_key('asset', asset7_vals[0]) - asset8_key = course2_key.make_asset_key('asset', asset8_vals[0]) - asset9_key = course2_key.make_asset_key('asset', asset9_vals[0]) - asset3_md = AssetMetadata(asset3_key, **asset3) - asset4_md = AssetMetadata(asset4_key, **asset4) - asset5_md = AssetMetadata(asset5_key, **non_existent_asset) # pylint: disable=W0612 - asset6_md = AssetMetadata(asset6_key, **asset6) # pylint: disable=W0612 - asset7_md = AssetMetadata(asset7_key, **asset7) - asset8_md = AssetMetadata(asset8_key, **asset8) - asset9_md = AssetMetadata(asset9_key, **asset9) - - if store is not None: - # Sleeps are to ensure that edited_on order is correct. - if course1_key: - store.save_asset_metadata(course1_key, asset1_md, ModuleStoreEnum.UserID.test) - sleep(0.0001) - store.save_asset_metadata(course1_key, asset2_md, ModuleStoreEnum.UserID.test * 2) - sleep(0.0001) - if course2_key: - store.save_asset_metadata(course2_key, asset3_md, ModuleStoreEnum.UserID.test * 3) - sleep(0.0001) - store.save_asset_metadata(course2_key, asset4_md, ModuleStoreEnum.UserID.test * 4) - sleep(0.0001) - # 5 & 6 are not saved on purpose! - store.save_asset_metadata(course2_key, asset7_md, ModuleStoreEnum.UserID.test * 7) - sleep(0.0001) - store.save_asset_metadata(course2_key, asset8_md, ModuleStoreEnum.UserID.test * 8) - sleep(0.0001) - store.save_asset_metadata(course2_key, asset9_md, ModuleStoreEnum.UserID.test * 9) + for i, asset in enumerate(all_asset_data): + asset_dict = dict(zip(asset_fields[1:], asset[1:])) + if i in (0, 1) and course1_key: + asset_key = course1_key.make_asset_key('asset', asset[0]) + asset_md = AssetMetadata(asset_key, **asset_dict) + if store is not None: + store.save_asset_metadata(course1_key, asset_md, asset[4]) + elif course2_key: + asset_key = course2_key.make_asset_key('asset', asset[0]) + asset_md = AssetMetadata(asset_key, **asset_dict) + # Don't save assets 5 and 6. + if store is not None and i not in (4, 5): + store.save_asset_metadata(course2_key, asset_md, asset[4]) def setup_thumbnails(self, course1_key, course2_key, store=None): """ Setup thumbs. Save in store if given """ thumbnail_fields = ('filename', 'internal_name') - thumbnail1_vals = ('cat_thumb.jpg', 'XYXYXYXYXYXY') - thumbnail2_vals = ('kitten_thumb.jpg', '123ABC123ABC') - thumbnail3_vals = ('puppy_thumb.jpg', 'ADAM12ADAM12') - thumbnail4_vals = ('meerkat_thumb.jpg', 'CHIPSPONCH14') - thumbnail5_vals = ('corgi_thumb.jpg', 'RON8LDXFFFF10') + all_thumbnail_data = ( + ('cat_thumb.jpg', 'XYXYXYXYXYXY'), + ('kitten_thumb.jpg', '123ABC123ABC'), + ('puppy_thumb.jpg', 'ADAM12ADAM12'), + ('meerkat_thumb.jpg', 'CHIPSPONCH14'), + ('corgi_thumb.jpg', 'RON8LDXFFFF10'), + ) - thumbnail1 = dict(zip(thumbnail_fields[1:], thumbnail1_vals[1:])) - thumbnail2 = dict(zip(thumbnail_fields[1:], thumbnail2_vals[1:])) - thumbnail3 = dict(zip(thumbnail_fields[1:], thumbnail3_vals[1:])) - thumbnail4 = dict(zip(thumbnail_fields[1:], thumbnail4_vals[1:])) - non_existent_thumbnail = dict(zip(thumbnail_fields[1:], thumbnail5_vals[1:])) + for i, thumb in enumerate(all_thumbnail_data): + thumb_dict = dict(zip(thumbnail_fields[1:], thumb[1:])) + if i in (0, 1) and course1_key: + thumb_key = course1_key.make_asset_key('thumbnail', thumb[0]) + thumb_md = AssetThumbnailMetadata(thumb_key, **thumb_dict) + if store is not None: + store.save_asset_thumbnail_metadata(course1_key, thumb_md, ModuleStoreEnum.UserID.test) + elif course2_key: + thumb_key = course2_key.make_asset_key('thumbnail', thumb[0]) + thumb_md = AssetThumbnailMetadata(thumb_key, **thumb_dict) + # Don't save assets 5 and 6. + if store is not None and i not in (4, 5): + store.save_asset_thumbnail_metadata(course2_key, thumb_md, ModuleStoreEnum.UserID.test) - # Asset6 and thumbnail6 have equivalent information on purpose. - thumbnail6_vals = ('asset.txt', 'JJJCCC747858') - thumbnail6 = dict(zip(thumbnail_fields[1:], thumbnail6_vals[1:])) - - if course1_key: - thumb1_key = course1_key.make_asset_key('thumbnail', thumbnail1_vals[0]) - thumb2_key = course1_key.make_asset_key('thumbnail', thumbnail2_vals[0]) - thumb1_md = AssetThumbnailMetadata(thumb1_key, **thumbnail1) - thumb2_md = AssetThumbnailMetadata(thumb2_key, **thumbnail2) - if course2_key: - thumb3_key = course2_key.make_asset_key('thumbnail', thumbnail3_vals[0]) - thumb4_key = course2_key.make_asset_key('thumbnail', thumbnail4_vals[0]) - thumb5_key = course2_key.make_asset_key('thumbnail', thumbnail5_vals[0]) - thumb6_key = course2_key.make_asset_key('thumbnail', thumbnail6_vals[0]) - thumb3_md = AssetThumbnailMetadata(thumb3_key, **thumbnail3) - thumb4_md = AssetThumbnailMetadata(thumb4_key, **thumbnail4) - thumb5_md = AssetThumbnailMetadata(thumb5_key, **non_existent_thumbnail) # pylint: disable=W0612 - thumb6_md = AssetThumbnailMetadata(thumb6_key, **thumbnail6) # pylint: disable=W0612 - - if store is not None: - if course1_key: - store.save_asset_thumbnail_metadata(course1_key, thumb1_md, ModuleStoreEnum.UserID.test) - store.save_asset_thumbnail_metadata(course1_key, thumb2_md, ModuleStoreEnum.UserID.test) - if course2_key: - store.save_asset_thumbnail_metadata(course2_key, thumb3_md, ModuleStoreEnum.UserID.test) - store.save_asset_thumbnail_metadata(course2_key, thumb4_md, ModuleStoreEnum.UserID.test) - # thumb5 and thumb6 are not saved on purpose! @ddt.data(*MODULESTORE_SETUPS) def test_save_one_and_confirm(self, storebuilder): @@ -430,22 +374,22 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): expected_sorts_by_2 = ( ( - ('displayname', 'ascending'), + ('displayname', ModuleStoreEnum.SortOrder.ascending), ('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'), (2, 2, 1) ), ( - ('displayname', 'descending'), + ('displayname', ModuleStoreEnum.SortOrder.descending), ('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'), (2, 2, 1) ), ( - ('uploadDate', 'ascending'), + ('uploadDate', ModuleStoreEnum.SortOrder.ascending), ('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'), (2, 2, 1) ), ( - ('uploadDate', 'descending'), + ('uploadDate', ModuleStoreEnum.SortOrder.descending), ('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'), (2, 2, 1) ), @@ -460,7 +404,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): self.assertEquals(asset_page[1].asset_id.path, sort_test[1][(2 * i) + 1]) # Now fetch everything. - asset_page = store.get_all_asset_metadata(course2.id, start=0, sort=('displayname', 'ascending')) + asset_page = store.get_all_asset_metadata(course2.id, start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)) self.assertEquals(len(asset_page), 5) self.assertEquals(asset_page[0].asset_id.path, 'code.tgz') self.assertEquals(asset_page[1].asset_id.path, 'demo.swf') @@ -469,11 +413,11 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp') # Some odd conditions. - asset_page = store.get_all_asset_metadata(course2.id, start=100, sort=('uploadDate', 'ascending')) + asset_page = store.get_all_asset_metadata(course2.id, start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending)) self.assertEquals(len(asset_page), 0) - asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=0, sort=('displayname', 'ascending')) + asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)) self.assertEquals(len(asset_page), 0) - asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=-12345, sort=('displayname', 'descending')) + asset_page = store.get_all_asset_metadata(course2.id, start=3, maxresults=-12345, sort=('displayname', ModuleStoreEnum.SortOrder.descending)) self.assertEquals(len(asset_page), 2) @ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())])) @@ -512,11 +456,11 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) self.assertEquals(len(store.get_all_asset_metadata(course1.id)), 2) self.assertEquals(len(store.get_all_asset_thumbnail_metadata(course1.id)), 2) - all_assets = store.get_all_asset_metadata(course2.id, sort=('displayname', 'ascending')) + all_assets = store.get_all_asset_metadata(course2.id, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)) self.assertEquals(len(all_assets), 2) self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') - all_thumbnails = store.get_all_asset_thumbnail_metadata(course2.id, sort=('uploadDate', 'descending')) + all_thumbnails = store.get_all_asset_thumbnail_metadata(course2.id, sort=('uploadDate', ModuleStoreEnum.SortOrder.descending)) self.assertEquals(len(all_thumbnails), 2) self.assertEquals(all_thumbnails[0].asset_id.path, 'kitten_thumb.jpg') self.assertEquals(all_thumbnails[1].asset_id.path, 'cat_thumb.jpg') diff --git a/docs/shared/requirements.txt b/docs/shared/requirements.txt index 330a7a1a21..860e55356d 100644 --- a/docs/shared/requirements.txt +++ b/docs/shared/requirements.txt @@ -56,7 +56,6 @@ PyYAML==3.10 requests==2.3.0 Shapely==1.2.16 sorl-thumbnail==11.12 -sortedcontainers==0.9.2 South==0.7.6 sympy==0.7.1 xmltodict==0.4.1 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index d58d506cbb..b4324a5241 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -76,6 +76,7 @@ scipy==0.14.0 Shapely==1.2.16 singledispatch==3.4.0.2 sorl-thumbnail==11.12 +sortedcontainers==0.9.2 South==0.7.6 stevedore==0.14.1 sure==1.2.3