diff --git a/common/lib/xmodule/xmodule/assetstore/__init__.py b/common/lib/xmodule/xmodule/assetstore/__init__.py index c371205b1c..6c0464a405 100644 --- a/common/lib/xmodule/xmodule/assetstore/__init__.py +++ b/common/lib/xmodule/xmodule/assetstore/__init__.py @@ -87,7 +87,7 @@ class AssetMetadata(object): else: self.fields[attr] = val - def to_mongo(self): + def to_storable(self): """ Converts metadata properties into a MongoDB-storable dict. """ @@ -106,7 +106,7 @@ class AssetMetadata(object): } @contract(asset_doc='dict|None') - def from_mongo(self, asset_doc): + def from_storable(self, asset_doc): """ Fill in all metadata fields from a MongoDB document. diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 6c89d5520c..84968eff6b 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -336,7 +336,7 @@ class ModuleStoreAssetInterface(object): info = asset_key.block_type mdata = AssetMetadata(asset_key, asset_key.path, **kwargs) all_assets = course_assets[info] - mdata.from_mongo(all_assets[asset_idx]) + mdata.from_storable(all_assets[asset_idx]) return mdata @contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='tuple(str,(int,>=1,<=2))|None',) @@ -392,7 +392,7 @@ class ModuleStoreAssetInterface(object): for idx in xrange(start_idx, end_idx, step_incr): raw_asset = all_assets[idx] new_asset = AssetMetadata(course_key.make_asset_key(asset_type, raw_asset['filename'])) - new_asset.from_mongo(raw_asset) + new_asset.from_storable(raw_asset) ret_assets.append(new_asset) return ret_assets diff --git a/common/lib/xmodule/xmodule/modulestore/mixed.py b/common/lib/xmodule/xmodule/modulestore/mixed.py index 8fc565537b..b284c6bddd 100644 --- a/common/lib/xmodule/xmodule/modulestore/mixed.py +++ b/common/lib/xmodule/xmodule/modulestore/mixed.py @@ -391,11 +391,21 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase): source_course_key (CourseKey): identifier of course to copy from dest_course_key (CourseKey): identifier of course to copy to """ - # When implementing this in https://openedx.atlassian.net/browse/PLAT-78 , consider this: - # Check the modulestores of both the source and dest course_keys. If in different modulestores, - # export all asset data from one modulestore and import it into the dest one. - store = self._get_modulestore_for_courseid(source_course_key) - return store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id) + source_store = self._get_modulestore_for_courseid(source_course_key) + dest_store = self._get_modulestore_for_courseid(dest_course_key) + if source_store != dest_store: + with self.bulk_operations(dest_course_key): + # Get all the asset metadata in the source course. + all_assets = source_store.get_all_asset_metadata(source_course_key, 'asset') + # Store it all in the dest course. + for asset in all_assets: + new_asset_key = dest_course_key.make_asset_key('asset', asset.asset_id.path) + copied_asset = AssetMetadata(new_asset_key) + copied_asset.from_storable(asset.to_storable()) + dest_store.save_asset_metadata(copied_asset, user_id) + else: + # Courses in the same modulestore can be handled by the modulestore itself. + source_store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id) @contract(asset_key='AssetKey', attr=str) def set_asset_metadata_attr(self, asset_key, attr, value, user_id): diff --git a/common/lib/xmodule/xmodule/modulestore/mongo/base.py b/common/lib/xmodule/xmodule/modulestore/mongo/base.py index 3bb2480eb7..f1c66545b7 100644 --- a/common/lib/xmodule/xmodule/modulestore/mongo/base.py +++ b/common/lib/xmodule/xmodule/modulestore/mongo/base.py @@ -1504,7 +1504,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)}) # Translate metadata to Mongo format. - metadata_to_insert = asset_metadata.to_mongo() + metadata_to_insert = asset_metadata.to_storable() if asset_idx is None: # Add new metadata sorted into the list. all_assets.add(metadata_to_insert) @@ -1562,11 +1562,11 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo # Form an AssetMetadata. all_assets = course_assets[asset_key.block_type] md = AssetMetadata(asset_key, asset_key.path) - md.from_mongo(all_assets[asset_idx]) + md.from_storable(all_assets[asset_idx]) md.update(attr_dict) # Generate a Mongo doc from the metadata and update the course asset info. - all_assets[asset_idx] = md.to_mongo() + all_assets[asset_idx] = md.to_storable() self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {asset_key.block_type: all_assets}}) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 8756a39842..f6fa21271f 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -2182,7 +2182,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): """ The guts of saving a new or updated asset """ - metadata_to_insert = asset_metadata.to_mongo() + metadata_to_insert = asset_metadata.to_storable() def _internal_method(all_assets, asset_idx): """ @@ -2218,11 +2218,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): # Form an AssetMetadata. mdata = AssetMetadata(asset_key, asset_key.path) - mdata.from_mongo(all_assets[asset_idx]) + mdata.from_storable(all_assets[asset_idx]) mdata.update(attr_dict) # Generate a Mongo doc from the metadata and update the course asset info. - all_assets[asset_idx] = mdata.to_mongo() + all_assets[asset_idx] = mdata.to_storable() return all_assets self._update_course_assets(user_id, asset_key, _internal_method) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py index be5d2b931f..cd224bc2fc 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py @@ -12,7 +12,8 @@ from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.tests.factories import CourseFactory from xmodule.modulestore.tests.test_cross_modulestore_import_export import ( - MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder + MIXED_MODULESTORE_BOTH_SETUP, MODULESTORE_SETUPS, MongoContentstoreBuilder, + XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder ) @@ -414,9 +415,9 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): store.get_all_asset_metadata(course_key, 'asset') @ddt.data(*MODULESTORE_SETUPS) - def test_copy_all_assets(self, storebuilder): + def test_copy_all_assets_same_modulestore(self, storebuilder): """ - Create a course with assets and such, copy it all to another course, and check on it. + Create a course with assets, copy them all to another course in the same modulestore, and check on it. """ with MongoContentstoreBuilder().build() as contentstore: with storebuilder.build(contentstore) as store: @@ -433,3 +434,30 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): self.assertEquals(len(all_assets), 2) self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') + + @ddt.data( + ('mongo', 'split'), + ('split', 'mongo'), + ) + @ddt.unpack + def test_copy_all_assets_cross_modulestore(self, from_store, to_store): + """ + Create a course with assets, copy them all to another course in a different modulestore, and check on it. + """ + mixed_builder = MIXED_MODULESTORE_BOTH_SETUP + with MongoContentstoreBuilder().build() as contentstore: + with mixed_builder.build(contentstore) as mixed_store: + with mixed_store.default_store(from_store): + course1 = CourseFactory.create(modulestore=mixed_store) + with mixed_store.default_store(to_store): + course2 = CourseFactory.create(modulestore=mixed_store) + self.setup_assets(course1.id, None, mixed_store) + self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2) + self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0) + mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102) + all_assets = mixed_store.get_all_asset_metadata( + course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(all_assets), 2) + self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') + self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py index c8ab6bd577..7fdae012b1 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py @@ -265,13 +265,20 @@ class MongoContentstoreBuilder(object): def __repr__(self): return 'MongoContentstoreBuilder()' - -MODULESTORE_SETUPS = ( - MongoModulestoreBuilder(), -# VersioningModulestoreBuilder(), # FIXME LMS-11227 +MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([ + ('draft', MongoModulestoreBuilder()), + ('split', VersioningModulestoreBuilder()) +]) +MIXED_MODULESTORE_SETUPS = ( MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())]), MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())]), ) +DIRECT_MODULESTORE_SETUPS = ( + MongoModulestoreBuilder(), +# VersioningModulestoreBuilder(), # FUTUREDO: LMS-11227 +) +MODULESTORE_SETUPS = DIRECT_MODULESTORE_SETUPS + MIXED_MODULESTORE_SETUPS + CONTENTSTORE_SETUPS = (MongoContentstoreBuilder(),) COURSE_DATA_NAMES = ( 'toy',