Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enable cross-modulestore copying of course asset metadata.

Browse Source 
https://openedx.atlassian.net/browse/PLAT-38
This commit is contained in:
John Eskew
2014-11-12 16:45:32 -05:00
parent 51927808f0
commit a2cc8d994a
7 changed files with 67 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/lib/xmodule/xmodule/assetstore/__init__.py
View File

@@ -87,7 +87,7 @@ class AssetMetadata(object):
else:
self.fields[attr] = val
def to_mongo(self):
def to_storable(self):
"""
Converts metadata properties into a MongoDB-storable dict.
"""
@@ -106,7 +106,7 @@ class AssetMetadata(object):
}
@contract(asset_doc='dict|None')
def from_mongo(self, asset_doc):
def from_storable(self, asset_doc):
"""
Fill in all metadata fields from a MongoDB document.

4
common/lib/xmodule/xmodule/modulestore/__init__.py
View File

@@ -336,7 +336,7 @@ class ModuleStoreAssetInterface(object):
info = asset_key.block_type
mdata = AssetMetadata(asset_key, asset_key.path, **kwargs)
all_assets = course_assets[info]
mdata.from_mongo(all_assets[asset_idx])
mdata.from_storable(all_assets[asset_idx])
return mdata
@contract(course_key='CourseKey', start='int | None', maxresults='int | None', sort='tuple(str,(int,>=1,<=2))|None',)
@@ -392,7 +392,7 @@ class ModuleStoreAssetInterface(object):
for idx in xrange(start_idx, end_idx, step_incr):
raw_asset = all_assets[idx]
new_asset = AssetMetadata(course_key.make_asset_key(asset_type, raw_asset['filename']))
new_asset.from_mongo(raw_asset)
new_asset.from_storable(raw_asset)
ret_assets.append(new_asset)
return ret_assets

20
common/lib/xmodule/xmodule/modulestore/mixed.py
View File

@@ -391,11 +391,21 @@ class MixedModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase):
source_course_key (CourseKey): identifier of course to copy from
dest_course_key (CourseKey): identifier of course to copy to
"""
# When implementing this in https://openedx.atlassian.net/browse/PLAT-78 , consider this:
# Check the modulestores of both the source and dest course_keys. If in different modulestores,
# export all asset data from one modulestore and import it into the dest one.
store = self._get_modulestore_for_courseid(source_course_key)
return store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id)
source_store = self._get_modulestore_for_courseid(source_course_key)
dest_store = self._get_modulestore_for_courseid(dest_course_key)
if source_store != dest_store:
with self.bulk_operations(dest_course_key):
# Get all the asset metadata in the source course.
all_assets = source_store.get_all_asset_metadata(source_course_key, 'asset')
# Store it all in the dest course.
for asset in all_assets:
new_asset_key = dest_course_key.make_asset_key('asset', asset.asset_id.path)
copied_asset = AssetMetadata(new_asset_key)
copied_asset.from_storable(asset.to_storable())
dest_store.save_asset_metadata(copied_asset, user_id)
else:
# Courses in the same modulestore can be handled by the modulestore itself.
source_store.copy_all_asset_metadata(source_course_key, dest_course_key, user_id)
@contract(asset_key='AssetKey', attr=str)
def set_asset_metadata_attr(self, asset_key, attr, value, user_id):

6
common/lib/xmodule/xmodule/modulestore/mongo/base.py
View File

@@ -1504,7 +1504,7 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
asset_metadata.update({'edited_by': user_id, 'edited_on': datetime.now(UTC)})
# Translate metadata to Mongo format.
metadata_to_insert = asset_metadata.to_mongo()
metadata_to_insert = asset_metadata.to_storable()
if asset_idx is None:
# Add new metadata sorted into the list.
all_assets.add(metadata_to_insert)
@@ -1562,11 +1562,11 @@ class MongoModuleStore(ModuleStoreDraftAndPublished, ModuleStoreWriteBase, Mongo
# Form an AssetMetadata.
all_assets = course_assets[asset_key.block_type]
md = AssetMetadata(asset_key, asset_key.path)
md.from_mongo(all_assets[asset_idx])
md.from_storable(all_assets[asset_idx])
md.update(attr_dict)
# Generate a Mongo doc from the metadata and update the course asset info.
all_assets[asset_idx] = md.to_mongo()
all_assets[asset_idx] = md.to_storable()
self.asset_collection.update({'_id': course_assets['_id']}, {"$set": {asset_key.block_type: all_assets}})

6
common/lib/xmodule/xmodule/modulestore/split_mongo/split.py
View File

@@ -2182,7 +2182,7 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
"""
The guts of saving a new or updated asset
"""
metadata_to_insert = asset_metadata.to_mongo()
metadata_to_insert = asset_metadata.to_storable()
def _internal_method(all_assets, asset_idx):
"""
@@ -2218,11 +2218,11 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase):
# Form an AssetMetadata.
mdata = AssetMetadata(asset_key, asset_key.path)
mdata.from_mongo(all_assets[asset_idx])
mdata.from_storable(all_assets[asset_idx])
mdata.update(attr_dict)
# Generate a Mongo doc from the metadata and update the course asset info.
all_assets[asset_idx] = mdata.to_mongo()
all_assets[asset_idx] = mdata.to_storable()
return all_assets
self._update_course_assets(user_id, asset_key, _internal_method)

34
common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py
View File

@@ -12,7 +12,8 @@ from xmodule.modulestore import ModuleStoreEnum
from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
MODULESTORE_SETUPS, MongoContentstoreBuilder, XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder
MIXED_MODULESTORE_BOTH_SETUP, MODULESTORE_SETUPS, MongoContentstoreBuilder,
XmlModulestoreBuilder, MixedModulestoreBuilder, MongoModulestoreBuilder
)
@@ -414,9 +415,9 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
store.get_all_asset_metadata(course_key, 'asset')
@ddt.data(*MODULESTORE_SETUPS)
def test_copy_all_assets(self, storebuilder):
def test_copy_all_assets_same_modulestore(self, storebuilder):
"""
Create a course with assets and such, copy it all to another course, and check on it.
Create a course with assets, copy them all to another course in the same modulestore, and check on it.
"""
with MongoContentstoreBuilder().build() as contentstore:
with storebuilder.build(contentstore) as store:
@@ -433,3 +434,30 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
self.assertEquals(len(all_assets), 2)
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
@ddt.data(
('mongo', 'split'),
('split', 'mongo'),
)
@ddt.unpack
def test_copy_all_assets_cross_modulestore(self, from_store, to_store):
"""
Create a course with assets, copy them all to another course in a different modulestore, and check on it.
"""
mixed_builder = MIXED_MODULESTORE_BOTH_SETUP
with MongoContentstoreBuilder().build() as contentstore:
with mixed_builder.build(contentstore) as mixed_store:
with mixed_store.default_store(from_store):
course1 = CourseFactory.create(modulestore=mixed_store)
with mixed_store.default_store(to_store):
course2 = CourseFactory.create(modulestore=mixed_store)
self.setup_assets(course1.id, None, mixed_store)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0)
mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102)
all_assets = mixed_store.get_all_asset_metadata(
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(all_assets), 2)
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')

15
common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py
View File

@@ -265,13 +265,20 @@ class MongoContentstoreBuilder(object):
def __repr__(self):
return 'MongoContentstoreBuilder()'
MODULESTORE_SETUPS = (
MongoModulestoreBuilder(),
# VersioningModulestoreBuilder(), # FIXME LMS-11227
MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([
('draft', MongoModulestoreBuilder()),
('split', VersioningModulestoreBuilder())
])
MIXED_MODULESTORE_SETUPS = (
MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())]),
MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())]),
)
DIRECT_MODULESTORE_SETUPS = (
MongoModulestoreBuilder(),
# VersioningModulestoreBuilder(), # FUTUREDO: LMS-11227
)
MODULESTORE_SETUPS = DIRECT_MODULESTORE_SETUPS + MIXED_MODULESTORE_SETUPS
CONTENTSTORE_SETUPS = (MongoContentstoreBuilder(),)
COURSE_DATA_NAMES = (
'toy',