diff --git a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py index f241e92085..81f3a474ea 100644 --- a/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/perf_tests/test_asset_import_export.py @@ -105,49 +105,43 @@ class CrossStoreXMLRoundtrip(unittest.TestCase): make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) - # Construct the contentstore for storing the first import - with MongoContentstoreBuilder().build() as source_content: - # Construct the modulestore for storing the first import (using the previously created contentstore) - with source_ms.build(source_content) as source_store: - # Construct the contentstore for storing the second import - with MongoContentstoreBuilder().build() as dest_content: - # Construct the modulestore for storing the second import (using the second contentstore) - with dest_ms.build(dest_content) as dest_store: - source_course_key = source_store.make_course_key('a', 'course', 'course') - dest_course_key = dest_store.make_course_key('a', 'course', 'course') + with source_ms.build() as (source_content, source_store): + with dest_ms.build() as (dest_content, dest_store): + source_course_key = source_store.make_course_key('a', 'course', 'course') + dest_course_key = dest_store.make_course_key('a', 'course', 'course') - with CodeBlockTimer("initial_import"): - import_from_xml( - source_store, - 'test_user', - TEST_DATA_ROOT, - course_dirs=TEST_COURSE, - static_content_store=source_content, - target_course_id=source_course_key, - create_course_if_not_present=True, - raise_on_failure=True, - ) + with CodeBlockTimer("initial_import"): + import_from_xml( + source_store, + 'test_user', + TEST_DATA_ROOT, + course_dirs=TEST_COURSE, + static_content_store=source_content, + target_course_id=source_course_key, + create_course_if_not_present=True, + raise_on_failure=True, + ) - with CodeBlockTimer("export"): - export_to_xml( - source_store, - source_content, - source_course_key, - self.export_dir, - 'exported_source_course', - ) + with CodeBlockTimer("export"): + export_to_xml( + source_store, + source_content, + source_course_key, + self.export_dir, + 'exported_source_course', + ) - with CodeBlockTimer("second_import"): - import_from_xml( - dest_store, - 'test_user', - self.export_dir, - course_dirs=['exported_source_course'], - static_content_store=dest_content, - target_course_id=dest_course_key, - create_course_if_not_present=True, - raise_on_failure=True, - ) + with CodeBlockTimer("second_import"): + import_from_xml( + dest_store, + 'test_user', + self.export_dir, + course_dirs=['exported_source_course'], + static_content_store=dest_content, + target_course_id=dest_course_key, + create_course_if_not_present=True, + raise_on_failure=True, + ) @ddt.ddt @@ -193,47 +187,44 @@ class FindAssetTest(unittest.TestCase): make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) - # Construct the contentstore for storing the first import - with MongoContentstoreBuilder().build() as source_content: - # Construct the modulestore for storing the first import (using the previously created contentstore) - with source_ms.build(source_content) as source_store: - source_course_key = source_store.make_course_key('a', 'course', 'course') - asset_key = source_course_key.make_asset_key( - AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif' + with source_ms.build() as (source_content, source_store): + source_course_key = source_store.make_course_key('a', 'course', 'course') + asset_key = source_course_key.make_asset_key( + AssetMetadata.GENERAL_ASSET_TYPE, 'silly_cat_picture.gif' + ) + + with CodeBlockTimer("initial_import"): + import_from_xml( + source_store, + 'test_user', + TEST_DATA_ROOT, + course_dirs=TEST_COURSE, + static_content_store=source_content, + target_course_id=source_course_key, + create_course_if_not_present=True, + raise_on_failure=True, ) - with CodeBlockTimer("initial_import"): - import_from_xml( - source_store, - 'test_user', - TEST_DATA_ROOT, - course_dirs=TEST_COURSE, - static_content_store=source_content, - target_course_id=source_course_key, - create_course_if_not_present=True, - raise_on_failure=True, + with CodeBlockTimer("find_nonexistent_asset"): + # More correct would be using the AssetManager.find() - but since the test + # has created its own test modulestore, the AssetManager can't be used. + __ = source_store.find_asset_metadata(asset_key) + + # Perform get_all_asset_metadata for each sort. + for sort in ALL_SORTS: + with CodeBlockTimer("get_asset_list:{}-{}".format( + sort[0], + 'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc' + )): + # Grab two ranges of 50 assets using different sorts. + # Why 50? That's how many are displayed on the current Studio "Files & Uploads" page. + start_middle = num_assets / 2 + __ = source_store.get_all_asset_metadata( + source_course_key, 'asset', start=0, sort=sort, maxresults=50 + ) + __ = source_store.get_all_asset_metadata( + source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50 ) - - with CodeBlockTimer("find_nonexistent_asset"): - # More correct would be using the AssetManager.find() - but since the test - # has created its own test modulestore, the AssetManager can't be used. - __ = source_store.find_asset_metadata(asset_key) - - # Perform get_all_asset_metadata for each sort. - for sort in ALL_SORTS: - with CodeBlockTimer("get_asset_list:{}-{}".format( - sort[0], - 'asc' if sort[1] == ModuleStoreEnum.SortOrder.ascending else 'desc' - )): - # Grab two ranges of 50 assets using different sorts. - # Why 50? That's how many are displayed on the current Studio "Files & Uploads" page. - start_middle = num_assets / 2 - __ = source_store.get_all_asset_metadata( - source_course_key, 'asset', start=0, sort=sort, maxresults=50 - ) - __ = source_store.get_all_asset_metadata( - source_course_key, 'asset', start=start_middle, sort=sort, maxresults=50 - ) @ddt.ddt @@ -265,48 +256,45 @@ class TestModulestoreAssetSize(unittest.TestCase): make_asset_xml(num_assets, ASSET_XML_PATH) validate_xml(ASSET_XSD_PATH, ASSET_XML_PATH) - # Construct the contentstore for storing the first import - with MongoContentstoreBuilder().build() as source_content: - # Construct the modulestore for storing the first import (using the previously created contentstore) - with source_ms.build(source_content) as source_store: - source_course_key = source_store.make_course_key('a', 'course', 'course') + with source_ms.build() as (source_content, source_store): + source_course_key = source_store.make_course_key('a', 'course', 'course') - import_from_xml( - source_store, - 'test_user', - TEST_DATA_ROOT, - course_dirs=TEST_COURSE, - static_content_store=source_content, - target_course_id=source_course_key, - create_course_if_not_present=True, - raise_on_failure=True, - ) + import_from_xml( + source_store, + 'test_user', + TEST_DATA_ROOT, + course_dirs=TEST_COURSE, + static_content_store=source_content, + target_course_id=source_course_key, + create_course_if_not_present=True, + raise_on_failure=True, + ) - asset_collection = source_ms.asset_collection() - # Ensure the asset collection exists. - if asset_collection.name in asset_collection.database.collection_names(): + asset_collection = source_ms.asset_collection() + # Ensure the asset collection exists. + if asset_collection.name in asset_collection.database.collection_names(): - # Map gets the size of each structure. - mapper = Code(""" - function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) } - """) - - # Reduce finds the largest structure size and returns only it. - reducer = Code(""" - function(key, values) { - var max_size = 0; - for (var i=0; i < values.length; i++) { - if (values[i] > max_size) { - max_size = values[i]; - } - } - return max_size; - } + # Map gets the size of each structure. + mapper = Code(""" + function() { emit("size", (this == null) ? 0 : Object.bsonsize(this)) } """) - results = asset_collection.map_reduce(mapper, reducer, "size_results") - result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format( - self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()] - ) - with open("bson_sizes.txt", "a") as f: - f.write(result_str) + # Reduce finds the largest structure size and returns only it. + reducer = Code(""" + function(key, values) { + var max_size = 0; + for (var i=0; i < values.length; i++) { + if (values[i] > max_size) { + max_size = values[i]; + } + } + return max_size; + } + """) + + results = asset_collection.map_reduce(mapper, reducer, "size_results") + result_str = "{} - Store: {:<15} - Num Assets: {:>6} - Result: {}\n".format( + self.test_run_time, SHORT_NAME_MAP[source_ms], num_assets, [r for r in results.find()] + ) + with open("bson_sizes.txt", "a") as f: + f.write(result_str) diff --git a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py index 324f317698..3868a5830e 100644 --- a/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py +++ b/common/lib/xmodule/xmodule/modulestore/split_mongo/split.py @@ -402,13 +402,18 @@ class SplitBulkWriteMixin(BulkOperationsMixin): bulk_write_record = self._get_bulk_ops_record(course_key) if bulk_write_record.active: + # Only query for the definitions that aren't already cached. for definition in bulk_write_record.definitions.values(): definition_id = definition.get('_id') if definition_id in ids: ids.remove(definition_id) definitions.append(definition) - definitions.extend(self.db_connection.get_definitions(list(ids))) + # Query the db for the definitions. + defs_from_db = self.db_connection.get_definitions(list(ids)) + # Add the retrieved definitions to the cache. + bulk_write_record.definitions.update({d.get('_id'): d for d in defs_from_db}) + definitions.extend(defs_from_db) return definitions def update_definition(self, course_key, definition): @@ -683,23 +688,29 @@ class SplitMongoModuleStore(SplitBulkWriteMixin, ModuleStoreWriteBase): new_module_data ) - if not lazy: - # Load all descendants by id + # This code supports lazy loading, where the descendent definitions aren't loaded + # until they're actually needed. + # However, assume that depth == 0 means no depth is specified and depth != 0 means + # a depth *is* specified. If a non-zero depth is specified, force non-lazy definition + # loading in order to populate the definition cache for later access. + load_definitions_now = depth != 0 or not lazy + if load_definitions_now: + # Non-lazy loading: Load all descendants by id. descendent_definitions = self.get_definitions( course_key, [ - block['definition'] + block.definition for block in new_module_data.itervalues() ] ) - # turn into a map + # Turn definitions into a map. definitions = {definition['_id']: definition for definition in descendent_definitions} for block in new_module_data.itervalues(): if block.definition in definitions: definition = definitions[block.definition] - # convert_fields was being done here, but it gets done later in the runtime's xblock_from_json + # convert_fields gets done later in the runtime's xblock_from_json block.fields.update(definition.get('fields')) block.definition_loaded = True diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_asides.py b/common/lib/xmodule/xmodule/modulestore/tests/test_asides.py index 4a9e98114c..80ca48cf6d 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_asides.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_asides.py @@ -34,7 +34,7 @@ class TestAsidesXmlStore(TestCase): """ Check that the xml modulestore read in all the asides with their values """ - with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as store: + with XmlModulestoreBuilder().build(course_ids=['edX/aside_test/2012_Fall']) as (__, store): def check_block(block): """ Check whether block has the expected aside w/ its fields and then recurse to the block's children diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py index d25dfd2bb7..29c77f471b 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py @@ -161,155 +161,146 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): """ Save the metadata in each store and retrieve it singularly, as all assets, and after deleting all. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - asset_filename = 'burnside.jpg' - new_asset_loc = course.id.make_asset_key('asset', asset_filename) - # Save the asset's metadata. - new_asset_md = self._make_asset_metadata(new_asset_loc) - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - # Find the asset's metadata and confirm it's the same. - found_asset_md = store.find_asset_metadata(new_asset_loc) - self.assertIsNotNone(found_asset_md) - self.assertEquals(new_asset_md, found_asset_md) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + asset_filename = 'burnside.jpg' + new_asset_loc = course.id.make_asset_key('asset', asset_filename) + # Save the asset's metadata. + new_asset_md = self._make_asset_metadata(new_asset_loc) + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + # Find the asset's metadata and confirm it's the same. + found_asset_md = store.find_asset_metadata(new_asset_loc) + self.assertIsNotNone(found_asset_md) + self.assertEquals(new_asset_md, found_asset_md) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) @ddt.data(*MODULESTORE_SETUPS) def test_delete(self, storebuilder): """ Delete non-existent and existent metadata """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') - # Attempt to delete an asset that doesn't exist. - self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') + # Attempt to delete an asset that doesn't exist. + self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) - new_asset_md = self._make_asset_metadata(new_asset_loc) - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) + new_asset_md = self._make_asset_metadata(new_asset_loc) + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) @ddt.data(*MODULESTORE_SETUPS) def test_find_non_existing_assets(self, storebuilder): """ Find a non-existent asset in an existing course. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') - # Find existing asset metadata. - asset_md = store.find_asset_metadata(new_asset_loc) - self.assertIsNone(asset_md) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') + # Find existing asset metadata. + asset_md = store.find_asset_metadata(new_asset_loc) + self.assertIsNone(asset_md) @ddt.data(*MODULESTORE_SETUPS) def test_get_all_non_existing_assets(self, storebuilder): """ Get all assets in an existing course when no assets exist. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - # Find existing asset metadata. - asset_md = store.get_all_asset_metadata(course.id, 'asset') - self.assertEquals(asset_md, []) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + # Find existing asset metadata. + asset_md = store.get_all_asset_metadata(course.id, 'asset') + self.assertEquals(asset_md, []) @ddt.data(*MODULESTORE_SETUPS) def test_find_assets_in_non_existent_course(self, storebuilder): """ Find asset metadata from a non-existent course. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - fake_course_id = CourseKey.from_string("{}nothere/{}nothere/{}nothere".format( - course.id.org, course.id.course, course.id.run - )) - new_asset_loc = fake_course_id.make_asset_key('asset', 'burnside.jpg') - # Find asset metadata from non-existent course. - with self.assertRaises(ItemNotFoundError): - store.find_asset_metadata(new_asset_loc) - with self.assertRaises(ItemNotFoundError): - store.get_all_asset_metadata(fake_course_id, 'asset') + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + fake_course_id = CourseKey.from_string("{}nothere/{}nothere/{}nothere".format( + course.id.org, course.id.course, course.id.run + )) + new_asset_loc = fake_course_id.make_asset_key('asset', 'burnside.jpg') + # Find asset metadata from non-existent course. + with self.assertRaises(ItemNotFoundError): + store.find_asset_metadata(new_asset_loc) + with self.assertRaises(ItemNotFoundError): + store.get_all_asset_metadata(fake_course_id, 'asset') @ddt.data(*MODULESTORE_SETUPS) def test_add_same_asset_twice(self, storebuilder): """ Add an asset's metadata, then add it again. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') - new_asset_md = self._make_asset_metadata(new_asset_loc) - # Add asset metadata. - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) - # Add *the same* asset metadata. - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - # Still one here? - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') + new_asset_md = self._make_asset_metadata(new_asset_loc) + # Add asset metadata. + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) + # Add *the same* asset metadata. + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + # Still one here? + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1) @ddt.data(*MODULESTORE_SETUPS) def test_different_asset_types(self, storebuilder): """ Test saving assets with other asset types. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('vrml', 'pyramid.vrml') - new_asset_md = self._make_asset_metadata(new_asset_loc) - # Add asset metadata. - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'vrml')), 1) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('vrml', 'pyramid.vrml') + new_asset_md = self._make_asset_metadata(new_asset_loc) + # Add asset metadata. + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'vrml')), 1) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) @ddt.data(*MODULESTORE_SETUPS) def test_asset_types_with_other_field_names(self, storebuilder): """ Test saving assets using an asset type of 'course_id'. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('course_id', 'just_to_see_if_it_still_works.jpg') - new_asset_md = self._make_asset_metadata(new_asset_loc) - # Add asset metadata. - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'course_id')), 1) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) - all_assets = store.get_all_asset_metadata(course.id, 'course_id') - self.assertEquals(all_assets[0].asset_id.path, new_asset_loc.path) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('course_id', 'just_to_see_if_it_still_works.jpg') + new_asset_md = self._make_asset_metadata(new_asset_loc) + # Add asset metadata. + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'course_id')), 1) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0) + all_assets = store.get_all_asset_metadata(course.id, 'course_id') + self.assertEquals(all_assets[0].asset_id.path, new_asset_loc.path) @ddt.data(*MODULESTORE_SETUPS) def test_lock_unlock_assets(self, storebuilder): """ Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') - new_asset_md = self._make_asset_metadata(new_asset_loc) - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') + new_asset_md = self._make_asset_metadata(new_asset_loc) + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - locked_state = new_asset_md.locked - # Flip the course asset's locked status. - store.set_asset_metadata_attr(new_asset_loc, "locked", not locked_state, ModuleStoreEnum.UserID.test) - # Find the same course and check its locked status. - updated_asset_md = store.find_asset_metadata(new_asset_loc) - self.assertIsNotNone(updated_asset_md) - self.assertEquals(updated_asset_md.locked, not locked_state) - # Now flip it back. - store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test) - reupdated_asset_md = store.find_asset_metadata(new_asset_loc) - self.assertIsNotNone(reupdated_asset_md) - self.assertEquals(reupdated_asset_md.locked, locked_state) + locked_state = new_asset_md.locked + # Flip the course asset's locked status. + store.set_asset_metadata_attr(new_asset_loc, "locked", not locked_state, ModuleStoreEnum.UserID.test) + # Find the same course and check its locked status. + updated_asset_md = store.find_asset_metadata(new_asset_loc) + self.assertIsNotNone(updated_asset_md) + self.assertEquals(updated_asset_md.locked, not locked_state) + # Now flip it back. + store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test) + reupdated_asset_md = store.find_asset_metadata(new_asset_loc) + self.assertIsNotNone(reupdated_asset_md) + self.assertEquals(reupdated_asset_md.locked, locked_state) ALLOWED_ATTRS = ( ('pathname', '/new/path'), @@ -340,98 +331,93 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): """ Save setting each attr one at a time """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') - new_asset_md = self._make_asset_metadata(new_asset_loc) - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - for attr, value in self.ALLOWED_ATTRS: - # Set the course asset's attr. - store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) - # Find the same course asset and check its changed attr. - updated_asset_md = store.find_asset_metadata(new_asset_loc) - self.assertIsNotNone(updated_asset_md) - self.assertIsNotNone(getattr(updated_asset_md, attr, None)) - self.assertEquals(getattr(updated_asset_md, attr, None), value) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') + new_asset_md = self._make_asset_metadata(new_asset_loc) + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + for attribute, value in self.ALLOWED_ATTRS: + # Set the course asset's attribute. + store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test) + # Find the same course asset and check its changed attribute. + updated_asset_md = store.find_asset_metadata(new_asset_loc) + self.assertIsNotNone(updated_asset_md) + self.assertIsNotNone(getattr(updated_asset_md, attribute, None)) + self.assertEquals(getattr(updated_asset_md, attribute, None), value) @ddt.data(*MODULESTORE_SETUPS) def test_set_disallowed_attrs(self, storebuilder): """ setting disallowed attrs should fail """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') - new_asset_md = self._make_asset_metadata(new_asset_loc) - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - for attr, value in self.DISALLOWED_ATTRS: - original_attr_val = getattr(new_asset_md, attr) - # Set the course asset's attr. - store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) - # Find the same course and check its changed attr. - updated_asset_md = store.find_asset_metadata(new_asset_loc) - self.assertIsNotNone(updated_asset_md) - self.assertIsNotNone(getattr(updated_asset_md, attr, None)) - # Make sure that the attr is unchanged from its original value. - self.assertEquals(getattr(updated_asset_md, attr, None), original_attr_val) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') + new_asset_md = self._make_asset_metadata(new_asset_loc) + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + for attribute, value in self.DISALLOWED_ATTRS: + original_attr_val = getattr(new_asset_md, attribute) + # Set the course asset's attribute. + store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test) + # Find the same course and check its changed attribute. + updated_asset_md = store.find_asset_metadata(new_asset_loc) + self.assertIsNotNone(updated_asset_md) + self.assertIsNotNone(getattr(updated_asset_md, attribute, None)) + # Make sure that the attribute is unchanged from its original value. + self.assertEquals(getattr(updated_asset_md, attribute, None), original_attr_val) @ddt.data(*MODULESTORE_SETUPS) def test_set_unknown_attrs(self, storebuilder): """ setting unknown attrs should fail """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') - new_asset_md = self._make_asset_metadata(new_asset_loc) - store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) - for attr, value in self.UNKNOWN_ATTRS: - # Set the course asset's attr. - store.set_asset_metadata_attr(new_asset_loc, attr, value, ModuleStoreEnum.UserID.test) - # Find the same course and check its changed attr. - updated_asset_md = store.find_asset_metadata(new_asset_loc) - self.assertIsNotNone(updated_asset_md) - # Make sure the unknown field was *not* added. - with self.assertRaises(AttributeError): - self.assertEquals(getattr(updated_asset_md, attr), value) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg') + new_asset_md = self._make_asset_metadata(new_asset_loc) + store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test) + for attribute, value in self.UNKNOWN_ATTRS: + # Set the course asset's attribute. + store.set_asset_metadata_attr(new_asset_loc, attribute, value, ModuleStoreEnum.UserID.test) + # Find the same course and check its changed attribute. + updated_asset_md = store.find_asset_metadata(new_asset_loc) + self.assertIsNotNone(updated_asset_md) + # Make sure the unknown field was *not* added. + with self.assertRaises(AttributeError): + self.assertEquals(getattr(updated_asset_md, attribute), value) @ddt.data(*MODULESTORE_SETUPS) def test_save_one_different_asset(self, storebuilder): """ saving and deleting things which are not 'asset' """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - asset_key = course.id.make_asset_key('different', 'burn.jpg') - new_asset_thumbnail = self._make_asset_thumbnail_metadata( - self._make_asset_metadata(asset_key) - ) - store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) - self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + asset_key = course.id.make_asset_key('different', 'burn.jpg') + new_asset_thumbnail = self._make_asset_thumbnail_metadata( + self._make_asset_metadata(asset_key) + ) + store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) + self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0) @ddt.data(*MODULESTORE_SETUPS) def test_find_different(self, storebuilder): """ finding things which are of type other than 'asset' """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - asset_key = course.id.make_asset_key('different', 'burn.jpg') - new_asset_thumbnail = self._make_asset_thumbnail_metadata( - self._make_asset_metadata(asset_key) - ) - store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + asset_key = course.id.make_asset_key('different', 'burn.jpg') + new_asset_thumbnail = self._make_asset_thumbnail_metadata( + self._make_asset_metadata(asset_key) + ) + store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) - self.assertIsNotNone(store.find_asset_metadata(asset_key)) - unknown_asset_key = course.id.make_asset_key('different', 'nosuchfile.jpg') - self.assertIsNone(store.find_asset_metadata(unknown_asset_key)) + self.assertIsNotNone(store.find_asset_metadata(asset_key)) + unknown_asset_key = course.id.make_asset_key('different', 'nosuchfile.jpg') + self.assertIsNone(store.find_asset_metadata(unknown_asset_key)) def _check_asset_values(self, assets, orig): """ @@ -447,37 +433,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): getting all things which are of type other than 'asset' """ # pylint: disable=bad-continuation - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) - # Save 'em. - for asset_type, filename in self.alls: - asset_key = course.id.make_asset_key(asset_type, filename) - new_asset = self._make_asset_thumbnail_metadata( - self._make_asset_metadata(asset_key) - ) - store.save_asset_metadata(new_asset, ModuleStoreEnum.UserID.test) - - # Check 'em. - for asset_type, asset_list in ( - ('different', self.differents), - ('vrml', self.vrmls), - ('asset', self.regular_assets), - ): - assets = store.get_all_asset_metadata(course.id, asset_type) - self.assertEquals(len(assets), len(asset_list)) - self._check_asset_values(assets, asset_list) - - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0) - self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4) - - assets = store.get_all_asset_metadata( - course.id, None, start=0, maxresults=-1, - sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + # Save 'em. + for asset_type, filename in self.alls: + asset_key = course.id.make_asset_key(asset_type, filename) + new_asset = self._make_asset_thumbnail_metadata( + self._make_asset_metadata(asset_key) ) - self.assertEquals(len(assets), len(self.alls)) - self._check_asset_values(assets, self.alls) + store.save_asset_metadata(new_asset, ModuleStoreEnum.UserID.test) + + # Check 'em. + for asset_type, asset_list in ( + ('different', self.differents), + ('vrml', self.vrmls), + ('asset', self.regular_assets), + ): + assets = store.get_all_asset_metadata(course.id, asset_type) + self.assertEquals(len(assets), len(asset_list)) + self._check_asset_values(assets, asset_list) + + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0) + self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4) + + assets = store.get_all_asset_metadata( + course.id, None, start=0, maxresults=-1, + sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(assets), len(self.alls)) + self._check_asset_values(assets, self.alls) @ddt.data(*MODULESTORE_SETUPS) def test_save_metadata_list(self, storebuilder): @@ -485,40 +470,39 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): Save a list of asset metadata all at once. """ # pylint: disable=bad-continuation - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) - # Make a list of AssetMetadata objects. - md_list = [] - for asset_type, filename in self.alls: - asset_key = course.id.make_asset_key(asset_type, filename) - md_list.append(self._make_asset_thumbnail_metadata( - self._make_asset_metadata(asset_key) - )) + # Make a list of AssetMetadata objects. + md_list = [] + for asset_type, filename in self.alls: + asset_key = course.id.make_asset_key(asset_type, filename) + md_list.append(self._make_asset_thumbnail_metadata( + self._make_asset_metadata(asset_key) + )) - # Save 'em. - store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test) + # Save 'em. + store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test) - # Check 'em. - for asset_type, asset_list in ( - ('different', self.differents), - ('vrml', self.vrmls), - ('asset', self.regular_assets), - ): - assets = store.get_all_asset_metadata(course.id, asset_type) - self.assertEquals(len(assets), len(asset_list)) - self._check_asset_values(assets, asset_list) + # Check 'em. + for asset_type, asset_list in ( + ('different', self.differents), + ('vrml', self.vrmls), + ('asset', self.regular_assets), + ): + assets = store.get_all_asset_metadata(course.id, asset_type) + self.assertEquals(len(assets), len(asset_list)) + self._check_asset_values(assets, asset_list) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0) - self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0) + self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4) - assets = store.get_all_asset_metadata( - course.id, None, start=0, maxresults=-1, - sort=('displayname', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(assets), len(self.alls)) - self._check_asset_values(assets, self.alls) + assets = store.get_all_asset_metadata( + course.id, None, start=0, maxresults=-1, + sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(assets), len(self.alls)) + self._check_asset_values(assets, self.alls) @ddt.data(*MODULESTORE_SETUPS) def test_save_metadata_list_with_mismatched_asset(self, storebuilder): @@ -526,140 +510,137 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): Save a list of asset metadata all at once - but with one asset's metadata from a different course. """ # pylint: disable=bad-continuation - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course1 = CourseFactory.create(modulestore=store) - course2 = CourseFactory.create(modulestore=store) + with storebuilder.build() as (__, store): + course1 = CourseFactory.create(modulestore=store) + course2 = CourseFactory.create(modulestore=store) - # Make a list of AssetMetadata objects. - md_list = [] - for asset_type, filename in self.alls: - if asset_type == 'asset': - asset_key = course2.id.make_asset_key(asset_type, filename) - else: - asset_key = course1.id.make_asset_key(asset_type, filename) - md_list.append(self._make_asset_thumbnail_metadata( - self._make_asset_metadata(asset_key) - )) + # Make a list of AssetMetadata objects. + md_list = [] + for asset_type, filename in self.alls: + if asset_type == 'asset': + asset_key = course2.id.make_asset_key(asset_type, filename) + else: + asset_key = course1.id.make_asset_key(asset_type, filename) + md_list.append(self._make_asset_thumbnail_metadata( + self._make_asset_metadata(asset_key) + )) - # Save 'em. - store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test) + # Save 'em. + store.save_asset_metadata_list(md_list, ModuleStoreEnum.UserID.test) - # Check 'em. - for asset_type, asset_list in ( - ('different', self.differents), - ('vrml', self.vrmls), - ): - assets = store.get_all_asset_metadata(course1.id, asset_type) - self.assertEquals(len(assets), len(asset_list)) - self._check_asset_values(assets, asset_list) + # Check 'em. + for asset_type, asset_list in ( + ('different', self.differents), + ('vrml', self.vrmls), + ): + assets = store.get_all_asset_metadata(course1.id, asset_type) + self.assertEquals(len(assets), len(asset_list)) + self._check_asset_values(assets, asset_list) - self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) - self.assertEquals(len(store.get_all_asset_metadata(course1.id, None)), 3) + self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) + self.assertEquals(len(store.get_all_asset_metadata(course1.id, None)), 3) - assets = store.get_all_asset_metadata( - course1.id, None, start=0, maxresults=-1, - sort=('displayname', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(assets), len(self.differents + self.vrmls)) - self._check_asset_values(assets, self.differents + self.vrmls) + assets = store.get_all_asset_metadata( + course1.id, None, start=0, maxresults=-1, + sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(assets), len(self.differents + self.vrmls)) + self._check_asset_values(assets, self.differents + self.vrmls) @ddt.data(*MODULESTORE_SETUPS) def test_delete_all_different_type(self, storebuilder): """ deleting all assets of a given but not 'asset' type """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course = CourseFactory.create(modulestore=store) - asset_key = course.id.make_asset_key('different', 'burn_thumb.jpg') - new_asset_thumbnail = self._make_asset_thumbnail_metadata( - self._make_asset_metadata(asset_key) - ) - store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) + with storebuilder.build() as (__, store): + course = CourseFactory.create(modulestore=store) + asset_key = course.id.make_asset_key('different', 'burn_thumb.jpg') + new_asset_thumbnail = self._make_asset_thumbnail_metadata( + self._make_asset_metadata(asset_key) + ) + store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test) - self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) + self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1) @ddt.data(*MODULESTORE_SETUPS) def test_get_all_assets_with_paging(self, storebuilder): """ Save multiple metadata in each store and retrieve it singularly, as all assets, and after deleting all. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course1 = CourseFactory.create(modulestore=store) - course2 = CourseFactory.create(modulestore=store) - self.setup_assets(course1.id, course2.id, store) + with storebuilder.build() as (__, store): + course1 = CourseFactory.create(modulestore=store) + course2 = CourseFactory.create(modulestore=store) + self.setup_assets(course1.id, course2.id, store) - expected_sorts_by_2 = ( - ( - ('displayname', ModuleStoreEnum.SortOrder.ascending), - ('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'), - (2, 2, 1) - ), - ( - ('displayname', ModuleStoreEnum.SortOrder.descending), - ('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'), - (2, 2, 1) - ), - ( - ('uploadDate', ModuleStoreEnum.SortOrder.ascending), - ('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'), - (2, 2, 1) - ), - ( - ('uploadDate', ModuleStoreEnum.SortOrder.descending), - ('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'), - (2, 2, 1) - ), - ) - # First, with paging across all sorts. - for sort_test in expected_sorts_by_2: - for i in xrange(3): - asset_page = store.get_all_asset_metadata( - course2.id, 'asset', start=2 * i, maxresults=2, sort=sort_test[0] - ) - num_expected_results = sort_test[2][i] - expected_filename = sort_test[1][2 * i] - self.assertEquals(len(asset_page), num_expected_results) - self.assertEquals(asset_page[0].asset_id.path, expected_filename) - if num_expected_results == 2: - expected_filename = sort_test[1][(2 * i) + 1] - self.assertEquals(asset_page[1].asset_id.path, expected_filename) + expected_sorts_by_2 = ( + ( + ('displayname', ModuleStoreEnum.SortOrder.ascending), + ('code.tgz', 'demo.swf', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp'), + (2, 2, 1) + ), + ( + ('displayname', ModuleStoreEnum.SortOrder.descending), + ('weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'demo.swf', 'code.tgz'), + (2, 2, 1) + ), + ( + ('uploadDate', ModuleStoreEnum.SortOrder.ascending), + ('code.tgz', 'dog.png', 'roman_history.pdf', 'weather_patterns.bmp', 'demo.swf'), + (2, 2, 1) + ), + ( + ('uploadDate', ModuleStoreEnum.SortOrder.descending), + ('demo.swf', 'weather_patterns.bmp', 'roman_history.pdf', 'dog.png', 'code.tgz'), + (2, 2, 1) + ), + ) + # First, with paging across all sorts. + for sort_test in expected_sorts_by_2: + for i in xrange(3): + asset_page = store.get_all_asset_metadata( + course2.id, 'asset', start=2 * i, maxresults=2, sort=sort_test[0] + ) + num_expected_results = sort_test[2][i] + expected_filename = sort_test[1][2 * i] + self.assertEquals(len(asset_page), num_expected_results) + self.assertEquals(asset_page[0].asset_id.path, expected_filename) + if num_expected_results == 2: + expected_filename = sort_test[1][(2 * i) + 1] + self.assertEquals(asset_page[1].asset_id.path, expected_filename) - # Now fetch everything. - asset_page = store.get_all_asset_metadata( - course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(asset_page), 5) - self.assertEquals(asset_page[0].asset_id.path, 'code.tgz') - self.assertEquals(asset_page[1].asset_id.path, 'demo.swf') - self.assertEquals(asset_page[2].asset_id.path, 'dog.png') - self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf') - self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp') + # Now fetch everything. + asset_page = store.get_all_asset_metadata( + course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(asset_page), 5) + self.assertEquals(asset_page[0].asset_id.path, 'code.tgz') + self.assertEquals(asset_page[1].asset_id.path, 'demo.swf') + self.assertEquals(asset_page[2].asset_id.path, 'dog.png') + self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf') + self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp') - # Some odd conditions. - asset_page = store.get_all_asset_metadata( - course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(asset_page), 0) - asset_page = store.get_all_asset_metadata( - course2.id, 'asset', start=3, maxresults=0, - sort=('displayname', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(asset_page), 0) - asset_page = store.get_all_asset_metadata( - course2.id, 'asset', start=3, maxresults=-12345, - sort=('displayname', ModuleStoreEnum.SortOrder.descending) - ) - self.assertEquals(len(asset_page), 2) + # Some odd conditions. + asset_page = store.get_all_asset_metadata( + course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(asset_page), 0) + asset_page = store.get_all_asset_metadata( + course2.id, 'asset', start=3, maxresults=0, + sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(asset_page), 0) + asset_page = store.get_all_asset_metadata( + course2.id, 'asset', start=3, maxresults=-12345, + sort=('displayname', ModuleStoreEnum.SortOrder.descending) + ) + self.assertEquals(len(asset_page), 2) @ddt.data(XmlModulestoreBuilder(), MixedModulestoreBuilder([('xml', XmlModulestoreBuilder())])) def test_xml_not_yet_implemented(self, storebuilder): """ Test coverage which shows that for now xml read operations are not implemented """ - with storebuilder.build(None) as store: + with storebuilder.build(contentstore=None) as (__, store): course_key = store.make_course_key("org", "course", "run") asset_key = course_key.make_asset_key('asset', 'foo.jpg') self.assertEquals(store.find_asset_metadata(asset_key), None) @@ -670,38 +651,36 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): """ Create a course with assets, copy them all to another course in the same modulestore, and check on it. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course1 = CourseFactory.create(modulestore=store) - course2 = CourseFactory.create(modulestore=store) - self.setup_assets(course1.id, None, store) - self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) - self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) - store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) - self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) - all_assets = store.get_all_asset_metadata( - course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(all_assets), 2) - self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') - self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') + with storebuilder.build() as (__, store): + course1 = CourseFactory.create(modulestore=store) + course2 = CourseFactory.create(modulestore=store) + self.setup_assets(course1.id, None, store) + self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) + self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) + store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) + self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2) + all_assets = store.get_all_asset_metadata( + course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(all_assets), 2) + self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') + self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') @ddt.data(*MODULESTORE_SETUPS) def test_copy_all_assets_from_course_with_no_assets(self, storebuilder): """ Create a course with *no* assets, and try copy them all to another course in the same modulestore. """ - with MongoContentstoreBuilder().build() as contentstore: - with storebuilder.build(contentstore) as store: - course1 = CourseFactory.create(modulestore=store) - course2 = CourseFactory.create(modulestore=store) - store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) - self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) - self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) - all_assets = store.get_all_asset_metadata( - course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(all_assets), 0) + with storebuilder.build() as (__, store): + course1 = CourseFactory.create(modulestore=store) + course2 = CourseFactory.create(modulestore=store) + store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101) + self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0) + self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0) + all_assets = store.get_all_asset_metadata( + course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(all_assets), 0) @ddt.data( ('mongo', 'split'), @@ -713,19 +692,18 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): Create a course with assets, copy them all to another course in a different modulestore, and check on it. """ mixed_builder = MIXED_MODULESTORE_BOTH_SETUP - with MongoContentstoreBuilder().build() as contentstore: - with mixed_builder.build(contentstore) as mixed_store: - with mixed_store.default_store(from_store): - course1 = CourseFactory.create(modulestore=mixed_store) - with mixed_store.default_store(to_store): - course2 = CourseFactory.create(modulestore=mixed_store) - self.setup_assets(course1.id, None, mixed_store) - self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2) - self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0) - mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102) - all_assets = mixed_store.get_all_asset_metadata( - course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) - ) - self.assertEquals(len(all_assets), 2) - self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') - self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') + with mixed_builder.build() as (__, mixed_store): + with mixed_store.default_store(from_store): + course1 = CourseFactory.create(modulestore=mixed_store) + with mixed_store.default_store(to_store): + course2 = CourseFactory.create(modulestore=mixed_store) + self.setup_assets(course1.id, None, mixed_store) + self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2) + self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0) + mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102) + all_assets = mixed_store.get_all_asset_metadata( + course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending) + ) + self.assertEquals(len(all_assets), 2) + self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg') + self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg') diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py index ac4429cbd3..80da4b8a0d 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_cross_modulestore_import_export.py @@ -76,12 +76,66 @@ class MemoryCache(object): self._data[key] = value -class MongoModulestoreBuilder(object): +class MongoContentstoreBuilder(object): + """ + A builder class for a MongoContentStore. + """ + @contextmanager + def build(self): + """ + A contextmanager that returns a MongoContentStore, and deletes its contents + when the context closes. + """ + contentstore = MongoContentStore( + db='contentstore{}'.format(random.randint(0, 10000)), + collection='content', + **COMMON_DOCSTORE_CONFIG + ) + contentstore.ensure_indexes() + + try: + yield contentstore + finally: + # Delete the created database + contentstore._drop_database() # pylint: disable=protected-access + + def __repr__(self): + return 'MongoContentstoreBuilder()' + + +class StoreBuilderBase(object): + """ + Base class for all modulestore builders. + """ + @contextmanager + def build(self, **kwargs): + """ + Build the modulstore, optionally building the contentstore as well. + """ + contentstore = kwargs.pop('contentstore', None) + if not contentstore: + with self.build_without_contentstore() as (contentstore, modulestore): + yield contentstore, modulestore + else: + with self.build_with_contentstore(contentstore) as modulestore: + yield modulestore + + @contextmanager + def build_without_contentstore(self): + """ + Build both the contentstore and the modulestore. + """ + with MongoContentstoreBuilder().build() as contentstore: + with self.build_with_contentstore(contentstore) as modulestore: + yield contentstore, modulestore + + +class MongoModulestoreBuilder(StoreBuilderBase): """ A builder class for a DraftModuleStore. """ @contextmanager - def build(self, contentstore): + def build_with_contentstore(self, contentstore): """ A contextmanager that returns an isolated mongo modulestore, and then deletes all of its data at the end of the context. @@ -125,12 +179,12 @@ class MongoModulestoreBuilder(object): return 'MongoModulestoreBuilder()' -class VersioningModulestoreBuilder(object): +class VersioningModulestoreBuilder(StoreBuilderBase): """ A builder class for a VersioningModuleStore. """ @contextmanager - def build(self, contentstore): + def build_with_contentstore(self, contentstore): """ A contextmanager that returns an isolated versioning modulestore, and then deletes all of its data at the end of the context. @@ -170,13 +224,13 @@ class VersioningModulestoreBuilder(object): return 'SplitModulestoreBuilder()' -class XmlModulestoreBuilder(object): +class XmlModulestoreBuilder(StoreBuilderBase): """ A builder class for a XMLModuleStore. """ # pylint: disable=unused-argument @contextmanager - def build(self, contentstore=None, course_ids=None): + def build_with_contentstore(self, contentstore=None, course_ids=None): """ A contextmanager that returns an isolated xml modulestore @@ -194,7 +248,7 @@ class XmlModulestoreBuilder(object): yield modulestore -class MixedModulestoreBuilder(object): +class MixedModulestoreBuilder(StoreBuilderBase): """ A builder class for a MixedModuleStore. """ @@ -210,7 +264,7 @@ class MixedModulestoreBuilder(object): self.mixed_modulestore = None @contextmanager - def build(self, contentstore): + def build_with_contentstore(self, contentstore): """ A contextmanager that returns a mixed modulestore built on top of modulestores generated by other builder classes. @@ -221,7 +275,7 @@ class MixedModulestoreBuilder(object): """ names, generators = zip(*self.store_builders) - with nested(*(gen.build(contentstore) for gen in generators)) as modulestores: + with nested(*(gen.build_with_contentstore(contentstore) for gen in generators)) as modulestores: # Make the modulestore creation function just return the already-created modulestores store_iterator = iter(modulestores) create_modulestore_instance = lambda *args, **kwargs: store_iterator.next() @@ -261,32 +315,6 @@ class MixedModulestoreBuilder(object): return store.db_connection.structures -class MongoContentstoreBuilder(object): - """ - A builder class for a MongoContentStore. - """ - @contextmanager - def build(self): - """ - A contextmanager that returns a MongoContentStore, and deletes its contents - when the context closes. - """ - contentstore = MongoContentStore( - db='contentstore{}'.format(random.randint(0, 10000)), - collection='content', - **COMMON_DOCSTORE_CONFIG - ) - contentstore.ensure_indexes() - - try: - yield contentstore - finally: - # Delete the created database - contentstore._drop_database() - - def __repr__(self): - return 'MongoContentstoreBuilder()' - MIXED_MODULESTORE_BOTH_SETUP = MixedModulestoreBuilder([ ('draft', MongoModulestoreBuilder()), ('split', VersioningModulestoreBuilder()) @@ -345,11 +373,11 @@ class CrossStoreXMLRoundtrip(CourseComparisonTest, PartitionTestCase): # Construct the contentstore for storing the first import with source_content_builder.build() as source_content: # Construct the modulestore for storing the first import (using the previously created contentstore) - with source_builder.build(source_content) as source_store: + with source_builder.build(contentstore=source_content) as source_store: # Construct the contentstore for storing the second import with dest_content_builder.build() as dest_content: # Construct the modulestore for storing the second import (using the second contentstore) - with dest_builder.build(dest_content) as dest_store: + with dest_builder.build(contentstore=dest_content) as dest_store: source_course_key = source_store.make_course_key('a', 'course', 'course') dest_course_key = dest_store.make_course_key('a', 'course', 'course') diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py new file mode 100644 index 0000000000..6a23d3eb90 --- /dev/null +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_mongo_call_count.py @@ -0,0 +1,124 @@ +""" +Tests to verify correct number of MongoDB calls during course import/export and traversal +when using the Split modulestore. +""" + +from tempfile import mkdtemp +from shutil import rmtree +from unittest import TestCase +import ddt + +from xmodule.modulestore.xml_importer import import_from_xml +from xmodule.modulestore.xml_exporter import export_to_xml +from xmodule.modulestore.tests.factories import check_mongo_calls +from xmodule.modulestore.tests.test_cross_modulestore_import_export import ( + MixedModulestoreBuilder, VersioningModulestoreBuilder, + MongoModulestoreBuilder, TEST_DATA_DIR +) + +MIXED_OLD_MONGO_MODULESTORE_BUILDER = MixedModulestoreBuilder([('draft', MongoModulestoreBuilder())]) +MIXED_SPLIT_MODULESTORE_BUILDER = MixedModulestoreBuilder([('split', VersioningModulestoreBuilder())]) + + +@ddt.ddt +class CountMongoCallsXMLRoundtrip(TestCase): + """ + This class exists to test XML import and export to/from Split. + """ + + def setUp(self): + super(CountMongoCallsXMLRoundtrip, self).setUp() + self.export_dir = mkdtemp() + self.addCleanup(rmtree, self.export_dir, ignore_errors=True) + + @ddt.data( + (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 287, 780, 702, 702), + (MIXED_SPLIT_MODULESTORE_BUILDER, 37, 16, 190, 189), + ) + @ddt.unpack + def test_import_export(self, store_builder, export_reads, import_reads, first_import_writes, second_import_writes): + with store_builder.build() as (source_content, source_store): + with store_builder.build() as (dest_content, dest_store): + source_course_key = source_store.make_course_key('a', 'course', 'course') + dest_course_key = dest_store.make_course_key('a', 'course', 'course') + + # An extra import write occurs in the first Split import due to the mismatch between + # the course id and the wiki_slug in the test XML course. The course must be updated + # with the correct wiki_slug during import. + with check_mongo_calls(import_reads, first_import_writes): + import_from_xml( + source_store, + 'test_user', + TEST_DATA_DIR, + course_dirs=['manual-testing-complete'], + static_content_store=source_content, + target_course_id=source_course_key, + create_course_if_not_present=True, + raise_on_failure=True, + ) + + with check_mongo_calls(export_reads): + export_to_xml( + source_store, + source_content, + source_course_key, + self.export_dir, + 'exported_source_course', + ) + + with check_mongo_calls(import_reads, second_import_writes): + import_from_xml( + dest_store, + 'test_user', + self.export_dir, + course_dirs=['exported_source_course'], + static_content_store=dest_content, + target_course_id=dest_course_key, + create_course_if_not_present=True, + raise_on_failure=True, + ) + + +@ddt.ddt +class CountMongoCallsCourseTraversal(TestCase): + """ + Tests the number of Mongo calls made when traversing a course tree from the top course root + to the leaf nodes. + """ + + @ddt.data( + (MIXED_OLD_MONGO_MODULESTORE_BUILDER, None, 189), # The way this traversal *should* be done. + (MIXED_OLD_MONGO_MODULESTORE_BUILDER, 0, 387), # The pathological case - do *not* query a course this way! + (MIXED_SPLIT_MODULESTORE_BUILDER, None, 7), # The way this traversal *should* be done. + (MIXED_SPLIT_MODULESTORE_BUILDER, 0, 145) # The pathological case - do *not* query a course this way! + ) + @ddt.unpack + def test_number_mongo_calls(self, store, depth, num_mongo_calls): + with store.build() as (source_content, source_store): + + source_course_key = source_store.make_course_key('a', 'course', 'course') + + # First, import a course. + import_from_xml( + source_store, + 'test_user', + TEST_DATA_DIR, + course_dirs=['manual-testing-complete'], + static_content_store=source_content, + target_course_id=source_course_key, + create_course_if_not_present=True, + raise_on_failure=True, + ) + + # Course traversal modeled after the traversal done here: + # lms/djangoapps/mobile_api/video_outlines/serializers.py:BlockOutline + # Starting at the root course block, do a breadth-first traversal using + # get_children() to retrieve each block's children. + with check_mongo_calls(num_mongo_calls): + start_block = source_store.get_course(source_course_key, depth=depth) + stack = [start_block] + while stack: + curr_block = stack.pop() + if curr_block.has_children: + for block in reversed(curr_block.get_children()): + stack.append(block)