Fix CorruptGridFile in Python 3 tests BOM-375 (#21794)

This commit is contained in:
Jeremy Bowman
2019-09-26 13:43:26 -04:00
committed by GitHub
parent eb8bb72123
commit 4babc5ee1c
7 changed files with 119 additions and 96 deletions

View File

@@ -306,3 +306,9 @@ class CourseAssetsFromStorage(object):
Iterates over the items of the asset dict.
"""
return six.iteritems(self.asset_md)
def items(self):
"""
Iterates over the items of the asset dict. (Python 3 naming convention)
"""
return self.iteritems()

View File

@@ -36,7 +36,7 @@ class TestAssetXml(unittest.TestCase):
# Read in the XML schema definition and make a validator.
xsd_path = path(__file__).realpath().parent / xsd_filename
with open(xsd_path, 'r') as f:
with open(xsd_path, 'rb') as f:
schema_root = etree.XML(f.read())
schema = etree.XMLSchema(schema_root)
self.xmlparser = etree.XMLParser(schema=schema)

View File

@@ -126,6 +126,10 @@ class MongoContentStore(ContentStore):
try:
if as_stream:
fp = self.fs.get(content_id)
# Need to replace dict IDs with SON for chunk lookup to work under Python 3
# because field order can be different and mongo cares about the order
if isinstance(fp._id, dict):
fp._file['_id'] = content_id
thumbnail_location = getattr(fp, 'thumbnail_location', None)
if thumbnail_location:
thumbnail_location = location.course_key.make_asset_key(
@@ -141,6 +145,10 @@ class MongoContentStore(ContentStore):
)
else:
with self.fs.get(content_id) as fp:
# Need to replace dict IDs with SON for chunk lookup to work under Python 3
# because field order can be different and mongo cares about the order
if isinstance(fp._id, dict):
fp._file['_id'] = content_id
thumbnail_location = getattr(fp, 'thumbnail_location', None)
if thumbnail_location:
thumbnail_location = location.course_key.make_asset_key(
@@ -398,6 +406,10 @@ class MongoContentStore(ContentStore):
if isinstance(asset_key, six.string_types):
asset_key = AssetKey.from_string(asset_key)
__, asset_key = self.asset_db_key(asset_key)
# Need to replace dict IDs with SON for chunk lookup to work under Python 3
# because field order can be different and mongo cares about the order
if isinstance(source_content._id, dict):
source_content._file['_id'] = asset_key.copy()
asset_key['org'] = dest_course_key.org
asset_key['course'] = dest_course_key.course
if getattr(dest_course_key, 'deprecated', False): # remove the run if exists

View File

@@ -239,7 +239,10 @@ class CourseStructureCache(object):
pickled_data = zlib.decompress(compressed_pickled_data)
tagger.measure('uncompressed_size', len(pickled_data))
return pickle.loads(pickled_data)
if six.PY2:
return pickle.loads(pickled_data)
else:
return pickle.loads(pickled_data, encoding='latin-1')
def set(self, key, structure, course_context=None):
"""Given a structure, will pickle, compress, and write to cache."""
@@ -247,7 +250,7 @@ class CourseStructureCache(object):
return None
with TIMER.timer("CourseStructureCache.set", course_context) as tagger:
pickled_data = pickle.dumps(structure, pickle.HIGHEST_PROTOCOL)
pickled_data = pickle.dumps(structure, 2) # Protocol can't be incremented until cache is cleared
tagger.measure('uncompressed_size', len(pickled_data))
# 1 = Fastest (slightly larger results)

View File

@@ -89,9 +89,9 @@ class TestSortedAssetList(unittest.TestCase):
def test_find(self):
asset_key = self.course_key.make_asset_key('asset', 'asset.txt')
self.assertEquals(self.sorted_asset_list_by_filename.find(asset_key), 0)
self.assertEqual(self.sorted_asset_list_by_filename.find(asset_key), 0)
asset_key_last = self.course_key.make_asset_key('asset', 'weather_patterns.bmp')
self.assertEquals(
self.assertEqual(
self.sorted_asset_list_by_filename.find(asset_key_last), len(AssetStoreTestData.all_asset_data) - 1
)
@@ -190,8 +190,8 @@ class TestMongoAssetMetadataStorage(TestCase):
# Find the asset's metadata and confirm it's the same.
found_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(found_asset_md)
self.assertEquals(new_asset_md, found_asset_md)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEqual(new_asset_md, found_asset_md)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS)
def test_delete(self, storebuilder):
@@ -202,13 +202,13 @@ class TestMongoAssetMetadataStorage(TestCase):
course = CourseFactory.create(modulestore=store)
new_asset_loc = course.id.make_asset_key('asset', 'burnside.jpg')
# Attempt to delete an asset that doesn't exist.
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEqual(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 0)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
new_asset_md = self._make_asset_metadata(new_asset_loc)
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEqual(store.delete_asset_metadata(new_asset_loc, ModuleStoreEnum.UserID.test), 1)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_find_non_existing_assets(self, storebuilder):
@@ -231,7 +231,7 @@ class TestMongoAssetMetadataStorage(TestCase):
course = CourseFactory.create(modulestore=store)
# Find existing asset metadata.
asset_md = store.get_all_asset_metadata(course.id, 'asset')
self.assertEquals(asset_md, [])
self.assertEqual(asset_md, [])
@ddt.data(*MODULESTORE_SETUPS)
def test_find_assets_in_non_existent_course(self, storebuilder):
@@ -261,11 +261,11 @@ class TestMongoAssetMetadataStorage(TestCase):
new_asset_md = self._make_asset_metadata(new_asset_loc)
# Add asset metadata.
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
# Add *the same* asset metadata.
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
# Still one here?
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'asset')), 1)
@ddt.data(*MODULESTORE_SETUPS)
def test_different_asset_types(self, storebuilder):
@@ -278,8 +278,8 @@ class TestMongoAssetMetadataStorage(TestCase):
new_asset_md = self._make_asset_metadata(new_asset_loc)
# Add asset metadata.
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'vrml')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'vrml')), 1)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_asset_types_with_other_field_names(self, storebuilder):
@@ -292,10 +292,10 @@ class TestMongoAssetMetadataStorage(TestCase):
new_asset_md = self._make_asset_metadata(new_asset_loc)
# Add asset metadata.
store.save_asset_metadata(new_asset_md, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'course_id')), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'course_id')), 1)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'asset')), 0)
all_assets = store.get_all_asset_metadata(course.id, 'course_id')
self.assertEquals(all_assets[0].asset_id.path, new_asset_loc.path)
self.assertEqual(all_assets[0].asset_id.path, new_asset_loc.path)
@ddt.data(*MODULESTORE_SETUPS)
def test_lock_unlock_assets(self, storebuilder):
@@ -314,12 +314,12 @@ class TestMongoAssetMetadataStorage(TestCase):
# Find the same course and check its locked status.
updated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(updated_asset_md)
self.assertEquals(updated_asset_md.locked, not locked_state)
self.assertEqual(updated_asset_md.locked, not locked_state)
# Now flip it back.
store.set_asset_metadata_attr(new_asset_loc, "locked", locked_state, ModuleStoreEnum.UserID.test)
reupdated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(reupdated_asset_md)
self.assertEquals(reupdated_asset_md.locked, locked_state)
self.assertEqual(reupdated_asset_md.locked, locked_state)
ALLOWED_ATTRS = (
('pathname', '/new/path'),
@@ -362,7 +362,7 @@ class TestMongoAssetMetadataStorage(TestCase):
updated_asset_md = store.find_asset_metadata(new_asset_loc)
self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(getattr(updated_asset_md, attribute, None))
self.assertEquals(getattr(updated_asset_md, attribute, None), value)
self.assertEqual(getattr(updated_asset_md, attribute, None), value)
@ddt.data(*MODULESTORE_SETUPS)
def test_set_disallowed_attrs(self, storebuilder):
@@ -383,7 +383,7 @@ class TestMongoAssetMetadataStorage(TestCase):
self.assertIsNotNone(updated_asset_md)
self.assertIsNotNone(getattr(updated_asset_md, attribute, None))
# Make sure that the attribute is unchanged from its original value.
self.assertEquals(getattr(updated_asset_md, attribute, None), original_attr_val)
self.assertEqual(getattr(updated_asset_md, attribute, None), original_attr_val)
@ddt.data(*MODULESTORE_SETUPS)
def test_set_unknown_attrs(self, storebuilder):
@@ -403,7 +403,7 @@ class TestMongoAssetMetadataStorage(TestCase):
self.assertIsNotNone(updated_asset_md)
# Make sure the unknown field was *not* added.
with self.assertRaises(AttributeError):
self.assertEquals(getattr(updated_asset_md, attribute), value)
self.assertEqual(getattr(updated_asset_md, attribute), value)
@ddt.data(*MODULESTORE_SETUPS)
def test_save_one_different_asset(self, storebuilder):
@@ -417,9 +417,9 @@ class TestMongoAssetMetadataStorage(TestCase):
self._make_asset_metadata(asset_key)
)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
self.assertEquals(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'different')), 1)
self.assertEqual(store.delete_asset_metadata(asset_key, ModuleStoreEnum.UserID.test), 1)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'different')), 0)
@ddt.data(*MODULESTORE_SETUPS)
def test_find_different(self, storebuilder):
@@ -443,8 +443,8 @@ class TestMongoAssetMetadataStorage(TestCase):
Check asset type/path values.
"""
for idx, asset in enumerate(orig):
self.assertEquals(assets[idx].asset_id.asset_type, asset[0])
self.assertEquals(assets[idx].asset_id.path, asset[1])
self.assertEqual(assets[idx].asset_id.asset_type, asset[0])
self.assertEqual(assets[idx].asset_id.path, asset[1])
@ddt.data(*MODULESTORE_SETUPS)
def test_get_multiple_types(self, storebuilder):
@@ -470,17 +470,17 @@ class TestMongoAssetMetadataStorage(TestCase):
('asset', self.regular_assets),
):
assets = store.get_all_asset_metadata(course.id, asset_type)
self.assertEquals(len(assets), len(asset_list))
self.assertEqual(len(assets), len(asset_list))
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
course.id, None, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(assets), len(self.alls))
self.assertEqual(len(assets), len(self.alls))
self._check_asset_values(assets, self.alls)
@ddt.data(*MODULESTORE_SETUPS)
@@ -510,17 +510,17 @@ class TestMongoAssetMetadataStorage(TestCase):
('asset', self.regular_assets),
):
assets = store.get_all_asset_metadata(course.id, asset_type)
self.assertEquals(len(assets), len(asset_list))
self.assertEqual(len(assets), len(asset_list))
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course.id, None)), 4)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'not_here')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course.id, None)), 4)
assets = store.get_all_asset_metadata(
course.id, None, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(assets), len(self.alls))
self.assertEqual(len(assets), len(self.alls))
self._check_asset_values(assets, self.alls)
@ddt.data(*MODULESTORE_SETUPS)
@@ -553,17 +553,17 @@ class TestMongoAssetMetadataStorage(TestCase):
('vrml', self.vrmls),
):
assets = store.get_all_asset_metadata(course1.id, asset_type)
self.assertEquals(len(assets), len(asset_list))
self.assertEqual(len(assets), len(asset_list))
self._check_asset_values(assets, asset_list)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, None)), 3)
self.assertEqual(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course1.id, None)), 3)
assets = store.get_all_asset_metadata(
course1.id, None, start=0, maxresults=-1,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(assets), len(self.differents + self.vrmls))
self.assertEqual(len(assets), len(self.differents + self.vrmls))
self._check_asset_values(assets, self.differents + self.vrmls)
@ddt.data(*MODULESTORE_SETUPS)
@@ -579,7 +579,7 @@ class TestMongoAssetMetadataStorage(TestCase):
)
store.save_asset_metadata(new_asset_thumbnail, ModuleStoreEnum.UserID.test)
self.assertEquals(len(store.get_all_asset_metadata(course.id, 'different')), 1)
self.assertEqual(len(store.get_all_asset_metadata(course.id, 'different')), 1)
@ddt.data(*MODULESTORE_SETUPS)
def test_get_all_assets_with_paging(self, storebuilder):
@@ -621,38 +621,38 @@ class TestMongoAssetMetadataStorage(TestCase):
)
num_expected_results = sort_test[2][i]
expected_filename = sort_test[1][2 * i]
self.assertEquals(len(asset_page), num_expected_results)
self.assertEquals(asset_page[0].asset_id.path, expected_filename)
self.assertEqual(len(asset_page), num_expected_results)
self.assertEqual(asset_page[0].asset_id.path, expected_filename)
if num_expected_results == 2:
expected_filename = sort_test[1][(2 * i) + 1]
self.assertEquals(asset_page[1].asset_id.path, expected_filename)
self.assertEqual(asset_page[1].asset_id.path, expected_filename)
# Now fetch everything.
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=0, sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(asset_page), 5)
self.assertEquals(asset_page[0].asset_id.path, 'code.tgz')
self.assertEquals(asset_page[1].asset_id.path, 'demo.swf')
self.assertEquals(asset_page[2].asset_id.path, 'dog.png')
self.assertEquals(asset_page[3].asset_id.path, 'roman_history.pdf')
self.assertEquals(asset_page[4].asset_id.path, 'weather_patterns.bmp')
self.assertEqual(len(asset_page), 5)
self.assertEqual(asset_page[0].asset_id.path, 'code.tgz')
self.assertEqual(asset_page[1].asset_id.path, 'demo.swf')
self.assertEqual(asset_page[2].asset_id.path, 'dog.png')
self.assertEqual(asset_page[3].asset_id.path, 'roman_history.pdf')
self.assertEqual(asset_page[4].asset_id.path, 'weather_patterns.bmp')
# Some odd conditions.
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=100, sort=('uploadDate', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(asset_page), 0)
self.assertEqual(len(asset_page), 0)
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=3, maxresults=0,
sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(asset_page), 0)
self.assertEqual(len(asset_page), 0)
asset_page = store.get_all_asset_metadata(
course2.id, 'asset', start=3, maxresults=-12345,
sort=('displayname', ModuleStoreEnum.SortOrder.descending)
)
self.assertEquals(len(asset_page), 2)
self.assertEqual(len(asset_page), 2)
@ddt.data('XML_MODULESTORE_BUILDER', 'MIXED_MODULESTORE_BUILDER')
def test_xml_not_yet_implemented(self, storebuilderName):
@@ -663,8 +663,8 @@ class TestMongoAssetMetadataStorage(TestCase):
with storebuilder.build(contentstore=None) as (__, store):
course_key = store.make_course_key("org", "course", "run")
asset_key = course_key.make_asset_key('asset', 'foo.jpg')
self.assertEquals(store.find_asset_metadata(asset_key), None)
self.assertEquals(store.get_all_asset_metadata(course_key, 'asset'), [])
self.assertEqual(store.find_asset_metadata(asset_key), None)
self.assertEqual(store.get_all_asset_metadata(course_key, 'asset'), [])
@ddt.data(*MODULESTORE_SETUPS)
def test_copy_all_assets_same_modulestore(self, storebuilder):
@@ -675,16 +675,16 @@ class TestMongoAssetMetadataStorage(TestCase):
course1 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store)
self.setup_assets(course1.id, None, store)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEqual(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEqual(len(store.get_all_asset_metadata(course1.id, 'asset')), 2)
all_assets = store.get_all_asset_metadata(
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(all_assets), 2)
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
self.assertEqual(len(all_assets), 2)
self.assertEqual(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEqual(all_assets[1].asset_id.path, 'shout.ogg')
@ddt.data(*MODULESTORE_SETUPS)
def test_copy_all_assets_from_course_with_no_assets(self, storebuilder):
@@ -695,12 +695,12 @@ class TestMongoAssetMetadataStorage(TestCase):
course1 = CourseFactory.create(modulestore=store)
course2 = CourseFactory.create(modulestore=store)
store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 101)
self.assertEquals(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEquals(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course1.id, 'asset')), 0)
self.assertEqual(len(store.get_all_asset_metadata(course2.id, 'asset')), 0)
all_assets = store.get_all_asset_metadata(
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(all_assets), 0)
self.assertEqual(len(all_assets), 0)
@ddt.data(
('mongo', 'split'),
@@ -718,12 +718,12 @@ class TestMongoAssetMetadataStorage(TestCase):
with mixed_store.default_store(to_store):
course2 = CourseFactory.create(modulestore=mixed_store)
self.setup_assets(course1.id, None, mixed_store)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEquals(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0)
self.assertEqual(len(mixed_store.get_all_asset_metadata(course1.id, 'asset')), 2)
self.assertEqual(len(mixed_store.get_all_asset_metadata(course2.id, 'asset')), 0)
mixed_store.copy_all_asset_metadata(course1.id, course2.id, ModuleStoreEnum.UserID.test * 102)
all_assets = mixed_store.get_all_asset_metadata(
course2.id, 'asset', sort=('displayname', ModuleStoreEnum.SortOrder.ascending)
)
self.assertEquals(len(all_assets), 2)
self.assertEquals(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEquals(all_assets[1].asset_id.path, 'shout.ogg')
self.assertEqual(len(all_assets), 2)
self.assertEqual(all_assets[0].asset_id.path, 'pic1.jpg')
self.assertEqual(all_assets[1].asset_id.path, 'shout.ogg')

View File

@@ -42,7 +42,7 @@ class TestXMLModuleStore(TestCase):
# uniquification of names, would raise a UnicodeError. It no longer does.
# Ensure that there really is a non-ASCII character in the course.
with open(os.path.join(DATA_DIR, "toy/sequential/vertical_sequential.xml")) as xmlf:
with open(os.path.join(DATA_DIR, "toy/sequential/vertical_sequential.xml"), 'rb') as xmlf:
xml = xmlf.read()
with self.assertRaises(UnicodeDecodeError):
xml.decode('ascii')