diff --git a/common/lib/xmodule/xmodule/assetstore/__init__.py b/common/lib/xmodule/xmodule/assetstore/__init__.py index f1e74ba2d3..c7bbe68f33 100644 --- a/common/lib/xmodule/xmodule/assetstore/__init__.py +++ b/common/lib/xmodule/xmodule/assetstore/__init__.py @@ -3,13 +3,19 @@ Classes representing asset metadata. """ from datetime import datetime +import dateutil.parser import pytz +import json from contracts import contract, new_contract from opaque_keys.edx.keys import AssetKey +from lxml import etree + new_contract('AssetKey', AssetKey) new_contract('datetime', datetime) new_contract('basestring', basestring) +new_contract('AssetElement', lambda x: isinstance(x, etree._Element) and x.tag == "asset") # pylint: disable=protected-access, no-member +new_contract('AssetsElement', lambda x: isinstance(x, etree._Element) and x.tag == "assets") # pylint: disable=protected-access, no-member class AssetMetadata(object): @@ -19,8 +25,10 @@ class AssetMetadata(object): """ TOP_LEVEL_ATTRS = ['basename', 'internal_name', 'locked', 'contenttype', 'thumbnail', 'fields'] - EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_on'] - ALLOWED_ATTRS = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS + EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_by_email', 'edited_on'] + CREATE_INFO_ATTRS = ['created_by', 'created_by_email', 'created_on'] + ATTRS_ALLOWED_TO_UPDATE = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS + ALL_ATTRS = ['asset_id'] + ATTRS_ALLOWED_TO_UPDATE + CREATE_INFO_ATTRS # Default type for AssetMetadata objects. A constant for convenience. ASSET_TYPE = 'asset' @@ -30,15 +38,15 @@ class AssetMetadata(object): locked='bool|None', contenttype='basestring|None', thumbnail='basestring|None', fields='dict|None', curr_version='basestring|None', prev_version='basestring|None', - created_by='int|None', created_on='datetime|None', - edited_by='int|None', edited_on='datetime|None') + created_by='int|None', created_by_email='basestring|None', created_on='datetime|None', + edited_by='int|None', edited_by_email='basestring|None', edited_on='datetime|None') def __init__(self, asset_id, basename=None, internal_name=None, locked=None, contenttype=None, thumbnail=None, fields=None, curr_version=None, prev_version=None, - created_by=None, created_on=None, - edited_by=None, edited_on=None, + created_by=None, created_by_email=None, created_on=None, + edited_by=None, edited_by_email=None, edited_on=None, field_decorator=None,): """ Construct a AssetMetadata object. @@ -53,7 +61,11 @@ class AssetMetadata(object): fields (dict): fields to save w/ the metadata curr_version (str): Current version of the asset. prev_version (str): Previous version of the asset. - edited_by (str): Username of last user to upload this asset. + created_by (int): User ID of initial user to upload this asset. + created_by_email (str): Email address of initial user to upload this asset. + created_on (datetime): Datetime of intial upload of this asset. + edited_by (int): User ID of last user to upload this asset. + edited_by_email (str): Email address of last user to upload this asset. edited_on (datetime): Datetime of last upload of this asset. field_decorator (function): used by strip_key to convert OpaqueKeys to the app's understanding. Not saved. @@ -68,9 +80,11 @@ class AssetMetadata(object): self.prev_version = prev_version now = datetime.now(pytz.utc) self.edited_by = edited_by + self.edited_by_email = edited_by_email self.edited_on = edited_on or now - # created_by and created_on should only be set here. + # created_by, created_by_email, and created_on should only be set here. self.created_by = created_by + self.created_by_email = created_by_email self.created_on = created_on or now self.fields = fields or {} @@ -80,20 +94,20 @@ class AssetMetadata(object): self.basename, self.internal_name, self.locked, self.contenttype, self.fields, self.curr_version, self.prev_version, - self.edited_by, self.edited_on, - self.created_by, self.created_on + self.created_by, self.created_by_email, self.created_on, + self.edited_by, self.edited_by_email, self.edited_on, )) def update(self, attr_dict): """ - Set the attributes on the metadata. Any which are not in ALLOWED_ATTRS get put into + Set the attributes on the metadata. Any which are not in ATTRS_ALLOWED_TO_UPDATE get put into fields. Arguments: attr_dict: Prop, val dictionary of all attributes to set. """ for attr, val in attr_dict.iteritems(): - if attr in self.ALLOWED_ATTRS: + if attr in self.ATTRS_ALLOWED_TO_UPDATE: setattr(self, attr, val) else: self.fields[attr] = val @@ -113,10 +127,12 @@ class AssetMetadata(object): 'edit_info': { 'curr_version': self.curr_version, 'prev_version': self.prev_version, - 'edited_by': self.edited_by, - 'edited_on': self.edited_on, 'created_by': self.created_by, - 'created_on': self.created_on + 'created_by_email': self.created_by_email, + 'created_on': self.created_on, + 'edited_by': self.edited_by, + 'edited_by_email': self.edited_by_email, + 'edited_on': self.edited_on } } @@ -137,7 +153,68 @@ class AssetMetadata(object): self.fields = asset_doc['fields'] self.curr_version = asset_doc['edit_info']['curr_version'] self.prev_version = asset_doc['edit_info']['prev_version'] - self.edited_by = asset_doc['edit_info']['edited_by'] - self.edited_on = asset_doc['edit_info']['edited_on'] self.created_by = asset_doc['edit_info']['created_by'] + self.created_by_email = asset_doc['edit_info']['created_by_email'] self.created_on = asset_doc['edit_info']['created_on'] + self.edited_by = asset_doc['edit_info']['edited_by'] + self.edited_by_email = asset_doc['edit_info']['edited_by_email'] + self.edited_on = asset_doc['edit_info']['edited_on'] + + @contract(node='AssetElement') + def from_xml(self, node): + """ + Walk the etree XML node and fill in the asset metadata. + The node should be a top-level "asset" element. + """ + for child in node: + qname = etree.QName(child) + tag = qname.localname + if tag in self.ALL_ATTRS: + value = child.text + if tag == 'asset_id': + # Locator. + value = AssetKey.from_string(value) + elif tag == 'locked': + # Boolean. + value = True if value == "true" else False + elif tag in ('created_on', 'edited_on'): + # ISO datetime. + value = dateutil.parser.parse(value) + elif tag in ('created_by', 'edited_by'): + # Integer representing user id. + value = int(value) + elif tag == 'fields': + # Dictionary. + value = json.loads(value) + elif value == 'None': + # None. + value = None + setattr(self, tag, value) + + @contract(node='AssetElement') + def to_xml(self, node): + """ + Add the asset data as XML to the passed-in node. + The node should already be created as a top-level "asset" element. + """ + for attr in self.ALL_ATTRS: + child = etree.SubElement(node, attr) + value = getattr(self, attr) + if isinstance(value, bool): + value = "true" if value else "false" + elif isinstance(value, datetime): + value = value.isoformat() + else: + value = unicode(value) + child.text = value + + @staticmethod + @contract(node='AssetsElement', assets=list) + def add_all_assets_as_xml(node, assets): + """ + Take a list of AssetMetadata objects. Add them all to the node. + The node should already be created as a top-level "assets" element. + """ + for asset in assets: + asset_node = etree.SubElement(node, "asset") + asset.to_xml(asset_node) diff --git a/common/lib/xmodule/xmodule/assetstore/tests/assets.xsd b/common/lib/xmodule/xmodule/assetstore/tests/assets.xsd new file mode 100644 index 0000000000..8a05d353c8 --- /dev/null +++ b/common/lib/xmodule/xmodule/assetstore/tests/assets.xsd @@ -0,0 +1,48 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/common/lib/xmodule/xmodule/assetstore/tests/test_asset_xml.py b/common/lib/xmodule/xmodule/assetstore/tests/test_asset_xml.py new file mode 100644 index 0000000000..d01187904f --- /dev/null +++ b/common/lib/xmodule/xmodule/assetstore/tests/test_asset_xml.py @@ -0,0 +1,84 @@ +""" +Test for asset XML generation / parsing. +""" + +from path import path +from lxml import etree +from contracts import ContractNotRespected +import unittest + +from opaque_keys.edx.locator import CourseLocator +from xmodule.assetstore import AssetMetadata +from xmodule.modulestore.tests.test_assetstore import AssetStoreTestData + + +class TestAssetXml(unittest.TestCase): + """ + Tests for storing/querying course asset metadata. + """ + def setUp(self): + super(TestAssetXml, self).setUp() + + xsd_filename = "assets.xsd" + + self.course_id = CourseLocator('org1', 'course1', 'run1') + + self.course_assets = [] + for asset in AssetStoreTestData.all_asset_data: + asset_dict = dict(zip(AssetStoreTestData.asset_fields[1:], asset[1:])) + asset_md = AssetMetadata(self.course_id.make_asset_key('asset', asset[0]), **asset_dict) + self.course_assets.append(asset_md) + + # Read in the XML schema definition and make a validator. + #xsd_path = path(__file__).abspath().dirname() / xsd_filename + xsd_path = path(__file__).realpath().parent / xsd_filename + with open(xsd_path, 'r') as f: + schema_root = etree.XML(f.read()) + schema = etree.XMLSchema(schema_root) + self.xmlparser = etree.XMLParser(schema=schema) + + def test_export_single_asset_to_from_xml(self): + """ + Export a single AssetMetadata to XML and verify the structure and fields. + """ + asset_md = self.course_assets[0] + root = etree.Element("assets") + asset = etree.SubElement(root, "asset") + asset_md.to_xml(asset) + # If this line does *not* raise, the XML is valid. + etree.fromstring(etree.tostring(root), self.xmlparser) + new_asset_key = self.course_id.make_asset_key('tmp', 'tmp') + new_asset_md = AssetMetadata(new_asset_key) + new_asset_md.from_xml(asset) + # Compare asset_md to new_asset_md. + for attr in AssetMetadata.ALL_ATTRS: + orig_value = getattr(asset_md, attr) + new_value = getattr(new_asset_md, attr) + self.assertEqual(orig_value, new_value) + + def test_export_all_assets_to_xml(self): + """ + Export all AssetMetadatas to XML and verify the structure and fields. + """ + root = etree.Element("assets") + AssetMetadata.add_all_assets_as_xml(root, self.course_assets) + # If this line does *not* raise, the XML is valid. + etree.fromstring(etree.tostring(root), self.xmlparser) + + def test_wrong_node_type_all(self): + """ + Ensure full asset sections with the wrong tag are detected. + """ + root = etree.Element("glassets") + with self.assertRaises(ContractNotRespected): + AssetMetadata.add_all_assets_as_xml(root, self.course_assets) + + def test_wrong_node_type_single(self): + """ + Ensure single asset blocks with the wrong tag are detected. + """ + asset_md = self.course_assets[0] + root = etree.Element("assets") + asset = etree.SubElement(root, "smashset") + with self.assertRaises(ContractNotRespected): + asset_md.to_xml(asset) diff --git a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py index ea7a627d9a..52feb63ed7 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py @@ -17,6 +17,32 @@ from xmodule.modulestore.tests.test_cross_modulestore_import_export import ( ) +class AssetStoreTestData(object): + """ + Shared data for constructing test assets. + """ + now = datetime.now(pytz.utc) + user_id = 144 + user_email = "me@example.com" + + asset_fields = ( + 'filename', 'internal_name', 'basename', 'locked', + 'edited_by', 'edited_by_email', 'edited_on', 'created_by', 'created_by_email', 'created_on', + 'curr_version', 'prev_version' + ) + all_asset_data = ( + ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, user_id, user_email, now, user_id, user_email, now, '14', '13'), + ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, user_id, user_email, now, user_id, user_email, now, '1', None), + ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, user_id * 2, user_email, now, user_id * 2, user_email, now, 'AB', 'AA'), + ('dog.png', 'PUPY4242X', 'pictures/animals', True, user_id * 3, user_email, now, user_id * 3, user_email, now, '5', '4'), + ('not_here.txt', 'JJJCCC747', '/dev/null', False, user_id * 4, user_email, now, user_id * 4, user_email, now, '50', '49'), + ('asset.txt', 'JJJCCC747858', '/dev/null', False, user_id * 4, user_email, now, user_id * 4, user_email, now, '50', '49'), + ('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, user_id * 7, user_email, now, user_id * 7, user_email, now, '1.1', '1.01'), + ('weather_patterns.bmp', '928SJXX2EB', 'science', False, user_id * 8, user_email, now, user_id * 8, user_email, now, '52', '51'), + ('demo.swf', 'DFDFGGGG14', 'demos/easy', False, user_id * 9, user_email, now, user_id * 9, user_email, now, '5', '4'), + ) + + @ddt.ddt class TestMongoAssetMetadataStorage(unittest.TestCase): """ @@ -33,7 +59,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): """ if type(mdata1) != type(mdata2): self.fail(self._formatMessage(msg, u"{} is not same type as {}".format(mdata1, mdata2))) - for attr in mdata1.ALLOWED_ATTRS: + for attr in mdata1.ATTRS_ALLOWED_TO_UPDATE: self.assertEqual(getattr(mdata1, attr), getattr(mdata2, attr), msg) def _compare_datetimes(self, datetime1, datetime2, msg=None): @@ -68,27 +94,8 @@ class TestMongoAssetMetadataStorage(unittest.TestCase): """ Setup assets. Save in store if given """ - asset_fields = ( - 'filename', 'internal_name', 'basename', 'locked', - 'edited_by', 'edited_on', 'created_by', 'created_on', - 'curr_version', 'prev_version' - ) - now = datetime.now(pytz.utc) - user_id = ModuleStoreEnum.UserID.test - all_asset_data = ( - ('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, user_id, now, user_id, now, '14', '13'), - ('shout.ogg', 'KFMDONSKF39K', 'sounds', True, user_id, now, user_id, now, '1', None), - ('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, user_id * 2, now, user_id * 2, now, 'AB', 'AA'), - ('dog.png', 'PUPY4242X', 'pictures/animals', True, user_id * 3, now, user_id * 3, now, '5', '4'), - ('not_here.txt', 'JJJCCC747', '/dev/null', False, user_id * 4, now, user_id * 4, now, '50', '49'), - ('asset.txt', 'JJJCCC747858', '/dev/null', False, user_id * 4, now, user_id * 4, now, '50', '49'), - ('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, user_id * 7, now, user_id * 7, now, '1.1', '1.01'), - ('weather_patterns.bmp', '928SJXX2EB', 'science', False, user_id * 8, now, user_id * 8, now, '52', '51'), - ('demo.swf', 'DFDFGGGG14', 'demos/easy', False, user_id * 9, now, user_id * 9, now, '5', '4'), - ) - - for i, asset in enumerate(all_asset_data): - asset_dict = dict(zip(asset_fields[1:], asset[1:])) + for i, asset in enumerate(AssetStoreTestData.all_asset_data): + asset_dict = dict(zip(AssetStoreTestData.asset_fields[1:], asset[1:])) if i in (0, 1) and course1_key: asset_key = course1_key.make_asset_key('asset', asset[0]) asset_md = AssetMetadata(asset_key, **asset_dict)