Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #6051 from edx/jeskew/asset_xml_input_output

Browse Source 
Implement XML input/output for AssetMetadata.
This commit is contained in:
John Eskew
2014-11-26 15:12:24 -05:00
parent b56d074648 29dcd88eef
commit 59801de232
4 changed files with 255 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
common/lib/xmodule/xmodule/assetstore/__init__.py
View File

@@ -3,13 +3,19 @@ Classes representing asset metadata.
"""
from datetime import datetime
import dateutil.parser
import pytz
import json
from contracts import contract, new_contract
from opaque_keys.edx.keys import AssetKey
from lxml import etree
new_contract('AssetKey', AssetKey)
new_contract('datetime', datetime)
new_contract('basestring', basestring)
new_contract('AssetElement', lambda x: isinstance(x, etree._Element) and x.tag == "asset") # pylint: disable=protected-access, no-member
new_contract('AssetsElement', lambda x: isinstance(x, etree._Element) and x.tag == "assets") # pylint: disable=protected-access, no-member
class AssetMetadata(object):
@@ -19,8 +25,10 @@ class AssetMetadata(object):
"""
TOP_LEVEL_ATTRS = ['basename', 'internal_name', 'locked', 'contenttype', 'thumbnail', 'fields']
EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_on']
ALLOWED_ATTRS = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS
EDIT_INFO_ATTRS = ['curr_version', 'prev_version', 'edited_by', 'edited_by_email', 'edited_on']
CREATE_INFO_ATTRS = ['created_by', 'created_by_email', 'created_on']
ATTRS_ALLOWED_TO_UPDATE = TOP_LEVEL_ATTRS + EDIT_INFO_ATTRS
ALL_ATTRS = ['asset_id'] + ATTRS_ALLOWED_TO_UPDATE + CREATE_INFO_ATTRS
# Default type for AssetMetadata objects. A constant for convenience.
ASSET_TYPE = 'asset'
@@ -30,15 +38,15 @@ class AssetMetadata(object):
locked='bool|None', contenttype='basestring|None',
thumbnail='basestring|None', fields='dict|None',
curr_version='basestring|None', prev_version='basestring|None',
created_by='int|None', created_on='datetime|None',
edited_by='int|None', edited_on='datetime|None')
created_by='int|None', created_by_email='basestring|None', created_on='datetime|None',
edited_by='int|None', edited_by_email='basestring|None', edited_on='datetime|None')
def __init__(self, asset_id,
basename=None, internal_name=None,
locked=None, contenttype=None,
thumbnail=None, fields=None,
curr_version=None, prev_version=None,
created_by=None, created_on=None,
edited_by=None, edited_on=None,
created_by=None, created_by_email=None, created_on=None,
edited_by=None, edited_by_email=None, edited_on=None,
field_decorator=None,):
"""
Construct a AssetMetadata object.
@@ -53,7 +61,11 @@ class AssetMetadata(object):
fields (dict): fields to save w/ the metadata
curr_version (str): Current version of the asset.
prev_version (str): Previous version of the asset.
edited_by (str): Username of last user to upload this asset.
created_by (int): User ID of initial user to upload this asset.
created_by_email (str): Email address of initial user to upload this asset.
created_on (datetime): Datetime of intial upload of this asset.
edited_by (int): User ID of last user to upload this asset.
edited_by_email (str): Email address of last user to upload this asset.
edited_on (datetime): Datetime of last upload of this asset.
field_decorator (function): used by strip_key to convert OpaqueKeys to the app's understanding.
Not saved.
@@ -68,9 +80,11 @@ class AssetMetadata(object):
self.prev_version = prev_version
now = datetime.now(pytz.utc)
self.edited_by = edited_by
self.edited_by_email = edited_by_email
self.edited_on = edited_on or now
# created_by and created_on should only be set here.
# created_by, created_by_email, and created_on should only be set here.
self.created_by = created_by
self.created_by_email = created_by_email
self.created_on = created_on or now
self.fields = fields or {}
@@ -80,20 +94,20 @@ class AssetMetadata(object):
self.basename, self.internal_name,
self.locked, self.contenttype, self.fields,
self.curr_version, self.prev_version,
self.edited_by, self.edited_on,
self.created_by, self.created_on
self.created_by, self.created_by_email, self.created_on,
self.edited_by, self.edited_by_email, self.edited_on,
))
def update(self, attr_dict):
"""
Set the attributes on the metadata. Any which are not in ALLOWED_ATTRS get put into
Set the attributes on the metadata. Any which are not in ATTRS_ALLOWED_TO_UPDATE get put into
fields.
Arguments:
attr_dict: Prop, val dictionary of all attributes to set.
"""
for attr, val in attr_dict.iteritems():
if attr in self.ALLOWED_ATTRS:
if attr in self.ATTRS_ALLOWED_TO_UPDATE:
setattr(self, attr, val)
else:
self.fields[attr] = val
@@ -113,10 +127,12 @@ class AssetMetadata(object):
'edit_info': {
'curr_version': self.curr_version,
'prev_version': self.prev_version,
'edited_by': self.edited_by,
'edited_on': self.edited_on,
'created_by': self.created_by,
'created_on': self.created_on
'created_by_email': self.created_by_email,
'created_on': self.created_on,
'edited_by': self.edited_by,
'edited_by_email': self.edited_by_email,
'edited_on': self.edited_on
}
}
@@ -137,7 +153,68 @@ class AssetMetadata(object):
self.fields = asset_doc['fields']
self.curr_version = asset_doc['edit_info']['curr_version']
self.prev_version = asset_doc['edit_info']['prev_version']
self.edited_by = asset_doc['edit_info']['edited_by']
self.edited_on = asset_doc['edit_info']['edited_on']
self.created_by = asset_doc['edit_info']['created_by']
self.created_by_email = asset_doc['edit_info']['created_by_email']
self.created_on = asset_doc['edit_info']['created_on']
self.edited_by = asset_doc['edit_info']['edited_by']
self.edited_by_email = asset_doc['edit_info']['edited_by_email']
self.edited_on = asset_doc['edit_info']['edited_on']
@contract(node='AssetElement')
def from_xml(self, node):
"""
Walk the etree XML node and fill in the asset metadata.
The node should be a top-level "asset" element.
"""
for child in node:
qname = etree.QName(child)
tag = qname.localname
if tag in self.ALL_ATTRS:
value = child.text
if tag == 'asset_id':
# Locator.
value = AssetKey.from_string(value)
elif tag == 'locked':
# Boolean.
value = True if value == "true" else False
elif tag in ('created_on', 'edited_on'):
# ISO datetime.
value = dateutil.parser.parse(value)
elif tag in ('created_by', 'edited_by'):
# Integer representing user id.
value = int(value)
elif tag == 'fields':
# Dictionary.
value = json.loads(value)
elif value == 'None':
# None.
value = None
setattr(self, tag, value)
@contract(node='AssetElement')
def to_xml(self, node):
"""
Add the asset data as XML to the passed-in node.
The node should already be created as a top-level "asset" element.
"""
for attr in self.ALL_ATTRS:
child = etree.SubElement(node, attr)
value = getattr(self, attr)
if isinstance(value, bool):
value = "true" if value else "false"
elif isinstance(value, datetime):
value = value.isoformat()
else:
value = unicode(value)
child.text = value
@staticmethod
@contract(node='AssetsElement', assets=list)
def add_all_assets_as_xml(node, assets):
"""
Take a list of AssetMetadata objects. Add them all to the node.
The node should already be created as a top-level "assets" element.
"""
for asset in assets:
asset_node = etree.SubElement(node, "asset")
asset.to_xml(asset_node)

48
common/lib/xmodule/xmodule/assetstore/tests/assets.xsd Normal file
View File

@@ -0,0 +1,48 @@
<?xml version="1.0" encoding="UTF-8" ?>
<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="assets" type="assetListType" />
<xs:simpleType name="stringType">
<xs:restriction base="xs:string"/>
</xs:simpleType>
<xs:simpleType name="userIdType">
<xs:restriction base="xs:nonNegativeInteger"/>
</xs:simpleType>
<xs:simpleType name="datetimeType">
<xs:restriction base="xs:dateTime"/>
</xs:simpleType>
<xs:simpleType name="boolType">
<xs:restriction base="xs:boolean"/>
</xs:simpleType>
<xs:complexType name="assetListType">
<xs:sequence>
<xs:element name="asset" type="assetType" minOccurs="0" maxOccurs="unbounded" />
</xs:sequence>
</xs:complexType>
<xs:complexType name="assetType">
<xs:all>
<xs:element name="asset_id" type="stringType"/>
<xs:element name="contenttype" type="stringType"/>
<xs:element name="basename" type="stringType"/>
<xs:element name="internal_name" type="stringType"/>
<xs:element name="locked" type="boolType"/>
<xs:element name="thumbnail" type="stringType" minOccurs="0"/>
<xs:element name="created_on" type="datetimeType" />
<xs:element name="created_by" type="userIdType" />
<xs:element name="created_by_email" type="stringType" minOccurs="0"/>
<xs:element name="edited_on" type="datetimeType" />
<xs:element name="edited_by" type="userIdType" />
<xs:element name="edited_by_email" type="stringType" minOccurs="0"/>
<xs:element name="prev_version" type="stringType"/>
<xs:element name="curr_version" type="stringType"/>
<xs:element name="fields" type="stringType" minOccurs="0"/>
</xs:all>
</xs:complexType>
</xs:schema>

84
common/lib/xmodule/xmodule/assetstore/tests/test_asset_xml.py Normal file
View File

@@ -0,0 +1,84 @@
"""
Test for asset XML generation / parsing.
"""
from path import path
from lxml import etree
from contracts import ContractNotRespected
import unittest
from opaque_keys.edx.locator import CourseLocator
from xmodule.assetstore import AssetMetadata
from xmodule.modulestore.tests.test_assetstore import AssetStoreTestData
class TestAssetXml(unittest.TestCase):
"""
Tests for storing/querying course asset metadata.
"""
def setUp(self):
super(TestAssetXml, self).setUp()
xsd_filename = "assets.xsd"
self.course_id = CourseLocator('org1', 'course1', 'run1')
self.course_assets = []
for asset in AssetStoreTestData.all_asset_data:
asset_dict = dict(zip(AssetStoreTestData.asset_fields[1:], asset[1:]))
asset_md = AssetMetadata(self.course_id.make_asset_key('asset', asset[0]), **asset_dict)
self.course_assets.append(asset_md)
# Read in the XML schema definition and make a validator.
#xsd_path = path(__file__).abspath().dirname() / xsd_filename
xsd_path = path(__file__).realpath().parent / xsd_filename
with open(xsd_path, 'r') as f:
schema_root = etree.XML(f.read())
schema = etree.XMLSchema(schema_root)
self.xmlparser = etree.XMLParser(schema=schema)
def test_export_single_asset_to_from_xml(self):
"""
Export a single AssetMetadata to XML and verify the structure and fields.
"""
asset_md = self.course_assets[0]
root = etree.Element("assets")
asset = etree.SubElement(root, "asset")
asset_md.to_xml(asset)
# If this line does *not* raise, the XML is valid.
etree.fromstring(etree.tostring(root), self.xmlparser)
new_asset_key = self.course_id.make_asset_key('tmp', 'tmp')
new_asset_md = AssetMetadata(new_asset_key)
new_asset_md.from_xml(asset)
# Compare asset_md to new_asset_md.
for attr in AssetMetadata.ALL_ATTRS:
orig_value = getattr(asset_md, attr)
new_value = getattr(new_asset_md, attr)
self.assertEqual(orig_value, new_value)
def test_export_all_assets_to_xml(self):
"""
Export all AssetMetadatas to XML and verify the structure and fields.
"""
root = etree.Element("assets")
AssetMetadata.add_all_assets_as_xml(root, self.course_assets)
# If this line does *not* raise, the XML is valid.
etree.fromstring(etree.tostring(root), self.xmlparser)
def test_wrong_node_type_all(self):
"""
Ensure full asset sections with the wrong tag are detected.
"""
root = etree.Element("glassets")
with self.assertRaises(ContractNotRespected):
AssetMetadata.add_all_assets_as_xml(root, self.course_assets)
def test_wrong_node_type_single(self):
"""
Ensure single asset blocks with the wrong tag are detected.
"""
asset_md = self.course_assets[0]
root = etree.Element("assets")
asset = etree.SubElement(root, "smashset")
with self.assertRaises(ContractNotRespected):
asset_md.to_xml(asset)

51
common/lib/xmodule/xmodule/modulestore/tests/test_assetstore.py
View File

@@ -17,6 +17,32 @@ from xmodule.modulestore.tests.test_cross_modulestore_import_export import (
)
class AssetStoreTestData(object):
"""
Shared data for constructing test assets.
"""
now = datetime.now(pytz.utc)
user_id = 144
user_email = "me@example.com"
asset_fields = (
'filename', 'internal_name', 'basename', 'locked',
'edited_by', 'edited_by_email', 'edited_on', 'created_by', 'created_by_email', 'created_on',
'curr_version', 'prev_version'
)
all_asset_data = (
('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, user_id, user_email, now, user_id, user_email, now, '14', '13'),
('shout.ogg', 'KFMDONSKF39K', 'sounds', True, user_id, user_email, now, user_id, user_email, now, '1', None),
('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, user_id * 2, user_email, now, user_id * 2, user_email, now, 'AB', 'AA'),
('dog.png', 'PUPY4242X', 'pictures/animals', True, user_id * 3, user_email, now, user_id * 3, user_email, now, '5', '4'),
('not_here.txt', 'JJJCCC747', '/dev/null', False, user_id * 4, user_email, now, user_id * 4, user_email, now, '50', '49'),
('asset.txt', 'JJJCCC747858', '/dev/null', False, user_id * 4, user_email, now, user_id * 4, user_email, now, '50', '49'),
('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, user_id * 7, user_email, now, user_id * 7, user_email, now, '1.1', '1.01'),
('weather_patterns.bmp', '928SJXX2EB', 'science', False, user_id * 8, user_email, now, user_id * 8, user_email, now, '52', '51'),
('demo.swf', 'DFDFGGGG14', 'demos/easy', False, user_id * 9, user_email, now, user_id * 9, user_email, now, '5', '4'),
)
@ddt.ddt
class TestMongoAssetMetadataStorage(unittest.TestCase):
"""
@@ -33,7 +59,7 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
"""
if type(mdata1) != type(mdata2):
self.fail(self._formatMessage(msg, u"{} is not same type as {}".format(mdata1, mdata2)))
for attr in mdata1.ALLOWED_ATTRS:
for attr in mdata1.ATTRS_ALLOWED_TO_UPDATE:
self.assertEqual(getattr(mdata1, attr), getattr(mdata2, attr), msg)
def _compare_datetimes(self, datetime1, datetime2, msg=None):
@@ -68,27 +94,8 @@ class TestMongoAssetMetadataStorage(unittest.TestCase):
"""
Setup assets. Save in store if given
"""
asset_fields = (
'filename', 'internal_name', 'basename', 'locked',
'edited_by', 'edited_on', 'created_by', 'created_on',
'curr_version', 'prev_version'
)
now = datetime.now(pytz.utc)
user_id = ModuleStoreEnum.UserID.test
all_asset_data = (
('pic1.jpg', 'EKMND332DDBK', 'pix/archive', False, user_id, now, user_id, now, '14', '13'),
('shout.ogg', 'KFMDONSKF39K', 'sounds', True, user_id, now, user_id, now, '1', None),
('code.tgz', 'ZZB2333YBDMW', 'exercises/14', False, user_id * 2, now, user_id * 2, now, 'AB', 'AA'),
('dog.png', 'PUPY4242X', 'pictures/animals', True, user_id * 3, now, user_id * 3, now, '5', '4'),
('not_here.txt', 'JJJCCC747', '/dev/null', False, user_id * 4, now, user_id * 4, now, '50', '49'),
('asset.txt', 'JJJCCC747858', '/dev/null', False, user_id * 4, now, user_id * 4, now, '50', '49'),
('roman_history.pdf', 'JASDUNSADK', 'texts/italy', True, user_id * 7, now, user_id * 7, now, '1.1', '1.01'),
('weather_patterns.bmp', '928SJXX2EB', 'science', False, user_id * 8, now, user_id * 8, now, '52', '51'),
('demo.swf', 'DFDFGGGG14', 'demos/easy', False, user_id * 9, now, user_id * 9, now, '5', '4'),
)
for i, asset in enumerate(all_asset_data):
asset_dict = dict(zip(asset_fields[1:], asset[1:]))
for i, asset in enumerate(AssetStoreTestData.all_asset_data):
asset_dict = dict(zip(AssetStoreTestData.asset_fields[1:], asset[1:]))
if i in (0, 1) and course1_key:
asset_key = course1_key.make_asset_key('asset', asset[0])
asset_md = AssetMetadata(asset_key, **asset_dict)