From 4ebd70dcc4ec40eaace413616dca13d3212f6d1b Mon Sep 17 00:00:00 2001 From: Brian Wilson Date: Tue, 4 Dec 2012 03:54:36 -0500 Subject: [PATCH] add support for utf8 in sequences, verticals and custom modules. --- common/djangoapps/mitxmako/shortcuts.py | 7 +------ common/djangoapps/mitxmako/template.py | 7 +------ common/lib/xmodule/xmodule/error_module.py | 4 ++-- common/lib/xmodule/xmodule/html_module.py | 6 +----- common/lib/xmodule/xmodule/modulestore/xml.py | 4 ++-- common/lib/xmodule/xmodule/raw_module.py | 2 +- common/lib/xmodule/xmodule/seq_module.py | 2 +- common/lib/xmodule/xmodule/stringify.py | 4 ++-- common/lib/xmodule/xmodule/xml_module.py | 8 +++++--- common/test/data/full/chapter/Overview.xml | 2 +- common/test/data/full/html/FirstOrderSystemHint.xml | 2 +- common/test/data/full/html/Midterm_Exam_1123.html | 2 +- common/test/data/full/html/Week_13_Tutorials.html | 4 ++-- common/test/data/full/html/html_5555.html | 2 +- common/test/data/full/html/linearity_clarify.html | 2 +- common/test/data/full/html/linearity_clarify.xml | 2 +- common/test/data/full/html/schematic_tutorial.html | 6 +++--- common/test/data/full/html/units_hint.html | 4 ++-- common/test/data/full/info/updates.html | 5 +++-- .../sequential/Administrivia_and_Circuit_Elements.xml | 3 ++- common/test/data/full/vertical/vertical_89.xml | 1 + 21 files changed, 35 insertions(+), 44 deletions(-) diff --git a/common/djangoapps/mitxmako/shortcuts.py b/common/djangoapps/mitxmako/shortcuts.py index 6aee39906a..181d3befd5 100644 --- a/common/djangoapps/mitxmako/shortcuts.py +++ b/common/djangoapps/mitxmako/shortcuts.py @@ -42,12 +42,7 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'): context_dictionary.update(context) # fetch and render template template = middleware.lookup[namespace].get_template(template_name) -# return template.render_unicode(**context_dictionary) - - output = template.render_unicode(**context_dictionary) -# log.info(' render_to_string of "{0}" as "{1}r"'.format(type(output), output)) - return output -# return template.render(**context_dictionary) + return template.render_unicode(**context_dictionary) def render_to_response(template_name, dictionary, context_instance=None, namespace='main', **kwargs): diff --git a/common/djangoapps/mitxmako/template.py b/common/djangoapps/mitxmako/template.py index efeb282d04..ff83a9be4c 100644 --- a/common/djangoapps/mitxmako/template.py +++ b/common/djangoapps/mitxmako/template.py @@ -57,9 +57,4 @@ class Template(MakoTemplate): context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL context_dictionary['django_context'] = context_instance -# return super(Template, self).render_unicode(**context_dictionary) -# return super(Template, self).render(**context_dictionary) - - output = super(Template, self).render(**context_dictionary) - log.info(' render_to_string of "{0}" as "{1}"'.format(type(output), output)) - return output + return super(Template, self).render_unicode(**context_dictionary) diff --git a/common/lib/xmodule/xmodule/error_module.py b/common/lib/xmodule/xmodule/error_module.py index 65fceb77c7..2df47e05e6 100644 --- a/common/lib/xmodule/xmodule/error_module.py +++ b/common/lib/xmodule/xmodule/error_module.py @@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor): ''' try: xml = etree.fromstring(self.definition['data']['contents']) - return etree.tostring(xml) + return etree.tostring(xml, encoding='unicode') except etree.XMLSyntaxError: # still not valid. root = etree.Element('error') root.text = self.definition['data']['contents'] err_node = etree.SubElement(root, 'error_msg') err_node.text = self.definition['data']['error_msg'] - return etree.tostring(root) + return etree.tostring(root, encoding='unicode') class NonStaffErrorDescriptor(ErrorDescriptor): diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 6b73535dac..df77f43a17 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -30,11 +30,7 @@ class HtmlModule(XModule): def get_html(self): # cdodge: perform link substitutions for any references to course static content (e.g. images) - input = self.html - output = rewrite_links(self.html, self.rewrite_content_links) -# log.info(' HTMLModule converting markup from "{0}" as "{1}r"'.format(type(input), input)) -# log.info(' HTMLModule converting markup to "{0}" as "{1}r"'.format(type(output), output)) - return output + return rewrite_links(self.html, self.rewrite_content_links) def __init__(self, system, location, definition, descriptor, instance_state=None, shared_state=None, **kwargs): diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 6f8430917d..6b3ff9bff4 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -152,7 +152,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): make_name_unique(xml_data) descriptor = XModuleDescriptor.load_from_xml( - etree.tostring(xml_data), self, self.org, + etree.tostring(xml_data, encoding='unicode'), self, self.org, self.course, xmlstore.default_class) except Exception as err: print err, self.load_error_modules @@ -436,7 +436,7 @@ class XMLModuleStore(ModuleStoreBase): self.load_error_modules, ) - course_descriptor = system.process_xml(etree.tostring(course_data)) + course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode')) # NOTE: The descriptors end up loading somewhat bottom up, which # breaks metadata inheritance via get_children(). Instead diff --git a/common/lib/xmodule/xmodule/raw_module.py b/common/lib/xmodule/xmodule/raw_module.py index 5ff16098ac..efdd2e7ba0 100644 --- a/common/lib/xmodule/xmodule/raw_module.py +++ b/common/lib/xmodule/xmodule/raw_module.py @@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor): """ @classmethod def definition_from_xml(cls, xml_object, system): - return {'data': etree.tostring(xml_object, pretty_print=True)} + return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')} def definition_to_xml(self, resource_fs): try: diff --git a/common/lib/xmodule/xmodule/seq_module.py b/common/lib/xmodule/xmodule/seq_module.py index b625646e66..9aec275e90 100644 --- a/common/lib/xmodule/xmodule/seq_module.py +++ b/common/lib/xmodule/xmodule/seq_module.py @@ -124,7 +124,7 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor): children = [] for child in xml_object: try: - children.append(system.process_xml(etree.tostring(child)).location.url()) + children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url()) except: log.exception("Unable to load child when parsing Sequence. Continuing...") continue diff --git a/common/lib/xmodule/xmodule/stringify.py b/common/lib/xmodule/xmodule/stringify.py index 1e3fa91210..dab8ff0425 100644 --- a/common/lib/xmodule/xmodule/stringify.py +++ b/common/lib/xmodule/xmodule/stringify.py @@ -22,7 +22,7 @@ def stringify_children(node): # next element. parts = [node.text] for c in node.getchildren(): - parts.append(etree.tostring(c, with_tail=True)) + parts.append(etree.tostring(c, with_tail=True, encoding='unicode')) # filter removes possible Nones in texts and tails - return ''.join(filter(None, parts)) + return u''.join(filter(None, parts)) diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index e65a8c74ea..91768d8e11 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -13,7 +13,8 @@ import sys log = logging.getLogger(__name__) edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False, - remove_comments=True, remove_blank_text=True) + remove_comments=True, remove_blank_text=True, + encoding='utf-8') def name_to_pathname(name): """ @@ -206,6 +207,7 @@ class XmlDescriptor(XModuleDescriptor): definition_xml = cls.load_file(filepath, system.resources_fs, location) + log.info(' read definition XML: %s', definition_xml) definition_metadata = get_metadata_from_xml(definition_xml) cls.clean_metadata_from_xml(definition_xml) definition = cls.definition_from_xml(definition_xml, system) @@ -366,7 +368,7 @@ class XmlDescriptor(XModuleDescriptor): filepath = self.__class__._format_filepath(self.category, url_path) resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: - file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8', xml_declaration=True)) + file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8')) # And return just a pointer with the category and filename. record_object = etree.Element(self.category) @@ -381,7 +383,7 @@ class XmlDescriptor(XModuleDescriptor): record_object.set('org', self.location.org) record_object.set('course', self.location.course) - return etree.tostring(record_object, pretty_print=True) + return etree.tostring(record_object, pretty_print=True, encoding='utf-8') def definition_to_xml(self, resource_fs): """ diff --git a/common/test/data/full/chapter/Overview.xml b/common/test/data/full/chapter/Overview.xml index a11a11a1e0..8ad44b366c 100644 --- a/common/test/data/full/chapter/Overview.xml +++ b/common/test/data/full/chapter/Overview.xml @@ -2,7 +2,7 @@