add support for utf8 in sequences, verticals and custom modules.

common/lib/xmodule/xmodule/error_module.py

@@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor):
         '''
         try:
             xml = etree.fromstring(self.definition['data']['contents'])
-            return etree.tostring(xml)
+            return etree.tostring(xml, encoding='unicode')
         except etree.XMLSyntaxError:
             # still not valid.
             root = etree.Element('error')
             root.text = self.definition['data']['contents']
             err_node = etree.SubElement(root, 'error_msg')
             err_node.text = self.definition['data']['error_msg']
-            return etree.tostring(root)
+            return etree.tostring(root, encoding='unicode')
 
 
 class NonStaffErrorDescriptor(ErrorDescriptor):

common/lib/xmodule/xmodule/html_module.py

@@ -30,11 +30,7 @@ class HtmlModule(XModule):
     
     def get_html(self):
         # cdodge: perform link substitutions for any references to course static content (e.g. images)
-        input = self.html
-        output = rewrite_links(self.html, self.rewrite_content_links)
-#        log.info('  HTMLModule converting markup from "{0}" as  "{1}r"'.format(type(input), input))
-#        log.info('  HTMLModule converting markup to "{0}" as "{1}r"'.format(type(output), output))
-        return output
+        return rewrite_links(self.html, self.rewrite_content_links)
 
     def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -152,7 +152,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 make_name_unique(xml_data)
 
                 descriptor = XModuleDescriptor.load_from_xml(
-                    etree.tostring(xml_data), self, self.org,
+                    etree.tostring(xml_data, encoding='unicode'), self, self.org,
                     self.course, xmlstore.default_class)
             except Exception as err:
                 print err, self.load_error_modules
@@ -436,7 +436,7 @@ class XMLModuleStore(ModuleStoreBase):
                 self.load_error_modules,
             )
 
-            course_descriptor = system.process_xml(etree.tostring(course_data))
+            course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode'))
 
             # NOTE: The descriptors end up loading somewhat bottom up, which
             # breaks metadata inheritance via get_children().  Instead

common/lib/xmodule/xmodule/raw_module.py

@@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
     """
     @classmethod
     def definition_from_xml(cls, xml_object, system):
-        return {'data': etree.tostring(xml_object, pretty_print=True)}
+        return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')}
 
     def definition_to_xml(self, resource_fs):
         try:

common/lib/xmodule/xmodule/seq_module.py

@@ -124,7 +124,7 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         children = []
         for child in xml_object:
             try:
-                children.append(system.process_xml(etree.tostring(child)).location.url())
+                children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url())
             except:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
                 continue

common/lib/xmodule/xmodule/stringify.py

@@ -22,7 +22,7 @@ def stringify_children(node):
     #                 next element.
     parts = [node.text]
     for c in node.getchildren():
-        parts.append(etree.tostring(c, with_tail=True))
+        parts.append(etree.tostring(c, with_tail=True, encoding='unicode'))
 
     # filter removes possible Nones in texts and tails
-    return ''.join(filter(None, parts))
+    return u''.join(filter(None, parts))

common/lib/xmodule/xmodule/xml_module.py

@@ -13,7 +13,8 @@ import sys
 log = logging.getLogger(__name__)
 
 edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
-                                 remove_comments=True, remove_blank_text=True)
+                                 remove_comments=True, remove_blank_text=True,
+                                 encoding='utf-8')
 
 def name_to_pathname(name):
     """
@@ -206,6 +207,7 @@ class XmlDescriptor(XModuleDescriptor):
 
             definition_xml = cls.load_file(filepath, system.resources_fs, location)
 
+        log.info(' read definition XML: %s', definition_xml)
         definition_metadata = get_metadata_from_xml(definition_xml)
         cls.clean_metadata_from_xml(definition_xml)
         definition = cls.definition_from_xml(definition_xml, system)
@@ -366,7 +368,7 @@ class XmlDescriptor(XModuleDescriptor):
             filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
-                file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8', xml_declaration=True))
+                file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8'))
 
             # And return just a pointer with the category and filename.
             record_object = etree.Element(self.category)
@@ -381,7 +383,7 @@ class XmlDescriptor(XModuleDescriptor):
             record_object.set('org', self.location.org)
             record_object.set('course', self.location.course)
 
-        return etree.tostring(record_object, pretty_print=True)
+        return etree.tostring(record_object, pretty_print=True, encoding='utf-8')
 
     def definition_to_xml(self, resource_fs):
         """