add support for utf8 in sequences, verticals and custom modules.

common/djangoapps/mitxmako/shortcuts.py

@@ -42,12 +42,7 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
         context_dictionary.update(context)
     # fetch and render template
     template = middleware.lookup[namespace].get_template(template_name)
-#    return template.render_unicode(**context_dictionary)
-
-    output = template.render_unicode(**context_dictionary)
-#    log.info('  render_to_string of "{0}" as "{1}r"'.format(type(output), output))
-    return output
-#    return template.render(**context_dictionary)
+    return template.render_unicode(**context_dictionary)
 
 
 def render_to_response(template_name, dictionary, context_instance=None, namespace='main', **kwargs):

common/djangoapps/mitxmako/template.py

@@ -57,9 +57,4 @@ class Template(MakoTemplate):
         context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
         context_dictionary['django_context'] = context_instance
                 
-#        return super(Template, self).render_unicode(**context_dictionary)
-#        return super(Template, self).render(**context_dictionary)
- 
-        output = super(Template, self).render(**context_dictionary)
-        log.info('  render_to_string of "{0}" as "{1}"'.format(type(output), output))       
-        return output
+        return super(Template, self).render_unicode(**context_dictionary)

common/lib/xmodule/xmodule/error_module.py

@@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor):
         '''
         try:
             xml = etree.fromstring(self.definition['data']['contents'])
-            return etree.tostring(xml)
+            return etree.tostring(xml, encoding='unicode')
         except etree.XMLSyntaxError:
             # still not valid.
             root = etree.Element('error')
             root.text = self.definition['data']['contents']
             err_node = etree.SubElement(root, 'error_msg')
             err_node.text = self.definition['data']['error_msg']
-            return etree.tostring(root)
+            return etree.tostring(root, encoding='unicode')
 
 
 class NonStaffErrorDescriptor(ErrorDescriptor):

common/lib/xmodule/xmodule/html_module.py

@@ -30,11 +30,7 @@ class HtmlModule(XModule):
     
     def get_html(self):
         # cdodge: perform link substitutions for any references to course static content (e.g. images)
-        input = self.html
-        output = rewrite_links(self.html, self.rewrite_content_links)
-#        log.info('  HTMLModule converting markup from "{0}" as  "{1}r"'.format(type(input), input))
-#        log.info('  HTMLModule converting markup to "{0}" as "{1}r"'.format(type(output), output))
-        return output
+        return rewrite_links(self.html, self.rewrite_content_links)
 
     def __init__(self, system, location, definition, descriptor,
                  instance_state=None, shared_state=None, **kwargs):

common/lib/xmodule/xmodule/modulestore/xml.py

@@ -152,7 +152,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
                 make_name_unique(xml_data)
 
                 descriptor = XModuleDescriptor.load_from_xml(
-                    etree.tostring(xml_data), self, self.org,
+                    etree.tostring(xml_data, encoding='unicode'), self, self.org,
                     self.course, xmlstore.default_class)
             except Exception as err:
                 print err, self.load_error_modules
@@ -436,7 +436,7 @@ class XMLModuleStore(ModuleStoreBase):
                 self.load_error_modules,
             )
 
-            course_descriptor = system.process_xml(etree.tostring(course_data))
+            course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode'))
 
             # NOTE: The descriptors end up loading somewhat bottom up, which
             # breaks metadata inheritance via get_children().  Instead

common/lib/xmodule/xmodule/raw_module.py

@@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
     """
     @classmethod
     def definition_from_xml(cls, xml_object, system):
-        return {'data': etree.tostring(xml_object, pretty_print=True)}
+        return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')}
 
     def definition_to_xml(self, resource_fs):
         try:

common/lib/xmodule/xmodule/seq_module.py

@@ -124,7 +124,7 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
         children = []
         for child in xml_object:
             try:
-                children.append(system.process_xml(etree.tostring(child)).location.url())
+                children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url())
             except:
                 log.exception("Unable to load child when parsing Sequence. Continuing...")
                 continue

common/lib/xmodule/xmodule/stringify.py

@@ -22,7 +22,7 @@ def stringify_children(node):
     #                 next element.
     parts = [node.text]
     for c in node.getchildren():
-        parts.append(etree.tostring(c, with_tail=True))
+        parts.append(etree.tostring(c, with_tail=True, encoding='unicode'))
 
     # filter removes possible Nones in texts and tails
-    return ''.join(filter(None, parts))
+    return u''.join(filter(None, parts))

common/lib/xmodule/xmodule/xml_module.py

@@ -13,7 +13,8 @@ import sys
 log = logging.getLogger(__name__)
 
 edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
-                                 remove_comments=True, remove_blank_text=True)
+                                 remove_comments=True, remove_blank_text=True,
+                                 encoding='utf-8')
 
 def name_to_pathname(name):
     """
@@ -206,6 +207,7 @@ class XmlDescriptor(XModuleDescriptor):
 
             definition_xml = cls.load_file(filepath, system.resources_fs, location)
 
+        log.info(' read definition XML: %s', definition_xml)
         definition_metadata = get_metadata_from_xml(definition_xml)
         cls.clean_metadata_from_xml(definition_xml)
         definition = cls.definition_from_xml(definition_xml, system)
@@ -366,7 +368,7 @@ class XmlDescriptor(XModuleDescriptor):
             filepath = self.__class__._format_filepath(self.category, url_path)
             resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
             with resource_fs.open(filepath, 'w') as file:
-                file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8', xml_declaration=True))
+                file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8'))
 
             # And return just a pointer with the category and filename.
             record_object = etree.Element(self.category)
@@ -381,7 +383,7 @@ class XmlDescriptor(XModuleDescriptor):
             record_object.set('org', self.location.org)
             record_object.set('course', self.location.course)
 
-        return etree.tostring(record_object, pretty_print=True)
+        return etree.tostring(record_object, pretty_print=True, encoding='utf-8')
 
     def definition_to_xml(self, resource_fs):
         """

common/test/data/full/chapter/Overview.xml

@@ -2,7 +2,7 @@
   <video url_name="welcome"/>
   <sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
   <vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
-    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
+    <html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab… </html>
     <html slug="html_5555" filename="html_5555"/>
     <problem filename="Lab_0_Using_the_Tools" slug="Lab_0_Using_the_Tools" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Lab 0: Using the Tools"/>
   </vertical>

common/test/data/full/html/FirstOrderSystemHint.xml

@@ -1,4 +1,4 @@
-Hint
+Hint…
 <br/><br/>
 Remember that the time evolution of any variable \(x(t)\) governed by
 a first-order system with a time-constant \(\tau\) for a time \(t) between an initial

common/test/data/full/html/Midterm_Exam_1123.html

@@ -13,7 +13,7 @@ When you open the next page, you will have started the examination.
 You do not need to start now: you will not be timed until you open the
 next page.  Once you have opened the next page page you must complete
 the exam and make your final submission within twenty four hours of starting
-the exam.  Gratuitous ≥ entity.
+the exam.
 </p>
     <p>
 You may use any notes, computational, or auxiliary materials that you

common/test/data/full/html/Week_13_Tutorials.html

@@ -4,14 +4,14 @@
     <section class="tutorials">
       <h2> Basic Tutorials </h2>
       <ul>
-        <li><a href="/section/wk13_solder">Soldering</a> -- Steve
+        <li><a href="/section/wk13_solder">Soldering</a> &mdash; Steve
        Finberg, one of the pioneers in from Draper Lab, talks about
        soldering. </li>
       </ul>
       <h2> Bonus Tutorials </h2>
       <ul>
         <li><a href="/section/wk13_FreqResp">Frequency Response
-       Curves</a> -- We explain several techniques for understanding
+       Curves</a> &mdash; We explain several techniques for understanding
        and approximating Bode plots. </li>
       </ul>
     </section>

common/test/data/full/html/html_5555.html

@@ -1 +1 @@
-    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab.   Gratuitous &ge; entity. </html>
+    <html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab. </html>

common/test/data/full/html/linearity_clarify.html

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/linearity_clarify.xml

@@ -34,6 +34,6 @@
     the Thevenin or Norton theorems to summarize the behavior at
     a pair of exposed terminals.
     </p><p>
-    Sorry for the confusion of words -- natural language is like
+    Sorry for the confusion of words &mdash; natural language is like
     that!
   </p>

common/test/data/full/html/schematic_tutorial.html

@@ -9,14 +9,14 @@ the right of the diagram area) and drag it onto the diagram.  Release
 the mouse when the component is in the correct position.
 </td>
 </tr>
-
+<!-- note that entities like &mdash; may be used. -->
 <tr>
 <td>Move a component</td>
 <td>Click to select a component in the diagram (it will turn green)
 and then drag it to its new location.  You can use shift-click to add
 a component to the current selection.  Or you can click somewhere in
 the diagram that is not on top of a component and drag out a selection
-rectangle -- components intersecting the rectangle will be added to
+rectangle &mdash; components intersecting the rectangle will be added to
 the current selection.
 </td>
 </tr>
@@ -63,7 +63,7 @@ engineeering notation:
 <td>Add a wire</td>
 <td>Wires start at connection points, the open circles that
 appear at the terminals of components or the ends of wires.
-Click on a connection point to start a wire -- a green wire
+Click on a connection point to start a wire &mdash; a green wire
 will appear with one end anchored at the starting point.
 Drag the mouse and release the mouse button when the other
 end of the wire is positioned as you wish.  Once a wire has

common/test/data/full/html/units_hint.html

@@ -1,4 +1,4 @@
-Hint
+Hint…
 <br/><br/>
 Be careful of units here.  Make sure you notice multipliers such
-as u, k, m, M.
+as u (or &mu;), k, m, M.

common/test/data/full/info/updates.html

@@ -9,8 +9,9 @@
 <li> <h2>May 2 </h2>
 <section class="update-description">
 <ul>
-<li> We have opened the show-answer button on the midterm. </li>
-<li> There was a four hour outage in posting ability on the discussion board Monday night. It has been fixed. We apologise for the inconvenience.</li>
+<!-- utf-8 characters are acceptable… as are HTML entities -->
+<li> We have opened the show-answer button on the midterm… </li>
+<li> There was a four hour outage in posting ability on the discussion board Monday night&hellip; It has been fixed. We apologise for the inconvenience.</li>
 </ul>
 </li>
 <li> <h2>April 30 </h2>

common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml

@@ -3,7 +3,8 @@
   <vertical slug="vertical_66" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
     <problem filename="S1E3_AC_power" slug="S1E3_AC_power" name="S1E3: AC power"/>
     <customtag tag="S1E3" slug="discuss_67" impl="discuss"/>
-    <html slug="html_68"> S1E4 has been removed… </html>
+    <!-- utf-8 characters acceptable, but not HTML entities -->
+    <html slug="html_68"> S1E4 has been removed…</html>
   </vertical>
   <vertical filename="vertical_89" slug="vertical_89" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
   <vertical slug="vertical_94" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">

common/test/data/full/vertical/vertical_89.xml

@@ -1,5 +1,6 @@
 <sequential>
   <html slug="html_90">
+    <!-- UTF-8 characters are acceptable… HTML entities are not -->
     <h1>Inline content…</h1>
   </html>
   <video youtube="1.50:vl9xrfxcr38,1.25:qxNX4REGqx4,1.0:BGU1poJDgOY,0.75:8rK9vnpystQ" slug="S1V14_Summary" name="S1V14: Summary"/>