diff --git a/common/djangoapps/mitxmako/shortcuts.py b/common/djangoapps/mitxmako/shortcuts.py index ba22f2db20..ebeb0fc180 100644 --- a/common/djangoapps/mitxmako/shortcuts.py +++ b/common/djangoapps/mitxmako/shortcuts.py @@ -12,10 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import logging - -log = logging.getLogger("mitx." + __name__) - from django.template import Context from django.http import HttpResponse @@ -42,7 +38,7 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'): context_dictionary.update(context) # fetch and render template template = middleware.lookup[namespace].get_template(template_name) - return template.render(**context_dictionary) + return template.render_unicode(**context_dictionary) def render_to_response(template_name, dictionary, context_instance=None, namespace='main', **kwargs): diff --git a/common/djangoapps/mitxmako/template.py b/common/djangoapps/mitxmako/template.py index 56096fe173..947dc8c1a4 100644 --- a/common/djangoapps/mitxmako/template.py +++ b/common/djangoapps/mitxmako/template.py @@ -54,5 +54,4 @@ class Template(MakoTemplate): context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL context_dictionary['django_context'] = context_instance - return super(Template, self).render(**context_dictionary) - + return super(Template, self).render_unicode(**context_dictionary) diff --git a/common/lib/xmodule/xmodule/error_module.py b/common/lib/xmodule/xmodule/error_module.py index 65fceb77c7..2df47e05e6 100644 --- a/common/lib/xmodule/xmodule/error_module.py +++ b/common/lib/xmodule/xmodule/error_module.py @@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor): ''' try: xml = etree.fromstring(self.definition['data']['contents']) - return etree.tostring(xml) + return etree.tostring(xml, encoding='unicode') except etree.XMLSyntaxError: # still not valid. root = etree.Element('error') root.text = self.definition['data']['contents'] err_node = etree.SubElement(root, 'error_msg') err_node.text = self.definition['data']['error_msg'] - return etree.tostring(root) + return etree.tostring(root, encoding='unicode') class NonStaffErrorDescriptor(ErrorDescriptor): diff --git a/common/lib/xmodule/xmodule/html_module.py b/common/lib/xmodule/xmodule/html_module.py index 2023ac7017..c11c7d22e7 100644 --- a/common/lib/xmodule/xmodule/html_module.py +++ b/common/lib/xmodule/xmodule/html_module.py @@ -7,15 +7,14 @@ from lxml import etree from lxml.html import rewrite_links from path import path -from .x_module import XModule from pkg_resources import resource_string -from .xml_module import XmlDescriptor, name_to_pathname -from .editing_module import EditingDescriptor -from .stringify import stringify_children -from .html_checker import check_html -from xmodule.modulestore import Location - from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent +from xmodule.editing_module import EditingDescriptor +from xmodule.html_checker import check_html +from xmodule.modulestore import Location +from xmodule.stringify import stringify_children +from xmodule.x_module import XModule +from xmodule.xml_module import XmlDescriptor, name_to_pathname log = logging.getLogger("mitx.courseware") @@ -123,7 +122,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): try: with system.resources_fs.open(filepath) as file: - html = file.read() + html = file.read().decode('utf-8') # Log a warning if we can't parse the file, but don't error if not check_html(html): msg = "Couldn't parse html in {0}.".format(filepath) @@ -164,7 +163,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor): resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: - file.write(self.definition['data']) + file.write(self.definition['data'].encode('utf-8')) # write out the relative name relname = path(pathname).basename() diff --git a/common/lib/xmodule/xmodule/modulestore/xml.py b/common/lib/xmodule/xmodule/modulestore/xml.py index 6f8430917d..6b3ff9bff4 100644 --- a/common/lib/xmodule/xmodule/modulestore/xml.py +++ b/common/lib/xmodule/xmodule/modulestore/xml.py @@ -152,7 +152,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem): make_name_unique(xml_data) descriptor = XModuleDescriptor.load_from_xml( - etree.tostring(xml_data), self, self.org, + etree.tostring(xml_data, encoding='unicode'), self, self.org, self.course, xmlstore.default_class) except Exception as err: print err, self.load_error_modules @@ -436,7 +436,7 @@ class XMLModuleStore(ModuleStoreBase): self.load_error_modules, ) - course_descriptor = system.process_xml(etree.tostring(course_data)) + course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode')) # NOTE: The descriptors end up loading somewhat bottom up, which # breaks metadata inheritance via get_children(). Instead diff --git a/common/lib/xmodule/xmodule/raw_module.py b/common/lib/xmodule/xmodule/raw_module.py index 5ff16098ac..efdd2e7ba0 100644 --- a/common/lib/xmodule/xmodule/raw_module.py +++ b/common/lib/xmodule/xmodule/raw_module.py @@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor): """ @classmethod def definition_from_xml(cls, xml_object, system): - return {'data': etree.tostring(xml_object, pretty_print=True)} + return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')} def definition_to_xml(self, resource_fs): try: diff --git a/common/lib/xmodule/xmodule/seq_module.py b/common/lib/xmodule/xmodule/seq_module.py index b625646e66..9aec275e90 100644 --- a/common/lib/xmodule/xmodule/seq_module.py +++ b/common/lib/xmodule/xmodule/seq_module.py @@ -124,7 +124,7 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor): children = [] for child in xml_object: try: - children.append(system.process_xml(etree.tostring(child)).location.url()) + children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url()) except: log.exception("Unable to load child when parsing Sequence. Continuing...") continue diff --git a/common/lib/xmodule/xmodule/stringify.py b/common/lib/xmodule/xmodule/stringify.py index 1e3fa91210..dab8ff0425 100644 --- a/common/lib/xmodule/xmodule/stringify.py +++ b/common/lib/xmodule/xmodule/stringify.py @@ -22,7 +22,7 @@ def stringify_children(node): # next element. parts = [node.text] for c in node.getchildren(): - parts.append(etree.tostring(c, with_tail=True)) + parts.append(etree.tostring(c, with_tail=True, encoding='unicode')) # filter removes possible Nones in texts and tails - return ''.join(filter(None, parts)) + return u''.join(filter(None, parts)) diff --git a/common/lib/xmodule/xmodule/template_module.py b/common/lib/xmodule/xmodule/template_module.py index 13eab038ec..d3fb0aab5e 100644 --- a/common/lib/xmodule/xmodule/template_module.py +++ b/common/lib/xmodule/xmodule/template_module.py @@ -58,7 +58,7 @@ class CustomTagDescriptor(RawDescriptor): params = dict(xmltree.items()) with system.resources_fs.open('custom_tags/{name}' .format(name=template_name)) as template: - return Template(template.read()).render(**params) + return Template(template.read().decode('utf-8')).render(**params) def __init__(self, system, definition, **kwargs): diff --git a/common/lib/xmodule/xmodule/xml_module.py b/common/lib/xmodule/xmodule/xml_module.py index ec755af4ef..bb5b44c67f 100644 --- a/common/lib/xmodule/xmodule/xml_module.py +++ b/common/lib/xmodule/xmodule/xml_module.py @@ -1,19 +1,20 @@ -from xmodule.x_module import (XModuleDescriptor, policy_key) -from xmodule.modulestore import Location -from lxml import etree import json import copy import logging -import traceback -from collections import namedtuple -from fs.errors import ResourceNotFoundError import os import sys +from collections import namedtuple +from lxml import etree + +from xmodule.x_module import (XModuleDescriptor, policy_key) +from xmodule.modulestore import Location log = logging.getLogger(__name__) +# assume all XML files are persisted as utf-8. edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False, - remove_comments=True, remove_blank_text=True) + remove_comments=True, remove_blank_text=True, + encoding='utf-8') def name_to_pathname(name): """ @@ -366,7 +367,7 @@ class XmlDescriptor(XModuleDescriptor): filepath = self.__class__._format_filepath(self.category, url_path) resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True) with resource_fs.open(filepath, 'w') as file: - file.write(etree.tostring(xml_object, pretty_print=True)) + file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8')) # And return just a pointer with the category and filename. record_object = etree.Element(self.category) @@ -381,7 +382,7 @@ class XmlDescriptor(XModuleDescriptor): record_object.set('org', self.location.org) record_object.set('course', self.location.course) - return etree.tostring(record_object, pretty_print=True) + return etree.tostring(record_object, pretty_print=True, encoding='utf-8') def definition_to_xml(self, resource_fs): """ diff --git a/common/test/data/full/about/faq.html b/common/test/data/full/about/faq.html index a5e54c9f15..a173e46753 100644 --- a/common/test/data/full/about/faq.html +++ b/common/test/data/full/about/faq.html @@ -6,7 +6,7 @@

No - anyone and everyone is welcome to take this course.

  • What textbook should I buy? -

    Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) -- Volume II, which was written specifically for this course.

    +

    Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) — Volume II, which was written specifically for this course.

  • Does Harvard award credentials or reports regarding my work in this course?

    Princeton does not award credentials or issue reports for student work in this course. However, Coursera could maintain a record of your score on the assessments and, with your permission, verify that score for authorized parties.

    diff --git a/common/test/data/full/chapter/Overview.xml b/common/test/data/full/chapter/Overview.xml index a11a11a1e0..8ad44b366c 100644 --- a/common/test/data/full/chapter/Overview.xml +++ b/common/test/data/full/chapter/Overview.xml @@ -2,7 +2,7 @@
  • OCW Problem 1-3 - Reverse engineer a black-box resistor network
    • -

    Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed.

    +

    Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. Gratuitous ≥ entity.

    diff --git a/common/test/data/full/html/html_5555.html b/common/test/data/full/html/html_5555.html index 44a015faa1..b8352b0b4f 100644 --- a/common/test/data/full/html/html_5555.html +++ b/common/test/data/full/html/html_5555.html @@ -1 +1 @@ - Lab Introduction or Interactive Lab Usage Handout for information on how to do the lab + Lab Introduction or Interactive Lab Usage Handout for information on how to do the lab. diff --git a/common/test/data/full/html/linearity_clarify.html b/common/test/data/full/html/linearity_clarify.html index 555f394c88..a349129ff8 100644 --- a/common/test/data/full/html/linearity_clarify.html +++ b/common/test/data/full/html/linearity_clarify.html @@ -34,6 +34,6 @@ the Thevenin or Norton theorems to summarize the behavior at a pair of exposed terminals.

    - Sorry for the confusion of words -- natural language is like + Sorry for the confusion of words — natural language is like that!

    diff --git a/common/test/data/full/html/linearity_clarify.xml b/common/test/data/full/html/linearity_clarify.xml index 066b22a110..400316a63e 100644 --- a/common/test/data/full/html/linearity_clarify.xml +++ b/common/test/data/full/html/linearity_clarify.xml @@ -34,6 +34,6 @@ the Thevenin or Norton theorems to summarize the behavior at a pair of exposed terminals.

    - Sorry for the confusion of words -- natural language is like + Sorry for the confusion of words — natural language is like that!

    diff --git a/common/test/data/full/html/schematic_tutorial.html b/common/test/data/full/html/schematic_tutorial.html index 991201ab15..fb0ecdaa95 100644 --- a/common/test/data/full/html/schematic_tutorial.html +++ b/common/test/data/full/html/schematic_tutorial.html @@ -9,14 +9,14 @@ the right of the diagram area) and drag it onto the diagram. Release the mouse when the component is in the correct position. - + Move a component Click to select a component in the diagram (it will turn green) and then drag it to its new location. You can use shift-click to add a component to the current selection. Or you can click somewhere in the diagram that is not on top of a component and drag out a selection -rectangle -- components intersecting the rectangle will be added to +rectangle — components intersecting the rectangle will be added to the current selection. @@ -63,7 +63,7 @@ engineeering notation: Add a wire Wires start at connection points, the open circles that appear at the terminals of components or the ends of wires. -Click on a connection point to start a wire -- a green wire +Click on a connection point to start a wire — a green wire will appear with one end anchored at the starting point. Drag the mouse and release the mouse button when the other end of the wire is positioned as you wish. Once a wire has diff --git a/common/test/data/full/html/units_hint.html b/common/test/data/full/html/units_hint.html index 02648b31e7..72e251a034 100644 --- a/common/test/data/full/html/units_hint.html +++ b/common/test/data/full/html/units_hint.html @@ -1,4 +1,4 @@ -Hint +Hint…

    Be careful of units here. Make sure you notice multipliers such -as u, k, m, M. +as u (or μ), k, m, M. diff --git a/common/test/data/full/info/updates.html b/common/test/data/full/info/updates.html index 6531ed417d..2604dc5d9e 100644 --- a/common/test/data/full/info/updates.html +++ b/common/test/data/full/info/updates.html @@ -9,8 +9,9 @@

  • May 2

  • April 30

    diff --git a/common/test/data/full/problem/Circuit_Sandbox.xml b/common/test/data/full/problem/Circuit_Sandbox.xml index 89625f447b..1582f3ff0b 100644 --- a/common/test/data/full/problem/Circuit_Sandbox.xml +++ b/common/test/data/full/problem/Circuit_Sandbox.xml @@ -1,6 +1,6 @@ -

    Here's a sandbox where you can experiment with all the components +

    Here's a sandbox where you can experiment with all the components we'll discuss in 6.002x. If you click on CHECK below, your diagram -will be saved on the server and you can return at some later time. +will be saved on the server and you can return at some later time…

    correct = ['correct']     diff --git a/common/test/data/full/problem/H1P3_Poor_Workmanship.xml b/common/test/data/full/problem/H1P3_Poor_Workmanship.xml index cf9db4053a..f32b9eb271 100644 --- a/common/test/data/full/problem/H1P3_Poor_Workmanship.xml +++ b/common/test/data/full/problem/H1P3_Poor_Workmanship.xml @@ -78,7 +78,8 @@ So the total heating power in Joe's shop was:
    -No wonder Joe was cold. + +No wonder Joe was cold… diff --git a/common/test/data/full/problem/Lab_0_Using_the_Tools.xml b/common/test/data/full/problem/Lab_0_Using_the_Tools.xml index b5f593c294..c270773da3 100644 --- a/common/test/data/full/problem/Lab_0_Using_the_Tools.xml +++ b/common/test/data/full/problem/Lab_0_Using_the_Tools.xml @@ -94,7 +94,7 @@ scope probes to nodes A, B and C and edit their properties so that the plots will be different colors. Now run a transient analysis for 5ms. Move the mouse over the plot until the marker (a vertical dashed line that follows the mouse when it's over the plot) is at approximately -1.25ms. Please report the measured voltages for nodes A, B and C. +1.25ms. Please report the measured voltages for nodes A, B and C…
    diff --git a/common/test/data/full/problem/Sample_Algebraic_Problem.xml b/common/test/data/full/problem/Sample_Algebraic_Problem.xml index 7bea1cc92e..85b9a2fcc4 100644 --- a/common/test/data/full/problem/Sample_Algebraic_Problem.xml +++ b/common/test/data/full/problem/Sample_Algebraic_Problem.xml @@ -6,7 +6,7 @@ z = "A*x^2 + sqrt(y)" Enter the algebraic expression \(A x^2 + \sqrt{y}\) in the box below. The entry is case sensitive. The product must be indicated with an asterisk, and the exponentation with a caret, so you must write -"A*x^2 + sqrt(y)". +"A*x^2 + sqrt(y)"… diff --git a/common/test/data/full/problem/Sample_Numeric_Problem.xml b/common/test/data/full/problem/Sample_Numeric_Problem.xml index f41881a028..fef9b4648c 100644 --- a/common/test/data/full/problem/Sample_Numeric_Problem.xml +++ b/common/test/data/full/problem/Sample_Numeric_Problem.xml @@ -1,6 +1,6 @@ Enter the numerical value of the expression \(x + y\) where -\(x = 3\) and \(y = 5\). +\(x = 3\) and \(y = 5\)… diff --git a/common/test/data/full/problem/choiceresponse_demo.xml b/common/test/data/full/problem/choiceresponse_demo.xml index f7d1fcf16c..7af7939d74 100644 --- a/common/test/data/full/problem/choiceresponse_demo.xml +++ b/common/test/data/full/problem/choiceresponse_demo.xml @@ -1,19 +1,20 @@ -

    Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to leftin the plane of your screen. A diagram of this situation is show below.

    +

    Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to left in the plane of your screen. A diagram of this situation is show below…

    a. The magnitude of the force experienced by the electron is proportional the product of which of the following? (Select all that apply.)

    -Magnetic field strength -Electric field strength -Electric charge of the electron -Radius of the electron -Mass of the electron -Velocity of the electron + +Magnetic field strength… +Electric field strength… +Electric charge of the electron… +Radius of the electron… +Mass of the electron… +Velocity of the electron… diff --git a/common/test/data/full/problem/codeinput_demo.xml b/common/test/data/full/problem/codeinput_demo.xml index 03d8fd8c31..a6662cb69c 100644 --- a/common/test/data/full/problem/codeinput_demo.xml +++ b/common/test/data/full/problem/codeinput_demo.xml @@ -2,7 +2,8 @@

    - Part 1: Function Types + + Part 1: Function Types…

    For each of the following functions, specify the type of its output. You can assume each function is called with an appropriate argument, as specified by its docstring.

    diff --git a/common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml b/common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml index 5c4c65f12d..26f8f5a08d 100644 --- a/common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml +++ b/common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml @@ -3,12 +3,13 @@ - S1E4 has been removed. + + S1E4 has been removed… diff --git a/common/test/data/full/vertical/vertical_89.xml b/common/test/data/full/vertical/vertical_89.xml index da15a6751a..c2b68b6bc2 100644 --- a/common/test/data/full/vertical/vertical_89.xml +++ b/common/test/data/full/vertical/vertical_89.xml @@ -1,6 +1,7 @@ -

    + +

    Inline content…