Andal.LND/edx-platform
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1100 from MITx/fix/brian/nonascii-html-xmodule

Browse Source 
Fix/brian/nonascii html xmodule
This commit is contained in:
David Ormsbee
2012-12-06 15:33:21 -08:00
parent 1d9d347df9 70f35b2e15
commit 61c17cabfa
37 changed files with 136 additions and 151 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
common/djangoapps/mitxmako/shortcuts.py
View File

@@ -12,10 +12,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
log = logging.getLogger("mitx." + __name__)
from django.template import Context
from django.http import HttpResponse
@@ -42,7 +38,7 @@ def render_to_string(template_name, dictionary, context=None, namespace='main'):
context_dictionary.update(context)
# fetch and render template
template = middleware.lookup[namespace].get_template(template_name)
return template.render(**context_dictionary)
return template.render_unicode(**context_dictionary)
def render_to_response(template_name, dictionary, context_instance=None, namespace='main', **kwargs):

3
common/djangoapps/mitxmako/template.py
View File

@@ -54,5 +54,4 @@ class Template(MakoTemplate):
context_dictionary['MITX_ROOT_URL'] = settings.MITX_ROOT_URL
context_dictionary['django_context'] = context_instance
return super(Template, self).render(**context_dictionary)
return super(Template, self).render_unicode(**context_dictionary)

4
common/lib/xmodule/xmodule/error_module.py
View File

@@ -149,14 +149,14 @@ class ErrorDescriptor(JSONEditingDescriptor):
'''
try:
xml = etree.fromstring(self.definition['data']['contents'])
return etree.tostring(xml)
return etree.tostring(xml, encoding='unicode')
except etree.XMLSyntaxError:
# still not valid.
root = etree.Element('error')
root.text = self.definition['data']['contents']
err_node = etree.SubElement(root, 'error_msg')
err_node.text = self.definition['data']['error_msg']
return etree.tostring(root)
return etree.tostring(root, encoding='unicode')
class NonStaffErrorDescriptor(ErrorDescriptor):

17
common/lib/xmodule/xmodule/html_module.py
View File

@@ -7,15 +7,14 @@ from lxml import etree
from lxml.html import rewrite_links
from path import path
from .x_module import XModule
from pkg_resources import resource_string
from .xml_module import XmlDescriptor, name_to_pathname
from .editing_module import EditingDescriptor
from .stringify import stringify_children
from .html_checker import check_html
from xmodule.modulestore import Location
from xmodule.contentstore.content import XASSET_SRCREF_PREFIX, StaticContent
from xmodule.editing_module import EditingDescriptor
from xmodule.html_checker import check_html
from xmodule.modulestore import Location
from xmodule.stringify import stringify_children
from xmodule.x_module import XModule
from xmodule.xml_module import XmlDescriptor, name_to_pathname
log = logging.getLogger("mitx.courseware")
@@ -123,7 +122,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
try:
with system.resources_fs.open(filepath) as file:
html = file.read()
html = file.read().decode('utf-8')
# Log a warning if we can't parse the file, but don't error
if not check_html(html):
msg = "Couldn't parse html in {0}.".format(filepath)
@@ -164,7 +163,7 @@ class HtmlDescriptor(XmlDescriptor, EditingDescriptor):
resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(self.definition['data'])
file.write(self.definition['data'].encode('utf-8'))
# write out the relative name
relname = path(pathname).basename()

4
common/lib/xmodule/xmodule/modulestore/xml.py
View File

@@ -152,7 +152,7 @@ class ImportSystem(XMLParsingSystem, MakoDescriptorSystem):
make_name_unique(xml_data)
descriptor = XModuleDescriptor.load_from_xml(
etree.tostring(xml_data), self, self.org,
etree.tostring(xml_data, encoding='unicode'), self, self.org,
self.course, xmlstore.default_class)
except Exception as err:
print err, self.load_error_modules
@@ -436,7 +436,7 @@ class XMLModuleStore(ModuleStoreBase):
self.load_error_modules,
)
course_descriptor = system.process_xml(etree.tostring(course_data))
course_descriptor = system.process_xml(etree.tostring(course_data, encoding='unicode'))
# NOTE: The descriptors end up loading somewhat bottom up, which
# breaks metadata inheritance via get_children(). Instead

2
common/lib/xmodule/xmodule/raw_module.py
View File

@@ -13,7 +13,7 @@ class RawDescriptor(XmlDescriptor, XMLEditingDescriptor):
"""
@classmethod
def definition_from_xml(cls, xml_object, system):
return {'data': etree.tostring(xml_object, pretty_print=True)}
return {'data': etree.tostring(xml_object, pretty_print=True,encoding='unicode')}
def definition_to_xml(self, resource_fs):
try:

2
common/lib/xmodule/xmodule/seq_module.py
View File

@@ -124,7 +124,7 @@ class SequenceDescriptor(MakoModuleDescriptor, XmlDescriptor):
children = []
for child in xml_object:
try:
children.append(system.process_xml(etree.tostring(child)).location.url())
children.append(system.process_xml(etree.tostring(child, encoding='unicode')).location.url())
except:
log.exception("Unable to load child when parsing Sequence. Continuing...")
continue

4
common/lib/xmodule/xmodule/stringify.py
View File

@@ -22,7 +22,7 @@ def stringify_children(node):
# next element.
parts = [node.text]
for c in node.getchildren():
parts.append(etree.tostring(c, with_tail=True))
parts.append(etree.tostring(c, with_tail=True, encoding='unicode'))
# filter removes possible Nones in texts and tails
return ''.join(filter(None, parts))
return u''.join(filter(None, parts))

2
common/lib/xmodule/xmodule/template_module.py
View File

@@ -58,7 +58,7 @@ class CustomTagDescriptor(RawDescriptor):
params = dict(xmltree.items())
with system.resources_fs.open('custom_tags/{name}'
.format(name=template_name)) as template:
return Template(template.read()).render(**params)
return Template(template.read().decode('utf-8')).render(**params)
def __init__(self, system, definition, **kwargs):

19
common/lib/xmodule/xmodule/xml_module.py
View File

@@ -1,19 +1,20 @@
from xmodule.x_module import (XModuleDescriptor, policy_key)
from xmodule.modulestore import Location
from lxml import etree
import json
import copy
import logging
import traceback
from collections import namedtuple
from fs.errors import ResourceNotFoundError
import os
import sys
from collections import namedtuple
from lxml import etree
from xmodule.x_module import (XModuleDescriptor, policy_key)
from xmodule.modulestore import Location
log = logging.getLogger(__name__)
# assume all XML files are persisted as utf-8.
edx_xml_parser = etree.XMLParser(dtd_validation=False, load_dtd=False,
remove_comments=True, remove_blank_text=True)
remove_comments=True, remove_blank_text=True,
encoding='utf-8')
def name_to_pathname(name):
"""
@@ -366,7 +367,7 @@ class XmlDescriptor(XModuleDescriptor):
filepath = self.__class__._format_filepath(self.category, url_path)
resource_fs.makedir(os.path.dirname(filepath), allow_recreate=True)
with resource_fs.open(filepath, 'w') as file:
file.write(etree.tostring(xml_object, pretty_print=True))
file.write(etree.tostring(xml_object, pretty_print=True, encoding='utf-8'))
# And return just a pointer with the category and filename.
record_object = etree.Element(self.category)
@@ -381,7 +382,7 @@ class XmlDescriptor(XModuleDescriptor):
record_object.set('org', self.location.org)
record_object.set('course', self.location.course)
return etree.tostring(record_object, pretty_print=True)
return etree.tostring(record_object, pretty_print=True, encoding='utf-8')
def definition_to_xml(self, resource_fs):
"""

2
common/test/data/full/about/faq.html
View File

@@ -6,7 +6,7 @@
<p>No - anyone and everyone is welcome to take this course.</p>
</li>
<li>What textbook should I buy?
<p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) -- Volume II, which was written specifically for this course.</p>
<p>Although the lectures are designed to be self-contained, we recommend (but do not require) that students refer to the book Worlds Together, Worlds Apart: A History of the World: From 1000 CE to the Present (W W Norton, 3rd edition) &mdash; Volume II, which was written specifically for this course.</p>
</li>
<li>Does Harvard award credentials or reports regarding my work in this course?
<p>Princeton does not award credentials or issue reports for student work in this course. However, Coursera could maintain a record of your score on the assessments and, with your permission, verify that score for authorized parties.</p>

2
common/test/data/full/chapter/Overview.xml
View File

@@ -2,7 +2,7 @@
<video url_name="welcome"/>
<sequential filename="System_Usage_Sequence" slug="System_Usage_Sequence" format="Lecture Sequence" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="System Usage Sequence"/>
<vertical slug="Lab0_Using_the_tools" format="Lab" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never" name="Lab0: Using the tools">
<html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
<html slug="html_19" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab… </html>
<html slug="html_5555" filename="html_5555"/>
<problem filename="Lab_0_Using_the_Tools" slug="Lab_0_Using_the_Tools" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="false" name="Lab 0: Using the Tools"/>
</vertical>

2
common/test/data/full/custom_tags/book
View File

@@ -1 +1 @@
More information given in <a href="/book/${page}">the text</a>.
More information given in… <a href="/book/${page}">the text</a>.

2
common/test/data/full/custom_tags/discuss
View File

@@ -1 +1 @@
<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag} </a>
<a href='https://6002x.mitx.mit.edu/discussion/questions/scope:all/sort:activity-desc/tags:${tag}/page:1/'> Discussion: ${tag}… </a>

2
common/test/data/full/custom_tags/slides
View File

@@ -1 +1 @@
Lecture Slides Handout [<a href="">Clean </a>][<a href="">Annotated</a>]
Lecture Slides Handout [<a href="">Clean… </a>][<a href="">Annotated…</a>]

2
common/test/data/full/html/FirstOrderSystemHint.xml
View File

@@ -1,4 +1,4 @@
Hint
Hint…
<br/><br/>
Remember that the time evolution of any variable \(x(t)\) governed by
a first-order system with a time-constant \(\tau\) for a time \(t) between an initial

4
common/test/data/full/html/Week_13_Tutorials.html
View File

@@ -4,14 +4,14 @@
<section class="tutorials">
<h2> Basic Tutorials </h2>
<ul>
<li><a href="/section/wk13_solder">Soldering</a> -- Steve
<li><a href="/section/wk13_solder">Soldering</a> &mdash; Steve
Finberg, one of the pioneers in from Draper Lab, talks about
soldering. </li>
</ul>
<h2> Bonus Tutorials </h2>
<ul>
<li><a href="/section/wk13_FreqResp">Frequency Response
Curves</a> -- We explain several techniques for understanding
Curves</a> &mdash; We explain several techniques for understanding
and approximating Bode plots. </li>
</ul>
</section>

2
common/test/data/full/html/Week_1_Tutorials.html
View File

@@ -41,7 +41,7 @@
<li><a href="/section/problem_1_3">OCW Problem 1-3 </a> - Reverse engineer a black-box resistor network</li>
</ul>
<hr/>
<p> Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. </p>
<p> Since the course has students from a diverse set of backgrounds, the first week's tutorials includes several extra segments, worked out with greater detail, to help bring everyone up to speed. Gratuitous &ge; entity.</p>
</section>
</body>
</html>

2
common/test/data/full/html/html_5555.html
View File

@@ -1 +1 @@
<html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab </html>
<html slug="html_5555" filename="html_5555> See the <a href="/section/labintro"> Lab Introduction </a> or <a href="/static/handouts/schematic_tutorial.pdf">Interactive Lab Usage Handout </a> for information on how to do the lab. </html>

2
common/test/data/full/html/linearity_clarify.html
View File

@@ -34,6 +34,6 @@
the Thevenin or Norton theorems to summarize the behavior at
a pair of exposed terminals.
</p><p>
Sorry for the confusion of words -- natural language is like
Sorry for the confusion of words &mdash; natural language is like
that!
</p>

2
common/test/data/full/html/linearity_clarify.xml
View File

@@ -34,6 +34,6 @@
the Thevenin or Norton theorems to summarize the behavior at
a pair of exposed terminals.
</p><p>
Sorry for the confusion of words -- natural language is like
Sorry for the confusion of words &mdash; natural language is like
that!
</p>

6
common/test/data/full/html/schematic_tutorial.html
View File

@@ -9,14 +9,14 @@ the right of the diagram area) and drag it onto the diagram. Release
the mouse when the component is in the correct position.
</td>
</tr>
<!-- note that entities like &mdash; may be used. -->
<tr>
<td>Move a component</td>
<td>Click to select a component in the diagram (it will turn green)
and then drag it to its new location. You can use shift-click to add
a component to the current selection. Or you can click somewhere in
the diagram that is not on top of a component and drag out a selection
rectangle -- components intersecting the rectangle will be added to
rectangle &mdash; components intersecting the rectangle will be added to
the current selection.
</td>
</tr>
@@ -63,7 +63,7 @@ engineeering notation:
<td>Add a wire</td>
<td>Wires start at connection points, the open circles that
appear at the terminals of components or the ends of wires.
Click on a connection point to start a wire -- a green wire
Click on a connection point to start a wire &mdash; a green wire
will appear with one end anchored at the starting point.
Drag the mouse and release the mouse button when the other
end of the wire is positioned as you wish. Once a wire has

4
common/test/data/full/html/units_hint.html
View File

@@ -1,4 +1,4 @@
Hint
Hint…
<br/><br/>
Be careful of units here. Make sure you notice multipliers such
as u, k, m, M.
as u (or &mu;), k, m, M.

5
common/test/data/full/info/updates.html
View File

@@ -9,8 +9,9 @@
<li> <h2>May 2 </h2>
<section class="update-description">
<ul>
<li> We have opened the show-answer button on the midterm. </li>
<li> There was a four hour outage in posting ability on the discussion board Monday night. It has been fixed. We apologise for the inconvenience.</li>
<!-- utf-8 characters are acceptable… as are HTML entities -->
<li> We have opened the show-answer button on the midterm… </li>
<li> There was a four hour outage in posting ability on the discussion board Monday night&hellip; It has been fixed. We apologise for the inconvenience.</li>
</ul>
</li>
<li> <h2>April 30 </h2>

4
common/test/data/full/problem/Circuit_Sandbox.xml
View File

@@ -1,6 +1,6 @@
<problem><startouttext/><p/>Here's a sandbox where you can experiment with all the components
<problem><!-- include ellipses to test non-ascii characters --><startouttext/><p/>Here's a sandbox where you can experiment with all the components
we'll discuss in 6.002x. If you click on CHECK below, your diagram
will be saved on the server and you can return at some later time.
will be saved on the server and you can return at some later time…
<endouttext/><schematicresponse><p/><center><schematic name="work" value="" width="800" height="600"/></center><answer type="loncapa/python">
correct = ['correct']
</answer></schematicresponse></problem>

3
common/test/data/full/problem/H1P3_Poor_Workmanship.xml
View File

@@ -78,7 +78,8 @@ So the total heating power in Joe's shop was:
<numericalresponse answer="$Pbad"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>
<startouttext/>
<br/>
No wonder Joe was cold.
<!-- add non-ascii utf-8 character here -->
No wonder Joe was cold…
<endouttext/>
</problem>

2
common/test/data/full/problem/Lab_0_Using_the_Tools.xml
View File

@@ -94,7 +94,7 @@ scope probes to nodes A, B and C and edit their properties so that the
plots will be different colors. Now run a transient analysis for 5ms.
Move the mouse over the plot until the marker (a vertical dashed line
that follows the mouse when it's over the plot) is at approximately
1.25ms. Please report the measured voltages for nodes A, B and C.
1.25ms. Please report the measured voltages for nodes A, B and C…
<br/>
<div style="margin-left: 4em;">

2
common/test/data/full/problem/Sample_Algebraic_Problem.xml
View File

@@ -6,7 +6,7 @@ z = "A*x^2 + sqrt(y)"
Enter the algebraic expression \(A x^2 + \sqrt{y}\) in the box below. The
entry is case sensitive. The product must be indicated with an
asterisk, and the exponentation with a caret, so you must write
"A*x^2 + sqrt(y)".
"A*x^2 + sqrt(y)"…
<endouttext/>
<formularesponse type="cs" samples="A,x,y@1,1,1:3,3,3#10" answer="$z"><responseparam description="Numerical Tolerance" type="tolerance" default="0.00001" name="tol"/><textline size="40"/></formularesponse>

2
common/test/data/full/problem/Sample_Numeric_Problem.xml
View File

@@ -1,6 +1,6 @@
<problem><startouttext/>
Enter the numerical value of the expression \(x + y\) where
\(x = 3\) and \(y = 5\).
\(x = 3\) and \(y = 5\)…
<endouttext/>
<numericalresponse answer="8"><responseparam type="tolerance" default="5%" name="tol" description="Numerical Tolerance"/><textline/></numericalresponse>

15
common/test/data/full/problem/choiceresponse_demo.xml
View File

@@ -1,19 +1,20 @@
<problem display_name="S3E2: Lorentz Force">
<startouttext/>
<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to leftin the plane of your screen. A diagram of this situation is show below.</p>
<p>Consider a hypothetical magnetic field pointing out of your computer screen. Now imagine an electron traveling from right to left in the plane of your screen. A diagram of this situation is show below…</p>
<center><img width="400" src="/static/images/LSQimages/LSQ_W01_8.png"/></center>
<p>a. The magnitude of the force experienced by the electron is proportional the product of which of the following? (Select all that apply.)</p>
<endouttext/>
<choiceresponse>
<checkboxgroup>
<choice correct="true"><text>Magnetic field strength</text></choice>
<choice correct="false"><text>Electric field strength</text></choice>
<choice correct="true"><text>Electric charge of the electron</text></choice>
<choice correct="false"><text>Radius of the electron</text></choice>
<choice correct="false"><text>Mass of the electron</text></choice>
<choice correct="true"><text>Velocity of the electron</text></choice>
<!-- include ellipses to test non-ascii characters -->
<choice correct="true"><text>Magnetic field strength…</text></choice>
<choice correct="false"><text>Electric field strength…</text></choice>
<choice correct="true"><text>Electric charge of the electron…</text></choice>
<choice correct="false"><text>Radius of the electron…</text></choice>
<choice correct="false"><text>Mass of the electron…</text></choice>
<choice correct="true"><text>Velocity of the electron…</text></choice>
</checkboxgroup>
</choiceresponse>

3
common/test/data/full/problem/codeinput_demo.xml
View File

@@ -2,7 +2,8 @@
<problem display_name="L4 Problem 1">
<text>
<p>
<b class="bfseries">Part 1: Function Types</b>
<!-- include ellipses to test non-ascii characters -->
<b class="bfseries">Part 1: Function Types…</b>
</p>
<p>
For each of the following functions, specify the type of its <b class="bfseries">output</b>. You can assume each function is called with an appropriate argument, as specified by its docstring. </p>

5
common/test/data/full/sequential/Administrivia_and_Circuit_Elements.xml
View File

@@ -3,12 +3,13 @@
<vertical slug="vertical_66" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
<problem filename="S1E3_AC_power" slug="S1E3_AC_power" name="S1E3: AC power"/>
<customtag tag="S1E3" slug="discuss_67" impl="discuss"/>
<html slug="html_68"> S1E4 has been removed. </html>
<!-- utf-8 characters acceptable, but not HTML entities -->
<html slug="html_68"> S1E4 has been removed…</html>
</vertical>
<vertical filename="vertical_89" slug="vertical_89" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never"/>
<vertical slug="vertical_94" graceperiod="1 day 12 hours 59 minutes 59 seconds" showanswer="attempted" rerandomize="never">
<video youtube="0.75:XNh13VZhThQ,1.0:XbDRmF6J0K0,1.25:JDty12WEQWk,1.50:wELKGj-5iyM" slug="What_s_next" name="What's next"/>
<html slug="html_95">Minor correction: Six elements (five resistors)</html>
<html slug="html_95">Minor correction: Six elements (five resistors)…</html>
<customtag tag="S1" slug="discuss_96" impl="discuss"/>
</vertical>
</sequential>

3
common/test/data/full/vertical/vertical_89.xml
View File

@@ -1,6 +1,7 @@
<sequential>
<html slug="html_90">
<h1> </h1>
<!-- UTF-8 characters are acceptable… HTML entities are not -->
<h1>Inline content…</h1>
</html>
<video youtube="1.50:vl9xrfxcr38,1.25:qxNX4REGqx4,1.0:BGU1poJDgOY,0.75:8rK9vnpystQ" slug="S1V14_Summary" name="S1V14: Summary"/>
<customtag tag="S1" slug="discuss_91" impl="discuss"/>

2
common/test/data/full/video/welcome.xml
View File

@@ -1 +1 @@
<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome"/>
<video youtube="0.75:izygArpw-Qo,1.0:p2Q6BrNhdh8,1.25:1EeWXzPdhSA,1.50:rABDYkeK0x8" format="Video" display_name="Welcome…"/>

10
common/test/data/toy/html/secret/toylab.html
View File

@@ -1,3 +1,13 @@
<b>Lab 2A: Superposition Experiment</b>
<<<<<<< Updated upstream
<p>Isn't the toy course great?</p>
<p>Let's add some markup that uses non-ascii characters.
For example, we should be able to write words like encyclop&aelig;dia, or foreign words like fran&ccedil;ais.
Looking beyond latin-1, we should handle math symbols: &pi;r&sup2 &le; &#8734.
And it shouldn't matter if we use entities or numeric codes &mdash; &Omega; &ne; &pi; &equiv; &#937; &#8800; &#960;.
</p>
=======
<p>Isn't the toy course great? — &le;</p>
>>>>>>> Stashed changes