Add tests for imports with unicode filenames

common/lib/xmodule/xmodule/modulestore/tests/test_mongo.py

@@ -4,6 +4,7 @@ from nose.tools import assert_equals, assert_raises, \
     assert_not_equals, assert_false
 # pylint: enable=E0611
 import pymongo
+import logging
 from uuid import uuid4
 
 from xblock.fields import Scope
@@ -19,6 +20,7 @@ from xmodule.contentstore.mongo import MongoContentStore
 
 from xmodule.modulestore.tests.test_modulestore import check_path_to_location
 
+log = logging.getLogger(__name__)
 
 HOST = 'localhost'
 PORT = 27017
@@ -59,7 +61,7 @@ class TestMongoModuleStore(object):
         #
         draft_store = DraftModuleStore(HOST, DB, COLLECTION, FS_ROOT, RENDER_TEMPLATE, default_class=DEFAULT_CLASS)
         # Explicitly list the courses to load (don't want the big one)
-        courses = ['toy', 'simple', 'simple_with_draft']
+        courses = ['toy', 'simple', 'simple_with_draft', 'test_unicode']
         import_from_xml(store, DATA_DIR, courses, draft_store=draft_store, static_content_store=content_store)
 
         # also test a course with no importing of static content
@@ -100,12 +102,14 @@ class TestMongoModuleStore(object):
     def test_get_courses(self):
         '''Make sure the course objects loaded properly'''
         courses = self.store.get_courses()
-        assert_equals(len(courses), 4)
+        assert_equals(len(courses), 5)
         courses.sort(key=lambda c: c.id)
         assert_equals(courses[0].id, 'edX/simple/2012_Fall')
         assert_equals(courses[1].id, 'edX/simple_with_draft/2012_Fall')
         assert_equals(courses[2].id, 'edX/test_import_course/2012_Fall')
-        assert_equals(courses[3].id, 'edX/toy/2012_Fall')
+        assert_equals(courses[3].id, 'edX/test_unicode/2012_Fall')
+        assert_equals(courses[4].id, 'edX/toy/2012_Fall')
+        log.debug(str(courses))
 
     def test_loads(self):
         assert_not_equals(
@@ -120,6 +124,22 @@ class TestMongoModuleStore(object):
             self.store.get_item("i4x://edX/toy/video/Welcome"),
             None)
 
+    def test_unicode_loads(self):
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/course/2012_Fall"),
+            None)
+        # All items with ascii-only filenames should load properly.
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
+            None)
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/video/Welcome"),
+            None)
+        assert_not_equals(
+            self.store.get_item("i4x://edX/test_unicode/chapter/Overview"),
+            None)
+
+
     def test_find_one(self):
         assert_not_equals(
             self.store._find_one(Location("i4x://edX/toy/course/2012_Fall")),
@@ -159,9 +179,9 @@ class TestMongoModuleStore(object):
             """
             Helper function for pulling out the name of a given static tab.
 
-            Assumes the information is desired for courses[1] ('toy' course).
+            Assumes the information is desired for courses[4] ('toy' course).
             """
-            return courses[2].tabs[index]['name']
+            return courses[4].tabs[index]['name']
 
         # There was a bug where model.save was not getting called after the static tab name
         # was set set for tabs that have a URL slug. 'Syllabus' and 'Resources' fall into that

common/lib/xmodule/xmodule/tests/test_import.py

@@ -369,6 +369,45 @@ class ImportTestCase(BaseCourseTestCase):
         html = modulestore.get_instance(course_id, loc)
         self.assertEquals(html.display_name, "Toy lab")
 
+    def test_unicode(self):
+        """Check that courses with unicode characters in filenames and in
+        org/course/name import properly. Currently, this means: (a) Having
+        files with unicode names does not prevent import; (b) if files are not
+        loaded because of unicode filenames, there are appropriate
+        exceptions/errors to that effect."""
+
+        print("Starting import")
+        modulestore = XMLModuleStore(DATA_DIR, course_dirs=['test_unicode'])
+        courses = modulestore.get_courses()
+        self.assertEquals(len(courses), 1)
+        course = courses[0]
+        course_id = course.id
+
+        print("course errors:")
+
+        # Expect to find an error/exception about characters in "®esources"
+        found = False
+        expect = "Invalid characters in '®esources'"
+        for (msg, err) in modulestore.get_item_errors(course.location):
+            msg_u = msg.encode("utf-8")
+            err_u = err.encode("utf-8")
+            print(msg_u)
+            print(err_u)
+            if max(msg_u.find(expect), err_u.find(expect)) >= 1:
+                found = True
+
+        self.assertTrue(found)
+        chapters = course.get_children()
+        self.assertEquals(len(chapters), 5)
+
+        ch2 = chapters[1]
+        self.assertEquals(ch2.url_name, "secret:magic")
+
+        print("Ch2 location: ", ch2.location)
+
+        also_ch2 = modulestore.get_instance(course_id, ch2.location)
+        self.assertEquals(ch2, also_ch2)
+
     def test_url_name_mangling(self):
         """
         Make sure that url_names are only mangled once.

common/test/data/test_unicode/about/end_date.html

@@ -0,0 +1 @@
+TBD

common/test/data/test_unicode/chapter/handout_container.xml

@@ -0,0 +1,3 @@
+<sequential>
+    <html> <a href="/static/handouts/sample_handout.txt"> handouts</a> </html>
+</sequential>

common/test/data/test_unicode/chapter/poll_test.xml

@@ -0,0 +1,7 @@
+<sequential>
+	<poll_question name="T1_changemind_poll_foo" display_name="Change your answer" reset="false">
+        <p>Have you changed your mind?</p>
+        <answer id="yes">Yes</answer>
+        <answer id="no">No</answer>
+    </poll_question>
+</sequential>

common/test/data/test_unicode/chapter/secret/magic.xml

@@ -0,0 +1,3 @@
+<chapter>
+    <video url_name="toyvideo" youtube_id_1_0="OEoXaMPEzfMA" display_name="toyvideo"/>
+</chapter>

common/test/data/test_unicode/course.xml

@@ -0,0 +1 @@
+<course org="edX" course="test_unicode" url_name="2012_Fall"/>

common/test/data/test_unicode/course/2012_Fall.xml

@@ -0,0 +1,20 @@
+<course>
+    <textbook title="Textbook" book_url="https://s3.amazonaws.com/edx-textbooks/guttag_computation_v3/"/>
+  <chapter display_name="Overview">
+    <videosequence url_name="Toy_Videos">
+      <html url_name="secret:toylab"/>
+      <html url_name="toyjumpto"/>
+      <html url_name="toyhtml"/>
+      <html url_name="nonportable"/>
+      <html url_name="nonportable_link"/>
+      <video url_name="Video_Resources" youtube_id_1_0="1bK-WdDi6Qw" display_name="Video Resources"/>
+    </videosequence>
+    <video url_name="Welcome" youtube_id_1_0="p2Q6BrNhdh8" display_name="Welcome"/>
+    <video url_name="video_123456789012" youtube_id_1_0="p2Q6BrNhdh8" display_name='Test Video'/>
+    <video url_name="video_4f66f493ac8f" youtube_id_1_0="p2Q6BrNhdh8"/>
+  </chapter>
+  <chapter url_name="secret:magic"/>
+  <chapter url_name="poll_test"/>
+  <chapter url_name="vertical_container"/>
+  <chapter url_name="handout_container"/>
+</course>

common/test/data/test_unicode/info/handouts.html

@@ -0,0 +1 @@
+<a href='/static/handouts/sample_handout.txt'>Sample</a>

common/test/data/test_unicode/policies/2012_Fall.json

@@ -0,0 +1,33 @@
+{
+    "course/2012_Fall": {
+        "graceperiod": "2 days 5 hours 59 minutes 59 seconds",
+        "start": "2015-07-17T12:00",
+        "display_name": "Toy Course",
+        "graded": "true",
+	"tabs": [
+          {"type": "courseware"},
+          {"type": "course_info", "name": "Course Info"},
+          {"type": "static_tab", "url_slug": "syllabus", "name": "ßyllabus"},
+          {"type": "static_tab", "url_slug": "resources", "name": "Resources"},
+          {"type": "discussion", "name": "∂iscussion"},
+          {"type": "wiki", "name": "∑iki"},
+          {"type": "progress", "name": "πrogress"}
+        ]
+    },
+    "chapter/Overview": {
+        "display_name": "O√erview"
+    },
+    "videosequence/Toy_Videos": {
+        "display_name": "†oy Videos",
+        "format": "Lecture Sequence"
+    },
+    "html/secret:toylab": {
+        "display_name": "Toy lab"
+    },
+    "video/Video_Resources": {
+        "display_name": "Video Resources"
+    },
+    "video/Welcome": {
+        "display_name": "Welcome"
+    }
+}

common/test/data/test_unicode/sequential/vertical_sequential.xml

@@ -0,0 +1,4 @@
+<sequential>
+    <vertical filename="vertical_test" slug="vertical_test" />
+    <html slug="unicode">…</html>
+</sequential>

common/test/data/test_unicode/static/unicø∂é.txt

@@ -0,0 +1,13 @@
+μῆνιν ἄειδε θεὰ Πηληϊάδεω Ἀχιλῆος
+οὐλομένην, ἣ μυρί᾽ Ἀχαιοῖς ἄλγε᾽ ἔθηκε,
+πολλὰς δ᾽ ἰφθίμους ψυχὰς Ἄϊδι προΐαψεν
+ἡρώων, αὐτοὺς δὲ ἑλώρια τεῦχε κύνεσσιν
+5οἰωνοῖσί τε πᾶσι, Διὸς δ᾽ ἐτελείετο βουλή,
+ἐξ οὗ δὴ τὰ πρῶτα διαστήτην ἐρίσαντε
+Ἀτρεΐδης τε ἄναξ ἀνδρῶν καὶ δῖος Ἀχιλλεύς.
+τίς τ᾽ ἄρ σφωε θεῶν ἔριδι ξυνέηκε μάχεσθαι;
+Λητοῦς καὶ Διὸς υἱός: ὃ γὰρ βασιλῆϊ χολωθεὶς
+10νοῦσον ἀνὰ στρατὸν ὄρσε κακήν, ὀλέκοντο δὲ λαοί,
+οὕνεκα τὸν Χρύσην ἠτίμασεν ἀρητῆρα
+Ἀτρεΐδης: ὃ γὰρ ἦλθε θοὰς ἐπὶ νῆας Ἀχαιῶν
+λυσόμενός τε θύγατρα φέρων τ᾽ ἀπερείσι᾽ ἄποινα,

common/test/data/test_unicode/static_import/∫hould_be_imπorted.html

@@ -0,0 +1 @@
+<p>this ƒîlë should be in the contentstore</p>

common/test/data/test_unicode/tabs/syllabus.html

@@ -0,0 +1 @@
+<h1>This is a syllabus</h1>

common/test/data/test_unicode/tabs/®esources.html

@@ -0,0 +1 @@
+<div>resources!</div>

common/test/data/test_unicode/vertical/vertical_test.xml

@@ -0,0 +1,3 @@
+<vertical>
+    <video url_name="separate_file_video"/>
+</vertical>

common/test/data/test_unicode/video/a_video.xml

@@ -0,0 +1 @@
+<video display_name="default" youtube_id_0_75="JMD_ifUUfsU" youtube_id_1_0="OEoXaMPEzfM" youtube_id_1_25="AKqURZnYqpk" youtube_id_1_5="DYpADpL7jAY" name="sample_video"/>