Merge pull request #2925 from edx/zub/bugfix/std1433-importunicodename

use unicode strings for slug and regex

cms/djangoapps/contentstore/tests/test_import.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # pylint: disable=E1101
 """
 Tests for import_from_xml using the mongo modulestore.
@@ -77,6 +78,25 @@ class ContentStoreImportTest(ModuleStoreTestCase):
 
         return module_store, content_store, course, course_location
 
+    def test_unicode_chars_in_course_name_import(self):
+        """
+        # Test that importing course with unicode 'id' and 'display name' doesn't give UnicodeEncodeError
+        """
+        module_store = modulestore('direct')
+        target_location = Location(['i4x', u'Юникода', 'unicode_course', 'course', u'échantillon'])
+        import_from_xml(
+            module_store,
+            'common/test/data/',
+            ['2014_Uni'],
+            target_location_namespace=target_location
+        )
+
+        course = module_store.get_item(target_location)
+        self.assertIsNotNone(course)
+
+        # test that course 'display_name' same as imported course 'display_name'
+        self.assertEqual(course.display_name, u"Φυσικά το όνομα Unicode")
+
     def test_static_import(self):
         '''
         Stuff in static_import should always be imported into contentstore

cms/djangoapps/contentstore/views/course.py

@@ -389,7 +389,7 @@ def create_new_course(request):
 
     # Set a unique wiki_slug for newly created courses. To maintain active wiki_slugs for existing xml courses this
     # cannot be changed in CourseDescriptor.
-    wiki_slug = "{0}.{1}.{2}".format(dest_location.org, dest_location.course, dest_location.name)
+    wiki_slug = u"{0}.{1}.{2}".format(dest_location.org, dest_location.course, dest_location.name)
     definition_data = {'wiki_slug': wiki_slug}
 
     modulestore('direct').create_and_save_xmodule(

common/lib/xmodule/xmodule/modulestore/store_utilities.py

@@ -12,7 +12,7 @@ def _prefix_only_url_replace_regex(prefix):
     To anyone contemplating making this more complicated:
     http://xkcd.com/1171/
     """
-    return r"""
+    return ur"""
         (?x)                      # flags=re.VERBOSE
         (?P<quote>\\?['"])        # the opening quotes
         (?P<prefix>{prefix})      # the prefix
@@ -28,7 +28,7 @@ def _prefix_and_category_url_replace_regex(prefix):
     To anyone contemplating making this more complicated:
     http://xkcd.com/1171/
     """
-    return r"""
+    return ur"""
         (?x)                      # flags=re.VERBOSE
         (?P<quote>\\?['"])        # the opening quotes
         (?P<prefix>{prefix})      # the prefix

common/lib/xmodule/xmodule/modulestore/xml_importer.py

@@ -312,7 +312,7 @@ def import_module(
         source_course_location, dest_course_location, allow_not_found=False,
         do_import_static=True):
 
-    logging.debug('processing import of module {}...'.format(module.location.url()))
+    logging.debug(u'processing import of module {}...'.format(module.location.url()))
 
     if do_import_static and 'data' in module.fields and isinstance(module.fields['data'], xblock.fields.String):
         # we want to convert all 'non-portable' links in the module_data
@@ -518,13 +518,13 @@ def remap_namespace(module, target_location_namespace):
         # If we are importing into a course with a different course_id and wiki_slug is equal to either of these default
         # values then remap it so that the wiki does not point to the old wiki.
         if original_location.course_id != target_location_namespace.course_id:
-            original_unique_wiki_slug = '{0}.{1}.{2}'.format(
+            original_unique_wiki_slug = u'{0}.{1}.{2}'.format(
                 original_location.org,
                 original_location.course,
                 original_location.name
             )
             if module.wiki_slug == original_unique_wiki_slug or module.wiki_slug == original_location.course:
-                module.wiki_slug = '{0}.{1}.{2}'.format(
+                module.wiki_slug = u'{0}.{1}.{2}'.format(
                     target_location_namespace.org,
                     target_location_namespace.course,
                     target_location_namespace.name,

common/test/data/2014_Uni/course.xml

@@ -0,0 +1 @@
+<course url_name="2014_Uni" org="&#1070;&#1085;&#1080;&#1082;&#1086;&#1076;&#1072;" course="&#233;chantillon"/>

common/test/data/2014_Uni/course/2014_Uni.xml

@@ -0,0 +1,3 @@
+<course display_name="Φυσικά το όνομα Unicode">
+  <wiki slug="Юникода.échantillon.2014_Uni"/>
+</course>

common/test/data/2014_Uni/policies/2014_Uni/policy.json

@@ -0,0 +1 @@
+{"course/2014_Uni": {"tabs": [{"type": "courseware", "name": "Courseware"}, {"type": "course_info", "name": "Course Info"}, {"type": "discussion", "name": "Discussion"}, {"type": "wiki", "name": "Wiki"}, {"type": "progress", "name": "Progress"}], "display_name": "\u03a6\u03c5\u03c3\u03b9\u03ba\u03ac \u03c4\u03bf \u03cc\u03bd\u03bf\u03bc\u03b1 Unicode", "discussion_topics": {"General": {"id": "i4x-\u042e\u043d\u0438\u043a\u043e\u0434\u0430-\u00e9chantillon-course-2014_Uni"}}}}

common/test/data/2014_Uni/policies/assets.json

@@ -0,0 +1 @@
+{}

lms/djangoapps/dashboard/tests/test_sysadmin.py

@@ -159,13 +159,13 @@ class TestSysadmin(SysadminBaseTestCase):
                                     {'action': 'create_user',
                                      'student_fullname': 'blah',
                                      'student_password': 'foozor', })
-        self.assertIn(_('Must provide username'), response.content)
+        self.assertIn(_('Must provide username'), response.content.decode('utf-8'))
         # no full name
         response = self.client.post(reverse('sysadmin'),
                                     {'action': 'create_user',
                                      'student_uname': 'test_cuser+sysadmin@edx.org',
                                      'student_password': 'foozor', })
-        self.assertIn(_('Must provide full name'), response.content)
+        self.assertIn(_('Must provide full name'), response.content.decode('utf-8'))
 
         # Test create valid user
         self.client.post(reverse('sysadmin'),
@@ -190,20 +190,20 @@ class TestSysadmin(SysadminBaseTestCase):
         # Try no username
         response = self.client.post(reverse('sysadmin'),
                                     {'action': 'del_user', })
-        self.assertIn(_('Must provide username'), response.content)
+        self.assertIn(_('Must provide username'), response.content.decode('utf-8'))
 
         # Try bad usernames
         response = self.client.post(reverse('sysadmin'),
                                     {'action': 'del_user',
                                      'student_uname': 'flabbergast@example.com',
                                      'student_fullname': 'enigma jones', })
-        self.assertIn(_('Cannot find user with email address'), response.content)
+        self.assertIn(_('Cannot find user with email address'), response.content.decode('utf-8'))
 
         response = self.client.post(reverse('sysadmin'),
                                     {'action': 'del_user',
                                      'student_uname': 'flabbergast',
                                      'student_fullname': 'enigma jones', })
-        self.assertIn(_('Cannot find user with username'), response.content)
+        self.assertIn(_('Cannot find user with username'), response.content.decode('utf-8'))
 
         self.client.post(reverse('sysadmin'),
                          {'action': 'del_user',
@@ -268,7 +268,7 @@ class TestSysadmin(SysadminBaseTestCase):
 
         self.assertIn('{0} test0'.format(_('Failed in authenticating')),
                       response.content)
-        self.assertIn(_('fixed password'), response.content)
+        self.assertIn(_('fixed password'), response.content.decode('utf-8'))
 
         self.assertTrue(self.client.login(username='test0',
                                           password=eamap.internal_password))
@@ -277,7 +277,7 @@ class TestSysadmin(SysadminBaseTestCase):
         self._setstaff_login()
         response = self.client.post(reverse('sysadmin'),
                                     {'action': 'repair_eamap', })
-        self.assertIn(_('All ok!'), response.content)
+        self.assertIn(_('All ok!'), response.content.decode('utf-8'))
 
     def test_xml_course_add_delete(self):
         """add and delete course from xml module store"""