Merge pull request #12607 from edx/shr/bug/PLAT-1040-video-module-non-ASCII-values-fixed

Fixed logging and unicode handling

common/lib/xmodule/xmodule/tests/test_video.py

@@ -758,6 +758,14 @@ class VideoExportTestCase(VideoDescriptorTestBase):
         with self.assertRaises(ValueError):
             self.descriptor.definition_to_xml(None)
 
+    def test_export_to_xml_unicode_characters(self):
+        """
+        Test XML export handles the unicode characters.
+        """
+        self.descriptor.display_name = '这是文'
+        xml = self.descriptor.definition_to_xml(None)
+        self.assertEqual(xml.get('display_name'), u'\u8fd9\u662f\u6587')
+
 
 class VideoDescriptorIndexingTestCase(unittest.TestCase):
     """

common/lib/xmodule/xmodule/video_module/video_module.py

@@ -38,7 +38,7 @@ from xmodule.exceptions import NotFoundError
 from xmodule.contentstore.content import StaticContent
 
 from .transcripts_utils import VideoTranscriptsMixin, Transcript, get_html5_ids
-from .video_utils import create_youtube_string, get_poster, rewrite_video_url
+from .video_utils import create_youtube_string, get_poster, rewrite_video_url, format_xml_exception_message
 from .bumper_utils import bumperize
 from .video_xfields import VideoFields
 from .video_handlers import VideoStudentViewHandlers, VideoStudioViewHandlers
@@ -563,14 +563,16 @@ class VideoDescriptor(VideoFields, VideoTranscriptsMixin, VideoStudioViewHandler
                 if key in self.fields and self.fields[key].is_set_on(self):
                     try:
                         xml.set(key, unicode(value))
-                    except ValueError as exception:
-                        exception_message = "{message}, Block-location:{location}, Key:{key}, Value:{value}".format(
-                            message=exception.message,
-                            location=unicode(self.location),
-                            key=key,
-                            value=unicode(value)
-                        )
-                        raise ValueError(exception_message)
+                    except UnicodeDecodeError:
+                        exception_message = format_xml_exception_message(self.location, key, value)
+                        log.exception(exception_message)
+                        # If exception is UnicodeDecodeError set value using unicode 'utf-8' scheme.
+                        log.info("Setting xml value using 'utf-8' scheme.")
+                        xml.set(key, unicode(value, 'utf-8'))
+                    except ValueError:
+                        exception_message = format_xml_exception_message(self.location, key, value)
+                        log.exception(exception_message)
+                        raise
 
         for source in self.html5_sources:
             ele = etree.Element('source')

common/lib/xmodule/xmodule/video_module/video_utils.py

@@ -98,6 +98,19 @@ def get_poster(video):
     return poster
 
 
+def format_xml_exception_message(location, key, value):
+    """
+    Generate exception message for VideoDescriptor class which will use for ValueError and UnicodeDecodeError
+    when setting xml attributes.
+    """
+    exception_message = "Block-location:{location}, Key:{key}, Value:{value}".format(
+        location=unicode(location),
+        key=key,
+        value=value
+    )
+    return exception_message
+
+
 def set_query_parameter(url, param_name, param_value):
     """
     Given a URL, set or replace a query parameter and return the