Merge pull request #17415 from mahyard/master

Prepend unicode signature (BOM) to csv reports

lms/djangoapps/instructor_task/models.py

@@ -12,6 +12,7 @@ file and check it in at the same time as your model changes. To do that,
 ASSUMPTIONS: modules have unique IDs, even across different module_types
 
 """
+import codecs
 import csv
 import hashlib
 import json
@@ -270,6 +271,8 @@ class DjangoStorageReportStore(ReportStore):
         strings), write the rows to the storage backend in csv format.
         """
         output_buffer = ContentFile('')
+        # Adding unicode signature (BOM) for MS Excel 2013 compatibility
+        output_buffer.write(codecs.BOM_UTF8)
         csvwriter = csv.writer(output_buffer)
         csvwriter.writerows(self._get_utf8_encoded_rows(rows))
         output_buffer.seek(0)

lms/djangoapps/instructor_task/tests/test_base.py

@@ -352,7 +352,7 @@ class TestReportMixin(object):
         report_path = report_store.path_to(self.course.id, report_csv_filename)
         with report_store.storage.open(report_path) as csv_file:
             # Expand the dict reader generator so we don't lose it's content
-            csv_rows = [row for row in unicodecsv.DictReader(csv_file)]
+            csv_rows = [row for row in unicodecsv.DictReader(csv_file, encoding='utf-8-sig')]
 
             if ignore_other_columns:
                 csv_rows = [
@@ -372,5 +372,5 @@ class TestReportMixin(object):
         report_csv_filename = report_store.links_for(self.course.id)[0][0]
         report_path = report_store.path_to(self.course.id, report_csv_filename)
         with report_store.storage.open(report_path) as csv_file:
-            rows = unicodecsv.reader(csv_file, encoding='utf-8')
+            rows = unicodecsv.reader(csv_file, encoding='utf-8-sig')
             return rows.next()

lms/djangoapps/instructor_task/tests/test_tasks_helper.py

@@ -1286,6 +1286,8 @@ class TestCourseSurveyReport(TestReportMixin, InstructorTaskCourseTestCase):
         report_path = report_store.path_to(self.course.id, report_csv_filename)
         with report_store.storage.open(report_path) as csv_file:
             csv_file_data = csv_file.read()
+            # Removing unicode signature (BOM) from the beginning
+            csv_file_data = csv_file_data.decode("utf-8-sig").encode("utf-8")
             for data in expected_data:
                 self.assertIn(data, csv_file_data)