Merge pull request #21284 from edx/aj/fix-japanees-chars-in-csv

Fix Japaness Characters not visible on Data Download Report

lms/djangoapps/instructor_analytics/basic.py

@@ -7,6 +7,7 @@ from __future__ import absolute_import
 
 import datetime
 import json
+import logging
 
 import six
 from django.conf import settings
@@ -16,7 +17,7 @@ from django.core.serializers.json import DjangoJSONEncoder
 from django.db.models import Count, Q
 from django.urls import reverse
 from edx_proctoring.api import get_exam_violation_report
-from opaque_keys.edx.keys import UsageKey, CourseKey
+from opaque_keys.edx.keys import CourseKey, UsageKey
 from six import text_type
 
 import xmodule.graders as xmgraders
@@ -35,6 +36,8 @@ from shoppingcart.models import (
 )
 from student.models import CourseEnrollment, CourseEnrollmentAllowed
 
+log = logging.getLogger(__name__)
+
 
 STUDENT_FEATURES = ('id', 'username', 'first_name', 'last_name', 'is_staff', 'email')
 PROFILE_FEATURES = ('name', 'language', 'location', 'year_of_birth', 'gender',
@@ -464,11 +467,75 @@ def list_problem_responses(course_key, problem_location, limit_responses=None):
         smdat = smdat[:limit_responses]
 
     return [
-        {'username': response.student.username, 'state': response.state}
+        {'username': response.student.username, 'state': get_response_state(response)}
         for response in smdat
     ]
 
 
+def get_response_state(response):
+    """
+    Returns state of a particular response as string.
+
+    This method also does necessary encoding for displaying unicode data correctly.
+    """
+    def get_transformer():
+        """
+        Returns state transformer depending upon the problem type.
+        """
+        problem_state_transformers = {
+            'openassessment': transform_ora_state,
+            'problem': transform_capa_state
+        }
+        problem_type = response.module_type
+        return problem_state_transformers.get(problem_type)
+
+    problem_state = response.state
+    problem_state_transformer = get_transformer()
+    if not problem_state_transformer:
+        return problem_state
+
+    state = json.loads(problem_state)
+    try:
+        transformed_state = problem_state_transformer(state)
+        return json.dumps(transformed_state, encoding='utf8', ensure_ascii=False)
+    except TypeError:
+        username = response.student.username
+        err_msg = (
+            u'Error occurred while attempting to load learner state '
+            u'{username} for state {state}.'.format(
+                username=username,
+                state=problem_state
+            )
+        )
+        log.error(err_msg)
+        return problem_state
+
+
+def transform_ora_state(state):
+    """
+    ORA problem state transformer transforms the problem states.
+
+    Some state variables values are json dumped strings which needs to be loaded
+    into a python object.
+    """
+    fields_to_transform = ['saved_response', 'saved_files_descriptions']
+
+    for field in fields_to_transform:
+        field_state = state.get(field)
+        if not field_state:
+            continue
+
+        state[field] = json.loads(field_state)
+    return state
+
+
+def transform_capa_state(state):
+    """
+    Transforms the CAPA problem state.
+    """
+    return state
+
+
 def course_registration_features(features, registration_codes, csv_type):
     """
     Return list of Course Registration Codes as dictionaries.

lms/djangoapps/instructor_analytics/tests/test_basic.py

@@ -1,9 +1,10 @@
+# coding=utf-8
 """
 Tests for instructor.basic
 """
-
 from __future__ import absolute_import
 
+import ddt
 import datetime
 import json
 
@@ -29,6 +30,7 @@ from lms.djangoapps.instructor_analytics.basic import (
     course_registration_features,
     enrolled_students_features,
     get_proctored_exam_results,
+    get_response_state,
     list_may_enroll,
     list_problem_responses,
     sale_order_record_features,
@@ -52,6 +54,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase
 from xmodule.modulestore.tests.factories import CourseFactory
 
 
+@ddt.ddt
 class TestAnalyticsBasic(ModuleStoreTestCase):
     """ Test basic analytics functions. """
 
@@ -74,6 +77,26 @@ class TestAnalyticsBasic(ModuleStoreTestCase):
                 email=student.email, course_id=self.course_key
             )
 
+    @ddt.data(
+        (u'あなた', u'スの中'),
+        (u"ГЂіи lіиэ ъэтшээи", u"Ђэаvэи аиↁ Ђэѓэ")
+    )
+    @ddt.unpack
+    def test_get_response_state_with_ora(self, files_descriptions, saved_response):
+        """
+        Tests that ORA response state is transformed expectedly when the problem
+        state contains unicode characters.
+        """
+        payload_state = json.dumps({
+            'saved_response': json.dumps({'parts': [{'text': saved_response}]}),
+            'saved_files_descriptions': json.dumps([files_descriptions]),
+        })
+        response = Mock(module_type='openassessment', student=Mock(username='staff'), state=payload_state)
+
+        transformed_state = json.loads(get_response_state(response))
+        self.assertEqual(transformed_state['saved_files_descriptions'][0], files_descriptions)
+        self.assertEqual(transformed_state['saved_response']['parts'][0]['text'], saved_response)
+
     def test_list_problem_responses(self):
         def result_factory(result_id):
             """