From 4df20733766a3bd75b527640f8ed7c35eadcc2f9 Mon Sep 17 00:00:00 2001 From: Shadi Naif Date: Sun, 7 Oct 2018 15:46:03 +0300 Subject: [PATCH] Fix exceptions raised when a lazy text is used in json dump --- .../contentstore/tests/test_contentstore.py | 2 +- .../contentstore/tests/test_i18n.py | 5 ++-- cms/envs/bok_choy.env.json | 1 - cms/envs/bok_choy.py | 7 +++++ cms/envs/bok_choy_docker.env.json | 1 - cms/envs/production.py | 12 +++++--- cms/envs/test.py | 9 ++++++ .../xmodule/xmodule/modulestore/__init__.py | 29 ++++--------------- .../lib/xmodule/xmodule/tests/test_export.py | 16 ++++++++++ .../tests/studio/test_studio_help.py | 7 +++-- lms/envs/bok_choy.env.json | 1 - lms/envs/bok_choy.py | 5 ++++ lms/envs/bok_choy_docker.env.json | 1 - lms/envs/production.py | 8 +++-- lms/envs/test.py | 10 +++++-- .../core/djangoapps/user_api/message_types.py | 4 +-- openedx/core/lib/json_utils.py | 29 +++++++++++++++++++ requirements/edx/base.in | 2 +- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/testing.txt | 2 +- 21 files changed, 107 insertions(+), 48 deletions(-) create mode 100644 openedx/core/lib/json_utils.py diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 70f99b0c1b..1ad872db72 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -1476,7 +1476,7 @@ class ContentStoreTest(ContentStoreTestCase): resp = self.client.get_html('/home/') self.assertContains( resp, - '

Studio Home

', + u'

{} Home

'.format(settings.STUDIO_SHORT_NAME), status_code=200, html=True ) diff --git a/cms/djangoapps/contentstore/tests/test_i18n.py b/cms/djangoapps/contentstore/tests/test_i18n.py index b33a4f7148..45034e3839 100644 --- a/cms/djangoapps/contentstore/tests/test_i18n.py +++ b/cms/djangoapps/contentstore/tests/test_i18n.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Tests for validate Internationalization and Module i18n service. """ @@ -207,7 +208,7 @@ class InternationalizationTest(ModuleStoreTestCase): resp = self.client.get_html('/home/') self.assertContains(resp, - '

Studio Home

', + u'

𝓒𝓽𝓾𝓭𝓲𝓸 Home

', status_code=200, html=True) @@ -223,7 +224,7 @@ class InternationalizationTest(ModuleStoreTestCase): ) self.assertContains(resp, - '

Studio Home

', + u'

𝓒𝓽𝓾𝓭𝓲𝓸 Home

', status_code=200, html=True) diff --git a/cms/envs/bok_choy.env.json b/cms/envs/bok_choy.env.json index 352f377200..cdb25a8fcb 100644 --- a/cms/envs/bok_choy.env.json +++ b/cms/envs/bok_choy.env.json @@ -91,7 +91,6 @@ "LOG_DIR": "** OVERRIDDEN **", "MEDIA_URL": "/media/", "MKTG_URL_LINK_MAP": {}, - "PLATFORM_NAME": "Γ©dX", "SERVER_EMAIL": "devops@example.com", "SESSION_COOKIE_DOMAIN": null, "SITE_NAME": "localhost", diff --git a/cms/envs/bok_choy.py b/cms/envs/bok_choy.py index 98c379a231..fff1ce11d9 100644 --- a/cms/envs/bok_choy.py +++ b/cms/envs/bok_choy.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Settings for Bok Choy tests that are used when running Studio. @@ -13,6 +14,7 @@ from the same directory. import os from path import Path as path +from django.utils.translation import ugettext_lazy from openedx.core.release import RELEASE_LINE ########################## Prod-like settings ################################### @@ -52,6 +54,11 @@ XBLOCK_SETTINGS.update({'VideoDescriptor': {'licensing_enabled': True}}) # Capture the console log via template includes, until webdriver supports log capture again CAPTURE_CONSOLE_LOG = True +PLATFORM_NAME = ugettext_lazy(u"Γ©dX") +PLATFORM_DESCRIPTION = ugettext_lazy(u"Open Γ©dX Platform") +STUDIO_NAME = ugettext_lazy(u"Your Platform 𝓒𝓽𝓾𝓭𝓲𝓸") +STUDIO_SHORT_NAME = ugettext_lazy(u"𝓒𝓽𝓾𝓭𝓲𝓸") + ############################ STATIC FILES ############################# # Enable debug so that static assets are served by Django diff --git a/cms/envs/bok_choy_docker.env.json b/cms/envs/bok_choy_docker.env.json index 373cd2cec1..d44346f75f 100644 --- a/cms/envs/bok_choy_docker.env.json +++ b/cms/envs/bok_choy_docker.env.json @@ -91,7 +91,6 @@ "LOG_DIR": "** OVERRIDDEN **", "MEDIA_URL": "/media/", "MKTG_URL_LINK_MAP": {}, - "PLATFORM_NAME": "Γ©dX", "SERVER_EMAIL": "devops@example.com", "SESSION_COOKIE_DOMAIN": null, "SITE_NAME": "localhost", diff --git a/cms/envs/production.py b/cms/envs/production.py index 88c873f107..8d4fa755d6 100644 --- a/cms/envs/production.py +++ b/cms/envs/production.py @@ -254,10 +254,14 @@ LOGGING = get_logger_config(LOG_DIR, service_variant=SERVICE_VARIANT) #theming start: -PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME', PLATFORM_NAME) -PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION', PLATFORM_DESCRIPTION) -STUDIO_NAME = ENV_TOKENS.get('STUDIO_NAME', STUDIO_NAME) -STUDIO_SHORT_NAME = ENV_TOKENS.get('STUDIO_SHORT_NAME', STUDIO_SHORT_NAME) + +# The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text +# values when the varibale is an empty string. Therefore, setting these variable as empty text in related +# json files will make the system reads thier values from django translation files +PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME') or PLATFORM_NAME +PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION') or PLATFORM_DESCRIPTION +STUDIO_NAME = ENV_TOKENS.get('STUDIO_NAME') or STUDIO_NAME +STUDIO_SHORT_NAME = ENV_TOKENS.get('STUDIO_SHORT_NAME') or STUDIO_SHORT_NAME # Event Tracking if "TRACKING_IGNORE_URL_PATTERNS" in ENV_TOKENS: diff --git a/cms/envs/test.py b/cms/envs/test.py index 543cff7d83..0951c0fa75 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -13,6 +13,8 @@ sessions. Assumes structure: # want to import all variables from base settings files # pylint: disable=wildcard-import, unused-wildcard-import +from django.utils.translation import ugettext_lazy + from .common import * import os from path import Path as path @@ -25,6 +27,7 @@ from openedx.core.lib.derived import derive_settings from lms.envs.test import ( WIKI_ENABLED, PLATFORM_NAME, + PLATFORM_DESCRIPTION, SITE_NAME, DEFAULT_FILE_STORAGE, MEDIA_ROOT, @@ -35,6 +38,12 @@ from lms.envs.test import ( ECOMMERCE_API_URL, ) + +# Include a non-ascii character in STUDIO_NAME and STUDIO_SHORT_NAME to uncover possible +# UnicodeEncodeErrors in tests. Also use lazy text to reveal possible json dumps errors +STUDIO_NAME = ugettext_lazy(u"Your Platform 𝓒𝓽𝓾𝓭𝓲𝓸") +STUDIO_SHORT_NAME = ugettext_lazy(u"𝓒𝓽𝓾𝓭𝓲𝓸") + # Allow all hosts during tests, we use a lot of different ones all over the codebase. ALLOWED_HOSTS = [ '*' diff --git a/common/lib/xmodule/xmodule/modulestore/__init__.py b/common/lib/xmodule/xmodule/modulestore/__init__.py index 4bba5cfdca..eb439a8aa5 100644 --- a/common/lib/xmodule/xmodule/modulestore/__init__.py +++ b/common/lib/xmodule/xmodule/modulestore/__init__.py @@ -5,7 +5,6 @@ that are stored in a database an accessible using their Location as an identifie import logging import re -import json import datetime from pytz import UTC @@ -22,11 +21,15 @@ from xblock.plugin import default_select from .exceptions import InvalidLocationError, InsufficientSpecificationError from xmodule.errortracker import make_error_tracker from xmodule.assetstore import AssetMetadata -from opaque_keys.edx.keys import CourseKey, UsageKey, AssetKey +from opaque_keys.edx.keys import CourseKey, AssetKey from opaque_keys.edx.locations import Location # For import backwards compatibility from xblock.runtime import Mixologist from xblock.core import XBlock +# The below import is not used within this module, but ir is still needed becuase +# other modules are imorting EdxJSONEncoder from here +from openedx.core.lib.json_utils import EdxJSONEncoder # pylint: disable=unused-import + log = logging.getLogger('edx.modulestore') new_contract('CourseKey', CourseKey) @@ -1430,25 +1433,3 @@ def prefer_xmodules(identifier, entry_points): return default_select(identifier, from_xmodule) else: return default_select(identifier, entry_points) - - -class EdxJSONEncoder(json.JSONEncoder): - """ - Custom JSONEncoder that handles `Location` and `datetime.datetime` objects. - - `Location`s are encoded as their url string form, and `datetime`s as - ISO date strings - """ - def default(self, obj): - if isinstance(obj, (CourseKey, UsageKey)): - return unicode(obj) - elif isinstance(obj, datetime.datetime): - if obj.tzinfo is not None: - if obj.utcoffset() is None: - return obj.isoformat() + 'Z' - else: - return obj.isoformat() - else: - return obj.isoformat() - else: - return super(EdxJSONEncoder, self).default(obj) diff --git a/common/lib/xmodule/xmodule/tests/test_export.py b/common/lib/xmodule/xmodule/tests/test_export.py index 9c5b797a9b..ae9bfed105 100644 --- a/common/lib/xmodule/xmodule/tests/test_export.py +++ b/common/lib/xmodule/xmodule/tests/test_export.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Tests of XML export """ @@ -10,6 +11,7 @@ import shutil import unittest from datetime import datetime, timedelta, tzinfo +from django.utils.translation import ugettext_lazy from fs.osfs import OSFS from path import Path as path from six import text_type @@ -212,3 +214,17 @@ class TestEdxJsonEncoder(unittest.TestCase): with self.assertRaises(TypeError): self.encoder.default({}) + + def test_encode_unicode_lazy_text(self): + """ + Verify that the encoding is functioning fine with lazy text + """ + + # Initializing a lazy text object with Unicode + unicode_text = u"Your π“Ÿπ“΅π“ͺ𝓽𝓯𝓸𝓻𝓢 Name Here" + lazy_text = ugettext_lazy(unicode_text) + + self.assertEquals( + unicode_text, + self.encoder.default(lazy_text) + ) diff --git a/common/test/acceptance/tests/studio/test_studio_help.py b/common/test/acceptance/tests/studio/test_studio_help.py index 3112e2c119..1ed9624bff 100644 --- a/common/test/acceptance/tests/studio/test_studio_help.py +++ b/common/test/acceptance/tests/studio/test_studio_help.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Test the Studio help links. """ @@ -189,7 +190,7 @@ class HomeHelpTest(StudioCourseTest): test=self, page=self.home_page, href=expected_url, - help_text='Getting Started with Your Platform Studio', + help_text=u'Getting Started with Your Platform 𝓒𝓽𝓾𝓭𝓲𝓸', as_list_item=True ) @@ -242,7 +243,7 @@ class NewCourseHelpTest(AcceptanceTest): test=self, page=self.dashboard_page, href=expected_url, - help_text='Getting Started with Your Platform Studio', + help_text=u'Getting Started with Your Platform 𝓒𝓽𝓾𝓭𝓲𝓸', as_list_item=True ) @@ -295,7 +296,7 @@ class NewLibraryHelpTest(AcceptanceTest): test=self, page=self.dashboard_page, href=expected_url, - help_text='Getting Started with Your Platform Studio', + help_text=u'Getting Started with Your Platform 𝓒𝓽𝓾𝓭𝓲𝓸', as_list_item=True ) diff --git a/lms/envs/bok_choy.env.json b/lms/envs/bok_choy.env.json index 5b644dcc9c..a2aed20071 100644 --- a/lms/envs/bok_choy.env.json +++ b/lms/envs/bok_choy.env.json @@ -115,7 +115,6 @@ "ROOT": "root", "SITEMAP.XML": "sitemap_xml" }, - "PLATFORM_NAME": "Γ©dX", "REGISTRATION_EXTENSION_FORM": "openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm", "REGISTRATION_EXTRA_FIELDS": { "level_of_education": "optional", diff --git a/lms/envs/bok_choy.py b/lms/envs/bok_choy.py index 74381db39f..e87c4ee056 100644 --- a/lms/envs/bok_choy.py +++ b/lms/envs/bok_choy.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ Settings for Bok Choy tests that are used when running LMS. @@ -14,6 +15,7 @@ import os from path import Path as path from tempfile import mkdtemp +from django.utils.translation import ugettext_lazy from openedx.core.release import RELEASE_LINE CONFIG_ROOT = path(__file__).abspath().dirname() @@ -52,6 +54,9 @@ update_module_store_settings( # Capture the console log via template includes, until webdriver supports log capture again CAPTURE_CONSOLE_LOG = True +PLATFORM_NAME = ugettext_lazy(u"Γ©dX") +PLATFORM_DESCRIPTION = ugettext_lazy(u"Open Γ©dX Platform") + ############################ STATIC FILES ############################# # Enable debug so that static assets are served by Django diff --git a/lms/envs/bok_choy_docker.env.json b/lms/envs/bok_choy_docker.env.json index b2a799da2f..8f68509bf0 100644 --- a/lms/envs/bok_choy_docker.env.json +++ b/lms/envs/bok_choy_docker.env.json @@ -115,7 +115,6 @@ "ROOT": "root", "SITEMAP.XML": "sitemap_xml" }, - "PLATFORM_NAME": "Γ©dX", "REGISTRATION_EXTENSION_FORM": "openedx.core.djangoapps.user_api.tests.test_helpers.TestCaseForm", "REGISTRATION_EXTRA_FIELDS": { "level_of_education": "optional", diff --git a/lms/envs/production.py b/lms/envs/production.py index 6f229d23bd..13fb2b3900 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -131,8 +131,12 @@ COURSE_MODE_DEFAULTS = ENV_TOKENS.get('COURSE_MODE_DEFAULTS', COURSE_MODE_DEFAUL MEDIA_ROOT = ENV_TOKENS.get('MEDIA_ROOT', MEDIA_ROOT) MEDIA_URL = ENV_TOKENS.get('MEDIA_URL', MEDIA_URL) -PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME', PLATFORM_NAME) -PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION', PLATFORM_DESCRIPTION) +# The following variables use (or) instead of the default value inside (get). This is to enforce using the Lazy Text +# values when the varibale is an empty string. Therefore, setting these variable as empty text in related +# json files will make the system reads thier values from django translation files +PLATFORM_NAME = ENV_TOKENS.get('PLATFORM_NAME') or PLATFORM_NAME +PLATFORM_DESCRIPTION = ENV_TOKENS.get('PLATFORM_DESCRIPTION') or PLATFORM_DESCRIPTION + # For displaying on the receipt. At Stanford PLATFORM_NAME != MERCHANT_NAME, but PLATFORM_NAME is a fine default PLATFORM_TWITTER_ACCOUNT = ENV_TOKENS.get('PLATFORM_TWITTER_ACCOUNT', PLATFORM_TWITTER_ACCOUNT) PLATFORM_FACEBOOK_ACCOUNT = ENV_TOKENS.get('PLATFORM_FACEBOOK_ACCOUNT', PLATFORM_FACEBOOK_ACCOUNT) diff --git a/lms/envs/test.py b/lms/envs/test.py index dae94ffb76..86be9082bc 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -13,6 +13,8 @@ sessions. Assumes structure: # want to import all variables from base settings files # pylint: disable=wildcard-import, unused-wildcard-import +from django.utils.translation import ugettext_lazy + from .common import * import os from path import Path as path @@ -397,8 +399,12 @@ FEATURES['CLASS_DASHBOARD'] = True import openid.oidutil openid.oidutil.log = lambda message, level=0: None -# Include a non-ascii character in PLATFORM_NAME to uncover possible UnicodeEncodeErrors in tests. -PLATFORM_NAME = u"Γ©dX" + +# Include a non-ascii character in PLATFORM_NAME and PLATFORM_DESCRIPTION to uncover possible +# UnicodeEncodeErrors in tests. Also use lazy text to reveal possible json dumps errors +PLATFORM_NAME = ugettext_lazy(u"Γ©dX") +PLATFORM_DESCRIPTION = ugettext_lazy(u"Open Γ©dX Platform") + SITE_NAME = "edx.org" # set up some testing for microsites diff --git a/openedx/core/djangoapps/user_api/message_types.py b/openedx/core/djangoapps/user_api/message_types.py index 0249ceca10..be0f0e42a3 100644 --- a/openedx/core/djangoapps/user_api/message_types.py +++ b/openedx/core/djangoapps/user_api/message_types.py @@ -4,11 +4,11 @@ Message Types for user_api emails from django.conf import settings -from edx_ace import message +from openedx.core.djangoapps.ace_common.message import BaseMessageType from openedx.core.djangoapps.site_configuration import helpers -class DeletionNotificationMessage(message.MessageType): +class DeletionNotificationMessage(BaseMessageType): """ Message to notify learners that their account is queued for deletion. """ diff --git a/openedx/core/lib/json_utils.py b/openedx/core/lib/json_utils.py new file mode 100644 index 0000000000..203e64577b --- /dev/null +++ b/openedx/core/lib/json_utils.py @@ -0,0 +1,29 @@ +""" +Helpers for json serialization +""" + +import datetime +from django.core.serializers.json import DjangoJSONEncoder +from opaque_keys.edx.keys import CourseKey, UsageKey + + +class EdxJSONEncoder(DjangoJSONEncoder): + """ + Custom JSONEncoder that handles `Location` and `datetime.datetime` objects. + + `Location`s are encoded as their url string form, and `datetime`s as + ISO date strings + """ + def default(self, o): # pylint: disable=method-hidden + if isinstance(o, (CourseKey, UsageKey)): + return unicode(o) + elif isinstance(o, datetime.datetime): + if o.tzinfo is not None: + if o.utcoffset() is None: + return o.isoformat() + 'Z' + else: + return o.isoformat() + else: + return o.isoformat() + else: + return super(EdxJSONEncoder, self).default(o) diff --git a/requirements/edx/base.in b/requirements/edx/base.in index 3da58a3af1..6ed70056f8 100644 --- a/requirements/edx/base.in +++ b/requirements/edx/base.in @@ -64,7 +64,7 @@ django-webpack-loader # Used to wire webpack bundles into the djan djangorestframework-jwt django-xforwardedfor-middleware==2.0 # Middleware to use the X-Forwarded-For header as the request IP dogapi==1.2.1 # Python bindings to Datadog's API, for metrics gathering -edx-ace==0.1.9 +edx-ace==0.1.10 edx-analytics-data-api-client edx-ccx-keys edx-celeryutils diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 64a88d7fa9..87aef92412 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -109,7 +109,7 @@ dm.xmlsec.binding==1.3.3 # via python-saml docopt==0.6.2 docutils==0.14 # via botocore dogapi==1.2.1 -edx-ace==0.1.9 +edx-ace==0.1.10 edx-analytics-data-api-client==0.14.4 edx-ccx-keys==0.2.1 edx-celeryutils==0.2.7 diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 78ba074617..896343b3af 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -128,7 +128,7 @@ dm.xmlsec.binding==1.3.3 docopt==0.6.2 docutils==0.14 dogapi==1.2.1 -edx-ace==0.1.9 +edx-ace==0.1.10 edx-analytics-data-api-client==0.14.4 edx-ccx-keys==0.2.1 edx-celeryutils==0.2.7 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index de7effee78..e9578330ed 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -123,7 +123,7 @@ dm.xmlsec.binding==1.3.3 docopt==0.6.2 docutils==0.14 dogapi==1.2.1 -edx-ace==0.1.9 +edx-ace==0.1.10 edx-analytics-data-api-client==0.14.4 edx-ccx-keys==0.2.1 edx-celeryutils==0.2.7