From d312d0e1525ac80bfe2b7ad8e9ff825c73720de3 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Fri, 5 Jun 2015 13:29:15 -0400 Subject: [PATCH 1/5] Port django.middleware.locale.LocaleMiddleware from Django 1.8 --- cms/envs/common.py | 4 +- common/djangoapps/django_locale/__init__.py | 7 + common/djangoapps/django_locale/middleware.py | 83 +++++++++ common/djangoapps/django_locale/tests.py | 157 ++++++++++++++++++ common/djangoapps/django_locale/trans_real.py | 131 +++++++++++++++ lms/envs/common.py | 4 +- 6 files changed, 384 insertions(+), 2 deletions(-) create mode 100644 common/djangoapps/django_locale/__init__.py create mode 100644 common/djangoapps/django_locale/middleware.py create mode 100644 common/djangoapps/django_locale/tests.py create mode 100644 common/djangoapps/django_locale/trans_real.py diff --git a/cms/envs/common.py b/cms/envs/common.py index bb47f87384..2842af1874 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -302,7 +302,9 @@ MIDDLEWARE_CLASSES = ( 'embargo.middleware.EmbargoMiddleware', # Detects user-requested locale from 'accept-language' header in http request - 'django.middleware.locale.LocaleMiddleware', + # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671] + # 'django.middleware.locale.LocaleMiddleware', + 'django_locale.middleware.LocaleMiddleware', 'django.middleware.transaction.TransactionMiddleware', # needs to run after locale middleware (or anything that modifies the request context) diff --git a/common/djangoapps/django_locale/__init__.py b/common/djangoapps/django_locale/__init__.py new file mode 100644 index 0000000000..655019022e --- /dev/null +++ b/common/djangoapps/django_locale/__init__.py @@ -0,0 +1,7 @@ +""" +TODO: This module is imported from the stable Django 1.8 branch, as a +copy of https://github.com/django/django/blob/stable/1.8.x/django/middleware/locale.py. + +Remove this file and re-import this middleware from Django once the +codebase is upgraded with a modern version of Django. [PLAT-671] +""" diff --git a/common/djangoapps/django_locale/middleware.py b/common/djangoapps/django_locale/middleware.py new file mode 100644 index 0000000000..b0601a807e --- /dev/null +++ b/common/djangoapps/django_locale/middleware.py @@ -0,0 +1,83 @@ +# TODO: This file is imported from the stable Django 1.8 branch. Remove this file +# and re-import this middleware from Django once the codebase is upgraded. [PLAT-671] +# pylint: disable=invalid-name, missing-docstring +"This is the locale selecting middleware that will look at accept headers" + +from django.conf import settings +from django.core.urlresolvers import ( + LocaleRegexURLResolver, get_resolver, get_script_prefix, is_valid_path, +) +from django.http import HttpResponseRedirect +from django.utils import translation +from django.utils.cache import patch_vary_headers +# Override the Django 1.4 implementation with the 1.8 implementation +from django_locale.trans_real import get_language_from_request + + +class LocaleMiddleware(object): + """ + This is a very simple middleware that parses a request + and decides what translation object to install in the current + thread context. This allows pages to be dynamically + translated to the language the user desires (if the language + is available, of course). + """ + response_redirect_class = HttpResponseRedirect + + def __init__(self): + self._is_language_prefix_patterns_used = False + for url_pattern in get_resolver(None).url_patterns: + if isinstance(url_pattern, LocaleRegexURLResolver): + self._is_language_prefix_patterns_used = True + break + + def process_request(self, request): + check_path = self.is_language_prefix_patterns_used() + # This call is broken in Django 1.4: + # https://github.com/django/django/blob/stable/1.4.x/django/utils/translation/trans_real.py#L399 + # (we override parse_accept_lang_header to a fixed version in dark_lang.middleware) + language = get_language_from_request( + request, check_path=check_path) + translation.activate(language) + request.LANGUAGE_CODE = translation.get_language() + + def process_response(self, request, response): + language = translation.get_language() + language_from_path = translation.get_language_from_path(request.path_info) + if (response.status_code == 404 and not language_from_path + and self.is_language_prefix_patterns_used()): + urlconf = getattr(request, 'urlconf', None) + language_path = '/%s%s' % (language, request.path_info) + path_valid = is_valid_path(language_path, urlconf) + if (not path_valid and settings.APPEND_SLASH + and not language_path.endswith('/')): + path_valid = is_valid_path("%s/" % language_path, urlconf) + + if path_valid: + script_prefix = get_script_prefix() + language_url = "%s://%s%s" % ( + request.scheme, + request.get_host(), + # insert language after the script prefix and before the + # rest of the URL + request.get_full_path().replace( + script_prefix, + '%s%s/' % (script_prefix, language), + 1 + ) + ) + return self.response_redirect_class(language_url) + + if not (self.is_language_prefix_patterns_used() + and language_from_path): + patch_vary_headers(response, ('Accept-Language',)) + if 'Content-Language' not in response: + response['Content-Language'] = language + return response + + def is_language_prefix_patterns_used(self): + """ + Returns `True` if the `LocaleRegexURLResolver` is used + at root level of the urlpatterns, else it returns `False`. + """ + return self._is_language_prefix_patterns_used diff --git a/common/djangoapps/django_locale/tests.py b/common/djangoapps/django_locale/tests.py new file mode 100644 index 0000000000..cc40ce9d4a --- /dev/null +++ b/common/djangoapps/django_locale/tests.py @@ -0,0 +1,157 @@ +# pylint: disable=invalid-name, line-too-long, super-method-not-called +""" +Tests taken from Django upstream: +https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py +""" +from django.conf import settings +from django.test import TestCase, RequestFactory +from django_locale.trans_real import ( + parse_accept_lang_header, get_language_from_request, LANGUAGE_SESSION_KEY +) + +# Added to test middleware around dark lang +from django.contrib.auth.models import User +from django.test.utils import override_settings +from dark_lang.models import DarkLangConfig + + +# Adding to support test differences between Django and our own settings +@override_settings(LANGUAGES=[ + ('pt', 'Portuguese'), + ('pt-br', 'Portuguese-Brasil'), + ('es', 'Spanish'), + ('es-ar', 'Spanish (Argentina)'), + ('de', 'Deutch'), + ('zh-cn', 'Chinese (China)'), + ('ar-sa', 'Arabic (Saudi Arabia)'), +]) +class MiscTests(TestCase): + """ + Tests taken from Django upstream: + https://github.com/django/django/blob/e6b34193c5c7d117ededdab04bb16caf8864f07c/tests/regressiontests/i18n/tests.py + """ + def setUp(self): + self.rf = RequestFactory() + # Added to test middleware around dark lang + user = User() + user.save() + DarkLangConfig( + released_languages='pt, pt-br, es, de, es-ar, zh-cn, ar-sa', + changed_by=user, + enabled=True + ).save() + + def test_parse_spec_http_header(self): + """ + Testing HTTP header parsing. First, we test that we can parse the + values according to the spec (and that we extract all the pieces in + the right order). + """ + p = parse_accept_lang_header + # Good headers. + self.assertEqual([('de', 1.0)], p('de')) + self.assertEqual([('en-AU', 1.0)], p('en-AU')) + self.assertEqual([('es-419', 1.0)], p('es-419')) + self.assertEqual([('*', 1.0)], p('*;q=1.00')) + self.assertEqual([('en-AU', 0.123)], p('en-AU;q=0.123')) + self.assertEqual([('en-au', 0.5)], p('en-au;q=0.5')) + self.assertEqual([('en-au', 1.0)], p('en-au;q=1.0')) + self.assertEqual([('da', 1.0), ('en', 0.5), ('en-gb', 0.25)], p('da, en-gb;q=0.25, en;q=0.5')) + self.assertEqual([('en-au-xx', 1.0)], p('en-au-xx')) + self.assertEqual([('de', 1.0), ('en-au', 0.75), ('en-us', 0.5), ('en', 0.25), ('es', 0.125), ('fa', 0.125)], p('de,en-au;q=0.75,en-us;q=0.5,en;q=0.25,es;q=0.125,fa;q=0.125')) + self.assertEqual([('*', 1.0)], p('*')) + self.assertEqual([('de', 1.0)], p('de;q=0.')) + self.assertEqual([('en', 1.0), ('*', 0.5)], p('en; q=1.0, * ; q=0.5')) + self.assertEqual([], p('')) + + # Bad headers; should always return []. + self.assertEqual([], p('en-gb;q=1.0000')) + self.assertEqual([], p('en;q=0.1234')) + self.assertEqual([], p('en;q=.2')) + self.assertEqual([], p('abcdefghi-au')) + self.assertEqual([], p('**')) + self.assertEqual([], p('en,,gb')) + self.assertEqual([], p('en-au;q=0.1.0')) + self.assertEqual([], p('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXZ,en')) + self.assertEqual([], p('da, en-gb;q=0.8, en;q=0.7,#')) + self.assertEqual([], p('de;q=2.0')) + self.assertEqual([], p('de;q=0.a')) + self.assertEqual([], p('12-345')) + self.assertEqual([], p('')) + + def test_parse_literal_http_header(self): + """ + Now test that we parse a literal HTTP header correctly. + """ + g = get_language_from_request + r = self.rf.get('/') + r.COOKIES = {} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt-br'} + self.assertEqual('pt-br', g(r)) + + r.META = {'HTTP_ACCEPT_LANGUAGE': 'pt'} + self.assertEqual('pt', g(r)) + + r.META = {'HTTP_ACCEPT_LANGUAGE': 'es,de'} + self.assertEqual('es', g(r)) + + r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-ar,de'} + self.assertEqual('es-ar', g(r)) + + # This test assumes there won't be a Django translation to a US + # variation of the Spanish language, a safe assumption. When the + # user sets it as the preferred language, the main 'es' + # translation should be selected instead. + r.META = {'HTTP_ACCEPT_LANGUAGE': 'es-us'} + self.assertEqual(g(r), 'es') + + # This tests the following scenario: there isn't a main language (zh) + # translation of Django but there is a translation to variation (zh_CN) + # the user sets zh-cn as the preferred language, it should be selected + # by Django without falling back nor ignoring it. + r.META = {'HTTP_ACCEPT_LANGUAGE': 'zh-cn,de'} + self.assertEqual(g(r), 'zh-cn') + + def test_logic_masked_by_darklang(self): + g = get_language_from_request + r = self.rf.get('/') + r.COOKIES = {} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'ar-qa'} + self.assertEqual('ar-sa', g(r)) + + r.session = {LANGUAGE_SESSION_KEY: 'es'} + self.assertEqual('es', g(r)) + + def test_parse_language_cookie(self): + """ + Now test that we parse language preferences stored in a cookie correctly. + """ + g = get_language_from_request + r = self.rf.get('/') + r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt-br'} + r.META = {} + self.assertEqual('pt-br', g(r)) + + r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'pt'} + r.META = {} + self.assertEqual('pt', g(r)) + + r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es'} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'} + self.assertEqual('es', g(r)) + + # This test assumes there won't be a Django translation to a US + # variation of the Spanish language, a safe assumption. When the + # user sets it as the preferred language, the main 'es' + # translation should be selected instead. + r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'es-us'} + r.META = {} + self.assertEqual(g(r), 'es') + + # This tests the following scenario: there isn't a main language (zh) + # translation of Django but there is a translation to variation (zh_CN) + # the user sets zh-cn as the preferred language, it should be selected + # by Django without falling back nor ignoring it. + r.COOKIES = {settings.LANGUAGE_COOKIE_NAME: 'zh-cn'} + r.META = {'HTTP_ACCEPT_LANGUAGE': 'de'} + self.assertEqual(g(r), 'zh-cn') diff --git a/common/djangoapps/django_locale/trans_real.py b/common/djangoapps/django_locale/trans_real.py new file mode 100644 index 0000000000..3ec6b6d026 --- /dev/null +++ b/common/djangoapps/django_locale/trans_real.py @@ -0,0 +1,131 @@ +"""Translation helper functions.""" +# Imported from Django 1.8 +# pylint: disable=invalid-name +import re +from django.conf import settings +from django.conf.locale import LANG_INFO +from django.utils import translation + + +# Format of Accept-Language header values. From RFC 2616, section 14.4 and 3.9. +# and RFC 3066, section 2.1 +accept_language_re = re.compile(r''' + ([A-Za-z]{1,8}(?:-[A-Za-z0-9]{1,8})*|\*) # "en", "en-au", "x-y-z", "*" + (?:\s*;\s*q=(0(?:\.\d{,3})?|1(?:.0{,3})?))? # Optional "q=1.00", "q=0.8" + (?:\s*,\s*|$) # Multiple accepts per header. + ''', re.VERBOSE) + + +language_code_re = re.compile(r'^[a-z]{1,8}(?:-[a-z0-9]{1,8})*$', re.IGNORECASE) + + +LANGUAGE_SESSION_KEY = '_language' + + +def parse_accept_lang_header(lang_string): + """ + Parses the lang_string, which is the body of an HTTP Accept-Language + header, and returns a list of (lang, q-value), ordered by 'q' values. + + Any format errors in lang_string results in an empty list being returned. + """ + # parse_accept_lang_header is broken until we are on Django 1.5 or greater + # See https://code.djangoproject.com/ticket/19381 + result = [] + pieces = accept_language_re.split(lang_string) + if pieces[-1]: + return [] + for i in range(0, len(pieces) - 1, 3): + first, lang, priority = pieces[i: i + 3] + if first: + return [] + priority = priority and float(priority) or 1.0 + result.append((lang, priority)) + result.sort(key=lambda k: k[1], reverse=True) + return result + + +def get_supported_language_variant(lang_code, strict=False): + """ + Returns the language-code that's listed in supported languages, possibly + selecting a more generic variant. Raises LookupError if nothing found. + If `strict` is False (the default), the function will look for an alternative + country-specific variant when the currently checked is not found. + lru_cache should have a maxsize to prevent from memory exhaustion attacks, + as the provided language codes are taken from the HTTP request. See also + . + """ + if lang_code: + # If 'fr-ca' is not supported, try special fallback or language-only 'fr'. + possible_lang_codes = [lang_code] + try: + # TODO skip this, or import updated LANG_INFO format from __future__ + # (fallback option wasn't added until + # https://github.com/django/django/commit/5dcdbe95c749d36072f527e120a8cb463199ae0d) + possible_lang_codes.extend(LANG_INFO[lang_code]['fallback']) + except KeyError: + pass + generic_lang_code = lang_code.split('-')[0] + possible_lang_codes.append(generic_lang_code) + supported_lang_codes = dict(settings.LANGUAGES) + + for code in possible_lang_codes: + # Note: django 1.4 implementation of check_for_language is OK to use + if code in supported_lang_codes and translation.check_for_language(code): + return code + if not strict: + # if fr-fr is not supported, try fr-ca. + for supported_code in supported_lang_codes: + if supported_code.startswith(generic_lang_code + '-'): + return supported_code + raise LookupError(lang_code) + + +def get_language_from_request(request, check_path=False): + """ + Analyzes the request to find what language the user wants the system to + show. Only languages listed in settings.LANGUAGES are taken into account. + If the user requests a sublanguage where we have a main language, we send + out the main language. + If check_path is True, the URL path prefix will be checked for a language + code, otherwise this is skipped for backwards compatibility. + """ + if check_path: + # Note: django 1.4 implementation of get_language_from_path is OK to use + lang_code = translation.get_language_from_path(request.path_info) + if lang_code is not None: + return lang_code + + supported_lang_codes = dict(settings.LANGUAGES) + + if hasattr(request, 'session'): + lang_code = request.session.get(LANGUAGE_SESSION_KEY) + # Note: django 1.4 implementation of check_for_language is OK to use + if lang_code in supported_lang_codes and lang_code is not None and translation.check_for_language(lang_code): + return lang_code + + lang_code = request.COOKIES.get(settings.LANGUAGE_COOKIE_NAME) + + try: + return get_supported_language_variant(lang_code) + except LookupError: + pass + + accept = request.META.get('HTTP_ACCEPT_LANGUAGE', '') + # broken in 1.4, so defined above + for accept_lang, unused in parse_accept_lang_header(accept): + if accept_lang == '*': + break + + if not language_code_re.search(accept_lang): + continue + + try: + return get_supported_language_variant(accept_lang) + except LookupError: + continue + + try: + return get_supported_language_variant(settings.LANGUAGE_CODE) + except LookupError: + return settings.LANGUAGE_CODE diff --git a/lms/envs/common.py b/lms/envs/common.py index 0b53750b48..4ceca8457b 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1156,7 +1156,9 @@ MIDDLEWARE_CLASSES = ( 'lang_pref.middleware.LanguagePreferenceMiddleware', # Detects user-requested locale from 'accept-language' header in http request - 'django.middleware.locale.LocaleMiddleware', + # TODO: Re-import the Django version once we upgrade to Django 1.8 [PLAT-671] + # 'django.middleware.locale.LocaleMiddleware', + 'django_locale.middleware.LocaleMiddleware', 'django.middleware.transaction.TransactionMiddleware', # 'debug_toolbar.middleware.DebugToolbarMiddleware', From 182beed2b6e176197de4463b0b4ef893a947c1c3 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Fri, 5 Jun 2015 15:38:13 -0400 Subject: [PATCH 2/5] Port django.utils.translation.trans_real.parse_accept_lang_header from Django 1.8 Add to dark_lang middleware --- common/djangoapps/dark_lang/middleware.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/common/djangoapps/dark_lang/middleware.py b/common/djangoapps/dark_lang/middleware.py index b18d064969..804d72aa2e 100644 --- a/common/djangoapps/dark_lang/middleware.py +++ b/common/djangoapps/dark_lang/middleware.py @@ -12,10 +12,12 @@ the SessionMiddleware. """ from django.conf import settings -from django.utils.translation.trans_real import parse_accept_lang_header - from dark_lang.models import DarkLangConfig +# TODO re-import this once we're on Django 1.5 or greater. [PLAT-671] +# from django.utils.translation.trans_real import parse_accept_lang_header +from django_locale.trans_real import parse_accept_lang_header + def dark_parse_accept_lang_header(accept): ''' From e4cd2982d488b18af4046eec39a213faa2afa857 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Fri, 5 Jun 2015 15:39:32 -0400 Subject: [PATCH 3/5] Store released dark_lang codes as all lower-case --- common/djangoapps/dark_lang/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/djangoapps/dark_lang/models.py b/common/djangoapps/dark_lang/models.py index 1daec994e8..e61ea41fb2 100644 --- a/common/djangoapps/dark_lang/models.py +++ b/common/djangoapps/dark_lang/models.py @@ -25,7 +25,7 @@ class DarkLangConfig(ConfigurationModel): if not self.released_languages.strip(): # pylint: disable=no-member return [] - languages = [lang.strip() for lang in self.released_languages.split(',')] # pylint: disable=no-member + languages = [lang.lower().strip() for lang in self.released_languages.split(',')] # pylint: disable=no-member # Put in alphabetical order languages.sort() return languages From 7b09ab5e0001d96f48e4391196156108041fb970 Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Fri, 5 Jun 2015 23:46:21 -0400 Subject: [PATCH 4/5] dark_lang: only allow released langs in accept header LOC-72, LOC-85 Only return languages we've actually released LOC-85 Perform fuzzy matching to greedily serve the best released language LOC-72 --- common/djangoapps/dark_lang/middleware.py | 29 ++++--- common/djangoapps/dark_lang/tests.py | 92 ++++++++++++++++++++++- 2 files changed, 106 insertions(+), 15 deletions(-) diff --git a/common/djangoapps/dark_lang/middleware.py b/common/djangoapps/dark_lang/middleware.py index 804d72aa2e..7a7a6b8868 100644 --- a/common/djangoapps/dark_lang/middleware.py +++ b/common/djangoapps/dark_lang/middleware.py @@ -83,11 +83,17 @@ class DarkLangMiddleware(object): self._clean_accept_headers(request) self._activate_preview_language(request) - def _is_released(self, lang_code): - """ - ``True`` iff one of the values in ``self.released_langs`` is a prefix of ``lang_code``. - """ - return any(lang_code.lower().startswith(released_lang.lower()) for released_lang in self.released_langs) + def _fuzzy_match(self, lang_code): + """Returns a fuzzy match for lang_code""" + if lang_code in self.released_langs: + return lang_code + + lang_prefix = lang_code.partition('-')[0] + for released_lang in self.released_langs: + released_prefix = released_lang.partition('-')[0] + if lang_prefix == released_prefix: + return released_lang + return None def _format_accept_value(self, lang, priority=1.0): """ @@ -104,12 +110,13 @@ class DarkLangMiddleware(object): if accept is None or accept == '*': return - new_accept = ", ".join( - self._format_accept_value(lang, priority) - for lang, priority - in dark_parse_accept_lang_header(accept) - if self._is_released(lang) - ) + new_accept = [] + for lang, priority in dark_parse_accept_lang_header(accept): + fuzzy_code = self._fuzzy_match(lang.lower()) + if fuzzy_code: + new_accept.append(self._format_accept_value(fuzzy_code, priority)) + + new_accept = ", ".join(new_accept) request.META['HTTP_ACCEPT_LANGUAGE'] = new_accept diff --git a/common/djangoapps/dark_lang/tests.py b/common/djangoapps/dark_lang/tests.py index 6dd0b41882..b7210088e9 100644 --- a/common/djangoapps/dark_lang/tests.py +++ b/common/djangoapps/dark_lang/tests.py @@ -4,8 +4,10 @@ Tests of DarkLangMiddleware from django.contrib.auth.models import User from django.http import HttpRequest +import ddt from django.test import TestCase from mock import Mock +import unittest from dark_lang.middleware import DarkLangMiddleware from dark_lang.models import DarkLangConfig @@ -23,6 +25,7 @@ def set_if_set(dct, key, value): dct[key] = value +@ddt.ddt class DarkLangMiddlewareTests(TestCase): """ Tests of DarkLangMiddleware @@ -82,6 +85,10 @@ class DarkLangMiddlewareTests(TestCase): def test_wildcard_accept(self): self.assertAcceptEquals('*', self.process_request(accept='*')) + def test_malformed_accept(self): + self.assertAcceptEquals('', self.process_request(accept='xxxxxxxxxxxx')) + self.assertAcceptEquals('', self.process_request(accept='en;q=1.0, es-419:q-0.8')) + def test_released_accept(self): self.assertAcceptEquals( 'rel;q=1.0', @@ -123,14 +130,17 @@ class DarkLangMiddlewareTests(TestCase): ) def test_accept_released_territory(self): + # We will munge 'rel-ter' to be 'rel', so the 'rel-ter' + # user will actually receive the released language 'rel' + # (Otherwise, the user will actually end up getting the server default) self.assertAcceptEquals( - 'rel-ter;q=1.0, rel;q=0.5', + 'rel;q=1.0, rel;q=0.5', self.process_request(accept='rel-ter;q=1.0, rel;q=0.5') ) def test_accept_mixed_case(self): self.assertAcceptEquals( - 'rel-TER;q=1.0, REL;q=0.5', + 'rel;q=1.0, rel;q=0.5', self.process_request(accept='rel-TER;q=1.0, REL;q=0.5') ) @@ -140,11 +150,85 @@ class DarkLangMiddlewareTests(TestCase): enabled=True ).save() + # Since we have only released "rel-ter", the requested code "rel" will + # fuzzy match to "rel-ter", in addition to "rel-ter" exact matching "rel-ter" self.assertAcceptEquals( - 'rel-ter;q=1.0', + 'rel-ter;q=1.0, rel-ter;q=0.5', self.process_request(accept='rel-ter;q=1.0, rel;q=0.5') ) + @ddt.data( + ('es;q=1.0, pt;q=0.5', 'es-419;q=1.0'), # 'es' should get 'es-419', not English + ('es-AR;q=1.0, pt;q=0.5', 'es-419;q=1.0'), # 'es-AR' should get 'es-419', not English + ) + @ddt.unpack + def test_partial_match_es419(self, accept_header, expected): + # Release es-419 + DarkLangConfig( + released_languages=('es-419, en'), + changed_by=self.user, + enabled=True + ).save() + + self.assertAcceptEquals( + expected, + self.process_request(accept=accept_header) + ) + + def test_partial_match_esar_es(self): + # If I release 'es', 'es-AR' should get 'es', not English + DarkLangConfig( + released_languages=('es, en'), + changed_by=self.user, + enabled=True + ).save() + + self.assertAcceptEquals( + 'es;q=1.0', + self.process_request(accept='es-AR;q=1.0, pt;q=0.5') + ) + + @ddt.data( + # Test condition: If I release 'es-419, es, es-es'... + ('es;q=1.0, pt;q=0.5', 'es;q=1.0'), # 1. es should get es + ('es-419;q=1.0, pt;q=0.5', 'es-419;q=1.0'), # 2. es-419 should get es-419 + ('es-es;q=1.0, pt;q=0.5', 'es-es;q=1.0'), # 3. es-es should get es-es + ) + @ddt.unpack + def test_exact_match_gets_priority(self, accept_header, expected): + # Release 'es-419, es, es-es' + DarkLangConfig( + released_languages=('es-419, es, es-es'), + changed_by=self.user, + enabled=True + ).save() + self.assertAcceptEquals( + expected, + self.process_request(accept=accept_header) + ) + + @unittest.skip("This won't work until fallback is implemented for LA country codes. See LOC-86") + @ddt.data( + 'es-AR', # Argentina + 'es-PY', # Paraguay + ) + def test_partial_match_es_la(self, latin_america_code): + # We need to figure out the best way to implement this. There are a ton of LA country + # codes that ought to fall back to 'es-419' rather than 'es-es'. + # http://unstats.un.org/unsd/methods/m49/m49regin.htm#americas + # If I release 'es, es-419' + # Latin American codes should get es-419 + DarkLangConfig( + released_languages=('es, es-419'), + changed_by=self.user, + enabled=True + ).save() + + self.assertAcceptEquals( + 'es-419;q=1.0', + self.process_request(accept='{};q=1.0, pt;q=0.5'.format(latin_america_code)) + ) + def assertSessionLangEquals(self, value, request): """ Assert that the 'django_language' set in request.session is equal to value @@ -224,6 +308,6 @@ class DarkLangMiddlewareTests(TestCase): ).save() self.assertAcceptEquals( - 'zh-CN;q=1.0, zh-TW;q=0.5, zh-HK;q=0.3', + 'zh-cn;q=1.0, zh-tw;q=0.5, zh-hk;q=0.3', self.process_request(accept='zh-Hans;q=1.0, zh-Hant-TW;q=0.5, zh-HK;q=0.3') ) From ed2f73e6d2116aa9d07d1b2bff090917c1baa4fa Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Mon, 8 Jun 2015 14:35:52 -0400 Subject: [PATCH 5/5] Add i18n regression tests (LOC-72, LOC-85) --- lms/djangoapps/courseware/tests/test_i18n.py | 59 +++++++++++++++++--- 1 file changed, 52 insertions(+), 7 deletions(-) diff --git a/lms/djangoapps/courseware/tests/test_i18n.py b/lms/djangoapps/courseware/tests/test_i18n.py index a67442e64b..4e4c147865 100644 --- a/lms/djangoapps/courseware/tests/test_i18n.py +++ b/lms/djangoapps/courseware/tests/test_i18n.py @@ -4,37 +4,59 @@ Tests i18n in courseware import re from nose.plugins.attrib import attr +from django.contrib.auth.models import User from django.test import TestCase -from django.test.utils import override_settings + +from dark_lang.models import DarkLangConfig -@attr('shard_1') -@override_settings(LANGUAGES=[('eo', 'Esperanto'), ('ar', 'Arabic')]) -class I18nTestCase(TestCase): +class BaseI18nTestCase(TestCase): """ - Tests for i18n + Base utilities for i18n test classes to derive from """ def assert_tag_has_attr(self, content, tag, attname, value): """Assert that a tag in `content` has a certain value in a certain attribute.""" - regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d ]+)['"][^>]*>""".format(tag=tag, attname=attname) + regex = r"""<{tag} [^>]*\b{attname}=['"]([\w\d\- ]+)['"][^>]*>""".format(tag=tag, attname=attname) match = re.search(regex, content) - self.assertTrue(match, "Couldn't find desired tag in %r" % content) + self.assertTrue(match, "Couldn't find desired tag '%s' with attr '%s' in %r" % (tag, attname, content)) attvalues = match.group(1).split() self.assertIn(value, attvalues) + def release_languages(self, languages): + """ + Release a set of languages using the dark lang interface. + languages is a list of comma-separated lang codes, eg, 'ar, es-419' + """ + user = User() + user.save() + DarkLangConfig( + released_languages=languages, + changed_by=user, + enabled=True + ).save() + + +@attr('shard_1') +class I18nTestCase(BaseI18nTestCase): + """ + Tests for i18n + """ def test_default_is_en(self): + self.release_languages('fr') response = self.client.get('/') self.assert_tag_has_attr(response.content, "html", "lang", "en") self.assertEqual(response['Content-Language'], 'en') self.assert_tag_has_attr(response.content, "body", "class", "lang_en") def test_esperanto(self): + self.release_languages('fr, eo') response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='eo') self.assert_tag_has_attr(response.content, "html", "lang", "eo") self.assertEqual(response['Content-Language'], 'eo') self.assert_tag_has_attr(response.content, "body", "class", "lang_eo") def test_switching_languages_bidi(self): + self.release_languages('ar, eo') response = self.client.get('/') self.assert_tag_has_attr(response.content, "html", "lang", "en") self.assertEqual(response['Content-Language'], 'en') @@ -46,3 +68,26 @@ class I18nTestCase(TestCase): self.assertEqual(response['Content-Language'], 'ar') self.assert_tag_has_attr(response.content, "body", "class", "lang_ar") self.assert_tag_has_attr(response.content, "body", "class", "rtl") + + +@attr('shard_1') +class I18nRegressionTests(BaseI18nTestCase): + """ + Tests for i18n + """ + def test_es419_acceptance(self): + # Regression test; LOC-72, and an issue with Django + self.release_languages('es-419') + response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='es-419') + self.assert_tag_has_attr(response.content, "html", "lang", "es-419") + + def test_unreleased_lang_resolution(self): + # Regression test; LOC-85 + self.release_languages('fa') + + # We've released 'fa', AND we have language files for 'fa-ir' but + # we want to keep 'fa-ir' as a dark language. Requesting 'fa-ir' + # in the http request (NOT with the ?preview-lang query param) should + # receive files for 'fa' + response = self.client.get('/', HTTP_ACCEPT_LANGUAGE='fa-ir') + self.assert_tag_has_attr(response.content, "html", "lang", "fa")