From 3ed3fea2fbb14b6c64a406a9226f34b4c2136484 Mon Sep 17 00:00:00 2001 From: Matt Drayer Date: Tue, 16 Feb 2016 15:04:54 -0500 Subject: [PATCH] mattdrayer/xblock-translations: Add I18N/L10N support to XBlocks via the runtime * mattdrayer: Latest proto code * mattdrayer: Add translation.py * asadiqbal08: Xblock translation ugettext update, remove translation.py * mattdrayer: Additional I18N work -- starting to see some translations! * asadiqbal08: Trying to make xBlock message catalog files path dynamic * mattdrayer: Remove unnecessary modifications * mattdrayer: Cleaned up implementation * mattdrayer: Moved import statement * asadiqbal08: update as suggested * asadiqbal08: xblock its own domain * asadiqbal08: translation: secure none object * asadiqbal08: pylint * asadiqbal08: get locale from xblock * asadiqbal08: update * mattdrayer: Determine XBlock locale path within runtime service * mattdrayer: Determine module location via the runtime * mattdrayer: Remove ModuleI18nService reference * asadiqbal08: override the service in studio * asadiqbal08: remove import * asadiqbal08: update the Modulei18nService * asadiqbal08: update the Modulei18nService * mattdrayer: Remove redundant __class__ reference * asadiqbal08: update the docstring * asadiqbal08: tests * mattdrayer: Remove specific ugettext override from ModuleI18nService * mattdrayer: Move service operation to base class * mattdrayer: Address quality violations * asadiqbal08: Investigating the test failure issue on jenkins and solved * asadiqbal08: First utilizing the parent class method * mattdrayer: Use recommended callable approach * asadiqbal08: remove unused code * asadiqbal08: Updated the test to use cms preview module system runtime in order to get i18n service. * asadiqbal08: Pylint quality * asadiqbal08: update the service call to check xblock declarations * asadiqbal08: update doc string * asadiqbal08: i18n callable test in studio * asadiqbal08: test lms runtime for modulei18n service * asadiqbal08: add doc strings * asadiqbal08: Rename locale and domain to Flask-Babel convention --- .../contentstore/tests/test_i18n.py | 154 +++++++++++++++++- cms/djangoapps/contentstore/views/preview.py | 5 +- .../lib/xmodule/xmodule/modulestore/django.py | 34 +++- common/lib/xmodule/xmodule/x_module.py | 23 +++ lms/djangoapps/courseware/module_render.py | 3 +- lms/djangoapps/lms_xblock/runtime.py | 22 +-- .../lms_xblock/test/test_runtime.py | 62 +++++++ 7 files changed, 285 insertions(+), 18 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_i18n.py b/cms/djangoapps/contentstore/tests/test_i18n.py index d3203e3b89..d1a53135ef 100644 --- a/cms/djangoapps/contentstore/tests/test_i18n.py +++ b/cms/djangoapps/contentstore/tests/test_i18n.py @@ -1,9 +1,159 @@ +""" +Tests for validate Internationalization and Module i18n service. +""" +import mock +import gettext from unittest import skip - from django.contrib.auth.models import User - from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from contentstore.tests.utils import AjaxEnabledTestClient +from xmodule.modulestore.django import ModuleI18nService +from django.utils import translation +from django.utils.translation import get_language +from django.conf import settings +from xmodule.modulestore.tests.factories import ItemFactory, CourseFactory +from contentstore.views.preview import _preview_module_system + + +class FakeTranslations(ModuleI18nService): + """A test GNUTranslations class that takes a map of msg -> translations.""" + + def __init__(self, translations): # pylint: disable=super-init-not-called + self.translations = translations + + def ugettext(self, msgid): + """ + Mock override for ugettext translation operation + """ + return self.translations.get(msgid, msgid) + + @staticmethod + def translator(locales_map): # pylint: disable=method-hidden + """Build mock translator for the given locales. + Returns a mock gettext.translation function that uses + individual TestTranslations to translate in the given locales. + :param locales_map: A map from locale name to a translations map. + { + 'es': {'Hi': 'Hola', 'Bye': 'Adios'}, + 'zh': {'Hi': 'Ni Hao', 'Bye': 'Zaijian'} + } + """ + def _translation(domain, localedir=None, languages=None): # pylint: disable=unused-argument + """ + return gettext.translation for given language + """ + if languages: + language = languages[0] + if language in locales_map: + return FakeTranslations(locales_map[language]) + return gettext.NullTranslations() + return _translation + + +class TestModuleI18nService(ModuleStoreTestCase): + """ Test ModuleI18nService """ + + def setUp(self): + """ Setting up tests """ + super(TestModuleI18nService, self).setUp() + self.test_language = 'dummy language' + self.request = mock.Mock() + self.course = CourseFactory.create() + self.field_data = mock.Mock() + self.descriptor = ItemFactory(category="pure", parent=self.course) + self.runtime = _preview_module_system( + self.request, + self.descriptor, + self.field_data, + ) + self.addCleanup(translation.activate, settings.LANGUAGE_CODE) + + def get_module_i18n_service(self, descriptor): + """ + return the module i18n service. + """ + i18n_service = self.runtime.service(descriptor, 'i18n') + self.assertIsNotNone(i18n_service) + self.assertIsInstance(i18n_service, ModuleI18nService) + return i18n_service + + def test_django_service_translation_works(self): + """ + Test django translation service works fine. + """ + + def wrap_with_xyz(func): + """ + A decorator function that just adds 'XYZ ' to the front of all strings + """ + def new_func(*args, **kwargs): + """ custom function """ + output = func(*args, **kwargs) + return "XYZ " + output + return new_func + + old_lang = translation.get_language() + + i18n_service = self.get_module_i18n_service(self.descriptor) + + # Activate french, so that if the fr files haven't been loaded, they will be loaded now. + translation.activate("fr") + french_translation = translation.trans_real._active.value # pylint: disable=protected-access + + # wrap the ugettext functions so that 'TEST ' will prefix each translation + french_translation.ugettext = wrap_with_xyz(french_translation.ugettext) + self.assertEqual(i18n_service.ugettext(self.test_language), 'XYZ dummy language') + + # Turn back on our old translations + translation.activate(old_lang) + del old_lang + self.assertEqual(i18n_service.ugettext(self.test_language), 'dummy language') + + @mock.patch('django.utils.translation.ugettext', mock.Mock(return_value='XYZ-TEST-LANGUAGE')) + def test_django_translator_in_use_with_empty_block(self): + """ + Test: Django default translator should in use if we have an empty block + """ + i18n_service = ModuleI18nService(None) + self.assertEqual(i18n_service.ugettext(self.test_language), 'XYZ-TEST-LANGUAGE') + + @mock.patch('django.utils.translation.ugettext', mock.Mock(return_value='XYZ-TEST-LANGUAGE')) + def test_message_catalog_translations(self): + """ + Test: Message catalog from FakeTranslation should return required translations. + """ + _translator = FakeTranslations.translator( + { + 'es': {'Hello': 'es-hello-world'}, + 'fr': {'Hello': 'fr-hello-world'}, + }, + ) + localedir = '/translations' + translation.activate("es") + with mock.patch('gettext.translation', return_value=_translator(domain='text', localedir=localedir, + languages=[get_language()])): + i18n_service = self.get_module_i18n_service(self.descriptor) + self.assertEqual(i18n_service.ugettext('Hello'), 'es-hello-world') + + translation.activate("ar") + with mock.patch('gettext.translation', return_value=_translator(domain='text', localedir=localedir, + languages=[get_language()])): + i18n_service = self.get_module_i18n_service(self.descriptor) + self.assertEqual(i18n_service.ugettext('Hello'), 'Hello') + self.assertNotEqual(i18n_service.ugettext('Hello'), 'fr-hello-world') + self.assertNotEqual(i18n_service.ugettext('Hello'), 'es-hello-world') + + translation.activate("fr") + with mock.patch('gettext.translation', return_value=_translator(domain='text', localedir=localedir, + languages=[get_language()])): + i18n_service = self.get_module_i18n_service(self.descriptor) + self.assertEqual(i18n_service.ugettext('Hello'), 'fr-hello-world') + + def test_i18n_service_callable(self): + """ + Test: i18n service should be callable in studio. + """ + self.assertTrue(callable(self.runtime._services.get('i18n'))) # pylint: disable=protected-access class InternationalizationTest(ModuleStoreTestCase): diff --git a/cms/djangoapps/contentstore/views/preview.py b/cms/djangoapps/contentstore/views/preview.py index d697b9e959..b9d39e417b 100644 --- a/cms/djangoapps/contentstore/views/preview.py +++ b/cms/djangoapps/contentstore/views/preview.py @@ -104,6 +104,9 @@ class PreviewModuleSystem(ModuleSystem): # pylint: disable=abstract-method # they are being rendered for preview (i.e. in Studio) is_author_mode = True + def __init__(self, **kwargs): + super(PreviewModuleSystem, self).__init__(**kwargs) + def handler_url(self, block, handler_name, suffix='', query='', thirdparty=False): return reverse('preview_handler', kwargs={ 'usage_key_string': unicode(block.scope_ids.usage_id), @@ -236,8 +239,8 @@ def _preview_module_system(request, descriptor, field_data): # Get the raw DescriptorSystem, not the CombinedSystem descriptor_runtime=descriptor._runtime, # pylint: disable=protected-access services={ - "i18n": ModuleI18nService(), "field-data": field_data, + "i18n": ModuleI18nService, "library_tools": LibraryToolsService(modulestore()), "settings": SettingsService(), "user": DjangoXBlockUserService(request.user), diff --git a/common/lib/xmodule/xmodule/modulestore/django.py b/common/lib/xmodule/xmodule/modulestore/django.py index a881082049..c7ddd95a1c 100644 --- a/common/lib/xmodule/xmodule/modulestore/django.py +++ b/common/lib/xmodule/xmodule/modulestore/django.py @@ -7,9 +7,11 @@ Passes settings.MODULESTORE as kwargs to MongoModuleStore from __future__ import absolute_import from importlib import import_module +import gettext import logging - +from pkg_resources import resource_filename import re + from django.conf import settings # This configuration must be executed BEFORE any additional Django imports. Otherwise, the imports may fail due to @@ -20,6 +22,7 @@ if not settings.configured: from django.core.cache import caches, InvalidCacheBackendError import django.dispatch import django.utils +from django.utils.translation import get_language, to_locale from pymongo import ReadPreference from xmodule.contentstore.django import contentstore @@ -28,7 +31,6 @@ from xmodule.modulestore.mixed import MixedModuleStore from xmodule.util.django import get_current_request_hostname import xblock.reference.plugins - try: # We may not always have the request_cache module available from request_cache.middleware import RequestCache @@ -243,9 +245,35 @@ class ModuleI18nService(object): i18n service. """ + def __init__(self, block=None): + """ + Attempt to load an XBlock-specific GNU gettext translator using the XBlock's own domain + translation catalog, currently expected to be found at: + /conf/locale//LC_MESSAGES/.po|mo + If we can't locate the domain translation catalog then we fall-back onto + django.utils.translation, which will point to the system's own domain translation catalog + This effectively achieves translations by coincidence for an XBlock which does not provide + its own dedicated translation catalog along with its implementation. + """ + self.translator = django.utils.translation + if block: + xblock_resource = block.unmixed_class.__module__ + xblock_locale_dir = '/translations' + xblock_locale_path = resource_filename(xblock_resource, xblock_locale_dir) + xblock_domain = 'text' + selected_language = get_language() + try: + self.translator = gettext.translation( + xblock_domain, + xblock_locale_path, + [to_locale(selected_language if selected_language else settings.LANGUAGE_CODE)] + ) + except IOError: + # Fall back to the default Django translator if the XBlock translator is not found. + pass def __getattr__(self, name): - return getattr(django.utils.translation, name) + return getattr(self.translator, name) def strftime(self, *args, **kwargs): """ diff --git a/common/lib/xmodule/xmodule/x_module.py b/common/lib/xmodule/xmodule/x_module.py index c6b54c8e24..d8a1fd8649 100644 --- a/common/lib/xmodule/xmodule/x_module.py +++ b/common/lib/xmodule/xmodule/x_module.py @@ -24,6 +24,7 @@ from xblock.fields import ( String, Dict, ScopeIds, Reference, ReferenceList, ReferenceValueDict, UserScope ) + from xblock.fragment import Fragment from xblock.runtime import Runtime, IdReader, IdGenerator from xmodule import course_metadata_utils @@ -1776,6 +1777,28 @@ class ModuleSystem(MetricsMixin, ConfigurableFragmentWrapper, Runtime): def publish(self, block, event_type, event): pass + def service(self, block, service_name): + """ + Runtime-specific override for the XBlock service manager. If a service is not currently + instantiated and is declared as a critical requirement, an attempt is made to load the + module. + + Arguments: + block (an XBlock): this block's class will be examined for service + decorators. + service_name (string): the name of the service requested. + + Returns: + An object implementing the requested service, or None. + """ + # getting the service from parent module. making sure of block service declarations. + service = super(ModuleSystem, self).service(block=block, service_name=service_name) + # Passing the block to service if it is callable e.g. ModuleI18nService. It is the responsibility of calling + # service to handle the passing argument. + if callable(service): + return service(block) + return service + class CombinedSystem(object): """ diff --git a/lms/djangoapps/courseware/module_render.py b/lms/djangoapps/courseware/module_render.py index 2d8295598b..171cdad2fd 100644 --- a/lms/djangoapps/courseware/module_render.py +++ b/lms/djangoapps/courseware/module_render.py @@ -75,7 +75,7 @@ from xmodule.error_module import ErrorDescriptor, NonStaffErrorDescriptor from xmodule.exceptions import NotFoundError, ProcessingError from xmodule.lti_module import LTIModule from xmodule.mixin import wrap_with_license -from xmodule.modulestore.django import modulestore, ModuleI18nService +from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.x_module import XModuleDescriptor from .field_overrides import OverrideFieldData @@ -713,7 +713,6 @@ def get_module_system_for_user(user, student_data, # TODO # pylint: disable=to wrappers=block_wrappers, get_real_user=user_by_anonymous_id, services={ - 'i18n': ModuleI18nService(), 'fs': FSService(), 'field-data': field_data, 'user': DjangoXBlockUserService(user, user_is_staff=user_is_staff), diff --git a/lms/djangoapps/lms_xblock/runtime.py b/lms/djangoapps/lms_xblock/runtime.py index 7fc1ddcedd..1206eef977 100644 --- a/lms/djangoapps/lms_xblock/runtime.py +++ b/lms/djangoapps/lms_xblock/runtime.py @@ -1,20 +1,21 @@ """ Module implementing `xblock.runtime.Runtime` functionality for the LMS """ - import re -import xblock.reference.plugins from django.core.urlresolvers import reverse from django.conf import settings -from request_cache.middleware import RequestCache -from lms.djangoapps.lms_xblock.models import XBlockAsidesConfig + from openedx.core.djangoapps.user_api.course_tag import api as user_course_tag_api -from xmodule.modulestore.django import modulestore -from xmodule.services import SettingsService +from request_cache.middleware import RequestCache +import xblock.reference.plugins from xmodule.library_tools import LibraryToolsService -from xmodule.x_module import ModuleSystem +from xmodule.modulestore.django import modulestore, ModuleI18nService from xmodule.partitions.partitions_service import PartitionService +from xmodule.services import SettingsService +from xmodule.x_module import ModuleSystem + +from lms.djangoapps.lms_xblock.models import XBlockAsidesConfig def _quote_slashes(match): @@ -201,16 +202,17 @@ class LmsModuleSystem(ModuleSystem): # pylint: disable=abstract-method def __init__(self, **kwargs): request_cache_dict = RequestCache.get_request_cache().data services = kwargs.setdefault('services', {}) - services['user_tags'] = UserTagsService(self) + services['fs'] = xblock.reference.plugins.FSService() + services['i18n'] = ModuleI18nService + services['library_tools'] = LibraryToolsService(modulestore()) services['partitions'] = LmsPartitionService( user=kwargs.get('user'), course_id=kwargs.get('course_id'), track_function=kwargs.get('track_function', None), cache=request_cache_dict ) - services['library_tools'] = LibraryToolsService(modulestore()) - services['fs'] = xblock.reference.plugins.FSService() services['settings'] = SettingsService() + services['user_tags'] = UserTagsService(self) self.request_token = kwargs.pop('request_token', None) super(LmsModuleSystem, self).__init__(**kwargs) diff --git a/lms/djangoapps/lms_xblock/test/test_runtime.py b/lms/djangoapps/lms_xblock/test/test_runtime.py index e844c92069..46bfe72f23 100644 --- a/lms/djangoapps/lms_xblock/test/test_runtime.py +++ b/lms/djangoapps/lms_xblock/test/test_runtime.py @@ -11,6 +11,10 @@ from urlparse import urlparse from opaque_keys.edx.locations import BlockUsageLocator, CourseLocator, SlashSeparatedCourseKey from lms.djangoapps.lms_xblock.runtime import quote_slashes, unquote_slashes, LmsModuleSystem from xblock.fields import ScopeIds +from xmodule.modulestore.django import ModuleI18nService +from xmodule.modulestore.tests.factories import CourseFactory +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xblock.exceptions import NoSuchServiceError TEST_STRINGS = [ '', @@ -181,3 +185,61 @@ class TestUserServiceAPI(TestCase): # Try to get tag in wrong scope with self.assertRaises(ValueError): self.runtime.service(self.mock_block, 'user_tags').get_tag('fake_scope', self.key) + + +class TestI18nService(ModuleStoreTestCase): + """ Test ModuleI18nService """ + + def setUp(self): + """ Setting up tests """ + super(TestI18nService, self).setUp() + self.course = CourseFactory.create() + self.test_language = 'dummy language' + self.runtime = LmsModuleSystem( + static_url='/static', + track_function=Mock(), + get_module=Mock(), + render_template=Mock(), + replace_urls=str, + course_id=self.course.id, + descriptor_runtime=Mock(), + ) + + self.mock_block = Mock() + self.mock_block.service_declaration.return_value = 'need' + + def test_module_i18n_lms_service(self): + """ + Test: module i18n service in LMS + """ + i18n_service = self.runtime.service(self.mock_block, 'i18n') + self.assertIsNotNone(i18n_service) + self.assertIsInstance(i18n_service, ModuleI18nService) + + def test_no_service_exception_with_none_declaration_(self): + """ + Test: NoSuchServiceError should be raised block declaration returns none + """ + self.mock_block.service_declaration.return_value = None + with self.assertRaises(NoSuchServiceError): + self.runtime.service(self.mock_block, 'i18n') + + def test_no_service_exception_(self): + """ + Test: NoSuchServiceError should be raised if i18n service is none. + """ + self.runtime._services['i18n'] = None # pylint: disable=protected-access + with self.assertRaises(NoSuchServiceError): + self.runtime.service(self.mock_block, 'i18n') + + def test_i18n_service_callable(self): + """ + Test: _services dict should contain the callable i18n service in LMS. + """ + self.assertTrue(callable(self.runtime._services.get('i18n'))) # pylint: disable=protected-access + + def test_i18n_service_not_callable(self): + """ + Test: i18n service should not be callable in LMS after initialization. + """ + self.assertFalse(callable(self.runtime.service(self.mock_block, 'i18n')))