diff --git a/.gitignore b/.gitignore index 10cc4812a8..3ddab93528 100644 --- a/.gitignore +++ b/.gitignore @@ -30,14 +30,14 @@ codekit-config.json ### Internationalization artifacts *.mo +*.po +!django.po +!django.mo +!djangojs.po +!djangojs.mo conf/locale/en/LC_MESSAGES/*.po -!messages.po -### Remove when we have real Esperanto translations. For now, ignore -### dummy Esperanto files. -conf/locale/eo/* -## Remove when we officially support these languages. -conf/locale/fr -conf/locale/ko_KR +conf/locale/en/LC_MESSAGES/*.mo +conf/locale/messages.mo ### Testing artifacts .testids/ diff --git a/cms/envs/common.py b/cms/envs/common.py index 4abea6ce13..9a83610efe 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ This is the common settings file, intended to set sane defaults. If you have a piece of configuration that's dependent on a set of feature flags being set, @@ -254,10 +255,7 @@ STATICFILES_DIRS = [ TIME_ZONE = 'America/New_York' # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name LANGUAGE_CODE = 'en' # http://www.i18nguy.com/unicode/language-identifiers.html -LANGUAGES = ( - ('eo', 'Esperanto'), -) - +LANGUAGES = lms.envs.common.LANGUAGES USE_I18N = True USE_L10N = True diff --git a/conf/locale/config.yaml b/conf/locale/config.yaml index 195cfdf0e0..35d0272a4b 100644 --- a/conf/locale/config.yaml +++ b/conf/locale/config.yaml @@ -1,33 +1,55 @@ # Configuration for i18n workflow. locales: - - en + - en # English - Source Language + - ach # Acoli + - ar # Arabic + - bg_BG # Bulgarian (Bulgaria) + - bn # Bengali + - bn_BD # Bengali (Bangladesh) + - cs # Czech + - cy # Welsh + - de_DE # German (Germany) + - en@lolcat # LOLCAT English + - en@pirate # Pirate English + - es_419 # Spanish (Latin America) + - es_EC # Spanish (Ecuador) + - es_ES # Spanish (Spain) + - es_MX # Spanish (Mexico) + - es_US # Spanish (United States) + - et_EE # Estonian (Estonia) + - fa # Persian + - fa_IR # Persian (Iran) + - fi_FI # Finnish (Finland) + - fr # French + - gl # Galician + - he # Hebrew + - hi # Hindi + - hy_AM # Armenian (Armenia) + - id # Indonesian + - it_IT # Italian (Italy) + - ja_JP # Japanese (Japan) + - km_KH # Khmer (Cambodia) + - ko_KR # Korean (Korea) + - lt_LT # Lithuanian (Lithuania) + - ml # Malayalam + - nb # Norwegian Bokmål + - nl_NL # Dutch (Netherlands) + - pl # Polish + - pt_BR # Portuguese (Brazil) + - pt_PT # Portuguese (Portugal) + - ru # Russian + - si # Sinhala + - sk # Slovak + - sl # Slovenian + - th # Thai + - tr_TR # Turkish (Turkey) + - uk # Ukranian + - vi # Vietnamese + - zh_CN # Chinese (China) + - zh_CN.GB2312 # Chinese (China) (GB2312) + - zh_TW # Chinese (Taiwan) - # More languages we might want someday, these have started on Transifex. - # fr - # ko_KR - # ru - # es_419 - # ja_JP - # pt_BR - # zh_CN - # zh_TW - # ar - # es_ES - # fa_IR - # tr_TR - # de_DE - # id - # hi - # vi - # pt_PT - # lt_LT - # gl - # it_IT - # cs - # et_EE - # nb - # sk # The locale used for fake-accented English, for testing. dummy-locale: eo diff --git a/i18n/config.py b/i18n/config.py index 3828578b5b..4551239166 100644 --- a/i18n/config.py +++ b/i18n/config.py @@ -69,5 +69,11 @@ class Configuration(object): """ return self.get_messages_dir(self.source_locale) + @property + def translated_locales(self): + """ + Returns the set of locales to be translated (ignoring the source_locale). + """ + return sorted(set(self.locales) - set([self.source_locale])) CONFIGURATION = Configuration(LOCALE_DIR.joinpath('config.yaml').normpath()) diff --git a/i18n/extract.py b/i18n/extract.py index 5145e7b651..70ef33c3b5 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -32,7 +32,7 @@ BABEL_CONFIG = BASE_DIR.relpathto(LOCALE_DIR.joinpath('babel.cfg')) # Use relpath to reduce noise in logs BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako.po')) -SOURCE_WARN = 'This English source file is machine-generated. Do not check it into git.' +EDX_MARKER = "edX translation file" LOG = logging.getLogger(__name__) @@ -104,8 +104,8 @@ def fix_header(po): po.metadata_is_fuzzy = [] # remove [u'fuzzy'] header = po.header fixes = ( - ('SOME DESCRIPTIVE TITLE', 'edX translation file\n' + SOURCE_WARN), - ('Translations template for PROJECT.', 'edX translation file\n' + SOURCE_WARN), + ('SOME DESCRIPTIVE TITLE', EDX_MARKER), + ('Translations template for PROJECT.', EDX_MARKER), ('YEAR', '%s' % datetime.utcnow().year), ('ORGANIZATION', 'edX'), ("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"), diff --git a/i18n/generate.py b/i18n/generate.py index 1d38273fea..b1ee991102 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -103,7 +103,7 @@ def validate_files(dir, files_to_merge): def main(): logging.basicConfig(stream=sys.stdout, level=logging.INFO) - for locale in CONFIGURATION.locales: + for locale in CONFIGURATION.translated_locales: merge_files(locale) # Dummy text is not required. Don't raise exception if files are missing. merge_files(CONFIGURATION.dummy_locale, fail_if_missing=False) diff --git a/i18n/tests/test_compiled_messages.py b/i18n/tests/test_compiled_messages.py index 5b5bda906e..bac8089a83 100644 --- a/i18n/tests/test_compiled_messages.py +++ b/i18n/tests/test_compiled_messages.py @@ -13,7 +13,6 @@ from unittest import TestCase from i18n.config import CONFIGURATION, LOCALE_DIR - @ddt.ddt class TestCompiledMessages(TestCase): """ @@ -22,7 +21,7 @@ class TestCompiledMessages(TestCase): PO_FILES = ['django.po', 'djangojs.po'] - @ddt.data(*CONFIGURATION.locales) + @ddt.data(*CONFIGURATION.translated_locales) def test_translated_messages(self, locale): message_dir = LOCALE_DIR / locale / 'LC_MESSAGES' for pofile_name in self.PO_FILES: diff --git a/i18n/tests/test_generate.py b/i18n/tests/test_generate.py index 2ed2836427..6fa67c16ca 100644 --- a/i18n/tests/test_generate.py +++ b/i18n/tests/test_generate.py @@ -8,6 +8,7 @@ from unittest import TestCase from polib import pofile from pytz import UTC +from i18n import extract from i18n import generate from i18n.config import CONFIGURATION @@ -27,6 +28,7 @@ class TestGenerate(TestCase): """ Tests merge script on English source files. """ + extract.main() filename = os.path.join(CONFIGURATION.source_messages_dir, random_name()) generate.merge(CONFIGURATION.source_locale, target=filename) self.assertTrue(os.path.exists(filename)) @@ -41,7 +43,7 @@ class TestGenerate(TestCase): after start of test suite) """ generate.main() - for locale in CONFIGURATION.locales: + for locale in CONFIGURATION.translated_locales: for filename in ('django', 'djangojs'): mofile = filename+'.mo' path = os.path.join(CONFIGURATION.get_messages_dir(locale), mofile) diff --git a/i18n/transifex.py b/i18n/transifex.py index b23432f905..c5df793325 100755 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -4,11 +4,11 @@ import sys from polib import pofile from i18n.config import CONFIGURATION -from i18n.extract import SOURCE_WARN from i18n.execute import execute +from i18n.extract import EDX_MARKER -TRANSIFEX_HEADER = 'Translations in this file have been downloaded from %s' -TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-studio/' +TRANSIFEX_HEADER = 'edX community translations have been downloaded from %s' +TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-platform/' def push(): @@ -26,9 +26,9 @@ def clean_translated_locales(): Strips out the warning from all translated po files about being an English source file. """ - for locale in CONFIGURATION.locales: - if locale != CONFIGURATION.source_locale: - clean_locale(locale) + for locale in CONFIGURATION.translated_locales: + clean_locale(locale) + def clean_locale(locale): """ @@ -40,18 +40,20 @@ def clean_locale(locale): for filename in ('django-partial.po', 'djangojs-partial.po', 'mako.po'): clean_file(dirname.joinpath(filename)) -def clean_file(file): + +def clean_file(filename): """ Strips out the warning from a translated po file about being an English source file. Replaces warning with a note about coming from Transifex. """ - po = pofile(file) - if po.header.find(SOURCE_WARN) != -1: + po = pofile(filename) + if po.header.find(EDX_MARKER) != -1: new_header = get_new_header(po) - new = po.header.replace(SOURCE_WARN, new_header) + new = po.header.replace(EDX_MARKER, new_header) po.header = new po.save() + def get_new_header(po): team = po.metadata.get('Language-Team', None) if not team: @@ -59,6 +61,7 @@ def get_new_header(po): else: return TRANSIFEX_HEADER % team + if __name__ == '__main__': if len(sys.argv) < 2: raise Exception("missing argument: push or pull") diff --git a/lms/envs/common.py b/lms/envs/common.py index 543a47737b..5bc8f27d32 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- """ This is the common settings file, intended to set sane defaults. If you have a piece of configuration that's dependent on a set of feature flags being set, @@ -499,8 +500,56 @@ FAVICON_PATH = 'images/favicon.ico' TIME_ZONE = 'America/New_York' # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name LANGUAGE_CODE = 'en' # http://www.i18nguy.com/unicode/language-identifiers.html +# Sourced from http://www.localeplanet.com/icu/ and wikipedia LANGUAGES = ( - ('eo', 'Esperanto'), + ('eo', u'Dummy Language (Esperanto)'), # Dummy languaged used for testing + + ('ach', u'Acholi'), # Acoli + ('ar', u'العربية'), # Arabic + ('bg-bg', u'български (България)'), # Bulgarian (Bulgaria) + ('bn', u'বাংলা'), # Bengali + ('bn-bd', u'বাংলা (বাংলাদেশ)'), # Bengali (Bangladesh) + ('cs', u'Čeština'), # Czech + ('cy', u'Cymraeg'), # Welsh + ('de-de', u'Deutsch (Deutschland)'), # German (Germany) + ('en@lolcat', u'LOLCAT English'), # LOLCAT English + ('en@pirate', u'Pirate English'), # Pirate English + ('es-419', u'Español (Latinoamérica)'), # Spanish (Latin America) + ('es-ec', u'Español (Ecuador)'), # Spanish (Ecuador) + ('es-es', u'Español (España)'), # Spanish (Spain) + ('es-mx', u'Español (México)'), # Spanish (Mexico) + ('es-us', u'Español (Estados Unidos)'), # Spanish (United States) + ('et-ee', u'Eesti (Eesti)'), # Estonian (Estonia) + ('fa', u'فارسی'), # Persian + ('fa-ir', u'فارسی (ایران)'), # Persian (Iran) + ('fi-fi', u'Suomi (Suomi)'), # Finnish (Finland) + ('fr', u'Français'), # French + ('gl', u'Galego'), # Galician + ('he', u'עברית'), # Hebrew + ('hi', u'हिन्दी'), # Hindi + ('hy-am', u'Հայերէն (Հայաստանի Հանրապետութիւն)'), # Armenian (Armenia) + ('id', u'Bahasa Indonesia'), # Indonesian + ('it-it', u'Italiano (Italia)'), # Italian (Italy) + ('ja-jp', u'日本語(日本)'), # Japanese (Japan) + ('km-kh', u'ភាសាខ្មែរ (កម្ពុជា)'), # Khmer (Cambodia) + ('ko-kr', u'한국어(대한민국)'), # Korean (Korea) + ('lt-lt', u'Lietuvių (Lietuva)'), # Lithuanian (Lithuania) + ('ml', u'മലയാളം'), # Malayalam + ('nb', u'Norsk bokmål'), # Norwegian Bokmål + ('nl-nl', u'Nederlands (Nederland)'), # Dutch (Netherlands) + ('pl', u'Polski'), # Polish + ('pt-br', u'Português (Brasil)'), # Portuguese (Brazil) + ('pt-pt', u'Português (Portugal)'), # Portuguese (Portugal) + ('ru', u'Русский'), # Russian + ('si', u'සිංහල'), # Sinhala + ('sk', u'Slovenčina'), # Slovak + ('sl', u'Slovenščina'), # Slovenian + ('th', u'ไทย'), # Thai + ('tr-tr', u'Türkçe (Türkiye)'), # Turkish (Turkey) + ('uk', u'Українська'), # Uknranian + ('vi', u'Tiếng Việt'), # Vietnamese + ('zh-cn', u'大陆简体'), # Chinese (China) + ('zh-tw', u'台灣正體'), # Chinese (Taiwan) ) USE_I18N = True diff --git a/rakelib/i18n.rake b/rakelib/i18n.rake index 9f1dac4dd3..e5ff74c1d6 100644 --- a/rakelib/i18n.rake +++ b/rakelib/i18n.rake @@ -57,7 +57,7 @@ namespace :i18n do end desc "Run tests for the internationalization library" - task :test => ["i18n:validate:gettext", "i18n:extract", "i18n:generate"] do + task :test do test = File.join(REPO_ROOT, "i18n", "tests") pythonpath_prefix = "PYTHONPATH=#{REPO_ROOT}/i18n:$PYTHONPATH" sh("#{pythonpath_prefix} nosetests #{test}")