diff --git a/.gitignore b/.gitignore index d01baf055a..f1784a48f3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.swp *.orig *.DS_Store +*.mo :2e_* :2e# .AppleDouble @@ -22,6 +23,8 @@ reports/ *.egg-info Gemfile.lock .env/ +conf/locale/en/LC_MESSAGES/*.po +!messages.po lms/static/sass/*.css cms/static/sass/*.css lms/lib/comment_client/python diff --git a/.tx/config b/.tx/config new file mode 100644 index 0000000000..540c4732af --- /dev/null +++ b/.tx/config @@ -0,0 +1,26 @@ +[main] +host = https://www.transifex.com + +[edx-studio.django-partial] +file_filter = conf/locale//LC_MESSAGES/django-partial.po +source_file = conf/locale/en/LC_MESSAGES/django-partial.po +source_lang = en +type = PO + +[edx-studio.djangojs] +file_filter = conf/locale//LC_MESSAGES/djangojs.po +source_file = conf/locale/en/LC_MESSAGES/djangojs.po +source_lang = en +type = PO + +[edx-studio.mako] +file_filter = conf/locale//LC_MESSAGES/mako.po +source_file = conf/locale/en/LC_MESSAGES/mako.po +source_lang = en +type = PO + +[edx-studio.messages] +file_filter = conf/locale//LC_MESSAGES/messages.po +source_file = conf/locale/en/LC_MESSAGES/messages.po +source_lang = en +type = PO diff --git a/conf/locale/config b/conf/locale/config index 2d01e1ea43..58f8da0513 100644 --- a/conf/locale/config +++ b/conf/locale/config @@ -1 +1,4 @@ -{"locales" : ["en"]} +{ + "locales" : ["en", "es"], + "dummy-locale" : "fr" +} diff --git a/conf/locale/en/LC_MESSAGES/messages.po b/conf/locale/en/LC_MESSAGES/messages.po index 1bb8bf6d7f..e5961753c5 100644 --- a/conf/locale/en/LC_MESSAGES/messages.po +++ b/conf/locale/en/LC_MESSAGES/messages.po @@ -1 +1,20 @@ +# edX translation file +# Copyright (C) 2013 edX +# This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE. +# +msgid "" +msgstr "" +"Project-Id-Version: EdX Studio\n" +"Report-Msgid-Bugs-To: translation_team@edx.org\n" +"POT-Creation-Date: 2013-05-02 13:13-0400\n" +"PO-Revision-Date: 2013-05-02 13:27-0400\n" +"Last-Translator: \n" +"Language-Team: translation team \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: en\n" + # empty +msgid "This is a key string." +msgstr "" diff --git a/i18n/config.py b/i18n/config.py new file mode 100644 index 0000000000..4f246ed942 --- /dev/null +++ b/i18n/config.py @@ -0,0 +1,77 @@ +import os, json +from path import path + +# BASE_DIR is the working directory to execute django-admin commands from. +# Typically this should be the 'mitx' directory. +BASE_DIR = path(__file__).abspath().dirname().joinpath('..').normpath() + +# LOCALE_DIR contains the locale files. +# Typically this should be 'mitx/conf/locale' +LOCALE_DIR = BASE_DIR.joinpath('conf', 'locale') + +class Configuration: + """ + # Reads localization configuration in json format + + """ + _source_locale = 'en' + + def __init__(self, filename): + self._filename = filename + self._config = self.read_config(filename) + + def read_config(self, filename): + """ + Returns data found in config file (as dict), or raises exception if file not found + """ + if not os.path.exists(filename): + raise Exception("Configuration file cannot be found: %s" % filename) + with open(filename) as stream: + return json.load(stream) + + @property + def locales(self): + """ + Returns a list of locales declared in the configuration file, + e.g. ['en', 'fr', 'es'] + Each locale is a string. + """ + return self._config['locales'] + + @property + def source_locale(self): + """ + Returns source language. + Source language is English. + """ + return self._source_locale + + @property + def dummy_locale(self): + """ + Returns a locale to use for the dummy text, e.g. 'fr'. + Throws exception if no dummy-locale is declared. + The locale is a string. + """ + dummy = self._config.get('dummy-locale', None) + if not dummy: + raise Exception('Could not read dummy-locale from configuration file.') + return dummy + + def get_messages_dir(self, locale): + """ + Returns the name of the directory holding the po files for locale. + Example: mitx/conf/locale/fr/LC_MESSAGES + """ + return LOCALE_DIR.joinpath(locale, 'LC_MESSAGES') + + @property + def source_messages_dir(self): + """ + Returns the name of the directory holding the source-language po files (English). + Example: mitx/conf/locale/en/LC_MESSAGES + """ + return self.get_messages_dir(self.source_locale) + + +CONFIGURATION = Configuration(LOCALE_DIR.joinpath('config').normpath()) diff --git a/i18n/execute.py b/i18n/execute.py index 3c3416b65d..8e7f0f52de 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -1,69 +1,30 @@ -import os, subprocess, logging, json +import os, subprocess, logging -def init_module(): - """ - Initializes module parameters - """ - global BASE_DIR, LOCALE_DIR, CONFIG_FILENAME, SOURCE_MSGS_DIR, SOURCE_LOCALE, LOG +from config import CONFIGURATION, BASE_DIR - # BASE_DIR is the working directory to execute django-admin commands from. - # Typically this should be the 'mitx' directory. - BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') +LOG = logging.getLogger(__name__) - # Source language is English - SOURCE_LOCALE = 'en' - - # LOCALE_DIR contains the locale files. - # Typically this should be 'mitx/conf/locale' - LOCALE_DIR = BASE_DIR + '/conf/locale' - - # CONFIG_FILENAME contains localization configuration in json format - CONFIG_FILENAME = LOCALE_DIR + '/config' - - # SOURCE_MSGS_DIR contains the English po files. - SOURCE_MSGS_DIR = messages_dir(SOURCE_LOCALE) - - # Default logger. - LOG = get_logger() - - -def messages_dir(locale): - """ - Returns the name of the directory holding the po files for locale. - Example: mitx/conf/locale/en/LC_MESSAGES - """ - return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES') - -def get_logger(): - """Returns a default logger""" - log = logging.getLogger(__name__) - log.setLevel(logging.INFO) - log_handler = logging.StreamHandler() - log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')) - log.addHandler(log_handler) - return log - -# Run this after defining messages_dir and get_logger, because it depends on these. -init_module() - -def execute (command, working_directory=BASE_DIR, log=LOG): +def execute(command, working_directory=BASE_DIR): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. - Output is logged to log. + Output is ignored. """ - log.info(command) + LOG.info(command) subprocess.call(command.split(' '), cwd=working_directory) - -def get_config(): - """Returns data found in config file, or returns None if file not found""" - config_path = os.path.abspath(CONFIG_FILENAME) - if not os.path.exists(config_path): - log.warn("Configuration file cannot be found: %s" % \ - os.path.relpath(config_path, BASE_DIR)) - return None - with open(config_path) as stream: - return json.load(stream) + + +def call(command, working_directory=BASE_DIR): + """ + Executes shell command in a given working_directory. + Command is a string to pass to the shell. + Returns a tuple of two strings: (stdout, stderr) + + """ + LOG.info(command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_directory) + out, err = p.communicate() + return (out, err) def create_dir_if_necessary(pathname): dirname = os.path.dirname(pathname) @@ -71,16 +32,16 @@ def create_dir_if_necessary(pathname): os.makedirs(dirname) -def remove_file(filename, log=LOG, verbose=True): +def remove_file(filename, verbose=True): """ Attempt to delete filename. + log is boolean. If true, removal is logged. Log a warning if file does not exist. Logging filenames are releative to BASE_DIR to cut down on noise in output. """ if verbose: - log.info('Deleting file %s' % os.path.relpath(filename, BASE_DIR)) + LOG.info('Deleting file %s' % os.path.relpath(filename, BASE_DIR)) if not os.path.exists(filename): - log.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR)) + LOG.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR)) else: os.remove(filename) - diff --git a/i18n/extract.py b/i18n/extract.py index c6fedd3bfa..c28c3868e2 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow @@ -15,28 +15,35 @@ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow """ -import os +import os, sys, logging from datetime import datetime from polib import pofile -from execute import execute, create_dir_if_necessary, remove_file, \ - BASE_DIR, LOCALE_DIR, SOURCE_MSGS_DIR, LOG +from config import BASE_DIR, LOCALE_DIR, CONFIGURATION +from execute import execute, create_dir_if_necessary, remove_file # BABEL_CONFIG contains declarations for Babel to extract strings from mako template files # Use relpath to reduce noise in logs -BABEL_CONFIG = os.path.relpath(LOCALE_DIR + '/babel.cfg', BASE_DIR) +BABEL_CONFIG = BASE_DIR.relpathto(LOCALE_DIR.joinpath('babel.cfg')) # Strings from mako template files are written to BABEL_OUT # Use relpath to reduce noise in logs -BABEL_OUT = os.path.relpath(SOURCE_MSGS_DIR + '/mako.po', BASE_DIR) +BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako.po')) +SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' + +LOG = logging.getLogger(__name__) def main (): + logging.basicConfig(stream=sys.stdout, level=logging.INFO) create_dir_if_necessary(LOCALE_DIR) - generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') + source_msgs_dir = CONFIGURATION.source_messages_dir + remove_file(source_msgs_dir.joinpath('django.po')) + generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') for filename in generated_files: - remove_file(os.path.join(SOURCE_MSGS_DIR, filename)) + remove_file(source_msgs_dir.joinpath(filename)) + # Extract strings from mako templates babel_mako_cmd = 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT) @@ -52,13 +59,13 @@ def main (): execute(make_django_cmd, working_directory=BASE_DIR) # makemessages creates 'django.po'. This filename is hardcoded. # Rename it to django-partial.po to enable merging into django.po later. - os.rename(os.path.join(SOURCE_MSGS_DIR, 'django.po'), - os.path.join(SOURCE_MSGS_DIR, 'django-partial.po')) + os.rename(source_msgs_dir.joinpath('django.po'), + source_msgs_dir.joinpath('django-partial.po')) execute(make_djangojs_cmd, working_directory=BASE_DIR) for filename in generated_files: LOG.info('Cleaning %s' % filename) - po = pofile(os.path.join(SOURCE_MSGS_DIR, filename)) + po = pofile(source_msgs_dir.joinpath(filename)) # replace default headers with edX headers fix_header(po) # replace default metadata with edX metadata @@ -79,10 +86,11 @@ def fix_header(po): """ Replace default headers with edX headers """ + po.metadata_is_fuzzy = [] # remove [u'fuzzy'] header = po.header fixes = ( - ('SOME DESCRIPTIVE TITLE', 'edX translation file'), - ('Translations template for PROJECT.', 'edX translation file'), + ('SOME DESCRIPTIVE TITLE', 'edX translation file\n' + SOURCE_WARN), + ('Translations template for PROJECT.', 'edX translation file\n' + SOURCE_WARN), ('YEAR', '%s' % datetime.utcnow().year), ('ORGANIZATION', 'edX'), ("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"), @@ -119,10 +127,9 @@ def fix_metadata(po): 'Report-Msgid-Bugs-To': 'translation_team@edx.org', 'Project-Id-Version': '0.1a', 'Language' : 'en', + 'Last-Translator' : '', 'Language-Team': 'translation team ', } - if po.metadata.has_key('Last-Translator'): - del po.metadata['Last-Translator'] po.metadata.update(fixes) def strip_key_strings(po): diff --git a/i18n/generate.py b/i18n/generate.py index ddbaadfa70..65c65c00d6 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow @@ -13,50 +13,71 @@ languages to generate. """ -import os -from execute import execute, get_config, messages_dir, remove_file, \ - BASE_DIR, LOG, SOURCE_LOCALE +import os, sys, logging +from polib import pofile -def merge(locale, target='django.po'): +from config import BASE_DIR, CONFIGURATION +from execute import execute + +LOG = logging.getLogger(__name__) + +def merge(locale, target='django.po', fail_if_missing=True): """ For the given locale, merge django-partial.po, messages.po, mako.po -> django.po + target is the resulting filename + If fail_if_missing is True, and the files to be merged are missing, + throw an Exception. + If fail_if_missing is False, and the files to be merged are missing, + just return silently. """ LOG.info('Merging locale={0}'.format(locale)) - locale_directory = messages_dir(locale) + locale_directory = CONFIGURATION.get_messages_dir(locale) files_to_merge = ('django-partial.po', 'messages.po', 'mako.po') - validate_files(locale_directory, files_to_merge) + try: + validate_files(locale_directory, files_to_merge) + except Exception, e: + if not fail_if_missing: + return + raise e # merged file is merged.po merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge) execute(merge_cmd, working_directory=locale_directory) + # clean up redunancies in the metadata + merged_filename = locale_directory.joinpath('merged.po') + clean_metadata(merged_filename) + # rename merged.po -> django.po (default) - merged_filename = os.path.join(locale_directory, 'merged.po') - django_filename = os.path.join(locale_directory, target) + django_filename = locale_directory.joinpath(target) os.rename(merged_filename, django_filename) # can't overwrite file on Windows +def clean_metadata(file): + """ + Clean up redundancies in the metadata caused by merging. + This reads in a PO file and simply saves it back out again. + """ + pofile(file).save() + def validate_files(dir, files_to_merge): """ Asserts that the given files exist. files_to_merge is a list of file names (no directories). - dir is the directory in which the files should appear. + dir is the directory (a path object from path.py) in which the files should appear. raises an Exception if any of the files are not in dir. """ for path in files_to_merge: - pathname = os.path.join(dir, path) - if not os.path.exists(pathname): - raise Exception("File not found: {0}".format(pathname)) + pathname = dir.joinpath(path) + if not pathname.exists(): + raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname)) def main (): - configuration = get_config() - if configuration == None: - LOG.warn('Configuration file not found, using only English.') - locales = (SOURCE_LOCALE,) - else: - locales = configuration['locales'] - for locale in locales: - merge(locale) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + for locale in CONFIGURATION.locales: + merge(locale) + # Dummy text is not required. Don't raise exception if files are missing. + merge(CONFIGURATION.dummy_locale, fail_if_missing=False) compile_cmd = 'django-admin.py compilemessages' execute(compile_cmd, working_directory=BASE_DIR) diff --git a/i18n/make_dummy.py b/i18n/make_dummy.py index c8dcde861a..6c14edd45a 100755 --- a/i18n/make_dummy.py +++ b/i18n/make_dummy.py @@ -1,7 +1,13 @@ -#!/usr/bin/python +#!/usr/bin/env python # Generate test translation files from human-readable po files. # +# Dummy language is specified in configuration file (see config.py) +# two letter language codes reference: +# see http://www.loc.gov/standards/iso639-2/php/code_list.php +# +# Django will not localize in languages that django itself has not been +# localized for. So we are using a well-known language (default='fr'). # # po files can be generated with this: # django-admin.py makemessages --all --extension html -l en @@ -10,14 +16,15 @@ # # $ ./make_dummy.py # -# $ ./make_dummy.py mitx/conf/locale/en/LC_MESSAGES/django.po +# $ ./make_dummy.py ../conf/locale/en/LC_MESSAGES/django.po # # generates output to -# mitx/conf/locale/vr/LC_MESSAGES/django.po +# mitx/conf/locale/fr/LC_MESSAGES/django.po import os, sys import polib from dummy import Dummy +from config import CONFIGURATION from execute import create_dir_if_necessary def main(file, locale): @@ -41,27 +48,19 @@ def new_filename(original_filename, new_locale): orig_dir = os.path.dirname(original_filename) msgs_dir = os.path.basename(orig_dir) orig_file = os.path.basename(original_filename) - return os.path.join(orig_dir, - '/../..', - new_locale, - msgs_dir, - orig_file) - - -# Dummy language -# two letter language codes reference: -# see http://www.loc.gov/standards/iso639-2/php/code_list.php -# -# Django will not localize in languages that django itself has not been -# localized for. So we are using a well-known language: 'fr'. - -DEFAULT_LOCALE = 'fr' + return os.path.abspath(os.path.join(orig_dir, + '../..', + new_locale, + msgs_dir, + orig_file)) if __name__ == '__main__': + # required arg: file if len(sys.argv)<2: raise Exception("missing file argument") - if len(sys.argv)<2: - locale = DEFAULT_LOCALE + # optional arg: locale + if len(sys.argv)<3: + locale = CONFIGURATION.get_dummy_locale() else: locale = sys.argv[2] main(sys.argv[1], locale) diff --git a/i18n/tests/__init__.py b/i18n/tests/__init__.py index d60515c712..ee6283376e 100644 --- a/i18n/tests/__init__.py +++ b/i18n/tests/__init__.py @@ -1,4 +1,6 @@ +from test_config import TestConfiguration from test_extract import TestExtract from test_generate import TestGenerate from test_converter import TestConverter from test_dummy import TestDummy +import test_validate diff --git a/i18n/tests/test_config.py b/i18n/tests/test_config.py new file mode 100644 index 0000000000..bcec6ac354 --- /dev/null +++ b/i18n/tests/test_config.py @@ -0,0 +1,33 @@ +import os +from unittest import TestCase + +from config import Configuration, LOCALE_DIR, CONFIGURATION + +class TestConfiguration(TestCase): + """ + Tests functionality of i18n/config.py + """ + + def test_config(self): + config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'config')) + config = Configuration(config_filename) + self.assertEqual(config.source_locale, 'en') + + def test_no_config(self): + config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'no_such_file')) + with self.assertRaises(Exception): + Configuration(config_filename) + + def test_valid_configuration(self): + """ + Make sure we have a valid configuration file, + and that it contains an 'en' locale. + Also check values of dummy_locale and source_locale. + """ + self.assertIsNotNone(CONFIGURATION) + locales = CONFIGURATION.locales + self.assertIsNotNone(locales) + self.assertIsInstance(locales, list) + self.assertIn('en', locales) + self.assertEqual('fr', CONFIGURATION.dummy_locale) + self.assertEqual('en', CONFIGURATION.source_locale) diff --git a/i18n/tests/test_extract.py b/i18n/tests/test_extract.py index b14ae9872d..7e8b1a9d2b 100644 --- a/i18n/tests/test_extract.py +++ b/i18n/tests/test_extract.py @@ -4,7 +4,7 @@ from nose.plugins.skip import SkipTest from datetime import datetime, timedelta import extract -from execute import SOURCE_MSGS_DIR +from config import CONFIGURATION # Make sure setup runs only once SETUP_HAS_RUN = False @@ -39,7 +39,7 @@ class TestExtract(TestCase): Fails assertion if one of the files doesn't exist. """ for filename in self.generated_files: - path = os.path.join(SOURCE_MSGS_DIR, filename) + path = os.path.join(CONFIGURATION.source_messages_dir, filename) exists = os.path.exists(path) self.assertTrue(exists, msg='Missing file: %s' % filename) if exists: diff --git a/i18n/tests/test_generate.py b/i18n/tests/test_generate.py index fc22988251..468858664f 100644 --- a/i18n/tests/test_generate.py +++ b/i18n/tests/test_generate.py @@ -1,9 +1,10 @@ -import os, string, random +import os, string, random, re +from polib import pofile from unittest import TestCase from datetime import datetime, timedelta import generate -from execute import get_config, messages_dir, SOURCE_MSGS_DIR, SOURCE_LOCALE +from config import CONFIGURATION class TestGenerate(TestCase): """ @@ -12,29 +13,16 @@ class TestGenerate(TestCase): generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') def setUp(self): - self.configuration = get_config() - # Subtract 1 second to help comparisons with file-modify time succeed, # since os.path.getmtime() is not millisecond-accurate self.start_time = datetime.now() - timedelta(seconds=1) - def test_configuration(self): - """ - Make sure we have a valid configuration file, - and that it contains an 'en' locale. - """ - self.assertIsNotNone(self.configuration) - locales = self.configuration['locales'] - self.assertIsNotNone(locales) - self.assertIsInstance(locales, list) - self.assertIn('en', locales) - def test_merge(self): """ Tests merge script on English source files. """ - filename = os.path.join(SOURCE_MSGS_DIR, random_name()) - generate.merge(SOURCE_LOCALE, target=filename) + filename = os.path.join(CONFIGURATION.source_messages_dir, random_name()) + generate.merge(CONFIGURATION.source_locale, target=filename) self.assertTrue(os.path.exists(filename)) os.remove(filename) @@ -47,13 +35,35 @@ class TestGenerate(TestCase): after start of test suite) """ generate.main() - for locale in self.configuration['locales']: - for filename in ('django.mo', 'djangojs.mo'): - path = os.path.join(messages_dir(locale), filename) + for locale in CONFIGURATION.locales: + for filename in ('django', 'djangojs'): + mofile = filename+'.mo' + path = os.path.join(CONFIGURATION.get_messages_dir(locale), mofile) exists = os.path.exists(path) - self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, filename)) + self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, mofile)) self.assertTrue(datetime.fromtimestamp(os.path.getmtime(path)) >= self.start_time, msg='File not recently modified: %s' % path) + self.assert_merge_headers(locale) + + def assert_merge_headers(self, locale): + """ + This is invoked by test_main to ensure that it runs after + calling generate.main(). + + There should be exactly three merge comment headers + in our merged .po file. This counts them to be sure. + A merge comment looks like this: + # #-#-#-#-# django-partial.po (0.1a) #-#-#-#-# + + """ + path = os.path.join(CONFIGURATION.get_messages_dir(locale), 'django.po') + po = pofile(path) + pattern = re.compile('^#-#-#-#-#', re.M) + match = pattern.findall(po.header) + self.assertEqual(len(match), 3, + msg="Found %s (should be 3) merge comments in the header for %s" % \ + (len(match), path)) + def random_name(size=6): """Returns random filename as string, like test-4BZ81W""" diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py new file mode 100644 index 0000000000..bef563faea --- /dev/null +++ b/i18n/tests/test_validate.py @@ -0,0 +1,34 @@ +import os, sys, logging +from unittest import TestCase +from nose.plugins.skip import SkipTest + +from config import LOCALE_DIR +from execute import call + +def test_po_files(root=LOCALE_DIR): + """ + This is a generator. It yields all of the .po files under root, and tests each one. + """ + log = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + + for (dirpath, dirnames, filenames) in os.walk(root): + for name in filenames: + (base, ext) = os.path.splitext(name) + if ext.lower() == '.po': + yield validate_po_file, os.path.join(dirpath, name), log + + +def validate_po_file(filename, log): + """ + Call GNU msgfmt -c on each .po file to validate its format. + Any errors caught by msgfmt are logged to log. + """ + # Skip this test for now because it's very noisy + raise SkipTest() + # Use relative paths to make output less noisy. + rfile = os.path.relpath(filename, LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], working_directory=LOCALE_DIR) + if err != '': + log.warn('\n'+err) + diff --git a/i18n/transifex.py b/i18n/transifex.py new file mode 100755 index 0000000000..ac203f3eea --- /dev/null +++ b/i18n/transifex.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +import os, sys +from polib import pofile +from config import CONFIGURATION +from extract import SOURCE_WARN +from execute import execute + +TRANSIFEX_HEADER = 'Translations in this file have been downloaded from %s' +TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-studio/' + +def push(): + execute('tx push -s') + +def pull(): + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: + execute('tx pull -l %s' % locale) + clean_translated_locales() + + +def clean_translated_locales(): + """ + Strips out the warning from all translated po files + about being an English source file. + """ + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: + clean_locale(locale) + +def clean_locale(locale): + """ + Strips out the warning from all of a locale's translated po files + about being an English source file. + Iterates over machine-generated files. + """ + dirname = CONFIGURATION.get_messages_dir(locale) + for filename in ('django-partial.po', 'djangojs.po', 'mako.po'): + clean_file(dirname.joinpath(filename)) + +def clean_file(file): + """ + Strips out the warning from a translated po file about being an English source file. + Replaces warning with a note about coming from Transifex. + """ + po = pofile(file) + if po.header.find(SOURCE_WARN) != -1: + new_header = get_new_header(po) + new = po.header.replace(SOURCE_WARN, new_header) + po.header = new + po.save() + +def get_new_header(po): + team = po.metadata.get('Language-Team', None) + if not team: + return TRANSIFEX_HEADER % TRANSIFEX_URL + else: + return TRANSIFEX_HEADER % team + +if __name__ == '__main__': + if len(sys.argv)<2: + raise Exception("missing argument: push or pull") + arg = sys.argv[1] + if arg == 'push': + push() + elif arg == 'pull': + pull() + else: + raise Exception("unknown argument: (%s)" % arg) + diff --git a/rakefile b/rakefile index 32d92a0349..04a7db4904 100644 --- a/rakefile +++ b/rakefile @@ -337,12 +337,6 @@ task :migrate, [:env] do |t, args| sh(django_admin(:lms, args.env, 'migrate')) end -desc "Run tests for the internationalization library" -task :test_i18n do - test = File.join(REPO_ROOT, "i18n", "tests") - sh("nosetests #{test}") -end - Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| task_name = "test_#{lib}" @@ -516,27 +510,76 @@ end # --- Internationalization tasks -desc "Extract localizable strings from sources" -task :extract_dev_strings do - sh(File.join(REPO_ROOT, "i18n", "extract.py")) -end +namespace :i18n do -desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." -task :generate_i18n do - if ARGV.last.downcase == 'extract' - Rake::Task["extract_dev_strings"].execute + desc "Extract localizable strings from sources" + task :extract => "i18n:validate:gettext" do + sh(File.join(REPO_ROOT, "i18n", "extract.py")) end - sh(File.join(REPO_ROOT, "i18n", "generate.py")) -end -desc "Simulate international translation by generating dummy strings corresponding to source strings." -task :dummy_i18n do - source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] - dummy_locale = 'fr' - cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") - for file in source_files do - sh("#{cmd} #{file} #{dummy_locale}") + desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." + task :generate => "i18n:validate:gettext" do + if ARGV.last.downcase == 'extract' + Rake::Task["i18n:extract"].execute + end + sh(File.join(REPO_ROOT, "i18n", "generate.py")) end + + desc "Simulate international translation by generating dummy strings corresponding to source strings." + task :dummy do + source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] + dummy_locale = 'fr' + cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") + for file in source_files do + sh("#{cmd} #{file} #{dummy_locale}") + end + end + + namespace :validate do + + desc "Make sure GNU gettext utilities are available" + task :gettext do + begin + select_executable('xgettext') + rescue + msg = "Cannot locate GNU gettext utilities, which are required by django for internationalization.\n" + msg += "(see https://docs.djangoproject.com/en/dev/topics/i18n/translation/#message-files)\n" + msg += "Try downloading them from http://www.gnu.org/software/gettext/" + abort(msg.red) + end + end + + desc "Make sure config file with username/password exists" + task :transifex_config do + config_file = "#{Dir.home}/.transifexrc" + if !File.file?(config_file) or File.size(config_file)==0 + msg ="Cannot connect to Transifex, config file is missing or empty: #{config_file}\n" + msg += "See http://help.transifex.com/features/client/#transifexrc" + abort(msg.red) + end + end + end + + namespace :transifex do + desc "Push source strings to Transifex for translation" + task :push => "i18n:validate:transifex_config" do + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} push") + end + + desc "Pull translated strings from Transifex" + task :pull => "i18n:validate:transifex_config" do + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} pull") + end + end + + desc "Run tests for the internationalization library" + task :test => "i18n:validate:gettext" do + test = File.join(REPO_ROOT, "i18n", "tests") + sh("nosetests #{test}") + end + end # --- Develop and public documentation --- diff --git a/requirements.txt b/requirements.txt index d3fdd46b81..c6ee47becb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ paramiko==1.9.0 path.py==3.0.1 Pillow==1.7.8 pip +polib==1.0.3 pygments==1.5 pygraphviz==1.1 pymongo==2.4.1