From 55dd0fc8bcb40efa2557653261d9ac6a29c3683f Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 3 May 2013 12:45:55 -0400 Subject: [PATCH 01/18] workaround for gettext parser bug --- .tx/config | 26 ++++++++++++++++ conf/locale/config | 2 +- i18n/execute.py | 21 +++++++++++-- i18n/extract.py | 4 +-- i18n/generate.py | 16 +++++++++- i18n/make_dummy.py | 18 +++++------ i18n/tests/__init__.py | 2 ++ i18n/tests/test_validate.py | 33 ++++++++++++++++++++ rakefile | 62 +++++++++++++++++++++++-------------- 9 files changed, 145 insertions(+), 39 deletions(-) create mode 100644 .tx/config create mode 100644 i18n/tests/test_validate.py diff --git a/.tx/config b/.tx/config new file mode 100644 index 0000000000..540c4732af --- /dev/null +++ b/.tx/config @@ -0,0 +1,26 @@ +[main] +host = https://www.transifex.com + +[edx-studio.django-partial] +file_filter = conf/locale//LC_MESSAGES/django-partial.po +source_file = conf/locale/en/LC_MESSAGES/django-partial.po +source_lang = en +type = PO + +[edx-studio.djangojs] +file_filter = conf/locale//LC_MESSAGES/djangojs.po +source_file = conf/locale/en/LC_MESSAGES/djangojs.po +source_lang = en +type = PO + +[edx-studio.mako] +file_filter = conf/locale//LC_MESSAGES/mako.po +source_file = conf/locale/en/LC_MESSAGES/mako.po +source_lang = en +type = PO + +[edx-studio.messages] +file_filter = conf/locale//LC_MESSAGES/messages.po +source_file = conf/locale/en/LC_MESSAGES/messages.po +source_lang = en +type = PO diff --git a/conf/locale/config b/conf/locale/config index 2d01e1ea43..8afaaa9482 100644 --- a/conf/locale/config +++ b/conf/locale/config @@ -1 +1 @@ -{"locales" : ["en"]} +{"locales" : ["en", "fr", "es"]} diff --git a/i18n/execute.py b/i18n/execute.py index 3c3416b65d..0cd152fb58 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -50,14 +50,29 @@ def execute (command, working_directory=BASE_DIR, log=LOG): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. - Output is logged to log. + The command is logged to log, output is ignored. """ - log.info(command) + if log: + log.info(command) subprocess.call(command.split(' '), cwd=working_directory) + + +def call(command, working_directory=BASE_DIR, log=LOG): + """ + Executes shell command in a given working_directory. + Command is a string to pass to the shell. + Returns a tuple of two strings: (stdout, stderr) + """ + if log: + log.info(command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_directory) + out, err = p.communicate() + return (out, err) + def get_config(): """Returns data found in config file, or returns None if file not found""" - config_path = os.path.abspath(CONFIG_FILENAME) + config_path = os.path.normpath(CONFIG_FILENAME) if not os.path.exists(config_path): log.warn("Configuration file cannot be found: %s" % \ os.path.relpath(config_path, BASE_DIR)) diff --git a/i18n/extract.py b/i18n/extract.py index c6fedd3bfa..c3d4368a67 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -79,6 +79,7 @@ def fix_header(po): """ Replace default headers with edX headers """ + po.metadata_is_fuzzy = [] # remove [u'fuzzy'] header = po.header fixes = ( ('SOME DESCRIPTIVE TITLE', 'edX translation file'), @@ -119,10 +120,9 @@ def fix_metadata(po): 'Report-Msgid-Bugs-To': 'translation_team@edx.org', 'Project-Id-Version': '0.1a', 'Language' : 'en', + 'Last-Translator' : '', 'Language-Team': 'translation team ', } - if po.metadata.has_key('Last-Translator'): - del po.metadata['Last-Translator'] po.metadata.update(fixes) def strip_key_strings(po): diff --git a/i18n/generate.py b/i18n/generate.py index ddbaadfa70..40a6cc88ca 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -14,6 +14,8 @@ """ import os +from polib import pofile + from execute import execute, get_config, messages_dir, remove_file, \ BASE_DIR, LOG, SOURCE_LOCALE @@ -30,11 +32,23 @@ def merge(locale, target='django.po'): merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge) execute(merge_cmd, working_directory=locale_directory) - # rename merged.po -> django.po (default) + # clean up redunancies in the metadata merged_filename = os.path.join(locale_directory, 'merged.po') + clean_metadata(merged_filename) + + # rename merged.po -> django.po (default) django_filename = os.path.join(locale_directory, target) os.rename(merged_filename, django_filename) # can't overwrite file on Windows +def clean_metadata(file): + """ + Clean up redundancies in the metadata caused by merging. + This reads in a PO file and simply saves it back out again. + """ + po = pofile(file) + po.save() + + def validate_files(dir, files_to_merge): """ Asserts that the given files exist. diff --git a/i18n/make_dummy.py b/i18n/make_dummy.py index c8dcde861a..9c8c3289ce 100755 --- a/i18n/make_dummy.py +++ b/i18n/make_dummy.py @@ -10,15 +10,15 @@ # # $ ./make_dummy.py # -# $ ./make_dummy.py mitx/conf/locale/en/LC_MESSAGES/django.po +# $ ./make_dummy.py ../conf/locale/en/LC_MESSAGES/django.po # # generates output to -# mitx/conf/locale/vr/LC_MESSAGES/django.po +# mitx/conf/locale/fr/LC_MESSAGES/django.po import os, sys import polib from dummy import Dummy -from execute import create_dir_if_necessary +from execute import get_logger, create_dir_if_necessary def main(file, locale): """ @@ -41,11 +41,11 @@ def new_filename(original_filename, new_locale): orig_dir = os.path.dirname(original_filename) msgs_dir = os.path.basename(orig_dir) orig_file = os.path.basename(original_filename) - return os.path.join(orig_dir, - '/../..', - new_locale, - msgs_dir, - orig_file) + return os.path.abspath(os.path.join(orig_dir, + '../..', + new_locale, + msgs_dir, + orig_file)) # Dummy language @@ -60,7 +60,7 @@ DEFAULT_LOCALE = 'fr' if __name__ == '__main__': if len(sys.argv)<2: raise Exception("missing file argument") - if len(sys.argv)<2: + if len(sys.argv)<3: locale = DEFAULT_LOCALE else: locale = sys.argv[2] diff --git a/i18n/tests/__init__.py b/i18n/tests/__init__.py index d60515c712..88216df993 100644 --- a/i18n/tests/__init__.py +++ b/i18n/tests/__init__.py @@ -2,3 +2,5 @@ from test_extract import TestExtract from test_generate import TestGenerate from test_converter import TestConverter from test_dummy import TestDummy +from test_validate import TestValidate + diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py new file mode 100644 index 0000000000..ed746db78f --- /dev/null +++ b/i18n/tests/test_validate.py @@ -0,0 +1,33 @@ +import os +from unittest import TestCase +from nose.plugins.skip import SkipTest + +from execute import call, LOCALE_DIR, LOG + +class TestValidate(TestCase): + """ + Call GNU msgfmt -c on each .po file to validate its format. + """ + + def test_validate(self): + # Skip this test for now because it's very noisy + raise SkipTest() + for file in self.get_po_files(): + # Use relative paths to make output less noisy. + rfile = os.path.relpath(file, LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) + if err != '': + LOG.warn('\n'+err) + + def get_po_files(self, root=LOCALE_DIR): + """ + This is a generator. It yields all of the .po files under root. + """ + for (dirpath, dirnames, filenames) in os.walk(root): + for name in filenames: + (base, ext) = os.path.splitext(name) + if ext.lower() == '.po': + yield os.path.join(dirpath, name) + + + diff --git a/rakefile b/rakefile index 32d92a0349..5914b2f0ae 100644 --- a/rakefile +++ b/rakefile @@ -337,12 +337,6 @@ task :migrate, [:env] do |t, args| sh(django_admin(:lms, args.env, 'migrate')) end -desc "Run tests for the internationalization library" -task :test_i18n do - test = File.join(REPO_ROOT, "i18n", "tests") - sh("nosetests #{test}") -end - Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| task_name = "test_#{lib}" @@ -516,27 +510,49 @@ end # --- Internationalization tasks -desc "Extract localizable strings from sources" -task :extract_dev_strings do - sh(File.join(REPO_ROOT, "i18n", "extract.py")) -end +namespace :i18n do -desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." -task :generate_i18n do - if ARGV.last.downcase == 'extract' - Rake::Task["extract_dev_strings"].execute + desc "Extract localizable strings from sources" + task :extract do + sh(File.join(REPO_ROOT, "i18n", "extract.py")) end - sh(File.join(REPO_ROOT, "i18n", "generate.py")) -end -desc "Simulate international translation by generating dummy strings corresponding to source strings." -task :dummy_i18n do - source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] - dummy_locale = 'fr' - cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") - for file in source_files do - sh("#{cmd} #{file} #{dummy_locale}") + desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." + task :generate do + if ARGV.last.downcase == 'extract' + Rake::Task["i18n:extract"].execute + end + sh(File.join(REPO_ROOT, "i18n", "generate.py")) end + + desc "Simulate international translation by generating dummy strings corresponding to source strings." + task :dummy do + source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] + dummy_locale = 'fr' + cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") + for file in source_files do + sh("#{cmd} #{file} #{dummy_locale}") + end + end + + namespace :transifex do + desc "Push source strings to Transifex for translation" + task :push do + sh("tx push -s") + end + + desc "Pull transated strings from Transifex" + task :pull do + sh("tx pull") + end + end + + desc "Run tests for the internationalization library" + task :test do + test = File.join(REPO_ROOT, "i18n", "tests") + sh("nosetests #{test}") + end + end # --- Develop and public documentation --- From e6334584d68272c90b139cdec43eb1bec2d01c2e Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 3 May 2013 15:18:29 -0400 Subject: [PATCH 02/18] rakefile cleanup --- conf/locale/config | 2 +- rakefile | 22 +++++++++++++++++++--- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/conf/locale/config b/conf/locale/config index 8afaaa9482..2d01e1ea43 100644 --- a/conf/locale/config +++ b/conf/locale/config @@ -1 +1 @@ -{"locales" : ["en", "fr", "es"]} +{"locales" : ["en"]} diff --git a/rakefile b/rakefile index 5914b2f0ae..f39cbacc41 100644 --- a/rakefile +++ b/rakefile @@ -510,6 +510,18 @@ end # --- Internationalization tasks +# Make sure config file with username/password exists +# Returns boolean: returns true if file exists and is nonzero length +def validate_transifex_config() + config_file = Dir.home + "/.transifexrc" + if !File.file?(config_file) or File.size(config_file)==0 + raise "Cannot connect to Transifex, config file is missing or empty: #{config_file}\n" + + "See http://help.transifex.com/features/client/#transifexrc\n" + return false + end + return true +end + namespace :i18n do desc "Extract localizable strings from sources" @@ -538,12 +550,16 @@ namespace :i18n do namespace :transifex do desc "Push source strings to Transifex for translation" task :push do - sh("tx push -s") + if validate_transifex_config() + sh("tx push -s") + end end - desc "Pull transated strings from Transifex" + desc "Pull translated strings from Transifex" task :pull do - sh("tx pull") + if validate_transifex_config() + sh("tx pull") + end end end From 03b9a9e22a6c8c69fd1f1b6275b334a873f54746 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 3 May 2013 15:42:39 -0400 Subject: [PATCH 03/18] tweak file reference --- rakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rakefile b/rakefile index f39cbacc41..240847812e 100644 --- a/rakefile +++ b/rakefile @@ -513,7 +513,7 @@ end # Make sure config file with username/password exists # Returns boolean: returns true if file exists and is nonzero length def validate_transifex_config() - config_file = Dir.home + "/.transifexrc" + config_file = "#{Dir.home}/.transifexrc" if !File.file?(config_file) or File.size(config_file)==0 raise "Cannot connect to Transifex, config file is missing or empty: #{config_file}\n" + "See http://help.transifex.com/features/client/#transifexrc\n" From c0278d0ff1c75647113a9282407ed9c35bc7bb7d Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Mon, 6 May 2013 11:29:27 -0400 Subject: [PATCH 04/18] refactor config file; fix duplicate merge --- conf/locale/config | 5 ++- i18n/config.py | 73 +++++++++++++++++++++++++++++++++++++ i18n/execute.py | 54 +++------------------------ i18n/extract.py | 25 +++++++------ i18n/generate.py | 15 ++------ i18n/tests/__init__.py | 1 + i18n/tests/test_config.py | 33 +++++++++++++++++ i18n/tests/test_extract.py | 4 +- i18n/tests/test_generate.py | 52 +++++++++++++++----------- i18n/tests/test_validate.py | 3 +- i18n/transifex.py | 23 ++++++++++++ rakefile | 6 ++- 12 files changed, 196 insertions(+), 98 deletions(-) create mode 100644 i18n/config.py create mode 100644 i18n/tests/test_config.py create mode 100755 i18n/transifex.py diff --git a/conf/locale/config b/conf/locale/config index 2d01e1ea43..67252b1fa0 100644 --- a/conf/locale/config +++ b/conf/locale/config @@ -1 +1,4 @@ -{"locales" : ["en"]} +{ + "locales" : ["en"], + "dummy-locale" : "fr" +} diff --git a/i18n/config.py b/i18n/config.py new file mode 100644 index 0000000000..f0d8e366d0 --- /dev/null +++ b/i18n/config.py @@ -0,0 +1,73 @@ +import os, json + +# BASE_DIR is the working directory to execute django-admin commands from. +# Typically this should be the 'mitx' directory. +BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') + +# LOCALE_DIR contains the locale files. +# Typically this should be 'mitx/conf/locale' +LOCALE_DIR = os.path.join(BASE_DIR, 'conf', 'locale') + +class Configuration: + """ + # Reads localization configuration in json format + + """ + _source_locale = 'en' + + def __init__(self, filename): + self.filename = filename + self.config = self.get_config(self.filename) + + def get_config(self, filename): + """ + Returns data found in config file (as dict), or raises exception if file not found + """ + if not os.path.exists(filename): + raise Exception("Configuration file cannot be found: %s" % filename) + with open(filename) as stream: + return json.load(stream) + + def get_locales(self): + """ + Returns a list of locales declared in the configuration file, + e.g. ['en', 'fr', 'es'] + Each locale is a string. + """ + return self.config['locales'] + + def get_source_locale(self): + """ + Returns source language. + Source language is English. + """ + return self._source_locale + + def get_dummy_locale(self): + """ + Returns a locale to use for the dummy text, e.g. 'fr'. + Throws exception if no dummy-locale is declared. + The locale is a string. + """ + dummy = self.config.get('dummy-locale', None) + if not dummy: + raise Exception('Could not read dummy-locale from configuration file.') + return dummy + + def get_messages_dir(self, locale): + """ + Returns the name of the directory holding the po files for locale. + Example: mitx/conf/locale/fr/LC_MESSAGES + """ + return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES') + + def get_source_messages_dir(self): + """ + Returns the name of the directory holding the source-language po files (English). + Example: mitx/conf/locale/en/LC_MESSAGES + """ + return self.get_messages_dir(self.get_source_locale()) + + +CONFIGURATION = Configuration(os.path.normpath(os.path.join(LOCALE_DIR, 'config'))) + diff --git a/i18n/execute.py b/i18n/execute.py index 0cd152fb58..4c47680101 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -1,40 +1,8 @@ -import os, subprocess, logging, json +import os, subprocess, logging +from config import CONFIGURATION, BASE_DIR -def init_module(): - """ - Initializes module parameters - """ - global BASE_DIR, LOCALE_DIR, CONFIG_FILENAME, SOURCE_MSGS_DIR, SOURCE_LOCALE, LOG - # BASE_DIR is the working directory to execute django-admin commands from. - # Typically this should be the 'mitx' directory. - BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') - - # Source language is English - SOURCE_LOCALE = 'en' - - # LOCALE_DIR contains the locale files. - # Typically this should be 'mitx/conf/locale' - LOCALE_DIR = BASE_DIR + '/conf/locale' - - # CONFIG_FILENAME contains localization configuration in json format - CONFIG_FILENAME = LOCALE_DIR + '/config' - - # SOURCE_MSGS_DIR contains the English po files. - SOURCE_MSGS_DIR = messages_dir(SOURCE_LOCALE) - - # Default logger. - LOG = get_logger() - - -def messages_dir(locale): - """ - Returns the name of the directory holding the po files for locale. - Example: mitx/conf/locale/en/LC_MESSAGES - """ - return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES') - -def get_logger(): +def get_default_logger(): """Returns a default logger""" log = logging.getLogger(__name__) log.setLevel(logging.INFO) @@ -43,8 +11,8 @@ def get_logger(): log.addHandler(log_handler) return log -# Run this after defining messages_dir and get_logger, because it depends on these. -init_module() +LOG = get_default_logger() + def execute (command, working_directory=BASE_DIR, log=LOG): """ @@ -69,17 +37,6 @@ def call(command, working_directory=BASE_DIR, log=LOG): out, err = p.communicate() return (out, err) - -def get_config(): - """Returns data found in config file, or returns None if file not found""" - config_path = os.path.normpath(CONFIG_FILENAME) - if not os.path.exists(config_path): - log.warn("Configuration file cannot be found: %s" % \ - os.path.relpath(config_path, BASE_DIR)) - return None - with open(config_path) as stream: - return json.load(stream) - def create_dir_if_necessary(pathname): dirname = os.path.dirname(pathname) if not os.path.exists(dirname): @@ -98,4 +55,3 @@ def remove_file(filename, log=LOG, verbose=True): log.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR)) else: os.remove(filename) - diff --git a/i18n/extract.py b/i18n/extract.py index c3d4368a67..ffac9b6270 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -18,9 +18,8 @@ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow import os from datetime import datetime from polib import pofile -from execute import execute, create_dir_if_necessary, remove_file, \ - BASE_DIR, LOCALE_DIR, SOURCE_MSGS_DIR, LOG - +from config import BASE_DIR, LOCALE_DIR, CONFIGURATION +from execute import execute, create_dir_if_necessary, remove_file, LOG # BABEL_CONFIG contains declarations for Babel to extract strings from mako template files # Use relpath to reduce noise in logs @@ -28,15 +27,19 @@ BABEL_CONFIG = os.path.relpath(LOCALE_DIR + '/babel.cfg', BASE_DIR) # Strings from mako template files are written to BABEL_OUT # Use relpath to reduce noise in logs -BABEL_OUT = os.path.relpath(SOURCE_MSGS_DIR + '/mako.po', BASE_DIR) +BABEL_OUT = os.path.relpath(CONFIGURATION.get_source_messages_dir() + '/mako.po', BASE_DIR) +SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' def main (): create_dir_if_necessary(LOCALE_DIR) - generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') + source_msgs_dir = CONFIGURATION.get_source_messages_dir() + remove_file(os.path.join(source_msgs_dir, 'django.po')) + generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') for filename in generated_files: - remove_file(os.path.join(SOURCE_MSGS_DIR, filename)) + remove_file(os.path.join(source_msgs_dir, filename)) + # Extract strings from mako templates babel_mako_cmd = 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT) @@ -52,13 +55,13 @@ def main (): execute(make_django_cmd, working_directory=BASE_DIR) # makemessages creates 'django.po'. This filename is hardcoded. # Rename it to django-partial.po to enable merging into django.po later. - os.rename(os.path.join(SOURCE_MSGS_DIR, 'django.po'), - os.path.join(SOURCE_MSGS_DIR, 'django-partial.po')) + os.rename(os.path.join(source_msgs_dir, 'django.po'), + os.path.join(source_msgs_dir, 'django-partial.po')) execute(make_djangojs_cmd, working_directory=BASE_DIR) for filename in generated_files: LOG.info('Cleaning %s' % filename) - po = pofile(os.path.join(SOURCE_MSGS_DIR, filename)) + po = pofile(os.path.join(source_msgs_dir, filename)) # replace default headers with edX headers fix_header(po) # replace default metadata with edX metadata @@ -82,8 +85,8 @@ def fix_header(po): po.metadata_is_fuzzy = [] # remove [u'fuzzy'] header = po.header fixes = ( - ('SOME DESCRIPTIVE TITLE', 'edX translation file'), - ('Translations template for PROJECT.', 'edX translation file'), + ('SOME DESCRIPTIVE TITLE', 'edX translation file\n' + SOURCE_WARN), + ('Translations template for PROJECT.', 'edX translation file\n' + SOURCE_WARN), ('YEAR', '%s' % datetime.utcnow().year), ('ORGANIZATION', 'edX'), ("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"), diff --git a/i18n/generate.py b/i18n/generate.py index 40a6cc88ca..e43efc268a 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -16,15 +16,15 @@ import os from polib import pofile -from execute import execute, get_config, messages_dir, remove_file, \ - BASE_DIR, LOG, SOURCE_LOCALE +from config import BASE_DIR, CONFIGURATION +from execute import execute, remove_file, LOG def merge(locale, target='django.po'): """ For the given locale, merge django-partial.po, messages.po, mako.po -> django.po """ LOG.info('Merging locale={0}'.format(locale)) - locale_directory = messages_dir(locale) + locale_directory = CONFIGURATION.get_messages_dir(locale) files_to_merge = ('django-partial.po', 'messages.po', 'mako.po') validate_files(locale_directory, files_to_merge) @@ -62,15 +62,8 @@ def validate_files(dir, files_to_merge): raise Exception("File not found: {0}".format(pathname)) def main (): - configuration = get_config() - if configuration == None: - LOG.warn('Configuration file not found, using only English.') - locales = (SOURCE_LOCALE,) - else: - locales = configuration['locales'] - for locale in locales: + for locale in CONFIGURATION.get_locales(): merge(locale) - compile_cmd = 'django-admin.py compilemessages' execute(compile_cmd, working_directory=BASE_DIR) diff --git a/i18n/tests/__init__.py b/i18n/tests/__init__.py index 88216df993..d8fce19df7 100644 --- a/i18n/tests/__init__.py +++ b/i18n/tests/__init__.py @@ -1,3 +1,4 @@ +from test_config import TestConfiguration from test_extract import TestExtract from test_generate import TestGenerate from test_converter import TestConverter diff --git a/i18n/tests/test_config.py b/i18n/tests/test_config.py new file mode 100644 index 0000000000..aea8f0bca3 --- /dev/null +++ b/i18n/tests/test_config.py @@ -0,0 +1,33 @@ +import os +from unittest import TestCase + +from config import Configuration, LOCALE_DIR, CONFIGURATION + +class TestConfiguration(TestCase): + """ + Tests functionality of i18n/config.py + """ + + def test_config(self): + config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'config')) + config = Configuration(config_filename) + self.assertEqual(config.get_source_locale(), 'en') + + def test_no_config(self): + config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'no_such_file')) + with self.assertRaises(Exception): + Configuration(config_filename) + + def test_valid_configuration(self): + """ + Make sure we have a valid configuration file, + and that it contains an 'en' locale. + Also check values of dummy_locale and source_locale. + """ + self.assertIsNotNone(CONFIGURATION) + locales = CONFIGURATION.get_locales() + self.assertIsNotNone(locales) + self.assertIsInstance(locales, list) + self.assertIn('en', locales) + self.assertEqual('fr', CONFIGURATION.get_dummy_locale()) + self.assertEqual('en', CONFIGURATION.get_source_locale()) diff --git a/i18n/tests/test_extract.py b/i18n/tests/test_extract.py index b14ae9872d..a9faa2bdd8 100644 --- a/i18n/tests/test_extract.py +++ b/i18n/tests/test_extract.py @@ -4,7 +4,7 @@ from nose.plugins.skip import SkipTest from datetime import datetime, timedelta import extract -from execute import SOURCE_MSGS_DIR +from config import CONFIGURATION # Make sure setup runs only once SETUP_HAS_RUN = False @@ -39,7 +39,7 @@ class TestExtract(TestCase): Fails assertion if one of the files doesn't exist. """ for filename in self.generated_files: - path = os.path.join(SOURCE_MSGS_DIR, filename) + path = os.path.join(CONFIGURATION.get_source_messages_dir(), filename) exists = os.path.exists(path) self.assertTrue(exists, msg='Missing file: %s' % filename) if exists: diff --git a/i18n/tests/test_generate.py b/i18n/tests/test_generate.py index fc22988251..bac727f671 100644 --- a/i18n/tests/test_generate.py +++ b/i18n/tests/test_generate.py @@ -1,9 +1,10 @@ -import os, string, random +import os, string, random, re +from polib import pofile from unittest import TestCase from datetime import datetime, timedelta import generate -from execute import get_config, messages_dir, SOURCE_MSGS_DIR, SOURCE_LOCALE +from config import CONFIGURATION class TestGenerate(TestCase): """ @@ -12,29 +13,16 @@ class TestGenerate(TestCase): generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') def setUp(self): - self.configuration = get_config() - # Subtract 1 second to help comparisons with file-modify time succeed, # since os.path.getmtime() is not millisecond-accurate self.start_time = datetime.now() - timedelta(seconds=1) - def test_configuration(self): - """ - Make sure we have a valid configuration file, - and that it contains an 'en' locale. - """ - self.assertIsNotNone(self.configuration) - locales = self.configuration['locales'] - self.assertIsNotNone(locales) - self.assertIsInstance(locales, list) - self.assertIn('en', locales) - def test_merge(self): """ Tests merge script on English source files. """ - filename = os.path.join(SOURCE_MSGS_DIR, random_name()) - generate.merge(SOURCE_LOCALE, target=filename) + filename = os.path.join(CONFIGURATION.get_source_messages_dir(), random_name()) + generate.merge(CONFIGURATION.get_source_locale(), target=filename) self.assertTrue(os.path.exists(filename)) os.remove(filename) @@ -47,13 +35,35 @@ class TestGenerate(TestCase): after start of test suite) """ generate.main() - for locale in self.configuration['locales']: - for filename in ('django.mo', 'djangojs.mo'): - path = os.path.join(messages_dir(locale), filename) + for locale in CONFIGURATION.get_locales(): + for filename in ('django', 'djangojs'): + mofile = filename+'.mo' + path = os.path.join(CONFIGURATION.get_messages_dir(locale), mofile) exists = os.path.exists(path) - self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, filename)) + self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, mofile)) self.assertTrue(datetime.fromtimestamp(os.path.getmtime(path)) >= self.start_time, msg='File not recently modified: %s' % path) + self.assert_merge_headers(locale) + + def assert_merge_headers(self, locale): + """ + This is invoked by test_main to ensure that it runs after + calling generate.main(). + + There should be exactly three merge comment headers + in our merged .po file. This counts them to be sure. + A merge comment looks like this: + # #-#-#-#-# django-partial.po (0.1a) #-#-#-#-# + + """ + path = os.path.join(CONFIGURATION.get_messages_dir(locale), 'django.po') + po = pofile(path) + pattern = re.compile('^#-#-#-#-#', re.M) + match = pattern.findall(po.header) + self.assertEqual(len(match), 3, + msg="Found %s (should be 3) merge comments in the header for %s" % \ + (len(match), path)) + def random_name(size=6): """Returns random filename as string, like test-4BZ81W""" diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py index ed746db78f..64579fb563 100644 --- a/i18n/tests/test_validate.py +++ b/i18n/tests/test_validate.py @@ -2,7 +2,8 @@ import os from unittest import TestCase from nose.plugins.skip import SkipTest -from execute import call, LOCALE_DIR, LOG +from config import LOCALE_DIR +from execute import call, LOG class TestValidate(TestCase): """ diff --git a/i18n/transifex.py b/i18n/transifex.py new file mode 100755 index 0000000000..9def339262 --- /dev/null +++ b/i18n/transifex.py @@ -0,0 +1,23 @@ +#!/usr/bin/python + +import sys +from execute import execute + +def push(): + execute('tx push -s') + +def pull(): + execute('tx pull') + + +if __name__ == '__main__': + if len(sys.argv)<2: + raise Exception("missing argument: push or pull") + arg = sys.argv[1] + if arg == 'push': + push() + elif arg == 'pull': + pull() + else: + raise Exception("unknown argument: (%s)" % arg) + diff --git a/rakefile b/rakefile index 240847812e..acb9bf6ecc 100644 --- a/rakefile +++ b/rakefile @@ -551,14 +551,16 @@ namespace :i18n do desc "Push source strings to Transifex for translation" task :push do if validate_transifex_config() - sh("tx push -s") + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} push") end end desc "Pull translated strings from Transifex" task :pull do if validate_transifex_config() - sh("tx pull") + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} pull") end end end From bec2ca4086cb15079ee86f28b6a33ecdb1085283 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Mon, 6 May 2013 14:05:05 -0400 Subject: [PATCH 05/18] cleanup headers of files pulled from transifex. --- conf/locale/config | 2 +- i18n/make_dummy.py | 23 +++++++++++------------ i18n/transifex.py | 47 +++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/conf/locale/config b/conf/locale/config index 67252b1fa0..58f8da0513 100644 --- a/conf/locale/config +++ b/conf/locale/config @@ -1,4 +1,4 @@ { - "locales" : ["en"], + "locales" : ["en", "es"], "dummy-locale" : "fr" } diff --git a/i18n/make_dummy.py b/i18n/make_dummy.py index 9c8c3289ce..8d0fb95ef2 100755 --- a/i18n/make_dummy.py +++ b/i18n/make_dummy.py @@ -2,6 +2,12 @@ # Generate test translation files from human-readable po files. # +# Dummy language is specified in configuration file (see config.py) +# two letter language codes reference: +# see http://www.loc.gov/standards/iso639-2/php/code_list.php +# +# Django will not localize in languages that django itself has not been +# localized for. So we are using a well-known language (default='fr'). # # po files can be generated with this: # django-admin.py makemessages --all --extension html -l en @@ -18,7 +24,8 @@ import os, sys import polib from dummy import Dummy -from execute import get_logger, create_dir_if_necessary +from config import CONFIGURATION +from execute import create_dir_if_necessary def main(file, locale): """ @@ -47,21 +54,13 @@ def new_filename(original_filename, new_locale): msgs_dir, orig_file)) - -# Dummy language -# two letter language codes reference: -# see http://www.loc.gov/standards/iso639-2/php/code_list.php -# -# Django will not localize in languages that django itself has not been -# localized for. So we are using a well-known language: 'fr'. - -DEFAULT_LOCALE = 'fr' - if __name__ == '__main__': + # required arg: file if len(sys.argv)<2: raise Exception("missing file argument") + # optional arg: locale if len(sys.argv)<3: - locale = DEFAULT_LOCALE + locale = CONFIGURATION.get_dummy_locale() else: locale = sys.argv[2] main(sys.argv[1], locale) diff --git a/i18n/transifex.py b/i18n/transifex.py index 9def339262..812ecd666f 100755 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -1,15 +1,60 @@ #!/usr/bin/python -import sys +import os, sys +from polib import pofile +from config import CONFIGURATION +from extract import SOURCE_WARN from execute import execute +TRANSIFEX_HEADER = 'Translations in this file have been downloaded from %s' +TRANSIFEX_URL = 'https://www.transifex.com/projects/p/edx-studio/' + def push(): execute('tx push -s') def pull(): execute('tx pull') + clean_translated_locales() +def clean_translated_locales(): + """ + Strips out the warning from all translated po files + about being an English source file. + """ + for locale in CONFIGURATION.get_locales(): + if locale != CONFIGURATION.get_source_locale(): + clean_locale(locale) + +def clean_locale(locale): + """ + Strips out the warning from all of a locale's translated po files + about being an English source file. + Iterates over machine-generated files. + """ + dirname = CONFIGURATION.get_messages_dir(locale) + for filename in ('django-partial.po', 'djangojs.po', 'mako.po'): + clean_file(os.path.join(dirname, filename)) + +def clean_file(file): + """ + Strips out the warning from a translated po file about being an English source file. + Replaces warning with a note about coming from Transifex. + """ + po = pofile(file) + if po.header.find(SOURCE_WARN) != -1: + new_header = get_new_header(po) + new = po.header.replace(SOURCE_WARN, new_header) + po.header = new + po.save() + +def get_new_header(po): + team = po.metadata.get('Language-Team', None) + if not team: + return TRANSIFEX_HEADER % TRANSIFEX_URL + else: + return TRANSIFEX_HEADER % team + if __name__ == '__main__': if len(sys.argv)<2: raise Exception("missing argument: push or pull") From 4620f50fb151e7a235bc119a92d4adec794a871d Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Tue, 7 May 2013 13:24:38 -0400 Subject: [PATCH 06/18] addressed Cale's comments; switched to path.py paths --- i18n/config.py | 34 +++++++++++++++++------------ i18n/execute.py | 2 +- i18n/extract.py | 16 +++++++------- i18n/generate.py | 34 ++++++++++++++++++----------- i18n/tests/__init__.py | 3 +-- i18n/tests/test_config.py | 8 +++---- i18n/tests/test_extract.py | 2 +- i18n/tests/test_generate.py | 6 +++--- i18n/tests/test_validate.py | 43 +++++++++++++++++-------------------- i18n/transifex.py | 10 +++++---- requirements.txt | 1 + 11 files changed, 87 insertions(+), 72 deletions(-) diff --git a/i18n/config.py b/i18n/config.py index f0d8e366d0..461b0dfd15 100644 --- a/i18n/config.py +++ b/i18n/config.py @@ -1,12 +1,14 @@ import os, json +from path import path # BASE_DIR is the working directory to execute django-admin commands from. # Typically this should be the 'mitx' directory. -BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') +#BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') +BASE_DIR = path(__file__).abspath().dirname().joinpath('..').normpath() # LOCALE_DIR contains the locale files. # Typically this should be 'mitx/conf/locale' -LOCALE_DIR = os.path.join(BASE_DIR, 'conf', 'locale') +LOCALE_DIR = BASE_DIR.joinpath('conf', 'locale') class Configuration: """ @@ -16,10 +18,10 @@ class Configuration: _source_locale = 'en' def __init__(self, filename): - self.filename = filename - self.config = self.get_config(self.filename) + self._filename = filename + self._config = self.read_config(filename) - def get_config(self, filename): + def read_config(self, filename): """ Returns data found in config file (as dict), or raises exception if file not found """ @@ -28,28 +30,31 @@ class Configuration: with open(filename) as stream: return json.load(stream) - def get_locales(self): + @property + def locales(self): """ Returns a list of locales declared in the configuration file, e.g. ['en', 'fr', 'es'] Each locale is a string. """ - return self.config['locales'] + return self._config['locales'] - def get_source_locale(self): + @property + def source_locale(self): """ Returns source language. Source language is English. """ return self._source_locale - def get_dummy_locale(self): + @property + def dummy_locale(self): """ Returns a locale to use for the dummy text, e.g. 'fr'. Throws exception if no dummy-locale is declared. The locale is a string. """ - dummy = self.config.get('dummy-locale', None) + dummy = self._config.get('dummy-locale', None) if not dummy: raise Exception('Could not read dummy-locale from configuration file.') return dummy @@ -59,15 +64,16 @@ class Configuration: Returns the name of the directory holding the po files for locale. Example: mitx/conf/locale/fr/LC_MESSAGES """ - return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES') + return LOCALE_DIR.joinpath(locale, 'LC_MESSAGES') - def get_source_messages_dir(self): + @property + def source_messages_dir(self): """ Returns the name of the directory holding the source-language po files (English). Example: mitx/conf/locale/en/LC_MESSAGES """ - return self.get_messages_dir(self.get_source_locale()) + return self.get_messages_dir(self.source_locale) -CONFIGURATION = Configuration(os.path.normpath(os.path.join(LOCALE_DIR, 'config'))) +CONFIGURATION = Configuration(LOCALE_DIR.joinpath('config').normpath()) diff --git a/i18n/execute.py b/i18n/execute.py index 4c47680101..6a7e12ac5d 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -14,7 +14,7 @@ def get_default_logger(): LOG = get_default_logger() -def execute (command, working_directory=BASE_DIR, log=LOG): +def execute(command, working_directory=BASE_DIR, log=LOG): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. diff --git a/i18n/extract.py b/i18n/extract.py index ffac9b6270..57da0bd76d 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -23,22 +23,22 @@ from execute import execute, create_dir_if_necessary, remove_file, LOG # BABEL_CONFIG contains declarations for Babel to extract strings from mako template files # Use relpath to reduce noise in logs -BABEL_CONFIG = os.path.relpath(LOCALE_DIR + '/babel.cfg', BASE_DIR) +BABEL_CONFIG = BASE_DIR.relpathto(LOCALE_DIR.joinpath('babel.cfg')) # Strings from mako template files are written to BABEL_OUT # Use relpath to reduce noise in logs -BABEL_OUT = os.path.relpath(CONFIGURATION.get_source_messages_dir() + '/mako.po', BASE_DIR) +BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako.po')) SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' def main (): create_dir_if_necessary(LOCALE_DIR) - source_msgs_dir = CONFIGURATION.get_source_messages_dir() + source_msgs_dir = CONFIGURATION.source_messages_dir - remove_file(os.path.join(source_msgs_dir, 'django.po')) + remove_file(source_msgs_dir.joinpath('django.po')) generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') for filename in generated_files: - remove_file(os.path.join(source_msgs_dir, filename)) + remove_file(source_msgs_dir.joinpath(filename)) # Extract strings from mako templates @@ -55,13 +55,13 @@ def main (): execute(make_django_cmd, working_directory=BASE_DIR) # makemessages creates 'django.po'. This filename is hardcoded. # Rename it to django-partial.po to enable merging into django.po later. - os.rename(os.path.join(source_msgs_dir, 'django.po'), - os.path.join(source_msgs_dir, 'django-partial.po')) + os.rename(source_msgs_dir.joinpath('django.po'), + source_msgs_dir.joinpath('django-partial.po')) execute(make_djangojs_cmd, working_directory=BASE_DIR) for filename in generated_files: LOG.info('Cleaning %s' % filename) - po = pofile(os.path.join(source_msgs_dir, filename)) + po = pofile(source_msgs_dir.joinpath(filename)) # replace default headers with edX headers fix_header(po) # replace default metadata with edX metadata diff --git a/i18n/generate.py b/i18n/generate.py index e43efc268a..0c7179b2c6 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -19,25 +19,35 @@ from polib import pofile from config import BASE_DIR, CONFIGURATION from execute import execute, remove_file, LOG -def merge(locale, target='django.po'): +def merge(locale, target='django.po', fail_if_missing=True): """ For the given locale, merge django-partial.po, messages.po, mako.po -> django.po + target is the resulting filename + If fail_if_missing is True, and the files to be merged are missing, + throw an Exception. + If fail_if_missing is False, and the files to be merged are missing, + just return silently. """ LOG.info('Merging locale={0}'.format(locale)) locale_directory = CONFIGURATION.get_messages_dir(locale) files_to_merge = ('django-partial.po', 'messages.po', 'mako.po') - validate_files(locale_directory, files_to_merge) + try: + validate_files(locale_directory, files_to_merge) + except Exception, e: + if not fail_if_missing: + return + raise e # merged file is merged.po merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge) execute(merge_cmd, working_directory=locale_directory) # clean up redunancies in the metadata - merged_filename = os.path.join(locale_directory, 'merged.po') + merged_filename = locale_directory.joinpath('merged.po') clean_metadata(merged_filename) # rename merged.po -> django.po (default) - django_filename = os.path.join(locale_directory, target) + django_filename = locale_directory.joinpath(target) os.rename(merged_filename, django_filename) # can't overwrite file on Windows def clean_metadata(file): @@ -45,25 +55,25 @@ def clean_metadata(file): Clean up redundancies in the metadata caused by merging. This reads in a PO file and simply saves it back out again. """ - po = pofile(file) - po.save() - + pofile(file).save() def validate_files(dir, files_to_merge): """ Asserts that the given files exist. files_to_merge is a list of file names (no directories). - dir is the directory in which the files should appear. + dir is the directory (a path object from path.py) in which the files should appear. raises an Exception if any of the files are not in dir. """ for path in files_to_merge: - pathname = os.path.join(dir, path) - if not os.path.exists(pathname): - raise Exception("File not found: {0}".format(pathname)) + pathname = dir.joinpath(path) + if not pathname.exists(): + raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname)) def main (): - for locale in CONFIGURATION.get_locales(): + for locale in CONFIGURATION.locales: merge(locale) + # Dummy text is not required. Don't raise exception if files are missing. + merge(CONFIGURATION.dummy_locale, fail_if_missing=False) compile_cmd = 'django-admin.py compilemessages' execute(compile_cmd, working_directory=BASE_DIR) diff --git a/i18n/tests/__init__.py b/i18n/tests/__init__.py index d8fce19df7..ee6283376e 100644 --- a/i18n/tests/__init__.py +++ b/i18n/tests/__init__.py @@ -3,5 +3,4 @@ from test_extract import TestExtract from test_generate import TestGenerate from test_converter import TestConverter from test_dummy import TestDummy -from test_validate import TestValidate - +import test_validate diff --git a/i18n/tests/test_config.py b/i18n/tests/test_config.py index aea8f0bca3..bcec6ac354 100644 --- a/i18n/tests/test_config.py +++ b/i18n/tests/test_config.py @@ -11,7 +11,7 @@ class TestConfiguration(TestCase): def test_config(self): config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'config')) config = Configuration(config_filename) - self.assertEqual(config.get_source_locale(), 'en') + self.assertEqual(config.source_locale, 'en') def test_no_config(self): config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'no_such_file')) @@ -25,9 +25,9 @@ class TestConfiguration(TestCase): Also check values of dummy_locale and source_locale. """ self.assertIsNotNone(CONFIGURATION) - locales = CONFIGURATION.get_locales() + locales = CONFIGURATION.locales self.assertIsNotNone(locales) self.assertIsInstance(locales, list) self.assertIn('en', locales) - self.assertEqual('fr', CONFIGURATION.get_dummy_locale()) - self.assertEqual('en', CONFIGURATION.get_source_locale()) + self.assertEqual('fr', CONFIGURATION.dummy_locale) + self.assertEqual('en', CONFIGURATION.source_locale) diff --git a/i18n/tests/test_extract.py b/i18n/tests/test_extract.py index a9faa2bdd8..7e8b1a9d2b 100644 --- a/i18n/tests/test_extract.py +++ b/i18n/tests/test_extract.py @@ -39,7 +39,7 @@ class TestExtract(TestCase): Fails assertion if one of the files doesn't exist. """ for filename in self.generated_files: - path = os.path.join(CONFIGURATION.get_source_messages_dir(), filename) + path = os.path.join(CONFIGURATION.source_messages_dir, filename) exists = os.path.exists(path) self.assertTrue(exists, msg='Missing file: %s' % filename) if exists: diff --git a/i18n/tests/test_generate.py b/i18n/tests/test_generate.py index bac727f671..468858664f 100644 --- a/i18n/tests/test_generate.py +++ b/i18n/tests/test_generate.py @@ -21,8 +21,8 @@ class TestGenerate(TestCase): """ Tests merge script on English source files. """ - filename = os.path.join(CONFIGURATION.get_source_messages_dir(), random_name()) - generate.merge(CONFIGURATION.get_source_locale(), target=filename) + filename = os.path.join(CONFIGURATION.source_messages_dir, random_name()) + generate.merge(CONFIGURATION.source_locale, target=filename) self.assertTrue(os.path.exists(filename)) os.remove(filename) @@ -35,7 +35,7 @@ class TestGenerate(TestCase): after start of test suite) """ generate.main() - for locale in CONFIGURATION.get_locales(): + for locale in CONFIGURATION.locales: for filename in ('django', 'djangojs'): mofile = filename+'.mo' path = os.path.join(CONFIGURATION.get_messages_dir(locale), mofile) diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py index 64579fb563..7f0cdd7a25 100644 --- a/i18n/tests/test_validate.py +++ b/i18n/tests/test_validate.py @@ -4,31 +4,28 @@ from nose.plugins.skip import SkipTest from config import LOCALE_DIR from execute import call, LOG + +def test_po_files(): + """ + This is a generator. It yields all of the .po files under root, and tests each one. + """ + for (dirpath, dirnames, filenames) in os.walk(LOCALE_DIR): + for name in filenames: + print name + (base, ext) = os.path.splitext(name) + if ext.lower() == '.po': + yield validate_po_file, os.path.join(dirpath, name) -class TestValidate(TestCase): + +def validate_po_file(filename): """ Call GNU msgfmt -c on each .po file to validate its format. """ - - def test_validate(self): - # Skip this test for now because it's very noisy - raise SkipTest() - for file in self.get_po_files(): - # Use relative paths to make output less noisy. - rfile = os.path.relpath(file, LOCALE_DIR) - (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) - if err != '': - LOG.warn('\n'+err) + # Skip this test for now because it's very noisy + raise SkipTest() + # Use relative paths to make output less noisy. + rfile = os.path.relpath(filename, LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) + if err != '': + LOG.warn('\n'+err) - def get_po_files(self, root=LOCALE_DIR): - """ - This is a generator. It yields all of the .po files under root. - """ - for (dirpath, dirnames, filenames) in os.walk(root): - for name in filenames: - (base, ext) = os.path.splitext(name) - if ext.lower() == '.po': - yield os.path.join(dirpath, name) - - - diff --git a/i18n/transifex.py b/i18n/transifex.py index 812ecd666f..d08a77b1c0 100755 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -13,7 +13,9 @@ def push(): execute('tx push -s') def pull(): - execute('tx pull') + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: + execute('tx pull -l %s' % locale) clean_translated_locales() @@ -22,8 +24,8 @@ def clean_translated_locales(): Strips out the warning from all translated po files about being an English source file. """ - for locale in CONFIGURATION.get_locales(): - if locale != CONFIGURATION.get_source_locale(): + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: clean_locale(locale) def clean_locale(locale): @@ -34,7 +36,7 @@ def clean_locale(locale): """ dirname = CONFIGURATION.get_messages_dir(locale) for filename in ('django-partial.po', 'djangojs.po', 'mako.po'): - clean_file(os.path.join(dirname, filename)) + clean_file(dirname.joinpath(filename)) def clean_file(file): """ diff --git a/requirements.txt b/requirements.txt index d3fdd46b81..c6ee47becb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ paramiko==1.9.0 path.py==3.0.1 Pillow==1.7.8 pip +polib==1.0.3 pygments==1.5 pygraphviz==1.1 pymongo==2.4.1 From cf8d6b89657295ed55fc7ca8a70bf3a7de3db17f Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Tue, 7 May 2013 14:10:59 -0400 Subject: [PATCH 07/18] factor logging out of library calls --- i18n/config.py | 1 - i18n/execute.py | 17 ++++------------- i18n/extract.py | 7 +++++-- i18n/generate.py | 12 ++++++++---- i18n/logger.py | 13 +++++++++++++ i18n/tests/test_validate.py | 15 +++++++++------ 6 files changed, 39 insertions(+), 26 deletions(-) create mode 100644 i18n/logger.py diff --git a/i18n/config.py b/i18n/config.py index 461b0dfd15..d78fc0ca45 100644 --- a/i18n/config.py +++ b/i18n/config.py @@ -76,4 +76,3 @@ class Configuration: CONFIGURATION = Configuration(LOCALE_DIR.joinpath('config').normpath()) - diff --git a/i18n/execute.py b/i18n/execute.py index 6a7e12ac5d..e3f3478d12 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -1,18 +1,9 @@ -import os, subprocess, logging +import os, subprocess + +from logger import get_logger from config import CONFIGURATION, BASE_DIR - -def get_default_logger(): - """Returns a default logger""" - log = logging.getLogger(__name__) - log.setLevel(logging.INFO) - log_handler = logging.StreamHandler() - log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')) - log.addHandler(log_handler) - return log - -LOG = get_default_logger() - +LOG = get_logger(__name__) def execute(command, working_directory=BASE_DIR, log=LOG): """ diff --git a/i18n/extract.py b/i18n/extract.py index 57da0bd76d..9b0ad3829c 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -18,8 +18,10 @@ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow import os from datetime import datetime from polib import pofile + +from logger import get_logger from config import BASE_DIR, LOCALE_DIR, CONFIGURATION -from execute import execute, create_dir_if_necessary, remove_file, LOG +from execute import execute, create_dir_if_necessary, remove_file # BABEL_CONFIG contains declarations for Babel to extract strings from mako template files # Use relpath to reduce noise in logs @@ -32,6 +34,7 @@ BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako. SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' def main (): + log = get_logger(__name__) create_dir_if_necessary(LOCALE_DIR) source_msgs_dir = CONFIGURATION.source_messages_dir @@ -60,7 +63,7 @@ def main (): execute(make_djangojs_cmd, working_directory=BASE_DIR) for filename in generated_files: - LOG.info('Cleaning %s' % filename) + log.info('Cleaning %s' % filename) po = pofile(source_msgs_dir.joinpath(filename)) # replace default headers with edX headers fix_header(po) diff --git a/i18n/generate.py b/i18n/generate.py index 0c7179b2c6..ffc88b64d0 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -16,10 +16,11 @@ import os from polib import pofile +from logger import get_logger from config import BASE_DIR, CONFIGURATION -from execute import execute, remove_file, LOG +from execute import execute, remove_file -def merge(locale, target='django.po', fail_if_missing=True): +def merge(locale, target='django.po', fail_if_missing=True, log=None): """ For the given locale, merge django-partial.po, messages.po, mako.po -> django.po target is the resulting filename @@ -28,7 +29,8 @@ def merge(locale, target='django.po', fail_if_missing=True): If fail_if_missing is False, and the files to be merged are missing, just return silently. """ - LOG.info('Merging locale={0}'.format(locale)) + if log: + log.info('Merging locale={0}'.format(locale)) locale_directory = CONFIGURATION.get_messages_dir(locale) files_to_merge = ('django-partial.po', 'messages.po', 'mako.po') try: @@ -70,10 +72,12 @@ def validate_files(dir, files_to_merge): raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname)) def main (): + log = get_logger(__name__) + for locale in CONFIGURATION.locales: merge(locale) # Dummy text is not required. Don't raise exception if files are missing. - merge(CONFIGURATION.dummy_locale, fail_if_missing=False) + merge(CONFIGURATION.dummy_locale, fail_if_missing=False, log=log) compile_cmd = 'django-admin.py compilemessages' execute(compile_cmd, working_directory=BASE_DIR) diff --git a/i18n/logger.py b/i18n/logger.py new file mode 100644 index 0000000000..20d767a032 --- /dev/null +++ b/i18n/logger.py @@ -0,0 +1,13 @@ +import logging + +def get_logger(name): + """ + Returns a default logger. + logging.basicConfig does not render to the console + """ + log = logging.getLogger() + log.setLevel(logging.INFO) + log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')) + log.addHandler(log_handler) + return log diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py index 7f0cdd7a25..6bb7164a50 100644 --- a/i18n/tests/test_validate.py +++ b/i18n/tests/test_validate.py @@ -2,24 +2,27 @@ import os from unittest import TestCase from nose.plugins.skip import SkipTest +from logger import get_logger from config import LOCALE_DIR -from execute import call, LOG +from execute import call -def test_po_files(): +def test_po_files(root=LOCALE_DIR): """ This is a generator. It yields all of the .po files under root, and tests each one. """ - for (dirpath, dirnames, filenames) in os.walk(LOCALE_DIR): + log = get_logger(__name__) + for (dirpath, dirnames, filenames) in os.walk(root): for name in filenames: print name (base, ext) = os.path.splitext(name) if ext.lower() == '.po': - yield validate_po_file, os.path.join(dirpath, name) + yield validate_po_file, os.path.join(dirpath, name), log -def validate_po_file(filename): +def validate_po_file(filename, log): """ Call GNU msgfmt -c on each .po file to validate its format. + Any errors caught by msgfmt are logged to log. """ # Skip this test for now because it's very noisy raise SkipTest() @@ -27,5 +30,5 @@ def validate_po_file(filename): rfile = os.path.relpath(filename, LOCALE_DIR) (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) if err != '': - LOG.warn('\n'+err) + log.warn('\n'+err) From 129c02f0b296dc5863eb7708cd67bd77ebd0124f Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Tue, 7 May 2013 14:12:58 -0400 Subject: [PATCH 08/18] forgot to remove stale comment --- i18n/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/i18n/config.py b/i18n/config.py index d78fc0ca45..4f246ed942 100644 --- a/i18n/config.py +++ b/i18n/config.py @@ -3,7 +3,6 @@ from path import path # BASE_DIR is the working directory to execute django-admin commands from. # Typically this should be the 'mitx' directory. -#BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') BASE_DIR = path(__file__).abspath().dirname().joinpath('..').normpath() # LOCALE_DIR contains the locale files. From d05ba84f1317820ce7265745b11e99e68c275c0c Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Wed, 8 May 2013 13:49:42 -0400 Subject: [PATCH 09/18] /usr/bin/env in shebang line --- i18n/extract.py | 2 +- i18n/generate.py | 2 +- i18n/make_dummy.py | 2 +- i18n/transifex.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/i18n/extract.py b/i18n/extract.py index 9b0ad3829c..2cb4ebe118 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow diff --git a/i18n/generate.py b/i18n/generate.py index ffc88b64d0..1deb1beeae 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python """ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow diff --git a/i18n/make_dummy.py b/i18n/make_dummy.py index 8d0fb95ef2..6c14edd45a 100755 --- a/i18n/make_dummy.py +++ b/i18n/make_dummy.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python # Generate test translation files from human-readable po files. # diff --git a/i18n/transifex.py b/i18n/transifex.py index d08a77b1c0..ac203f3eea 100755 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python import os, sys from polib import pofile From dc473e6f7b5f919844493e6782a017f56d589194 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Wed, 8 May 2013 14:07:57 -0400 Subject: [PATCH 10/18] more verbose messages.po --- conf/locale/en/LC_MESSAGES/messages.po | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/conf/locale/en/LC_MESSAGES/messages.po b/conf/locale/en/LC_MESSAGES/messages.po index 1bb8bf6d7f..e5961753c5 100644 --- a/conf/locale/en/LC_MESSAGES/messages.po +++ b/conf/locale/en/LC_MESSAGES/messages.po @@ -1 +1,20 @@ +# edX translation file +# Copyright (C) 2013 edX +# This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE. +# +msgid "" +msgstr "" +"Project-Id-Version: EdX Studio\n" +"Report-Msgid-Bugs-To: translation_team@edx.org\n" +"POT-Creation-Date: 2013-05-02 13:13-0400\n" +"PO-Revision-Date: 2013-05-02 13:27-0400\n" +"Last-Translator: \n" +"Language-Team: translation team \n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=UTF-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Language: en\n" + # empty +msgid "This is a key string." +msgstr "" From beb4b39b73f84dbd839f9802e1508c3e0abc2fbe Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Wed, 8 May 2013 14:25:31 -0400 Subject: [PATCH 11/18] fix logging --- i18n/extract.py | 6 +++--- i18n/generate.py | 6 +++--- i18n/logger.py | 13 ------------- i18n/tests/test_validate.py | 8 ++++---- 4 files changed, 10 insertions(+), 23 deletions(-) delete mode 100644 i18n/logger.py diff --git a/i18n/extract.py b/i18n/extract.py index 2cb4ebe118..c517de3b51 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -15,11 +15,10 @@ See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow """ -import os +import os, sys, logging from datetime import datetime from polib import pofile -from logger import get_logger from config import BASE_DIR, LOCALE_DIR, CONFIGURATION from execute import execute, create_dir_if_necessary, remove_file @@ -34,7 +33,8 @@ BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako. SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' def main (): - log = get_logger(__name__) + log = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) create_dir_if_necessary(LOCALE_DIR) source_msgs_dir = CONFIGURATION.source_messages_dir diff --git a/i18n/generate.py b/i18n/generate.py index 1deb1beeae..48470796a2 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -13,10 +13,9 @@ languages to generate. """ -import os +import os, sys, logging from polib import pofile -from logger import get_logger from config import BASE_DIR, CONFIGURATION from execute import execute, remove_file @@ -72,7 +71,8 @@ def validate_files(dir, files_to_merge): raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname)) def main (): - log = get_logger(__name__) + log = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) for locale in CONFIGURATION.locales: merge(locale) diff --git a/i18n/logger.py b/i18n/logger.py deleted file mode 100644 index 20d767a032..0000000000 --- a/i18n/logger.py +++ /dev/null @@ -1,13 +0,0 @@ -import logging - -def get_logger(name): - """ - Returns a default logger. - logging.basicConfig does not render to the console - """ - log = logging.getLogger() - log.setLevel(logging.INFO) - log_handler = logging.StreamHandler() - log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')) - log.addHandler(log_handler) - return log diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py index 6bb7164a50..67057a30e7 100644 --- a/i18n/tests/test_validate.py +++ b/i18n/tests/test_validate.py @@ -1,8 +1,7 @@ -import os +import os, sys, logging from unittest import TestCase from nose.plugins.skip import SkipTest -from logger import get_logger from config import LOCALE_DIR from execute import call @@ -10,10 +9,11 @@ def test_po_files(root=LOCALE_DIR): """ This is a generator. It yields all of the .po files under root, and tests each one. """ - log = get_logger(__name__) + log = logging.getLogger(__name__) + logging.basicConfig(stream=sys.stdout, level=logging.INFO) + for (dirpath, dirnames, filenames) in os.walk(root): for name in filenames: - print name (base, ext) = os.path.splitext(name) if ext.lower() == '.po': yield validate_po_file, os.path.join(dirpath, name), log From dfcbb73662ae6d845213ace78ba450c010ad61b7 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Wed, 8 May 2013 14:55:16 -0400 Subject: [PATCH 12/18] add guard code to ensure gnu gettext utilities are loaded before rake tests --- rakefile | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/rakefile b/rakefile index acb9bf6ecc..64afb2e389 100644 --- a/rakefile +++ b/rakefile @@ -522,19 +522,39 @@ def validate_transifex_config() return true end +# Make sure GNU gettext utilities are available +# Returns boolean: returns true if utilities are available, else returns false +def validate_gnu_gettext() + begin + select_executable('xgettext') + return true + rescue + puts "Error:".red + puts "Cannot locate GNU gettext utilities, which are required by django for internationalization.".red + puts "(see https://docs.djangoproject.com/en/dev/topics/i18n/translation/#message-files)".red + puts "Try downloading them from http://www.gnu.org/software/gettext/".red + return false + end +end + + namespace :i18n do desc "Extract localizable strings from sources" task :extract do - sh(File.join(REPO_ROOT, "i18n", "extract.py")) + if validate_gnu_gettext() + sh(File.join(REPO_ROOT, "i18n", "extract.py")) + end end desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." task :generate do - if ARGV.last.downcase == 'extract' - Rake::Task["i18n:extract"].execute + if validate_gnu_gettext() + if ARGV.last.downcase == 'extract' + Rake::Task["i18n:extract"].execute + end + sh(File.join(REPO_ROOT, "i18n", "generate.py")) end - sh(File.join(REPO_ROOT, "i18n", "generate.py")) end desc "Simulate international translation by generating dummy strings corresponding to source strings." @@ -567,8 +587,10 @@ namespace :i18n do desc "Run tests for the internationalization library" task :test do - test = File.join(REPO_ROOT, "i18n", "tests") - sh("nosetests #{test}") + if validate_gnu_gettext() + test = File.join(REPO_ROOT, "i18n", "tests") + sh("nosetests #{test}") + end end end From 165e7059c81c10005f6412594dc3167eaef13da6 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Wed, 8 May 2013 15:34:35 -0400 Subject: [PATCH 13/18] guard predicates are rake tasks, not functions --- rakefile | 88 ++++++++++++++++++++++++++------------------------------ 1 file changed, 40 insertions(+), 48 deletions(-) diff --git a/rakefile b/rakefile index 64afb2e389..3edbc39067 100644 --- a/rakefile +++ b/rakefile @@ -510,51 +510,21 @@ end # --- Internationalization tasks -# Make sure config file with username/password exists -# Returns boolean: returns true if file exists and is nonzero length -def validate_transifex_config() - config_file = "#{Dir.home}/.transifexrc" - if !File.file?(config_file) or File.size(config_file)==0 - raise "Cannot connect to Transifex, config file is missing or empty: #{config_file}\n" + - "See http://help.transifex.com/features/client/#transifexrc\n" - return false - end - return true -end - -# Make sure GNU gettext utilities are available -# Returns boolean: returns true if utilities are available, else returns false -def validate_gnu_gettext() - begin - select_executable('xgettext') - return true - rescue - puts "Error:".red - puts "Cannot locate GNU gettext utilities, which are required by django for internationalization.".red - puts "(see https://docs.djangoproject.com/en/dev/topics/i18n/translation/#message-files)".red - puts "Try downloading them from http://www.gnu.org/software/gettext/".red - return false - end -end - - namespace :i18n do desc "Extract localizable strings from sources" task :extract do - if validate_gnu_gettext() - sh(File.join(REPO_ROOT, "i18n", "extract.py")) - end + Rake::Task["i18n:validate:gettext"].execute + sh(File.join(REPO_ROOT, "i18n", "extract.py")) end desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." task :generate do - if validate_gnu_gettext() - if ARGV.last.downcase == 'extract' - Rake::Task["i18n:extract"].execute - end - sh(File.join(REPO_ROOT, "i18n", "generate.py")) + Rake::Task["i18n:validate:gettext"].execute + if ARGV.last.downcase == 'extract' + Rake::Task["i18n:extract"].execute end + sh(File.join(REPO_ROOT, "i18n", "generate.py")) end desc "Simulate international translation by generating dummy strings corresponding to source strings." @@ -567,30 +537,52 @@ namespace :i18n do end end + namespace :validate do + + desc "Make sure GNU gettext utilities are available" + task :gettext do + begin + select_executable('xgettext') + rescue + msg = "Cannot locate GNU gettext utilities, which are required by django for internationalization.\n" + msg += "(see https://docs.djangoproject.com/en/dev/topics/i18n/translation/#message-files)\n" + msg += "Try downloading them from http://www.gnu.org/software/gettext/" + abort(msg.red) + end + end + + desc "Make sure config file with username/password exists" + task :transifex_config do + config_file = "#{Dir.home}/.transifexrc" + if !File.file?(config_file) or File.size(config_file)==0 + msg ="Cannot connect to Transifex, config file is missing or empty: #{config_file}\n" + msg += "See http://help.transifex.com/features/client/#transifexrc" + abort(msg.red) + end + end + end + namespace :transifex do desc "Push source strings to Transifex for translation" task :push do - if validate_transifex_config() - cmd = File.join(REPO_ROOT, "i18n", "transifex.py") - sh("#{cmd} push") - end + Rake::Task["i18n:validate:transifex_config"].execute + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} push") end desc "Pull translated strings from Transifex" task :pull do - if validate_transifex_config() - cmd = File.join(REPO_ROOT, "i18n", "transifex.py") - sh("#{cmd} pull") - end + Rake::Task["i18n:validate:transifex_config"].execute + cmd = File.join(REPO_ROOT, "i18n", "transifex.py") + sh("#{cmd} pull") end end desc "Run tests for the internationalization library" task :test do - if validate_gnu_gettext() - test = File.join(REPO_ROOT, "i18n", "tests") - sh("nosetests #{test}") - end + Rake::Task["i18n:validate:gettext"].execute + test = File.join(REPO_ROOT, "i18n", "tests") + sh("nosetests #{test}") end end From 35d72f30b905dc2132d7d4bda428c48b97b782af Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 10 May 2013 14:40:30 -0400 Subject: [PATCH 14/18] update .gitignore; fix logger import in execute.py --- .gitignore | 3 +++ i18n/execute.py | 5 ++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index d01baf055a..f1784a48f3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ *.swp *.orig *.DS_Store +*.mo :2e_* :2e# .AppleDouble @@ -22,6 +23,8 @@ reports/ *.egg-info Gemfile.lock .env/ +conf/locale/en/LC_MESSAGES/*.po +!messages.po lms/static/sass/*.css cms/static/sass/*.css lms/lib/comment_client/python diff --git a/i18n/execute.py b/i18n/execute.py index e3f3478d12..e55e653ea7 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -1,9 +1,8 @@ -import os, subprocess +import os, subprocess, logging -from logger import get_logger from config import CONFIGURATION, BASE_DIR -LOG = get_logger(__name__) +LOG = logging.getLogger(__name__) def execute(command, working_directory=BASE_DIR, log=LOG): """ From f30f6207d5e46938daf674d8685b1236c63be138 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 10 May 2013 14:47:47 -0400 Subject: [PATCH 15/18] clean up rakefile syntax for task dependencies --- rakefile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/rakefile b/rakefile index 3edbc39067..cf9363d47b 100644 --- a/rakefile +++ b/rakefile @@ -513,14 +513,12 @@ end namespace :i18n do desc "Extract localizable strings from sources" - task :extract do - Rake::Task["i18n:validate:gettext"].execute + task :extract => "i18n:validate:gettext" do sh(File.join(REPO_ROOT, "i18n", "extract.py")) end desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." - task :generate do - Rake::Task["i18n:validate:gettext"].execute + task :generate => "i18n:validate:gettext" do if ARGV.last.downcase == 'extract' Rake::Task["i18n:extract"].execute end @@ -579,8 +577,7 @@ namespace :i18n do end desc "Run tests for the internationalization library" - task :test do - Rake::Task["i18n:validate:gettext"].execute + task :test => "i18n:validate:gettext" do test = File.join(REPO_ROOT, "i18n", "tests") sh("nosetests #{test}") end From a52cf85c0811d8bfd77b96676873550d0910da30 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 10 May 2013 14:50:49 -0400 Subject: [PATCH 16/18] rake task dependency for transifex --- rakefile | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/rakefile b/rakefile index cf9363d47b..04a7db4904 100644 --- a/rakefile +++ b/rakefile @@ -562,15 +562,13 @@ namespace :i18n do namespace :transifex do desc "Push source strings to Transifex for translation" - task :push do - Rake::Task["i18n:validate:transifex_config"].execute + task :push => "i18n:validate:transifex_config" do cmd = File.join(REPO_ROOT, "i18n", "transifex.py") sh("#{cmd} push") end desc "Pull translated strings from Transifex" - task :pull do - Rake::Task["i18n:validate:transifex_config"].execute + task :pull => "i18n:validate:transifex_config" do cmd = File.join(REPO_ROOT, "i18n", "transifex.py") sh("#{cmd} pull") end From 1c5815a8444a1f8e8fa406fbced565468fb3d731 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 10 May 2013 15:11:10 -0400 Subject: [PATCH 17/18] per-file log objects --- i18n/execute.py | 14 +++++++++----- i18n/tests/test_validate.py | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/i18n/execute.py b/i18n/execute.py index e55e653ea7..1ff439ef38 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -4,25 +4,28 @@ from config import CONFIGURATION, BASE_DIR LOG = logging.getLogger(__name__) -def execute(command, working_directory=BASE_DIR, log=LOG): +def execute(command, working_directory=BASE_DIR, log=True): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. - The command is logged to log, output is ignored. + log is boolean. If true, the command's invocation string is logged. + Output is ignored. """ if log: - log.info(command) + LOG.info(command) subprocess.call(command.split(' '), cwd=working_directory) -def call(command, working_directory=BASE_DIR, log=LOG): +def call(command, working_directory=BASE_DIR, log=True): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. Returns a tuple of two strings: (stdout, stderr) + log is boolean. If true, the command's invocation string is logged. + """ if log: - log.info(command) + LOG.info(command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_directory) out, err = p.communicate() return (out, err) @@ -36,6 +39,7 @@ def create_dir_if_necessary(pathname): def remove_file(filename, log=LOG, verbose=True): """ Attempt to delete filename. + log is boolean. If true, removal is logged. Log a warning if file does not exist. Logging filenames are releative to BASE_DIR to cut down on noise in output. """ diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py index 67057a30e7..7d970c8de2 100644 --- a/i18n/tests/test_validate.py +++ b/i18n/tests/test_validate.py @@ -28,7 +28,7 @@ def validate_po_file(filename, log): raise SkipTest() # Use relative paths to make output less noisy. rfile = os.path.relpath(filename, LOCALE_DIR) - (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], log=False, working_directory=LOCALE_DIR) if err != '': log.warn('\n'+err) From 1f7bf1f00a43ccf142b1f4b1517cbeb2acbcafd3 Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Sun, 12 May 2013 21:52:07 -0400 Subject: [PATCH 18/18] use global LOG instead of local log --- i18n/execute.py | 18 +++++++----------- i18n/extract.py | 5 +++-- i18n/generate.py | 12 ++++++------ i18n/tests/test_validate.py | 2 +- 4 files changed, 17 insertions(+), 20 deletions(-) diff --git a/i18n/execute.py b/i18n/execute.py index 1ff439ef38..8e7f0f52de 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -4,28 +4,24 @@ from config import CONFIGURATION, BASE_DIR LOG = logging.getLogger(__name__) -def execute(command, working_directory=BASE_DIR, log=True): +def execute(command, working_directory=BASE_DIR): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. - log is boolean. If true, the command's invocation string is logged. Output is ignored. """ - if log: - LOG.info(command) + LOG.info(command) subprocess.call(command.split(' '), cwd=working_directory) -def call(command, working_directory=BASE_DIR, log=True): +def call(command, working_directory=BASE_DIR): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. Returns a tuple of two strings: (stdout, stderr) - log is boolean. If true, the command's invocation string is logged. """ - if log: - LOG.info(command) + LOG.info(command) p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_directory) out, err = p.communicate() return (out, err) @@ -36,7 +32,7 @@ def create_dir_if_necessary(pathname): os.makedirs(dirname) -def remove_file(filename, log=LOG, verbose=True): +def remove_file(filename, verbose=True): """ Attempt to delete filename. log is boolean. If true, removal is logged. @@ -44,8 +40,8 @@ def remove_file(filename, log=LOG, verbose=True): Logging filenames are releative to BASE_DIR to cut down on noise in output. """ if verbose: - log.info('Deleting file %s' % os.path.relpath(filename, BASE_DIR)) + LOG.info('Deleting file %s' % os.path.relpath(filename, BASE_DIR)) if not os.path.exists(filename): - log.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR)) + LOG.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR)) else: os.remove(filename) diff --git a/i18n/extract.py b/i18n/extract.py index c517de3b51..c28c3868e2 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -32,8 +32,9 @@ BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako. SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' +LOG = logging.getLogger(__name__) + def main (): - log = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout, level=logging.INFO) create_dir_if_necessary(LOCALE_DIR) source_msgs_dir = CONFIGURATION.source_messages_dir @@ -63,7 +64,7 @@ def main (): execute(make_djangojs_cmd, working_directory=BASE_DIR) for filename in generated_files: - log.info('Cleaning %s' % filename) + LOG.info('Cleaning %s' % filename) po = pofile(source_msgs_dir.joinpath(filename)) # replace default headers with edX headers fix_header(po) diff --git a/i18n/generate.py b/i18n/generate.py index 48470796a2..65c65c00d6 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -17,9 +17,11 @@ import os, sys, logging from polib import pofile from config import BASE_DIR, CONFIGURATION -from execute import execute, remove_file +from execute import execute -def merge(locale, target='django.po', fail_if_missing=True, log=None): +LOG = logging.getLogger(__name__) + +def merge(locale, target='django.po', fail_if_missing=True): """ For the given locale, merge django-partial.po, messages.po, mako.po -> django.po target is the resulting filename @@ -28,8 +30,7 @@ def merge(locale, target='django.po', fail_if_missing=True, log=None): If fail_if_missing is False, and the files to be merged are missing, just return silently. """ - if log: - log.info('Merging locale={0}'.format(locale)) + LOG.info('Merging locale={0}'.format(locale)) locale_directory = CONFIGURATION.get_messages_dir(locale) files_to_merge = ('django-partial.po', 'messages.po', 'mako.po') try: @@ -71,13 +72,12 @@ def validate_files(dir, files_to_merge): raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname)) def main (): - log = logging.getLogger(__name__) logging.basicConfig(stream=sys.stdout, level=logging.INFO) for locale in CONFIGURATION.locales: merge(locale) # Dummy text is not required. Don't raise exception if files are missing. - merge(CONFIGURATION.dummy_locale, fail_if_missing=False, log=log) + merge(CONFIGURATION.dummy_locale, fail_if_missing=False) compile_cmd = 'django-admin.py compilemessages' execute(compile_cmd, working_directory=BASE_DIR) diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py index 7d970c8de2..bef563faea 100644 --- a/i18n/tests/test_validate.py +++ b/i18n/tests/test_validate.py @@ -28,7 +28,7 @@ def validate_po_file(filename, log): raise SkipTest() # Use relative paths to make output less noisy. rfile = os.path.relpath(filename, LOCALE_DIR) - (out, err) = call(['msgfmt','-c', rfile], log=False, working_directory=LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], working_directory=LOCALE_DIR) if err != '': log.warn('\n'+err)