From 55dd0fc8bcb40efa2557653261d9ac6a29c3683f Mon Sep 17 00:00:00 2001 From: Steve Strassmann Date: Fri, 3 May 2013 12:45:55 -0400 Subject: [PATCH] workaround for gettext parser bug --- .tx/config | 26 ++++++++++++++++ conf/locale/config | 2 +- i18n/execute.py | 21 +++++++++++-- i18n/extract.py | 4 +-- i18n/generate.py | 16 +++++++++- i18n/make_dummy.py | 18 +++++------ i18n/tests/__init__.py | 2 ++ i18n/tests/test_validate.py | 33 ++++++++++++++++++++ rakefile | 62 +++++++++++++++++++++++-------------- 9 files changed, 145 insertions(+), 39 deletions(-) create mode 100644 .tx/config create mode 100644 i18n/tests/test_validate.py diff --git a/.tx/config b/.tx/config new file mode 100644 index 0000000000..540c4732af --- /dev/null +++ b/.tx/config @@ -0,0 +1,26 @@ +[main] +host = https://www.transifex.com + +[edx-studio.django-partial] +file_filter = conf/locale//LC_MESSAGES/django-partial.po +source_file = conf/locale/en/LC_MESSAGES/django-partial.po +source_lang = en +type = PO + +[edx-studio.djangojs] +file_filter = conf/locale//LC_MESSAGES/djangojs.po +source_file = conf/locale/en/LC_MESSAGES/djangojs.po +source_lang = en +type = PO + +[edx-studio.mako] +file_filter = conf/locale//LC_MESSAGES/mako.po +source_file = conf/locale/en/LC_MESSAGES/mako.po +source_lang = en +type = PO + +[edx-studio.messages] +file_filter = conf/locale//LC_MESSAGES/messages.po +source_file = conf/locale/en/LC_MESSAGES/messages.po +source_lang = en +type = PO diff --git a/conf/locale/config b/conf/locale/config index 2d01e1ea43..8afaaa9482 100644 --- a/conf/locale/config +++ b/conf/locale/config @@ -1 +1 @@ -{"locales" : ["en"]} +{"locales" : ["en", "fr", "es"]} diff --git a/i18n/execute.py b/i18n/execute.py index 3c3416b65d..0cd152fb58 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -50,14 +50,29 @@ def execute (command, working_directory=BASE_DIR, log=LOG): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. - Output is logged to log. + The command is logged to log, output is ignored. """ - log.info(command) + if log: + log.info(command) subprocess.call(command.split(' '), cwd=working_directory) + + +def call(command, working_directory=BASE_DIR, log=LOG): + """ + Executes shell command in a given working_directory. + Command is a string to pass to the shell. + Returns a tuple of two strings: (stdout, stderr) + """ + if log: + log.info(command) + p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=working_directory) + out, err = p.communicate() + return (out, err) + def get_config(): """Returns data found in config file, or returns None if file not found""" - config_path = os.path.abspath(CONFIG_FILENAME) + config_path = os.path.normpath(CONFIG_FILENAME) if not os.path.exists(config_path): log.warn("Configuration file cannot be found: %s" % \ os.path.relpath(config_path, BASE_DIR)) diff --git a/i18n/extract.py b/i18n/extract.py index c6fedd3bfa..c3d4368a67 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -79,6 +79,7 @@ def fix_header(po): """ Replace default headers with edX headers """ + po.metadata_is_fuzzy = [] # remove [u'fuzzy'] header = po.header fixes = ( ('SOME DESCRIPTIVE TITLE', 'edX translation file'), @@ -119,10 +120,9 @@ def fix_metadata(po): 'Report-Msgid-Bugs-To': 'translation_team@edx.org', 'Project-Id-Version': '0.1a', 'Language' : 'en', + 'Last-Translator' : '', 'Language-Team': 'translation team ', } - if po.metadata.has_key('Last-Translator'): - del po.metadata['Last-Translator'] po.metadata.update(fixes) def strip_key_strings(po): diff --git a/i18n/generate.py b/i18n/generate.py index ddbaadfa70..40a6cc88ca 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -14,6 +14,8 @@ """ import os +from polib import pofile + from execute import execute, get_config, messages_dir, remove_file, \ BASE_DIR, LOG, SOURCE_LOCALE @@ -30,11 +32,23 @@ def merge(locale, target='django.po'): merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge) execute(merge_cmd, working_directory=locale_directory) - # rename merged.po -> django.po (default) + # clean up redunancies in the metadata merged_filename = os.path.join(locale_directory, 'merged.po') + clean_metadata(merged_filename) + + # rename merged.po -> django.po (default) django_filename = os.path.join(locale_directory, target) os.rename(merged_filename, django_filename) # can't overwrite file on Windows +def clean_metadata(file): + """ + Clean up redundancies in the metadata caused by merging. + This reads in a PO file and simply saves it back out again. + """ + po = pofile(file) + po.save() + + def validate_files(dir, files_to_merge): """ Asserts that the given files exist. diff --git a/i18n/make_dummy.py b/i18n/make_dummy.py index c8dcde861a..9c8c3289ce 100755 --- a/i18n/make_dummy.py +++ b/i18n/make_dummy.py @@ -10,15 +10,15 @@ # # $ ./make_dummy.py # -# $ ./make_dummy.py mitx/conf/locale/en/LC_MESSAGES/django.po +# $ ./make_dummy.py ../conf/locale/en/LC_MESSAGES/django.po # # generates output to -# mitx/conf/locale/vr/LC_MESSAGES/django.po +# mitx/conf/locale/fr/LC_MESSAGES/django.po import os, sys import polib from dummy import Dummy -from execute import create_dir_if_necessary +from execute import get_logger, create_dir_if_necessary def main(file, locale): """ @@ -41,11 +41,11 @@ def new_filename(original_filename, new_locale): orig_dir = os.path.dirname(original_filename) msgs_dir = os.path.basename(orig_dir) orig_file = os.path.basename(original_filename) - return os.path.join(orig_dir, - '/../..', - new_locale, - msgs_dir, - orig_file) + return os.path.abspath(os.path.join(orig_dir, + '../..', + new_locale, + msgs_dir, + orig_file)) # Dummy language @@ -60,7 +60,7 @@ DEFAULT_LOCALE = 'fr' if __name__ == '__main__': if len(sys.argv)<2: raise Exception("missing file argument") - if len(sys.argv)<2: + if len(sys.argv)<3: locale = DEFAULT_LOCALE else: locale = sys.argv[2] diff --git a/i18n/tests/__init__.py b/i18n/tests/__init__.py index d60515c712..88216df993 100644 --- a/i18n/tests/__init__.py +++ b/i18n/tests/__init__.py @@ -2,3 +2,5 @@ from test_extract import TestExtract from test_generate import TestGenerate from test_converter import TestConverter from test_dummy import TestDummy +from test_validate import TestValidate + diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py new file mode 100644 index 0000000000..ed746db78f --- /dev/null +++ b/i18n/tests/test_validate.py @@ -0,0 +1,33 @@ +import os +from unittest import TestCase +from nose.plugins.skip import SkipTest + +from execute import call, LOCALE_DIR, LOG + +class TestValidate(TestCase): + """ + Call GNU msgfmt -c on each .po file to validate its format. + """ + + def test_validate(self): + # Skip this test for now because it's very noisy + raise SkipTest() + for file in self.get_po_files(): + # Use relative paths to make output less noisy. + rfile = os.path.relpath(file, LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) + if err != '': + LOG.warn('\n'+err) + + def get_po_files(self, root=LOCALE_DIR): + """ + This is a generator. It yields all of the .po files under root. + """ + for (dirpath, dirnames, filenames) in os.walk(root): + for name in filenames: + (base, ext) = os.path.splitext(name) + if ext.lower() == '.po': + yield os.path.join(dirpath, name) + + + diff --git a/rakefile b/rakefile index 32d92a0349..5914b2f0ae 100644 --- a/rakefile +++ b/rakefile @@ -337,12 +337,6 @@ task :migrate, [:env] do |t, args| sh(django_admin(:lms, args.env, 'migrate')) end -desc "Run tests for the internationalization library" -task :test_i18n do - test = File.join(REPO_ROOT, "i18n", "tests") - sh("nosetests #{test}") -end - Dir["common/lib/*"].select{|lib| File.directory?(lib)}.each do |lib| task_name = "test_#{lib}" @@ -516,27 +510,49 @@ end # --- Internationalization tasks -desc "Extract localizable strings from sources" -task :extract_dev_strings do - sh(File.join(REPO_ROOT, "i18n", "extract.py")) -end +namespace :i18n do -desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." -task :generate_i18n do - if ARGV.last.downcase == 'extract' - Rake::Task["extract_dev_strings"].execute + desc "Extract localizable strings from sources" + task :extract do + sh(File.join(REPO_ROOT, "i18n", "extract.py")) end - sh(File.join(REPO_ROOT, "i18n", "generate.py")) -end -desc "Simulate international translation by generating dummy strings corresponding to source strings." -task :dummy_i18n do - source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] - dummy_locale = 'fr' - cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") - for file in source_files do - sh("#{cmd} #{file} #{dummy_locale}") + desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first." + task :generate do + if ARGV.last.downcase == 'extract' + Rake::Task["i18n:extract"].execute + end + sh(File.join(REPO_ROOT, "i18n", "generate.py")) end + + desc "Simulate international translation by generating dummy strings corresponding to source strings." + task :dummy do + source_files = Dir["#{REPO_ROOT}/conf/locale/en/LC_MESSAGES/*.po"] + dummy_locale = 'fr' + cmd = File.join(REPO_ROOT, "i18n", "make_dummy.py") + for file in source_files do + sh("#{cmd} #{file} #{dummy_locale}") + end + end + + namespace :transifex do + desc "Push source strings to Transifex for translation" + task :push do + sh("tx push -s") + end + + desc "Pull transated strings from Transifex" + task :pull do + sh("tx pull") + end + end + + desc "Run tests for the internationalization library" + task :test do + test = File.join(REPO_ROOT, "i18n", "tests") + sh("nosetests #{test}") + end + end # --- Develop and public documentation ---