diff --git a/i18n/config.py b/i18n/config.py index f0d8e366d0..461b0dfd15 100644 --- a/i18n/config.py +++ b/i18n/config.py @@ -1,12 +1,14 @@ import os, json +from path import path # BASE_DIR is the working directory to execute django-admin commands from. # Typically this should be the 'mitx' directory. -BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') +#BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..') +BASE_DIR = path(__file__).abspath().dirname().joinpath('..').normpath() # LOCALE_DIR contains the locale files. # Typically this should be 'mitx/conf/locale' -LOCALE_DIR = os.path.join(BASE_DIR, 'conf', 'locale') +LOCALE_DIR = BASE_DIR.joinpath('conf', 'locale') class Configuration: """ @@ -16,10 +18,10 @@ class Configuration: _source_locale = 'en' def __init__(self, filename): - self.filename = filename - self.config = self.get_config(self.filename) + self._filename = filename + self._config = self.read_config(filename) - def get_config(self, filename): + def read_config(self, filename): """ Returns data found in config file (as dict), or raises exception if file not found """ @@ -28,28 +30,31 @@ class Configuration: with open(filename) as stream: return json.load(stream) - def get_locales(self): + @property + def locales(self): """ Returns a list of locales declared in the configuration file, e.g. ['en', 'fr', 'es'] Each locale is a string. """ - return self.config['locales'] + return self._config['locales'] - def get_source_locale(self): + @property + def source_locale(self): """ Returns source language. Source language is English. """ return self._source_locale - def get_dummy_locale(self): + @property + def dummy_locale(self): """ Returns a locale to use for the dummy text, e.g. 'fr'. Throws exception if no dummy-locale is declared. The locale is a string. """ - dummy = self.config.get('dummy-locale', None) + dummy = self._config.get('dummy-locale', None) if not dummy: raise Exception('Could not read dummy-locale from configuration file.') return dummy @@ -59,15 +64,16 @@ class Configuration: Returns the name of the directory holding the po files for locale. Example: mitx/conf/locale/fr/LC_MESSAGES """ - return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES') + return LOCALE_DIR.joinpath(locale, 'LC_MESSAGES') - def get_source_messages_dir(self): + @property + def source_messages_dir(self): """ Returns the name of the directory holding the source-language po files (English). Example: mitx/conf/locale/en/LC_MESSAGES """ - return self.get_messages_dir(self.get_source_locale()) + return self.get_messages_dir(self.source_locale) -CONFIGURATION = Configuration(os.path.normpath(os.path.join(LOCALE_DIR, 'config'))) +CONFIGURATION = Configuration(LOCALE_DIR.joinpath('config').normpath()) diff --git a/i18n/execute.py b/i18n/execute.py index 4c47680101..6a7e12ac5d 100644 --- a/i18n/execute.py +++ b/i18n/execute.py @@ -14,7 +14,7 @@ def get_default_logger(): LOG = get_default_logger() -def execute (command, working_directory=BASE_DIR, log=LOG): +def execute(command, working_directory=BASE_DIR, log=LOG): """ Executes shell command in a given working_directory. Command is a string to pass to the shell. diff --git a/i18n/extract.py b/i18n/extract.py index ffac9b6270..57da0bd76d 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -23,22 +23,22 @@ from execute import execute, create_dir_if_necessary, remove_file, LOG # BABEL_CONFIG contains declarations for Babel to extract strings from mako template files # Use relpath to reduce noise in logs -BABEL_CONFIG = os.path.relpath(LOCALE_DIR + '/babel.cfg', BASE_DIR) +BABEL_CONFIG = BASE_DIR.relpathto(LOCALE_DIR.joinpath('babel.cfg')) # Strings from mako template files are written to BABEL_OUT # Use relpath to reduce noise in logs -BABEL_OUT = os.path.relpath(CONFIGURATION.get_source_messages_dir() + '/mako.po', BASE_DIR) +BABEL_OUT = BASE_DIR.relpathto(CONFIGURATION.source_messages_dir.joinpath('mako.po')) SOURCE_WARN = 'This English source file is machine-generated. Do not check it into github' def main (): create_dir_if_necessary(LOCALE_DIR) - source_msgs_dir = CONFIGURATION.get_source_messages_dir() + source_msgs_dir = CONFIGURATION.source_messages_dir - remove_file(os.path.join(source_msgs_dir, 'django.po')) + remove_file(source_msgs_dir.joinpath('django.po')) generated_files = ('django-partial.po', 'djangojs.po', 'mako.po') for filename in generated_files: - remove_file(os.path.join(source_msgs_dir, filename)) + remove_file(source_msgs_dir.joinpath(filename)) # Extract strings from mako templates @@ -55,13 +55,13 @@ def main (): execute(make_django_cmd, working_directory=BASE_DIR) # makemessages creates 'django.po'. This filename is hardcoded. # Rename it to django-partial.po to enable merging into django.po later. - os.rename(os.path.join(source_msgs_dir, 'django.po'), - os.path.join(source_msgs_dir, 'django-partial.po')) + os.rename(source_msgs_dir.joinpath('django.po'), + source_msgs_dir.joinpath('django-partial.po')) execute(make_djangojs_cmd, working_directory=BASE_DIR) for filename in generated_files: LOG.info('Cleaning %s' % filename) - po = pofile(os.path.join(source_msgs_dir, filename)) + po = pofile(source_msgs_dir.joinpath(filename)) # replace default headers with edX headers fix_header(po) # replace default metadata with edX metadata diff --git a/i18n/generate.py b/i18n/generate.py index e43efc268a..0c7179b2c6 100755 --- a/i18n/generate.py +++ b/i18n/generate.py @@ -19,25 +19,35 @@ from polib import pofile from config import BASE_DIR, CONFIGURATION from execute import execute, remove_file, LOG -def merge(locale, target='django.po'): +def merge(locale, target='django.po', fail_if_missing=True): """ For the given locale, merge django-partial.po, messages.po, mako.po -> django.po + target is the resulting filename + If fail_if_missing is True, and the files to be merged are missing, + throw an Exception. + If fail_if_missing is False, and the files to be merged are missing, + just return silently. """ LOG.info('Merging locale={0}'.format(locale)) locale_directory = CONFIGURATION.get_messages_dir(locale) files_to_merge = ('django-partial.po', 'messages.po', 'mako.po') - validate_files(locale_directory, files_to_merge) + try: + validate_files(locale_directory, files_to_merge) + except Exception, e: + if not fail_if_missing: + return + raise e # merged file is merged.po merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge) execute(merge_cmd, working_directory=locale_directory) # clean up redunancies in the metadata - merged_filename = os.path.join(locale_directory, 'merged.po') + merged_filename = locale_directory.joinpath('merged.po') clean_metadata(merged_filename) # rename merged.po -> django.po (default) - django_filename = os.path.join(locale_directory, target) + django_filename = locale_directory.joinpath(target) os.rename(merged_filename, django_filename) # can't overwrite file on Windows def clean_metadata(file): @@ -45,25 +55,25 @@ def clean_metadata(file): Clean up redundancies in the metadata caused by merging. This reads in a PO file and simply saves it back out again. """ - po = pofile(file) - po.save() - + pofile(file).save() def validate_files(dir, files_to_merge): """ Asserts that the given files exist. files_to_merge is a list of file names (no directories). - dir is the directory in which the files should appear. + dir is the directory (a path object from path.py) in which the files should appear. raises an Exception if any of the files are not in dir. """ for path in files_to_merge: - pathname = os.path.join(dir, path) - if not os.path.exists(pathname): - raise Exception("File not found: {0}".format(pathname)) + pathname = dir.joinpath(path) + if not pathname.exists(): + raise Exception("I18N: Cannot generate because file not found: {0}".format(pathname)) def main (): - for locale in CONFIGURATION.get_locales(): + for locale in CONFIGURATION.locales: merge(locale) + # Dummy text is not required. Don't raise exception if files are missing. + merge(CONFIGURATION.dummy_locale, fail_if_missing=False) compile_cmd = 'django-admin.py compilemessages' execute(compile_cmd, working_directory=BASE_DIR) diff --git a/i18n/tests/__init__.py b/i18n/tests/__init__.py index d8fce19df7..ee6283376e 100644 --- a/i18n/tests/__init__.py +++ b/i18n/tests/__init__.py @@ -3,5 +3,4 @@ from test_extract import TestExtract from test_generate import TestGenerate from test_converter import TestConverter from test_dummy import TestDummy -from test_validate import TestValidate - +import test_validate diff --git a/i18n/tests/test_config.py b/i18n/tests/test_config.py index aea8f0bca3..bcec6ac354 100644 --- a/i18n/tests/test_config.py +++ b/i18n/tests/test_config.py @@ -11,7 +11,7 @@ class TestConfiguration(TestCase): def test_config(self): config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'config')) config = Configuration(config_filename) - self.assertEqual(config.get_source_locale(), 'en') + self.assertEqual(config.source_locale, 'en') def test_no_config(self): config_filename = os.path.normpath(os.path.join(LOCALE_DIR, 'no_such_file')) @@ -25,9 +25,9 @@ class TestConfiguration(TestCase): Also check values of dummy_locale and source_locale. """ self.assertIsNotNone(CONFIGURATION) - locales = CONFIGURATION.get_locales() + locales = CONFIGURATION.locales self.assertIsNotNone(locales) self.assertIsInstance(locales, list) self.assertIn('en', locales) - self.assertEqual('fr', CONFIGURATION.get_dummy_locale()) - self.assertEqual('en', CONFIGURATION.get_source_locale()) + self.assertEqual('fr', CONFIGURATION.dummy_locale) + self.assertEqual('en', CONFIGURATION.source_locale) diff --git a/i18n/tests/test_extract.py b/i18n/tests/test_extract.py index a9faa2bdd8..7e8b1a9d2b 100644 --- a/i18n/tests/test_extract.py +++ b/i18n/tests/test_extract.py @@ -39,7 +39,7 @@ class TestExtract(TestCase): Fails assertion if one of the files doesn't exist. """ for filename in self.generated_files: - path = os.path.join(CONFIGURATION.get_source_messages_dir(), filename) + path = os.path.join(CONFIGURATION.source_messages_dir, filename) exists = os.path.exists(path) self.assertTrue(exists, msg='Missing file: %s' % filename) if exists: diff --git a/i18n/tests/test_generate.py b/i18n/tests/test_generate.py index bac727f671..468858664f 100644 --- a/i18n/tests/test_generate.py +++ b/i18n/tests/test_generate.py @@ -21,8 +21,8 @@ class TestGenerate(TestCase): """ Tests merge script on English source files. """ - filename = os.path.join(CONFIGURATION.get_source_messages_dir(), random_name()) - generate.merge(CONFIGURATION.get_source_locale(), target=filename) + filename = os.path.join(CONFIGURATION.source_messages_dir, random_name()) + generate.merge(CONFIGURATION.source_locale, target=filename) self.assertTrue(os.path.exists(filename)) os.remove(filename) @@ -35,7 +35,7 @@ class TestGenerate(TestCase): after start of test suite) """ generate.main() - for locale in CONFIGURATION.get_locales(): + for locale in CONFIGURATION.locales: for filename in ('django', 'djangojs'): mofile = filename+'.mo' path = os.path.join(CONFIGURATION.get_messages_dir(locale), mofile) diff --git a/i18n/tests/test_validate.py b/i18n/tests/test_validate.py index 64579fb563..7f0cdd7a25 100644 --- a/i18n/tests/test_validate.py +++ b/i18n/tests/test_validate.py @@ -4,31 +4,28 @@ from nose.plugins.skip import SkipTest from config import LOCALE_DIR from execute import call, LOG + +def test_po_files(): + """ + This is a generator. It yields all of the .po files under root, and tests each one. + """ + for (dirpath, dirnames, filenames) in os.walk(LOCALE_DIR): + for name in filenames: + print name + (base, ext) = os.path.splitext(name) + if ext.lower() == '.po': + yield validate_po_file, os.path.join(dirpath, name) -class TestValidate(TestCase): + +def validate_po_file(filename): """ Call GNU msgfmt -c on each .po file to validate its format. """ - - def test_validate(self): - # Skip this test for now because it's very noisy - raise SkipTest() - for file in self.get_po_files(): - # Use relative paths to make output less noisy. - rfile = os.path.relpath(file, LOCALE_DIR) - (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) - if err != '': - LOG.warn('\n'+err) + # Skip this test for now because it's very noisy + raise SkipTest() + # Use relative paths to make output less noisy. + rfile = os.path.relpath(filename, LOCALE_DIR) + (out, err) = call(['msgfmt','-c', rfile], log=None, working_directory=LOCALE_DIR) + if err != '': + LOG.warn('\n'+err) - def get_po_files(self, root=LOCALE_DIR): - """ - This is a generator. It yields all of the .po files under root. - """ - for (dirpath, dirnames, filenames) in os.walk(root): - for name in filenames: - (base, ext) = os.path.splitext(name) - if ext.lower() == '.po': - yield os.path.join(dirpath, name) - - - diff --git a/i18n/transifex.py b/i18n/transifex.py index 812ecd666f..d08a77b1c0 100755 --- a/i18n/transifex.py +++ b/i18n/transifex.py @@ -13,7 +13,9 @@ def push(): execute('tx push -s') def pull(): - execute('tx pull') + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: + execute('tx pull -l %s' % locale) clean_translated_locales() @@ -22,8 +24,8 @@ def clean_translated_locales(): Strips out the warning from all translated po files about being an English source file. """ - for locale in CONFIGURATION.get_locales(): - if locale != CONFIGURATION.get_source_locale(): + for locale in CONFIGURATION.locales: + if locale != CONFIGURATION.source_locale: clean_locale(locale) def clean_locale(locale): @@ -34,7 +36,7 @@ def clean_locale(locale): """ dirname = CONFIGURATION.get_messages_dir(locale) for filename in ('django-partial.po', 'djangojs.po', 'mako.po'): - clean_file(os.path.join(dirname, filename)) + clean_file(dirname.joinpath(filename)) def clean_file(file): """ diff --git a/requirements.txt b/requirements.txt index d3fdd46b81..c6ee47becb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,6 +33,7 @@ paramiko==1.9.0 path.py==3.0.1 Pillow==1.7.8 pip +polib==1.0.3 pygments==1.5 pygraphviz==1.1 pymongo==2.4.1