Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test_generate and test_extract

Browse Source 
both tests work
This commit is contained in:
Steve Strassmann
2013-04-26 15:01:59 -04:00
parent 87205e23da
commit d9381eccf2
7 changed files with 367 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
conf/locale/config
View File

@@ -1 +1 @@
{"locales" : ["en", "fr", "de"]}
{"locales" : ["en"]}

86
i18n/execute.py Normal file
View File

@@ -0,0 +1,86 @@
import os, subprocess, logging, json
def init_module():
"""
Initializes module parameters
"""
global BASE_DIR, LOCALE_DIR, CONFIG_FILENAME, SOURCE_MSGS_DIR, SOURCE_LOCALE, LOG
# BASE_DIR is the working directory to execute django-admin commands from.
# Typically this should be the 'mitx' directory.
BASE_DIR = os.path.normpath(os.path.dirname(os.path.abspath(__file__))+'/..')
# Source language is English
SOURCE_LOCALE = 'en'
# LOCALE_DIR contains the locale files.
# Typically this should be 'mitx/conf/locale'
LOCALE_DIR = BASE_DIR + '/conf/locale'
# CONFIG_FILENAME contains localization configuration in json format
CONFIG_FILENAME = LOCALE_DIR + '/config'
# SOURCE_MSGS_DIR contains the English po files.
SOURCE_MSGS_DIR = messages_dir(SOURCE_LOCALE)
# Default logger.
LOG = get_logger()
def messages_dir(locale):
"""
Returns the name of the directory holding the po files for locale.
Example: mitx/conf/locale/en/LC_MESSAGES
"""
return os.path.join(LOCALE_DIR, locale, 'LC_MESSAGES')
def get_logger():
"""Returns a default logger"""
log = logging.getLogger(__name__)
log.setLevel(logging.INFO)
log_handler = logging.StreamHandler()
log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s'))
log.addHandler(log_handler)
return log
# Run this after defining messages_dir and get_logger, because it depends on these.
init_module()
def execute (command, working_directory=BASE_DIR, log=LOG):
"""
Executes shell command in a given working_directory.
Command is a string to pass to the shell.
Output is logged to log.
"""
log.info(command)
subprocess.call(command.split(' '), cwd=working_directory)
def get_config():
"""Returns data found in config file, or returns None if file not found"""
config_path = os.path.abspath(CONFIG_FILENAME)
if not os.path.exists(config_path):
log.warn("Configuration file cannot be found: %s" % \
os.path.relpath(config_path, BASE_DIR))
return None
with open(config_path) as stream:
return json.load(stream)
def create_dir_if_necessary(pathname):
dirname = os.path.dirname(pathname)
if not os.path.exists(dirname):
os.makedirs(dirname)
def remove_file(filename, log=LOG, verbose=True):
"""
Attempt to delete filename.
Log a warning if file does not exist.
Logging filenames are releative to BASE_DIR to cut down on noise in output.
"""
if verbose:
log.info('Deleting file %s' % os.path.relpath(filename, BASE_DIR))
if not os.path.exists(filename):
log.warn("File does not exist: %s" % os.path.relpath(filename, BASE_DIR))
else:
os.remove(filename)

144
i18n/extract.py Executable file
View File

@@ -0,0 +1,144 @@
#!/usr/bin/python
"""
See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow
This task extracts all English strings from all source code
and produces three human-readable files:
conf/locale/en/LC_MESSAGES/django-partial.po
conf/locale/en/LC_MESSAGES/djangojs.po
conf/locale/en/LC_MESSAGES/mako.po
This task will clobber any existing django.po file.
This is because django-admin.py makemessages hardcodes this filename
and it cannot be overridden.
"""
import os
from datetime import datetime
from polib import pofile
from execute import execute, create_dir_if_necessary, remove_file, \
BASE_DIR, LOCALE_DIR, SOURCE_MSGS_DIR, LOG
# BABEL_CONFIG contains declarations for Babel to extract strings from mako template files
# Use relpath to reduce noise in logs
BABEL_CONFIG = os.path.relpath(LOCALE_DIR + '/babel.cfg', BASE_DIR)
# Strings from mako template files are written to BABEL_OUT
# Use relpath to reduce noise in logs
BABEL_OUT = os.path.relpath(SOURCE_MSGS_DIR + '/mako.po', BASE_DIR)
def main ():
create_dir_if_necessary(LOCALE_DIR)
generated_files = ('django-partial.po', 'djangojs.po', 'mako.po')
for filename in generated_files:
remove_file(os.path.join(SOURCE_MSGS_DIR, filename))
# Extract strings from mako templates
babel_mako_cmd = 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT)
# Extract strings from django source files
make_django_cmd = 'django-admin.py makemessages -l en --ignore=src/* --ignore=i18n/* ' \
+ '--extension html'
# Extract strings from javascript source files
make_djangojs_cmd = 'django-admin.py makemessages -l en -d djangojs --ignore=src/* ' \
+ '--ignore=i18n/* --extension js'
execute(babel_mako_cmd, working_directory=BASE_DIR)
execute(make_django_cmd, working_directory=BASE_DIR)
# makemessages creates 'django.po'. This filename is hardcoded.
# Rename it to django-partial.po to enable merging into django.po later.
os.rename(os.path.join(SOURCE_MSGS_DIR, 'django.po'),
os.path.join(SOURCE_MSGS_DIR, 'django-partial.po'))
execute(make_djangojs_cmd, working_directory=BASE_DIR)
for filename in generated_files:
LOG.info('Cleaning %s' % filename)
po = pofile(os.path.join(SOURCE_MSGS_DIR, filename))
# replace default headers with edX headers
fix_header(po)
# replace default metadata with edX metadata
fix_metadata(po)
# remove key strings which belong in messages.po
strip_key_strings(po)
po.save()
# By default, django-admin.py makemessages creates this header:
"""
SOME DESCRIPTIVE TITLE.
Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER
This file is distributed under the same license as the PACKAGE package.
FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
"""
def fix_header(po):
"""
Replace default headers with edX headers
"""
header = po.header
fixes = (
('SOME DESCRIPTIVE TITLE', 'edX translation file'),
('Translations template for PROJECT.', 'edX translation file'),
('YEAR', '%s' % datetime.utcnow().year),
('ORGANIZATION', 'edX'),
("THE PACKAGE'S COPYRIGHT HOLDER", "EdX"),
('This file is distributed under the same license as the PROJECT project.',
'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'),
('This file is distributed under the same license as the PACKAGE package.',
'This file is distributed under the GNU AFFERO GENERAL PUBLIC LICENSE.'),
('FIRST AUTHOR <EMAIL@ADDRESS>',
'EdX Team <info@edx.org>')
)
for (src, dest) in fixes:
header = header.replace(src, dest)
po.header = header
# By default, django-admin.py makemessages creates this metadata:
"""
{u'PO-Revision-Date': u'YEAR-MO-DA HO:MI+ZONE',
u'Language': u'',
u'Content-Transfer-Encoding': u'8bit',
u'Project-Id-Version': u'PACKAGE VERSION',
u'Report-Msgid-Bugs-To': u'',
u'Last-Translator': u'FULL NAME <EMAIL@ADDRESS>',
u'Language-Team': u'LANGUAGE <LL@li.org>',
u'POT-Creation-Date': u'2013-04-25 14:14-0400',
u'Content-Type': u'text/plain; charset=UTF-8',
u'MIME-Version': u'1.0'}
"""
def fix_metadata(po):
"""
Replace default metadata with edX metadata
"""
fixes = {'PO-Revision-Date': datetime.utcnow(),
'Report-Msgid-Bugs-To': 'translation_team@edx.org',
'Project-Id-Version': '0.1a',
'Language' : 'en',
'Language-Team': 'translation team <translation_team@edx.org>',
}
del po.metadata['Last-Translator']
po.metadata.update(fixes)
def strip_key_strings(po):
"""
Removes all entries in PO which are key strings.
These entries should appear only in messages.po, not in any other po files.
"""
newlist = [entry for entry in po if not is_key_string(entry.msgid)]
del po[:]
po += newlist
def is_key_string(string):
"""
returns True if string is a key string.
Key strings begin with underscore.
"""
return len(string)>1 and string[0]=='_'
if __name__ == '__main__':
main()

64
i18n/generate.py Executable file
View File

@@ -0,0 +1,64 @@
#!/usr/bin/python
"""
See https://edx-wiki.atlassian.net/wiki/display/ENG/PO+File+workflow
This task merges and compiles the human-readable .pofiles on the
local filesystem into machine-readable .mofiles. This is typically
necessary as part of the build process since these .mofiles are
needed by Django when serving the web app.
The configuration file (in mitx/conf/locale/config) specifies which
languages to generate.
"""
import os
from execute import execute, get_config, messages_dir, remove_file, \
BASE_DIR, LOG, SOURCE_LOCALE
def merge(locale, target='django.po'):
"""
For the given locale, merge django-partial.po, messages.po, mako.po -> django.po
"""
LOG.info('Merging locale=%s' % locale)
locale_directory = messages_dir(locale)
files_to_merge = ('django-partial.po', 'messages.po', 'mako.po')
validate_files(locale_directory, files_to_merge)
# merged file is merged.po
merge_cmd = 'msgcat -o merged.po ' + ' '.join(files_to_merge)
execute(merge_cmd, working_directory=locale_directory)
# rename merged.po -> django.po (default)
merged_filename = os.path.join(locale_directory, 'merged.po')
django_filename = os.path.join(locale_directory, target)
os.rename(merged_filename, django_filename) # can't overwrite file on Windows
def validate_files(dir, files_to_merge):
"""
Asserts that the given files exist.
files_to_merge is a list of file names (no directories).
dir is the directory in which the files should appear.
raises an Exception if any of the files are not in dir.
"""
for path in files_to_merge:
pathname = os.path.join(dir, path)
if not os.path.exists(pathname):
raise Exception("File not found: %s" % pathname)
def main ():
configuration = get_config()
if configuration == None:
LOG.warn('Configuration file not found, using only English.')
locales = (SOURCE_LOCALE,)
else:
locales = configuration['locales']
for locale in locales:
merge(locale)
compile_cmd = 'django-admin.py compilemessages'
execute(compile_cmd, working_directory=BASE_DIR)
if __name__ == '__main__':
main()

1
i18n/test/__init__.py
View File

@@ -1 +1,2 @@
from test_extract import TestExtract
from test_generate import TestGenerate

8
i18n/test/test_extract.py
View File

@@ -10,10 +10,14 @@ class TestExtract(TestCase):
"""
Tests functionality of i18n/extract.py
"""
generated_files = ('django.po', 'djangojs.po', 'mako.po')
generated_files = ('django-partial.po', 'djangojs.po', 'mako.po')
def setUp(self):
self.start_time = datetime.now()
# Subtract 1 second to help comparisons with file-modify time succeed,
# since os.path.getmtime() is not millisecond-accurate
self.start_time = datetime.now() - timedelta(seconds=1)
# Run extraction script. Warning, this takes 1 minute or more
extract.main()
def get_files (self):

65
i18n/test/test_generate.py Normal file
View File

@@ -0,0 +1,65 @@
import os
import string
import polib
import random
from unittest import TestCase
from datetime import datetime, timedelta
import generate
from execute import get_config, messages_dir, SOURCE_MSGS_DIR, SOURCE_LOCALE
class TestGenerate(TestCase):
"""
Tests functionality of i18n/generate.py
"""
generated_files = ('django-partial.po', 'djangojs.po', 'mako.po')
def setUp(self):
self.configuration = get_config()
# Subtract 1 second to help comparisons with file-modify time succeed,
# since os.path.getmtime() is not millisecond-accurate
self.start_time = datetime.now() - timedelta(seconds=1)
def test_configuration(self):
"""
Make sure we have a valid configuration file,
and that it contains an 'en' locale.
"""
self.assertIsNotNone(self.configuration)
locales = self.configuration['locales']
self.assertIsNotNone(locales)
self.assertIsInstance(locales, list)
self.assertIn('en', locales)
def test_merge(self):
"""
Tests merge script on English source files.
"""
filename = os.path.join(SOURCE_MSGS_DIR, random_name())
generate.merge(SOURCE_LOCALE, target=filename)
self.assertTrue(os.path.exists(filename))
os.remove(filename)
def test_main(self):
"""
Runs generate.main() which should merge source files,
then compile all sources in all configured languages.
Validates output by checking all .mo files in all configured languages.
.mo files should exist, and be recently created (modified
after start of test suite)
"""
generate.main()
for locale in self.configuration['locales']:
for filename in ('django.mo', 'djangojs.mo'):
path = os.path.join(messages_dir(locale), filename)
exists = os.path.exists(path)
self.assertTrue(exists, msg='Missing file in locale %s: %s' % (locale, filename))
self.assertTrue(datetime.fromtimestamp(os.path.getmtime(path)) >= self.start_time,
msg='File not recently modified: %s' % path)
def random_name(size=6):
"""Returns random filename as string, like test-4BZ81W"""
chars = string.ascii_uppercase + string.digits
return 'test-' + ''.join(random.choice(chars) for x in range(size))