From 563d71c14f4e51295f21894f03190221e02bfec9 Mon Sep 17 00:00:00 2001 From: David Baumgold Date: Tue, 3 Jun 2014 10:28:47 -0400 Subject: [PATCH 1/5] Extract i18n strings from *.underscore files using django-babel-underscore --- conf/locale/babel_mako.cfg | 6 +++ conf/locale/babel_third_party.cfg | 6 +++ i18n/extract.py | 74 ------------------------------- requirements/edx/base.txt | 1 + 4 files changed, 13 insertions(+), 74 deletions(-) diff --git a/conf/locale/babel_mako.cfg b/conf/locale/babel_mako.cfg index 065372d06d..ad359878aa 100644 --- a/conf/locale/babel_mako.cfg +++ b/conf/locale/babel_mako.cfg @@ -11,3 +11,9 @@ input_encoding = utf-8 input_encoding = utf-8 [mako: lms/templates/emails/**.txt] input_encoding = utf-8 + +[underscore: **.underscore] +input_encoding = utf-8 + +[extractors] +underscore = django_babel_underscore:extract diff --git a/conf/locale/babel_third_party.cfg b/conf/locale/babel_third_party.cfg index 8a6a50af06..b45e87657e 100644 --- a/conf/locale/babel_third_party.cfg +++ b/conf/locale/babel_third_party.cfg @@ -4,3 +4,9 @@ input_encoding = utf-8 [django: **/template/**.html] input_encoding = utf-8 + +[underscore: **.underscore] +input_encoding = utf-8 + +[extractors] +underscore = django_babel_underscore:extract diff --git a/i18n/extract.py b/i18n/extract.py index 3e91fe4fca..1f43df69ca 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -22,7 +22,6 @@ import os.path import logging import sys import argparse -import copy from path import path from polib import pofile @@ -85,9 +84,6 @@ def main(verbosity=1): make_djangojs_cmd = makemessages + ' -d djangojs --extension js' execute(make_djangojs_cmd, working_directory=BASE_DIR, stderr=stderr) - # Extract and megre strings from underscore files - extract_and_merge_underscore() - # makemessages creates 'django.po'. This filename is hardcoded. # Rename it to django-partial.po to enable merging into django.po later. os.rename( @@ -139,76 +135,6 @@ def main(verbosity=1): po.save() -def extract_and_merge_underscore(): - source_msgs_dir = CONFIGURATION.source_messages_dir - - # Extract strings from .underscore file by using grep and sed into - # a temp file 'underscore.po'. It is done by the following steps: - # - # 1. Extract all the patterns of gettext('...') or gettext("...") - # using grep's regexp "gettext\([\'\"][^\(\)]+\)", and grep will - # return each occurence as "::" - # 2. Replace all the single quotes in grep's output into double quotes - # by using two consequent sed's regexps s/\(\'/\(\"/ and s/\'\)/\"\)/ - # 3. Replace the starting './' of each line into '#: ' to make the filename - # looks like occurrence string already in .po files, by using sed's - # regexp s/^\.[/]/#\:\ / - # 4. Replace the first occurence of ':gettext(' (which is always the matched - # string returned by grep) into '\nmsgid ' by using sed's regexp - # s/\:gettext\(/\\nmsgid\ / - # 5. Replace the last occurence of ')' by '\nmsgstr ""\n' by using sed's - # regexp s/\)$/\\nmsgstr\ \"\"\\n/ - # - # For example, if step 1 returns a string like the following line: - # ./cms/templates/js/edit-textbook.underscore:25:gettext("Save") - # Then after steps 2 - 5, it will be converted into the following three lines: - # #: cms/templates/js/edit-textbook.underscore:25 - # msgid "Save" - # msgstr "" - # - extract_underscore_cmd = 'find -name *.underscore -exec {step1_cmd} \\; '\ - '| {step2_cmd_1} | {step2_cmd_2} | {step3_cmd} '\ - '| {step4_cmd} | {step5_cmd} > {output}' - extract_underscore_cmd = extract_underscore_cmd.format( - step1_cmd='grep -HnoE "gettext\\([\\\'\\"][^\\(\\)]+\\)" \'{}\'', - step2_cmd_1='sed s/\\(\\\'/\\(\\"/', - step2_cmd_2='sed s/\\\'\\)/\\"\\)/', - step3_cmd='sed s/^\\.[/]/#\\:\\ /', - step4_cmd='sed s/\\:gettext\\(/\\\\nmsgid\\ /', - step5_cmd='sed s/\\)$/\\\\nmsgstr\\ \\"\\"\\\\n/', - output=source_msgs_dir.joinpath('underscore.po') - ) - execute(extract_underscore_cmd, working_directory=BASE_DIR) - - # Construct a dictionary by using the string as key and occurrence as value - # from underscore.po. This dictionary is used for merging later - underscore_po = pofile(source_msgs_dir.joinpath('underscore.po')) - underscore_po_occurrences = {} - for msg in underscore_po: - if msg.msgid in underscore_po_occurrences: - if msg.occurrences[0] not in underscore_po_occurrences[msg.msgid]: - underscore_po_occurrences[msg.msgid].extend(msg.occurrences) - else: - underscore_po_occurrences[msg.msgid] = msg.occurrences - # The temp file can be safely deleted - os.remove(source_msgs_dir.joinpath('underscore.po')) - - # Merge the messages into djangojs.po - djangojs_po = pofile(source_msgs_dir.joinpath('djangojs.po')) - # Step 1: - # Append new occurrences from .underscore files for the strings already in djangojs.po - for msg in djangojs_po: - msg.occurrences.extend(underscore_po_occurrences.pop(msg.msgid, [])) - # Step 2: - # Append all the remaining strings into djangojs.po - for msgid in underscore_po_occurrences: - msg = copy.deepcopy(djangojs_po[0]) - msg.msgid = msgid - msg.occurrences = underscore_po_occurrences[msgid] - djangojs_po.append(msg) - djangojs_po.save(source_msgs_dir.joinpath('djangojs.po')) - - def fix_header(po): """ Replace default headers with edX headers diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 354b242c0b..b5ec03800c 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -12,6 +12,7 @@ boto==2.13.3 celery==3.0.19 dealer==0.2.3 distribute>=0.6.28, <0.7 +django-babel-underscore==0.1.0 django-celery==3.0.17 django-countries==1.5 django-extensions==1.2.5 From 644d4ac008332fd58c557ed978b99d5519c77efd Mon Sep 17 00:00:00 2001 From: louyihua Date: Wed, 4 Jun 2014 17:46:36 +0800 Subject: [PATCH 2/5] Extract i18n strings from *.underscore files using django-babel-underscore Change strings from mako.po into underscore.po, which is finally merged into djangojs.po. --- conf/locale/babel_mako.cfg | 6 ------ conf/locale/babel_third_party.cfg | 6 ------ conf/locale/babel_underscore.cfg | 6 ++++++ conf/locale/config.yaml | 1 + i18n/extract.py | 20 +++++++++++++++----- 5 files changed, 22 insertions(+), 17 deletions(-) create mode 100644 conf/locale/babel_underscore.cfg diff --git a/conf/locale/babel_mako.cfg b/conf/locale/babel_mako.cfg index ad359878aa..065372d06d 100644 --- a/conf/locale/babel_mako.cfg +++ b/conf/locale/babel_mako.cfg @@ -11,9 +11,3 @@ input_encoding = utf-8 input_encoding = utf-8 [mako: lms/templates/emails/**.txt] input_encoding = utf-8 - -[underscore: **.underscore] -input_encoding = utf-8 - -[extractors] -underscore = django_babel_underscore:extract diff --git a/conf/locale/babel_third_party.cfg b/conf/locale/babel_third_party.cfg index b45e87657e..8a6a50af06 100644 --- a/conf/locale/babel_third_party.cfg +++ b/conf/locale/babel_third_party.cfg @@ -4,9 +4,3 @@ input_encoding = utf-8 [django: **/template/**.html] input_encoding = utf-8 - -[underscore: **.underscore] -input_encoding = utf-8 - -[extractors] -underscore = django_babel_underscore:extract diff --git a/conf/locale/babel_underscore.cfg b/conf/locale/babel_underscore.cfg new file mode 100644 index 0000000000..0769f65f28 --- /dev/null +++ b/conf/locale/babel_underscore.cfg @@ -0,0 +1,6 @@ +# Use this configuration file for extracting strings from .underscore files. +[underscore: **.underscore] +input_encoding = utf-8 + +[extractors] +underscore = django_babel_underscore:extract diff --git a/conf/locale/config.yaml b/conf/locale/config.yaml index 1e32519701..17c07cc705 100644 --- a/conf/locale/config.yaml +++ b/conf/locale/config.yaml @@ -121,3 +121,4 @@ generate_merge: djangojs.po: - djangojs-partial.po - djangojs-studio.po + - underscore.po diff --git a/i18n/extract.py b/i18n/extract.py index 1f43df69ca..502a33ca55 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -58,19 +58,27 @@ def main(verbosity=1): } babel_verbosity = verbosity_map.get(verbosity, "") + if verbosity: + stderr = None + else: + stderr = DEVNULL + babel_mako_cmd = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}' babel_mako_cmd = babel_mako_cmd.format( verbosity=babel_verbosity, config=base(LOCALE_DIR, 'babel_mako.cfg'), output=base(CONFIGURATION.source_messages_dir, 'mako.po'), ) - if verbosity: - stderr = None - else: - stderr = DEVNULL - execute(babel_mako_cmd, working_directory=BASE_DIR, stderr=stderr) + babel_underscore_cmd = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}' + babel_underscore_cmd = babel_underscore_cmd.format( + verbosity=babel_verbosity, + config=base(LOCALE_DIR, 'babel_underscore.cfg'), + output=base(CONFIGURATION.source_messages_dir, 'underscore.po'), + ) + execute(babel_underscore_cmd, working_directory=BASE_DIR, stderr=stderr) + makemessages = "django-admin.py makemessages -l en -v{}".format(verbosity) ignores = " ".join('--ignore="{}/*"'.format(d) for d in CONFIGURATION.ignore_dirs) if ignores: @@ -100,6 +108,8 @@ def main(verbosity=1): files_to_clean = set() + files_to_clean.add(source_msgs_dir / "underscore.po") + # Extract strings from third-party applications. for app_name in CONFIGURATION.third_party: # Import the app to find out where it is. Then use pybabel to extract From ddfd50bb13d1c116663fc0c1bb5ef7e0ae01a3f4 Mon Sep 17 00:00:00 2001 From: David Baumgold Date: Wed, 4 Jun 2014 11:58:03 -0400 Subject: [PATCH 3/5] extract from underscore files under cms, lms, and common to avoid extracting from test_root and such --- conf/locale/babel_underscore.cfg | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conf/locale/babel_underscore.cfg b/conf/locale/babel_underscore.cfg index 0769f65f28..a1c83d6531 100644 --- a/conf/locale/babel_underscore.cfg +++ b/conf/locale/babel_underscore.cfg @@ -1,5 +1,9 @@ # Use this configuration file for extracting strings from .underscore files. -[underscore: **.underscore] +[underscore: cms/templates/**.underscore] +input_encoding = utf-8 +[underscore: lms/templates/**.underscore] +input_encoding = utf-8 +[underscore: common/templates/**.underscore] input_encoding = utf-8 [extractors] From fa0dee17549a4c88c8d8a203a6c46c63135d93fe Mon Sep 17 00:00:00 2001 From: David Baumgold Date: Wed, 4 Jun 2014 13:43:05 -0400 Subject: [PATCH 4/5] Don't search for underscore files under lms --- conf/locale/babel_underscore.cfg | 2 -- 1 file changed, 2 deletions(-) diff --git a/conf/locale/babel_underscore.cfg b/conf/locale/babel_underscore.cfg index a1c83d6531..a726b22e3a 100644 --- a/conf/locale/babel_underscore.cfg +++ b/conf/locale/babel_underscore.cfg @@ -1,8 +1,6 @@ # Use this configuration file for extracting strings from .underscore files. [underscore: cms/templates/**.underscore] input_encoding = utf-8 -[underscore: lms/templates/**.underscore] -input_encoding = utf-8 [underscore: common/templates/**.underscore] input_encoding = utf-8 From 12d8df970b76b4ee4117de9267fa5df90f71e2d5 Mon Sep 17 00:00:00 2001 From: louyihua Date: Thu, 5 Jun 2014 15:22:40 +0800 Subject: [PATCH 5/5] Babel underscore extraction update according to @nedbat 1. Add underscore.po into the merge section in config.yaml 2. Using one babel command template to construct two babel commands --- conf/locale/config.yaml | 4 ++++ i18n/extract.py | 10 ++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/conf/locale/config.yaml b/conf/locale/config.yaml index 17c07cc705..d7614f440a 100644 --- a/conf/locale/config.yaml +++ b/conf/locale/config.yaml @@ -108,6 +108,9 @@ segment: mako.po: mako-studio.po: - cms/* + underscore.po: + underscore-studio.po: + - cms/* # How should the generate step merge files? generate_merge: @@ -122,3 +125,4 @@ generate_merge: - djangojs-partial.po - djangojs-studio.po - underscore.po + - underscore-studio.po diff --git a/i18n/extract.py b/i18n/extract.py index 502a33ca55..0aeeaadff3 100755 --- a/i18n/extract.py +++ b/i18n/extract.py @@ -63,16 +63,16 @@ def main(verbosity=1): else: stderr = DEVNULL - babel_mako_cmd = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}' - babel_mako_cmd = babel_mako_cmd.format( + babel_cmd_template = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}' + + babel_mako_cmd = babel_cmd_template.format( verbosity=babel_verbosity, config=base(LOCALE_DIR, 'babel_mako.cfg'), output=base(CONFIGURATION.source_messages_dir, 'mako.po'), ) execute(babel_mako_cmd, working_directory=BASE_DIR, stderr=stderr) - babel_underscore_cmd = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}' - babel_underscore_cmd = babel_underscore_cmd.format( + babel_underscore_cmd = babel_cmd_template.format( verbosity=babel_verbosity, config=base(LOCALE_DIR, 'babel_underscore.cfg'), output=base(CONFIGURATION.source_messages_dir, 'underscore.po'), @@ -108,8 +108,6 @@ def main(verbosity=1): files_to_clean = set() - files_to_clean.add(source_msgs_dir / "underscore.po") - # Extract strings from third-party applications. for app_name in CONFIGURATION.third_party: # Import the app to find out where it is. Then use pybabel to extract