Merge pull request #3965 from edx/db/django-babel-underscore

Extract i18n strings from *.underscore files using django-babel-underscore

conf/locale/babel_underscore.cfg

@@ -0,0 +1,8 @@
+# Use this configuration file for extracting strings from .underscore files.
+[underscore: cms/templates/**.underscore]
+input_encoding = utf-8
+[underscore: common/templates/**.underscore]
+input_encoding = utf-8
+
+[extractors]
+underscore = django_babel_underscore:extract

conf/locale/config.yaml

@@ -108,6 +108,9 @@ segment:
     mako.po:
         mako-studio.po:
             - cms/*
+    underscore.po:
+        underscore-studio.po:
+            - cms/*
 
 # How should the generate step merge files?
 generate_merge:
@@ -121,3 +124,5 @@ generate_merge:
     djangojs.po:
         - djangojs-partial.po
         - djangojs-studio.po
+        - underscore.po
+        - underscore-studio.po

i18n/extract.py

@@ -22,7 +22,6 @@ import os.path
 import logging
 import sys
 import argparse
-import copy
 
 from path import path
 from polib import pofile
@@ -59,19 +58,27 @@ def main(verbosity=1):
     }
     babel_verbosity = verbosity_map.get(verbosity, "")
 
-    babel_mako_cmd = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}'
-    babel_mako_cmd = babel_mako_cmd.format(
-        verbosity=babel_verbosity,
-        config=base(LOCALE_DIR, 'babel_mako.cfg'),
-        output=base(CONFIGURATION.source_messages_dir, 'mako.po'),
-    )
     if verbosity:
         stderr = None
     else:
         stderr = DEVNULL
 
+    babel_cmd_template = 'pybabel {verbosity} extract -F {config} -c "Translators:" . -o {output}'
+
+    babel_mako_cmd = babel_cmd_template.format(
+        verbosity=babel_verbosity,
+        config=base(LOCALE_DIR, 'babel_mako.cfg'),
+        output=base(CONFIGURATION.source_messages_dir, 'mako.po'),
+    )
     execute(babel_mako_cmd, working_directory=BASE_DIR, stderr=stderr)
 
+    babel_underscore_cmd = babel_cmd_template.format(
+        verbosity=babel_verbosity,
+        config=base(LOCALE_DIR, 'babel_underscore.cfg'),
+        output=base(CONFIGURATION.source_messages_dir, 'underscore.po'),
+    )
+    execute(babel_underscore_cmd, working_directory=BASE_DIR, stderr=stderr)
+
     makemessages = "django-admin.py makemessages -l en -v{}".format(verbosity)
     ignores = " ".join('--ignore="{}/*"'.format(d) for d in CONFIGURATION.ignore_dirs)
     if ignores:
@@ -85,9 +92,6 @@ def main(verbosity=1):
     make_djangojs_cmd = makemessages + ' -d djangojs --extension js'
     execute(make_djangojs_cmd, working_directory=BASE_DIR, stderr=stderr)
 
-    # Extract and megre strings from underscore files
-    extract_and_merge_underscore()
-
     # makemessages creates 'django.po'. This filename is hardcoded.
     # Rename it to django-partial.po to enable merging into django.po later.
     os.rename(
@@ -139,76 +143,6 @@ def main(verbosity=1):
         po.save()
 
 
-def extract_and_merge_underscore():
-    source_msgs_dir = CONFIGURATION.source_messages_dir
-
-    # Extract strings from .underscore file by using grep and sed into
-    # a temp file 'underscore.po'. It is done by the following steps:
-    #
-    # 1. Extract all the patterns of gettext('...') or gettext("...")
-    #    using grep's regexp "gettext\([\'\"][^\(\)]+\)", and grep will
-    #    return each occurence as "<filename>:<line number>:<string>"
-    # 2. Replace all the single quotes in grep's output into double quotes
-    #    by using two consequent sed's regexps s/\(\'/\(\"/ and s/\'\)/\"\)/
-    # 3. Replace the starting './' of each line into '#: ' to make the filename
-    #    looks like occurrence string already in .po files, by using sed's
-    #    regexp s/^\.[/]/#\:\ /
-    # 4. Replace the first occurence of ':gettext(' (which is always the matched
-    #    string returned by grep) into '\nmsgid ' by using sed's regexp
-    #    s/\:gettext\(/\\nmsgid\ /
-    # 5. Replace the last occurence of ')' by '\nmsgstr ""\n' by using sed's
-    #    regexp s/\)$/\\nmsgstr\ \"\"\\n/
-    #
-    # For example, if step 1 returns a string like the following line:
-    # ./cms/templates/js/edit-textbook.underscore:25:gettext("Save")
-    # Then after steps 2 - 5, it will be converted into the following three lines:
-    # #: cms/templates/js/edit-textbook.underscore:25
-    # msgid "Save"
-    # msgstr ""
-    #
-    extract_underscore_cmd = 'find -name *.underscore -exec {step1_cmd} \\; '\
-                             '| {step2_cmd_1} | {step2_cmd_2} | {step3_cmd} '\
-                             '| {step4_cmd} | {step5_cmd} > {output}'
-    extract_underscore_cmd = extract_underscore_cmd.format(
-        step1_cmd='grep -HnoE "gettext\\([\\\'\\"][^\\(\\)]+\\)" \'{}\'',
-        step2_cmd_1='sed s/\\(\\\'/\\(\\"/',
-        step2_cmd_2='sed s/\\\'\\)/\\"\\)/',
-        step3_cmd='sed s/^\\.[/]/#\\:\\ /',
-        step4_cmd='sed s/\\:gettext\\(/\\\\nmsgid\\ /',
-        step5_cmd='sed s/\\)$/\\\\nmsgstr\\ \\"\\"\\\\n/',
-        output=source_msgs_dir.joinpath('underscore.po')
-    )
-    execute(extract_underscore_cmd, working_directory=BASE_DIR)
-
-    # Construct a dictionary by using the string as key and occurrence as value
-    # from underscore.po. This dictionary is used for merging later
-    underscore_po = pofile(source_msgs_dir.joinpath('underscore.po'))
-    underscore_po_occurrences = {}
-    for msg in underscore_po:
-        if msg.msgid in underscore_po_occurrences:
-            if msg.occurrences[0] not in underscore_po_occurrences[msg.msgid]:
-                underscore_po_occurrences[msg.msgid].extend(msg.occurrences)
-        else:
-            underscore_po_occurrences[msg.msgid] = msg.occurrences
-    # The temp file can be safely deleted
-    os.remove(source_msgs_dir.joinpath('underscore.po'))
-
-    # Merge the messages into djangojs.po
-    djangojs_po = pofile(source_msgs_dir.joinpath('djangojs.po'))
-    # Step 1:
-    # Append new occurrences from .underscore files for the strings already in djangojs.po 
-    for msg in djangojs_po:
-        msg.occurrences.extend(underscore_po_occurrences.pop(msg.msgid, []))
-    # Step 2:
-    # Append all the remaining strings into djangojs.po
-    for msgid in underscore_po_occurrences:
-        msg = copy.deepcopy(djangojs_po[0])
-        msg.msgid = msgid
-        msg.occurrences = underscore_po_occurrences[msgid]
-        djangojs_po.append(msg)
-    djangojs_po.save(source_msgs_dir.joinpath('djangojs.po'))
-
-
 def fix_header(po):
     """
     Replace default headers with edX headers

requirements/edx/base.txt

@@ -12,6 +12,7 @@ boto==2.13.3
 celery==3.0.19
 dealer==0.2.3
 distribute>=0.6.28, <0.7
+django-babel-underscore==0.1.0
 django-celery==3.0.17
 django-countries==1.5
 django-extensions==1.2.5