diff --git a/.gitignore b/.gitignore index 8fb170c30f..e7b0b16be8 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,13 @@ cover_html/ chromedriver.log /nbproject ghostdriver.log +/cms/doc/en/getting_started/ +/conf/locale/en +/conf/locale/fr +create-dev-env.hack.sh +distribute-0.6.36.tar.gz +i18n/googleTranslate.hack.py +i18n/mitx/conf/locale/fr/LC_MESSAGES/django.po +i18n/split.py +.gitignore + diff --git a/cms/djangoapps/contentstore/tests/test_i18n.py b/cms/djangoapps/contentstore/tests/test_i18n.py new file mode 100644 index 0000000000..e6d68ba004 --- /dev/null +++ b/cms/djangoapps/contentstore/tests/test_i18n.py @@ -0,0 +1,97 @@ +from unittest import skip + +from django.core.urlresolvers import reverse +from django.contrib.auth.models import User +from django.test.client import Client + +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase + +class InternationalizationTest(ModuleStoreTestCase): + """ + Tests to validate Internationalization. + """ + + def setUp(self): + """ + These tests need a user in the DB so that the django Test Client + can log them in. + They inherit from the ModuleStoreTestCase class so that the mongodb collection + will be cleared out before each test case execution and deleted + afterwards. + """ + self.uname = 'testuser' + self.email = 'test+courses@edx.org' + self.password = 'foo' + + # Create the use so we can log them in. + self.user = User.objects.create_user(self.uname, self.email, self.password) + + # Note that we do not actually need to do anything + # for registration if we directly mark them active. + self.user.is_active = True + # Staff has access to view all courses + self.user.is_staff = True + self.user.save() + + self.course_data = { + 'template': 'i4x://edx/templates/course/Empty', + 'org': 'MITx', + 'number': '999', + 'display_name': 'Robot Super Course', + } + + def test_course_plain_english(self): + """Test viewing the index page with no courses""" + self.client = Client() + self.client.login(username=self.uname, password=self.password) + + resp = self.client.get(reverse('index')) + self.assertContains(resp, + '

My Courses

', + status_code=200, + html=True) + + def test_course_explicit_english(self): + """Test viewing the index page with no courses""" + self.client = Client() + self.client.login(username=self.uname, password=self.password) + + resp = self.client.get(reverse('index'), + {}, + HTTP_ACCEPT_LANGUAGE='en' + ) + + self.assertContains(resp, + '

My Courses

', + status_code=200, + html=True) + + + # **** + # NOTE: + # **** + # + # This test will break when we replace this fake 'test' language + # with actual French. This test will need to be updated with + # actual French at that time. + + # Test temporarily disable since it depends on creation of dummy strings + @skip + def test_course_with_accents (self): + """Test viewing the index page with no courses""" + self.client = Client() + self.client.login(username=self.uname, password=self.password) + + resp = self.client.get(reverse('index'), + {}, + HTTP_ACCEPT_LANGUAGE='fr' + ) + + TEST_STRING = u'

' \ + + u'My \xc7\xf6\xfcrs\xe9s L#' \ + + u'

' + + self.assertContains(resp, + TEST_STRING, + status_code=200, + html=True) diff --git a/cms/envs/common.py b/cms/envs/common.py index ca08a56a70..3adab3d384 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -129,6 +129,9 @@ MIDDLEWARE_CLASSES = ( 'track.middleware.TrackMiddleware', 'mitxmako.middleware.MakoMiddleware', + # Detects user-requested locale from 'accept-language' header in http request + 'django.middleware.locale.LocaleMiddleware', + 'django.middleware.transaction.TransactionMiddleware' ) @@ -173,9 +176,13 @@ STATICFILES_DIRS = [ # Locale/Internationalization TIME_ZONE = 'America/New_York' # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name LANGUAGE_CODE = 'en' # http://www.i18nguy.com/unicode/language-identifiers.html + USE_I18N = True USE_L10N = True +# Localization strings (e.g. django.po) are under this directory +LOCALE_PATHS = (REPO_ROOT + '/conf/locale',) # mitx/conf/locale/ + # Tracking TRACK_MAX_EVENT = 10000 diff --git a/cms/static/js/base.js b/cms/static/js/base.js index 51d358d0eb..3a51d797ec 100644 --- a/cms/static/js/base.js +++ b/cms/static/js/base.js @@ -245,7 +245,7 @@ function showImportSubmit(e) { $('.submit-button').show(); $('.progress').show(); } else { - $('.error-block').html('File format not supported. Please upload a file with a tar.gz extension.').show(); + $('.error-block').html(gettext('File format not supported. Please upload a file with a tar.gz extension.')).show(); } } @@ -406,7 +406,7 @@ function showFileSelectionMenu(e) { } function startUpload(e) { - $('.upload-modal h1').html('Uploading…'); + $('.upload-modal h1').html(gettext('Uploading…')); $('.upload-modal .file-name').html($('.file-input').val().replace('C:\\fakepath\\', '')); $('.upload-modal .file-chooser').ajaxSubmit({ beforeSend: resetUploadBar, @@ -439,7 +439,7 @@ function displayFinishedUpload(xhr) { $('.upload-modal .embeddable').show(); $('.upload-modal .file-name').hide(); $('.upload-modal .progress-fill').html(resp.msg); - $('.upload-modal .choose-file-button').html('Load Another File').show(); + $('.upload-modal .choose-file-button').html(gettext('Load Another File')).show(); $('.upload-modal .progress-fill').width('100%'); // see if this id already exists, if so, then user must have updated an existing piece of content @@ -500,11 +500,11 @@ function toggleSock(e) { }); if($sock.hasClass('is-shown')) { - $btnLabel.text('Hide Studio Help'); + $btnLabel.text(gettext('Hide Studio Help')); } else { - $btnLabel.text('Looking for Help with Studio?'); + $btnLabel.text(gettext('Looking for Help with Studio?')); } } @@ -845,7 +845,15 @@ function saveSetSectionScheduleDate(e) { data: JSON.stringify({ 'id': id, 'metadata': {'start': start}}) }).success(function () { var $thisSection = $('.courseware-section[data-id="' + id + '"]'); - $thisSection.find('.section-published-date').html('Will Release: ' + input_date + ' at ' + input_time + ' UTCEdit'); + var format = gettext('Will Release: %(date)s at $(time)s UTC'); + var willReleaseAt = interpolate(format, [input_date, input_time], true); + $thisSection.find('.section-published-date').html( + '' + willReleaseAt + '' + + '' + + gettext('Edit') + ''); $thisSection.find('.section-published-date').animate({ 'background-color': 'rgb(182,37,104)' }, 300).animate({ diff --git a/cms/templates/base.html b/cms/templates/base.html index 4517790622..65e08b3cc5 100644 --- a/cms/templates/base.html +++ b/cms/templates/base.html @@ -30,6 +30,7 @@ <%include file="courseware_vendor_js.html"/> + diff --git a/cms/templates/index.html b/cms/templates/index.html index 916720f4e7..0f6e982b1d 100644 --- a/cms/templates/index.html +++ b/cms/templates/index.html @@ -1,6 +1,8 @@ +<%! from django.utils.translation import ugettext as _ %> + <%inherit file="base.html" /> -<%block name="title">My Courses +<%block name="title">${_("My Courses")} <%block name="bodyclass">is-signedin index dashboard <%block name="header_extras"> @@ -36,18 +38,18 @@
-

My Courses

+

${_("My Courses")}

% if user.is_active:
- \ No newline at end of file + diff --git a/cms/templates/widgets/footer.html b/cms/templates/widgets/footer.html index 7162dad50f..db7d5fb3f8 100644 --- a/cms/templates/widgets/footer.html +++ b/cms/templates/widgets/footer.html @@ -1,8 +1,10 @@ <%! from django.core.urlresolvers import reverse %> +<%! from django.utils.translation import ugettext as _ %> +
-
\ No newline at end of file + + diff --git a/cms/urls.py b/cms/urls.py index 06569e4178..30d9ccbf56 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -120,6 +120,17 @@ urlpatterns += ( url(r'^ux-alerts$', 'contentstore.views.ux_alerts', name='ux-alerts') ) +js_info_dict = { + 'domain': 'djangojs', + 'packages': ('cms',), + } + +urlpatterns += ( + # Serve catalog of localized strings to be rendered by Javascript + url(r'^jsi18n/$', 'django.views.i18n.javascript_catalog', js_info_dict), + ) + + if settings.ENABLE_JASMINE: # # Jasmine urlpatterns = urlpatterns + (url(r'^_jasmine/', include('django_jasmine.urls')),) diff --git a/conf/locale/babel.cfg b/conf/locale/babel.cfg new file mode 100644 index 0000000000..5b8333cf1e --- /dev/null +++ b/conf/locale/babel.cfg @@ -0,0 +1,19 @@ +# Extraction from Python source files +#[python: cms/**.py] +#[python: lms/**.py] +#[python: common/**.py] + +# Extraction from Javscript source files +#[javascript: cms/**.js] +#[javascript: lms/**.js] +#[javascript: common/static/js/capa/**.js] +#[javascript: common/static/js/course_groups/**.js] +# do not extract from common/static/js/vendor/** + +# Extraction from Mako templates +[mako: cms/templates/**.html] +input_encoding = utf-8 +[mako: lms/templates/**.html] +input_encoding = utf-8 +[mako: common/templates/**.html] +input_encoding = utf-8 diff --git a/conf/locale/config b/conf/locale/config new file mode 100644 index 0000000000..fe811ee02e --- /dev/null +++ b/conf/locale/config @@ -0,0 +1 @@ +{"locales" : ["en", "fr", "de"]} diff --git a/i18n/converter.py b/i18n/converter.py new file mode 100644 index 0000000000..63d8f83e00 --- /dev/null +++ b/i18n/converter.py @@ -0,0 +1,65 @@ +import re +import itertools + +class Converter: + """Converter is an abstract class that transforms strings. + It hides embedded tags (HTML or Python sequences) from transformation + + To implement Converter, provide implementation for inner_convert_string() + + Strategy: + 1. extract tags embedded in the string + a. use the index of each extracted tag to re-insert it later + b. replace tags in string with numbers (<0>, <1>, etc.) + c. save extracted tags in a separate list + 2. convert string + 3. re-insert the extracted tags + + """ + + # matches tags like these: + # HTML: , ,
, + # Python: %(date)s, %(name)s + tag_pattern = re.compile(r'(<[-\w" .:?=/]*>)|({[^}]*})|(%\([^)]*\)\w)', re.I) + + def convert(self, string): + """Returns: a converted tagged string + param: string (contains html tags) + + Don't replace characters inside tags + """ + (string, tags) = self.detag_string(string) + string = self.inner_convert_string(string) + string = self.retag_string(string, tags) + return string + + def detag_string(self, string): + """Extracts tags from string. + + returns (string, list) where + string: string has tags replaced by indices (
... => <0>, <1>, <2>, etc.) + list: list of the removed tags ('
', '', '') + """ + counter = itertools.count(0) + count = lambda m: '<%s>' % counter.next() + tags = self.tag_pattern.findall(string) + tags = [''.join(tag) for tag in tags] + (new, nfound) = self.tag_pattern.subn(count, string) + if len(tags) != nfound: + raise Exception('tags dont match:'+string) + return (new, tags) + + def retag_string(self, string, tags): + """substitutes each tag back into string, into occurrences of <0>, <1> etc""" + for (i, tag) in enumerate(tags): + p = '<%s>' % i + string = re.sub(p, tag, string, 1) + return string + + + # ------------------------------ + # Customize this in subclasses of Converter + + def inner_convert_string(self, string): + return string # do nothing by default + diff --git a/i18n/dummy.py b/i18n/dummy.py new file mode 100644 index 0000000000..798ee525b5 --- /dev/null +++ b/i18n/dummy.py @@ -0,0 +1,132 @@ +from converter import Converter + +# Creates new localization properties files in a dummy language +# Each property file is derived from the equivalent en_US file, except +# 1. Every vowel is replaced with an equivalent with extra accent marks +# 2. Every string is padded out to +30% length to simulate verbose languages (e.g. German) +# to see if layout and flows work properly +# 3. Every string is terminated with a '#' character to make it easier to detect truncation + + +# -------------------------------- +# Example use: +# >>> from dummy import Dummy +# >>> c = Dummy() +# >>> c.convert("hello my name is Bond, James Bond") +# u'h\xe9ll\xf6 my n\xe4m\xe9 \xefs B\xf6nd, J\xe4m\xe9s B\xf6nd Lorem i#' +# +# >>> c.convert('don\'t convert tag ids') +# u'd\xf6n\'t \xe7\xf6nv\xe9rt t\xe4g \xefds Lorem ipsu#' +# +# >>> c.convert('don\'t convert %(name)s tags on %(date)s') +# u"d\xf6n't \xe7\xf6nv\xe9rt %(name)s t\xe4gs \xf6n %(date)s Lorem ips#" + + +# Substitute plain characters with accented lookalikes. +# http://tlt.its.psu.edu/suggestions/international/web/codehtml.html#accent +TABLE = {'A': u'\xC0', + 'a': u'\xE4', + 'b': u'\xDF', + 'C': u'\xc7', + 'c': u'\xE7', + 'E': u'\xC9', + 'e': u'\xE9', + 'I': U'\xCC', + 'i': u'\xEF', + 'O': u'\xD8', + 'o': u'\xF6', + 'u': u'\xFC' + } + + + +# The print industry's standard dummy text, in use since the 1500s +# see http://www.lipsum.com/ +LOREM = ' Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed ' \ + 'do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad ' \ + 'minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ' \ + 'ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate ' \ + 'velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat ' \ + 'cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. ' + +# To simulate more verbose languages (like German), pad the length of a string +# by a multiple of PAD_FACTOR +PAD_FACTOR = 1.3 + + +class Dummy (Converter): + """ + A string converter that generates dummy strings with fake accents + and lorem ipsum padding. + """ + + def convert(self, string): + result = Converter.convert(self, string) + return self.pad(result) + + def inner_convert_string(self, string): + for (k,v) in TABLE.items(): + string = string.replace(k, v) + return string + + + def pad(self, string): + """add some lorem ipsum text to the end of string""" + size = len(string) + if size < 7: + target = size*3 + else: + target = int(size*PAD_FACTOR) + return string + self.terminate(LOREM[:(target-size)]) + + def terminate(self, string): + """replaces the final char of string with #""" + return string[:-1]+'#' + + def init_msgs(self, msgs): + """ + Make sure the first msg in msgs has a plural property. + msgs is list of instances of pofile.Msg + """ + if len(msgs)==0: + return + headers = msgs[0].get_property('msgstr') + has_plural = len([header for header in headers if header.find('Plural-Forms:') == 0])>0 + if not has_plural: + # Apply declaration for English pluralization rules + plural = "Plural-Forms: nplurals=2; plural=(n != 1);\\n" + headers.append(plural) + + + def convert_msg(self, msg): + """ + Takes one Msg object and converts it (adds a dummy translation to it) + msg is an instance of pofile.Msg + """ + source = msg.msgid + if len(source)==0: + # don't translate empty string + return + + plural = msg.msgid_plural + if len(plural)>0: + # translate singular and plural + foreign_single = self.convert(source) + foreign_plural = self.convert(plural) + plural = {'0': self.final_newline(source, foreign_single), + '1': self.final_newline(plural, foreign_plural)} + msg.msgstr_plural = plural + return + else: + foreign = self.convert(source) + msg.msgstr = self.final_newline(source, foreign) + + def final_newline(self, original, translated): + """ Returns a new translated string. + If last char of original is a newline, make sure translation + has a newline too. + """ + if len(original)>1: + if original[-1]=='\n' and translated[-1]!='\n': + return translated + '\n' + return translated diff --git a/i18n/make_dummy.py b/i18n/make_dummy.py new file mode 100755 index 0000000000..4ccfb0d5f1 --- /dev/null +++ b/i18n/make_dummy.py @@ -0,0 +1,68 @@ +#!/usr/bin/python + +# Generate test translation files from human-readable po files. +# +# +# po files can be generated with this: +# django-admin.py makemessages --all --extension html -l en + +# Usage: +# +# $ ./make_dummy.py +# +# $ ./make_dummy.py mitx/conf/locale/en/LC_MESSAGES/django.po +# +# generates output to +# mitx/conf/locale/vr/LC_MESSAGES/django.po + +import os, sys +import polib +from dummy import Dummy + +# Dummy language +# two letter language codes reference: +# see http://www.loc.gov/standards/iso639-2/php/code_list.php +# +# Django will not localize in languages that django itself has not been +# localized for. So we are using a well-known language: 'fr'. + +OUT_LANG = 'fr' + +def main(file): + """ + Takes a source po file, reads it, and writes out a new po file + containing a dummy translation. + """ + if not os.path.exists(file): + raise IOError('File does not exist: %s' % file) + pofile = polib.pofile(file) + converter = Dummy() + converter.init_msgs(pofile.translated_entries()) + for msg in pofile: + converter.convert_msg(msg) + new_file = new_filename(file, OUT_LANG) + create_dir_if_necessary(new_file) + pofile.save(new_file) + + + +def new_filename(original_filename, new_lang): + """Returns a filename derived from original_filename, using new_lang as the locale""" + orig_dir = os.path.dirname(original_filename) + msgs_dir = os.path.basename(orig_dir) + orig_file = os.path.basename(original_filename) + return '%s/%s/%s/%s' % (os.path.abspath(orig_dir + '/../..'), + new_lang, + msgs_dir, + orig_file) + + +def create_dir_if_necessary(pathname): + dirname = os.path.dirname(pathname) + if not os.path.exists(dirname): + os.makedirs(dirname) + +if __name__ == '__main__': + if len(sys.argv)<2: + raise Exception("missing file argument") + main(sys.argv[1]) diff --git a/i18n/update.py b/i18n/update.py new file mode 100755 index 0000000000..447dcf71d5 --- /dev/null +++ b/i18n/update.py @@ -0,0 +1,110 @@ +#!/usr/bin/python + +import os, subprocess, logging, json +from make_dummy import create_dir_if_necessary, main as dummy_main + +''' +Generate or update all translation files + Usage: + $ update.py + + + 1. extracts files from mako templates + 2. extracts files from django templates and python source files + 3. extracts files from django javascript files + 4. generates dummy text translations + 5. compiles po files to mo files + + Configuration (e.g. known languages) declared in mitx/conf/locale/config +''' + +# ----------------------------------- +# BASE_DIR is the working directory to execute django-admin commands from. +# Typically this should be the 'mitx' directory. +BASE_DIR = os.path.abspath(os.path.dirname(os.path.abspath(__file__))+'/..') + +# LOCALE_DIR contains the locale files. +# Typically this should be 'mitx/conf/locale' +LOCALE_DIR = BASE_DIR + '/conf/locale' + +# MSGS_DIR contains the English po files +MSGS_DIR = LOCALE_DIR + '/en/LC_MESSAGES' + +# CONFIG_FILENAME contains localization configuration in json format +CONFIG_FILENAME = LOCALE_DIR + '/config' + +# BABEL_CONFIG contains declarations for Babel to extract strings from mako template files +BABEL_CONFIG = LOCALE_DIR + '/babel.cfg' + +# Strings from mako template files are written to BABEL_OUT +BABEL_OUT = MSGS_DIR + '/mako.po' + +# These are the shell commands invoked by main() +COMMANDS = { + 'babel_mako': 'pybabel extract -F %s -c "TRANSLATORS:" . -o %s' % (BABEL_CONFIG, BABEL_OUT), + 'make_django': 'django-admin.py makemessages --all --ignore=src/* --extension html -l en', + 'make_djangojs': 'django-admin.py makemessages --all -d djangojs --ignore=src/* --extension js -l en', + 'msgcat' : 'msgcat -o merged.po django.po %s' % BABEL_OUT, + 'rename_django' : 'mv django.po django_old.po', + 'rename_merged' : 'mv merged.po django.po', + 'compile': 'django-admin.py compilemessages' + + } + +def execute (command_kwd, log, working_directory=BASE_DIR): + ''' + Executes command_kwd, which references a shell command in COMMANDS. + ''' + full_cmd = COMMANDS[command_kwd] + log.info('%s' % full_cmd) + subprocess.call(full_cmd.split(' '), cwd=working_directory) + +def make_log (): + '''returns a logger''' + log = logging.getLogger(__name__) + log.setLevel(logging.INFO) + log_handler = logging.StreamHandler() + log_handler.setFormatter(logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')) + log.addHandler(log_handler) + return log + +def get_config (): + '''Returns data found in config file, or returns None if file not found''' + config_path = os.path.abspath(CONFIG_FILENAME) + if not os.path.exists(config_path): + return None + with open(config_path) as stream: + return json.load(stream) + +def main (): + log = make_log() + create_dir_if_necessary(LOCALE_DIR) + log.info('Executing all commands from %s' % BASE_DIR) + + remove_files = ['django.po', 'djangojs.po', 'nonesuch'] + for filename in remove_files: + path = MSGS_DIR + '/' + filename + log.info('Deleting file %s' % path) + if not os.path.exists(path): + log.warn("File does not exist: %s" % path) + else: + os.remove(path) + + # Generate or update human-readable .po files from all source code. + execute('babel_mako', log=log) + execute('make_django', log=log) + execute('make_djangojs', log=log) + execute('msgcat', log=log, working_directory=MSGS_DIR) + execute('rename_django', log=log, working_directory=MSGS_DIR) + execute('rename_merged', log=log, working_directory=MSGS_DIR) + + # Generate dummy text files from the English .po files + log.info('Generating dummy text.') + dummy_main(LOCALE_DIR + '/en/LC_MESSAGES/django.po') + dummy_main(LOCALE_DIR + '/en/LC_MESSAGES/djangojs.po') + + # Generate machine-readable .mo files + execute('compile', log) + +if __name__ == '__main__': + main() diff --git a/lms/templates/discussion/_single_thread.html b/lms/templates/discussion/_single_thread.html index 0dec32ad47..e291bc955c 100644 --- a/lms/templates/discussion/_single_thread.html +++ b/lms/templates/discussion/_single_thread.html @@ -6,7 +6,7 @@
- %if thread['group_id'] + %if thread['group_id']:
This post visible only to group ${cohort_dictionary[thread['group_id']]}.
%endif @@ -35,4 +35,4 @@ -<%include file="_js_data.html" /> \ No newline at end of file +<%include file="_js_data.html" /> diff --git a/requirements.txt b/requirements.txt index 08aaecc71e..a25c2f6adf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -r repo-requirements.txt +Babel==0.9.6 beautifulsoup4==4.1.3 beautifulsoup==3.2.1 boto==2.6.0