diff --git a/i18n/execute.py b/i18n/execute.py
index 28222177c4..43bdec2deb 100644
--- a/i18n/execute.py
+++ b/i18n/execute.py
@@ -11,7 +11,7 @@ def execute(command, working_directory=BASE_DIR):
     Output is ignored.
     """
     LOG.info(command)
-    subprocess.check_output(command.split(' '), cwd=working_directory, stderr=subprocess.STDOUT)
+    subprocess.check_call(command, cwd=working_directory, stderr=sys.STDOUT, shell=True)
 
 
 def call(command, working_directory=BASE_DIR):
diff --git a/i18n/tests/test_compiled_messages.py b/i18n/tests/test_compiled_messages.py
new file mode 100644
index 0000000000..5b5bda906e
--- /dev/null
+++ b/i18n/tests/test_compiled_messages.py
@@ -0,0 +1,59 @@
+"""
+Test that the compiled .mo files match the translations in the
+uncompiled .po files.
+
+This is required because we are checking in the .mo files into
+the repo, but compiling them is a manual process. We want to make
+sure that we find out if someone forgets the compilation step.
+"""
+
+import ddt
+import polib
+from unittest import TestCase
+
+from i18n.config import CONFIGURATION, LOCALE_DIR
+
+
+@ddt.ddt
+class TestCompiledMessages(TestCase):
+    """
+    Test that mo files match their source po files
+    """
+
+    PO_FILES = ['django.po', 'djangojs.po']
+
+    @ddt.data(*CONFIGURATION.locales)
+    def test_translated_messages(self, locale):
+        message_dir = LOCALE_DIR / locale / 'LC_MESSAGES'
+        for pofile_name in self.PO_FILES:
+            pofile_path = message_dir / pofile_name
+            pofile = polib.pofile(pofile_path)
+            mofile = polib.mofile(pofile_path.stripext() + '.mo')
+
+            po_entries = {entry.msgid: entry for entry in pofile.translated_entries()}
+            mo_entries = {entry.msgid: entry for entry in mofile.translated_entries()}
+
+            # Check that there are no entries in po that aren't in mo, and vice-versa
+            self.assertEquals(po_entries.viewkeys(), mo_entries.viewkeys())
+
+            for entry_id, po_entry in po_entries.iteritems():
+                mo_entry = mo_entries[entry_id]
+                for attr in ('msgstr', 'msgid_plural', 'msgstr_plural', 'msgctxt', 'obsolete', 'encoding'):
+                    po_attr = getattr(po_entry, attr)
+                    mo_attr = getattr(mo_entry, attr)
+
+                    # The msgstr_plural in the mo_file is keyed on ints, but in the po_file it's
+                    # keyed on strings. This normalizes them.
+                    if attr == 'msgstr_plural':
+                        po_attr = {int(key): val for (key, val) in po_attr.items()}
+
+                    self.assertEquals(
+                        po_attr,
+                        mo_attr,
+                        "When comparing {} for entry {!r}, {!r} from the .po file doesn't match {!r} from the .mo file".format(
+                            attr,
+                            entry_id,
+                            po_attr,
+                            mo_attr,
+                        )
+                    )
diff --git a/rakelib/i18n.rake b/rakelib/i18n.rake
index b6c65dba42..44d00ad0ab 100644
--- a/rakelib/i18n.rake
+++ b/rakelib/i18n.rake
@@ -7,11 +7,8 @@ namespace :i18n do
     sh(File.join(REPO_ROOT, "i18n", "extract.py"))
   end
 
-  desc "Compile localizable strings from sources. With optional flag 'extract', will extract strings first."
-  task :generate => "i18n:validate:gettext" do
-    if ARGV.last.downcase == 'extract'
-      Rake::Task["i18n:extract"].execute
-    end
+  desc "Compile localizable strings from sources, extracting strings first."
+  task :generate => "i18n:extract" do
     sh(File.join(REPO_ROOT, "i18n", "generate.py"))
   end
 
diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt
index d285bed391..1f94f4ed89 100644
--- a/requirements/edx/base.txt
+++ b/requirements/edx/base.txt
@@ -89,7 +89,7 @@ sphinx==1.1.3
 
 # Used for Internationalization and localization
 Babel==1.3
-transifex-client==0.9.1
+transifex-client==0.10
 
 # Used for testing
 coverage==3.7