feat: atlas pull plugins translation

Makefile

@@ -56,8 +56,14 @@ endif
 push_translations: ## push source strings to Transifex for translation
 	i18n_tool transifex push
 
+pull_plugin_translations:  ## Pull translations from Transifex for edx_django_utils.plugins for both lms and cms
+	rm -rf conf/plugins-locale/plugins  # Clean up existing atlas translations
+	mkdir -p conf/plugins-locale/plugins
+	python manage.py lms pull_plugin_translations --verbose $(ATLAS_OPTIONS)
+	python manage.py lms compile_plugin_translations
+
 pull_xblock_translations:  ## pull xblock translations via atlas
-	rm -rf conf/plugins-locale  # Clean up existing atlas translations
+	rm -rf conf/plugins-locale/xblock.v1  # Clean up existing atlas translations
 	rm -rf lms/static/i18n/xblock.v1 cms/static/i18n/xblock.v1  # Clean up existing xblock compiled translations
 	mkdir -p conf/plugins-locale/xblock.v1/ lms/static/js/xblock.v1-i18n cms/static/js
 	python manage.py lms pull_xblock_translations --verbose $(ATLAS_OPTIONS)
@@ -76,6 +82,7 @@ ifeq ($(OPENEDX_ATLAS_PULL),)
 	i18n_tool validate --verbose
 else
 	make pull_xblock_translations
+	make pull_plugin_translations
 	find conf/locale -mindepth 1 -maxdepth 1 -type d -exec rm -r {} \;
 	atlas pull $(ATLAS_OPTIONS) translations/edx-platform/conf/locale:conf/locale
 	i18n_tool generate

common/djangoapps/xblock_django/management/commands/pull_xblock_translations.py

@@ -2,15 +2,13 @@
 Download the translations via atlas for the XBlocks.
 """
 
-from django.core.management.base import BaseCommand, CommandError
-
-from openedx.core.djangoapps.plugins.i18n_api import ATLAS_ARGUMENTS
+from openedx.core.djangoapps.plugins.i18n_api import BaseAtlasPullCommand
 from xmodule.modulestore import api as xmodule_api
 
 from ...translation import xblocks_atlas_pull
 
 
-class Command(BaseCommand):
+class Command(BaseAtlasPullCommand):
     """
     Pull the XBlock translations via atlas for the XBlocks.
 
@@ -19,33 +17,9 @@ class Command(BaseCommand):
      - https://github.com/openedx/openedx-atlas
     """
 
-    def add_arguments(self, parser):
-        for argument in ATLAS_ARGUMENTS:
-            parser.add_argument(*argument.get_args(), **argument.get_kwargs())
-
-        parser.add_argument(
-            '--verbose|-v',
-            action='store_true',
-            default=False,
-            dest='verbose',
-            help='Verbose output using `--verbose` argument for `atlas pull`.',
-        )
-
     def handle(self, *args, **options):
         xblock_translations_root = xmodule_api.get_python_locale_root()
-        if list(xblock_translations_root.listdir()):
-            raise CommandError(f'"{xblock_translations_root}" should be empty before running atlas pull.')
-
-        atlas_pull_options = []
-
-        for argument in ATLAS_ARGUMENTS:
-            option_value = options.get(argument.dest)
-            if option_value is not None:
-                atlas_pull_options += [argument.flag, option_value]
-
-        if options['verbose']:
-            atlas_pull_options += ['--verbose']
-        else:
-            atlas_pull_options += ['--silent']
+        self.ensure_empty_directory(xblock_translations_root)
 
+        atlas_pull_options = self.get_atlas_pull_options(**options)
         xblocks_atlas_pull(pull_options=atlas_pull_options)

openedx/core/djangoapps/plugins/apps.py

@@ -9,7 +9,8 @@ from django.apps import AppConfig
 from django.conf import settings
 from edx_django_utils.plugins import connect_plugin_receivers
 
-from openedx.core.djangoapps.plugins.constants import ProjectType
+from openedx.core.djangoapps.plugins.constants import ProjectType, SettingsType
+from edx_django_utils.plugins import PluginSettings
 
 
 class PluginsConfig(AppConfig):
@@ -19,7 +20,16 @@ class PluginsConfig(AppConfig):
 
     name = 'openedx.core.djangoapps.plugins'
 
-    plugin_app = {}
+    plugin_app = {
+        PluginSettings.CONFIG: {
+            ProjectType.LMS: {
+                SettingsType.PRODUCTION: {PluginSettings.RELATIVE_PATH: 'settings.production'},
+            },
+            ProjectType.CMS: {
+                SettingsType.PRODUCTION: {PluginSettings.RELATIVE_PATH: 'settings.production'},
+            },
+        }
+    }
 
     def ready(self):
         """

openedx/core/djangoapps/plugins/constants.py

@@ -37,3 +37,7 @@ class SettingsType():
     COMMON = 'common'
     DEVSTACK = 'devstack'
     TEST = 'test'
+
+
+# Locale root for IDA plugins for LMS and CMS, relative to settings.REPO_ROOT
+plugins_locale_root = 'conf/plugins-locale/plugins'

openedx/core/djangoapps/plugins/i18n_api.py

@@ -3,10 +3,14 @@ edx-platform specific i18n helpers for edx-django-utils plugins.
 """
 
 from dataclasses import dataclass, asdict
+from collections import defaultdict
 import os
 from pathlib import Path
 import subprocess
 
+from django.core.management import BaseCommand, CommandError
+from importlib_metadata import entry_points
+
 
 @dataclass
 class ArgparseArgument:
@@ -33,7 +37,7 @@ class ArgparseArgument:
         return [self.flag]
 
 
-# `atlas pull` arguments defintions.
+# `atlas pull` arguments definitions.
 #
 #  - https://github.com/openedx/openedx-atlas
 #
@@ -53,11 +57,63 @@ ATLAS_ARGUMENTS = [
     ArgparseArgument(
         flag='--branch',
         dest='branch',
-        help='Custom branch for "atlas pull" e.g. --branch=release/redwood . Default is "main".',
+        help='Deprecated option. Use --revision instead.',
+    ),
+    ArgparseArgument(
+        flag='--revision',
+        dest='revision',
+        help='Custom git revision for "atlas pull" e.g. --revision=release/redwood . Default is "main".',
     ),
 ]
 
 
+class BaseAtlasPullCommand(BaseCommand):
+    """
+    Base `atlas pull` Django command.
+    """
+
+    def add_arguments(self, parser):
+        """
+        Configure Django command arguments.
+        """
+        for argument in ATLAS_ARGUMENTS:
+            parser.add_argument(*argument.get_args(), **argument.get_kwargs())
+
+        parser.add_argument(
+            '--verbose|-v',
+            action='store_true',
+            default=False,
+            dest='verbose',
+            help='Verbose output using `--verbose` argument for `atlas pull`.',
+        )
+
+    def ensure_empty_directory(self, directory):
+        """
+        Ensure the pull directory is empty before running atlas pull.
+        """
+        plugin_translations_root = directory
+        if os.listdir(plugin_translations_root):
+            raise CommandError(f'"{plugin_translations_root}" should be empty before running atlas pull.')
+
+    def get_atlas_pull_options(self, **options):
+        """
+        Pass-through the Django command options to `atlas pull`.
+        """
+        atlas_pull_options = []
+
+        for argument in ATLAS_ARGUMENTS:
+            option_value = options.get(argument.dest)
+            if option_value is not None:
+                atlas_pull_options += [argument.flag, option_value]
+
+        if options['verbose']:
+            atlas_pull_options += ['--verbose']
+        else:
+            atlas_pull_options += ['--silent']
+
+        return atlas_pull_options
+
+
 def atlas_pull_by_modules(module_names, locale_root, pull_options):
     """
     Atlas pull translations by module name instead of repository name.
@@ -91,3 +147,40 @@ def compile_po_files(root_dir):
                     args=['msgfmt', '--check-format', '-o', str(po_file_path.with_suffix('.mo')), str(po_file_path)],
                     check=True,
                 )
+
+
+def get_installed_plugins_module_names():
+    """
+    Return the installed plugins Python module names.
+
+    This function excludes the built-in edx-platform plugins such as `lms`, `cms` and `openedx`.
+    """
+    # group (e.g 'lms.djangoapp') -> set for root module names (e.g {'edx_sga'})
+    root_modules = defaultdict(set)
+
+    for entry_point in entry_points():
+        module_name = entry_point.value
+        root_module = module_name.split('.')[0]  # e.g. `edx_sga` from `edx_sga.core.xblock`
+        root_modules[entry_point.group].add(root_module)
+
+    return (
+        # Return all lms.djangopapp and cms.djangoapp plugins
+        (root_modules['lms.djangoapp'] | root_modules['cms.djangoapp'])
+        # excluding the edx-platform built-in plugins which don't need atlas
+        - {'lms', 'cms', 'common', 'openedx', 'xmodule'}
+        # excluding XBlocks, which is handled by `pull_xblock_translations` command
+        - root_modules['xblock.v1']
+    )
+
+
+def plugin_translations_atlas_pull(pull_options, locale_root):
+    """
+    Atlas pull the translations for the installed non-XBlocks plugins.
+    """
+    module_names = get_installed_plugins_module_names()
+
+    atlas_pull_by_modules(
+        module_names=module_names,
+        locale_root=locale_root,
+        pull_options=pull_options,
+    )

openedx/core/djangoapps/plugins/management/commands/compile_plugin_translations.py

@@ -0,0 +1,19 @@
+"""
+Compile the translation files for the edx_django_utils.plugins.
+"""
+
+from django.core.management.base import BaseCommand
+from django.conf import settings
+
+
+from ...constants import plugins_locale_root
+
+from ... import i18n_api
+
+
+class Command(BaseCommand):
+    """
+    Compile the translation files for the edx_django_utils.plugins.
+    """
+    def handle(self, *args, **options):
+        i18n_api.compile_po_files(settings.REPO_ROOT / plugins_locale_root)

openedx/core/djangoapps/plugins/management/commands/pull_plugin_translations.py

@@ -0,0 +1,36 @@
+"""
+Download the translations via atlas for the edx-platform plugins (edx_django_utils.plugins).
+
+For the XBlock command check the `pull_xblock_translations` command.
+"""
+
+from django.conf import settings
+
+from openedx.core.djangoapps.plugins.i18n_api import BaseAtlasPullCommand
+
+from ...constants import plugins_locale_root
+
+from ...i18n_api import (
+    plugin_translations_atlas_pull,
+)
+
+
+class Command(BaseAtlasPullCommand):
+    """
+    Pull the edx_django_utils.plugins translations via atlas.
+
+    For detailed information about atlas pull options check the atlas documentation:
+
+     - https://github.com/openedx/openedx-atlas
+    """
+
+    def handle(self, *args, **options):
+        plugin_translations_root = settings.REPO_ROOT / plugins_locale_root
+        self.ensure_empty_directory(plugin_translations_root)
+
+        atlas_pull_options = self.get_atlas_pull_options(**options)
+
+        plugin_translations_atlas_pull(
+            pull_options=atlas_pull_options,
+            locale_root=plugin_translations_root,
+        )

openedx/core/djangoapps/plugins/settings/production.py

@@ -0,0 +1,17 @@
+"""
+Production environment variables for `edx_django_utils.plugins` plugins.
+"""
+
+from ..constants import plugins_locale_root
+
+
+def plugin_settings(settings):
+    """
+    Settings for the `edx_django_utils.plugins` plugins.
+    """
+    locale_root = settings.REPO_ROOT / plugins_locale_root
+    if locale_root.isdir():
+        for plugin_locale in locale_root.listdir():
+            # Add the plugin locale directory only if it's a non-empty directory
+            if plugin_locale.isdir() and plugin_locale.listdir():
+                settings.LOCALE_PATHS.append(plugin_locale)

openedx/core/djangoapps/plugins/tests/test_commands.py

@@ -0,0 +1,49 @@
+"""
+Tests for the plugins.i18n_api Django commands module.
+"""
+from unittest.mock import patch
+
+from django.core.management import call_command
+
+
+def test_pull_plugin_translations_command(settings, tmp_path):
+    """
+    Test the `pull_plugin_translations` Django command.
+    """
+    plugins_locale_root = tmp_path / 'conf/plugins-locale/plugins'
+    plugins_locale_root.mkdir(parents=True)
+    settings.REPO_ROOT = tmp_path
+
+    with patch('subprocess.run') as mock_run:
+        call_command(
+            'pull_plugin_translations',
+            verbose=True,
+            filter='ar,es_ES',
+            repository='custom_repo',
+        )
+
+    assert mock_run.call_count == 1, 'Expected to call `subprocess.run` once'
+    call_kwargs = mock_run.call_args.kwargs
+
+    assert call_kwargs['check'] is True
+    assert call_kwargs['cwd'] == plugins_locale_root
+    assert call_kwargs['args'][:8] == [
+        'atlas', 'pull', '--expand-glob',
+        '--filter', 'ar,es_ES',
+        '--repository', 'custom_repo',
+        '--verbose'
+    ], 'Pass arguments to atlas pull correctly'
+
+    assert 'translations/*/edx_proctoring/conf/locale:edx_proctoring' in call_kwargs['args'], (
+        'Pull edx-proctoring translations by Python module name using the "--expand-glob" option'
+    )
+
+
+def test_compile_plugin_translations_command(settings):
+    """
+    Test the `compile_plugin_translations` Django command.
+    """
+    with patch('openedx.core.djangoapps.plugins.i18n_api.compile_po_files') as mock_compile_po_files:
+        call_command('compile_plugin_translations')
+
+    mock_compile_po_files.assert_called_once_with(settings.REPO_ROOT / 'conf/plugins-locale/plugins')

openedx/core/djangoapps/plugins/tests/test_i18n_api.py

@@ -1,12 +1,17 @@
 """
 Tests for the plugins.i18n_api module.
 """
-
 from unittest.mock import patch
 
+import pytest
+from django.core.management import CommandError
+
 from ..i18n_api import (
     ArgparseArgument,
+    BaseAtlasPullCommand,
     atlas_pull_by_modules,
+    compile_po_files,
+    get_installed_plugins_module_names,
 )
 
 
@@ -50,3 +55,61 @@ def test_atlas_pull_by_modules():
         check=True,
         cwd=locale_root,
     )
+
+
+def test_compile_po_files(tmp_path):
+    """
+    Test the compile_po_files recursive call to `msgfmt`.
+    """
+    locale_root = tmp_path / 'locale'
+    locale_root.mkdir()
+    po_file_path = locale_root / 'test.po'
+    with open(po_file_path, 'w'):
+        # Creates an empty po file
+        pass
+
+    with patch('subprocess.run') as mock_run:
+        compile_po_files(locale_root)
+
+    mock_run.assert_called_once_with(
+        args=[
+            'msgfmt', '--check-format',
+            '-o', str(po_file_path.with_suffix('.mo')),
+            str(po_file_path),
+        ],
+        check=True,
+    )
+
+
+def test_base_atlas_pull_command(tmp_path):
+    """
+    Test the BaseAtlasPullCommand's methods.
+    """
+    command = BaseAtlasPullCommand()
+
+    assert command.ensure_empty_directory(tmp_path) is None, 'Should not raise an exception if the directory is empty'
+    with pytest.raises(CommandError):
+        with open(tmp_path / 'test.txt', 'w'):
+            # Directory is not empty anymore
+            pass
+        command.ensure_empty_directory(tmp_path)
+
+    assert command.get_atlas_pull_options(
+        filter='ar,jp_JP',
+        revision='custom_branch',
+        repository='my_org/custom_repo',
+        verbose=False,
+    ) == [
+        '--filter', 'ar,jp_JP', '--repository', 'my_org/custom_repo', '--revision', 'custom_branch', '--silent',
+    ], 'Flatten out the options into a list of arguments for atlas pull'
+
+
+def test_get_installed_plugins_module_names():
+    """
+    Test the get_installed_plugins_module_names helper.
+    """
+    plugins = get_installed_plugins_module_names()
+
+    assert 'drag_and_drop_v2' not in plugins, 'XBlocks have their own translation process'
+    assert 'edx_proctoring' in plugins, 'edx-proctoring should be included'
+    assert 'lms' not in plugins, 'lms and cms plugins are translated as part of the edx-platform itself'