diff --git a/cms/djangoapps/contentstore/features/pages.py b/cms/djangoapps/contentstore/features/pages.py index 9004a5a0a2..4149af0228 100644 --- a/cms/djangoapps/contentstore/features/pages.py +++ b/cms/djangoapps/contentstore/features/pages.py @@ -1,6 +1,7 @@ # pylint: disable=missing-docstring # pylint: disable=redefined-outer-name # pylint: disable=unused-argument +# pylint: disable=no-member from lettuce import step, world from nose.tools import assert_equal, assert_in diff --git a/cms/envs/aws.py b/cms/envs/aws.py index d68d194d1a..564ce9eb90 100644 --- a/cms/envs/aws.py +++ b/cms/envs/aws.py @@ -36,7 +36,6 @@ CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) # prefix. CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" - ############### ALWAYS THE SAME ################################ DEBUG = False @@ -561,6 +560,11 @@ COMPLETION_VIDEO_COMPLETE_PERCENTAGE = ENV_TOKENS.get( COMPLETION_VIDEO_COMPLETE_PERCENTAGE, ) +####################### Plugin Settings ########################## + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.CMS, SettingsType.AWS) + ########################## Derive Any Derived Settings ####################### derive_settings(__name__) diff --git a/cms/envs/common.py b/cms/envs/common.py index ec796329fc..efe84ee5e4 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1008,9 +1008,6 @@ INSTALLED_APPS = [ 'require', 'webpack_loader', - # Theming - 'openedx.core.djangoapps.theming.apps.ThemingConfig', - # Site configuration for theming and behavioral modification 'openedx.core.djangoapps.site_configuration', @@ -1123,10 +1120,6 @@ INSTALLED_APPS = [ # Waffle related utilities 'openedx.core.djangoapps.waffle_utils', - # Dynamic schedules - 'openedx.core.djangoapps.ace_common.apps.AceCommonConfig', - 'openedx.core.djangoapps.schedules.apps.SchedulesConfig', - # DRF filters 'django_filters', 'cms.djangoapps.api', @@ -1504,3 +1497,10 @@ ZENDESK_CUSTOM_FIELDS = {} # Once a user has watched this percentage of a video, mark it as complete: # (0.0 = 0%, 1.0 = 100%) COMPLETION_VIDEO_COMPLETE_PERCENTAGE = 0.95 + + +############## Installed Django Apps ######################### + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +INSTALLED_APPS.extend(DjangoAppRegistry.get_plugin_apps(ProjectType.CMS)) +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.CMS, SettingsType.COMMON) diff --git a/cms/envs/devstack.py b/cms/envs/devstack.py index b49f40631c..78b83ab947 100644 --- a/cms/envs/devstack.py +++ b/cms/envs/devstack.py @@ -143,6 +143,10 @@ JWT_AUTH.update({ 'JWT_AUDIENCE': 'lms-key', }) +##################################################################### +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.CMS, SettingsType.DEVSTACK) + ############################################################################### # See if the developer has any local overrides. if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')): diff --git a/cms/envs/test.py b/cms/envs/test.py index 166af8268c..9abe6e0e07 100644 --- a/cms/envs/test.py +++ b/cms/envs/test.py @@ -353,6 +353,11 @@ VIDEO_TRANSCRIPTS_SETTINGS = dict( DIRECTORY_PREFIX='video-transcripts/', ) +####################### Plugin Settings ########################## + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.CMS, SettingsType.TEST) + ########################## Derive Any Derived Settings ####################### derive_settings(__name__) diff --git a/cms/urls.py b/cms/urls.py index 1dee28ec66..df9bacee77 100644 --- a/cms/urls.py +++ b/cms/urls.py @@ -62,9 +62,6 @@ urlpatterns = [ # Darklang View to change the preview language (or dark language) url(r'^update_lang/', include('openedx.core.djangoapps.dark_lang.urls', namespace='dark_lang')), - # URLs for managing theming - url(r'^theming/', include('openedx.core.djangoapps.theming.urls', namespace='theming')), - # For redirecting to help pages. url(r'^help_token/', include('help_tokens.urls')), url(r'^api/', include('cms.djangoapps.api.urls', namespace='api')), @@ -259,3 +256,6 @@ urlpatterns += [ url(r'^404$', handler404), url(r'^500$', handler500), ] + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType +urlpatterns.extend(DjangoAppRegistry.get_plugin_url_patterns(ProjectType.CMS)) diff --git a/common/lib/xmodule/xmodule/tabs.py b/common/lib/xmodule/xmodule/tabs.py index ab16903470..e634bae879 100644 --- a/common/lib/xmodule/xmodule/tabs.py +++ b/common/lib/xmodule/xmodule/tabs.py @@ -8,7 +8,7 @@ from django.core.files.storage import get_storage_class from six import text_type from xblock.fields import List -from openedx.core.lib.api.plugins import PluginError +from openedx.core.lib.plugins import PluginError log = logging.getLogger("edx.courseware") diff --git a/lms/djangoapps/grades/apps.py b/lms/djangoapps/grades/apps.py index 66684d5051..7336695b21 100644 --- a/lms/djangoapps/grades/apps.py +++ b/lms/djangoapps/grades/apps.py @@ -7,6 +7,7 @@ Signal handlers are connected here. from django.apps import AppConfig from django.conf import settings from edx_proctoring.runtime import set_runtime_service +from openedx.core.djangolib.django_plugins import ProjectType, SettingsType, PluginURLs, PluginSettings class GradesConfig(AppConfig): @@ -15,6 +16,23 @@ class GradesConfig(AppConfig): """ name = u'lms.djangoapps.grades' + plugin_app = { + PluginURLs.CONFIG: { + ProjectType.LMS: { + PluginURLs.NAMESPACE: u'grades_api', + PluginURLs.REGEX: u'api/grades/', + PluginURLs.RELATIVE_PATH: u'api.urls', + } + }, + PluginSettings.CONFIG: { + ProjectType.LMS: { + SettingsType.AWS: {PluginSettings.RELATIVE_PATH: u'settings.aws'}, + SettingsType.COMMON: {PluginSettings.RELATIVE_PATH: u'settings.common'}, + SettingsType.TEST: {PluginSettings.RELATIVE_PATH: u'settings.test'}, + } + } + } + def ready(self): """ Connect handlers to recalculate grades. diff --git a/lms/djangoapps/grades/settings/__init__.py b/lms/djangoapps/grades/settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/lms/djangoapps/grades/settings/aws.py b/lms/djangoapps/grades/settings/aws.py new file mode 100644 index 0000000000..e929e15313 --- /dev/null +++ b/lms/djangoapps/grades/settings/aws.py @@ -0,0 +1,10 @@ +def plugin_settings(settings): + # Queue to use for updating persistent grades + settings.RECALCULATE_GRADES_ROUTING_KEY = settings.ENV_TOKENS.get( + 'RECALCULATE_GRADES_ROUTING_KEY', settings.LOW_PRIORITY_QUEUE, + ) + + # Queue to use for updating grades due to grading policy change + settings.POLICY_CHANGE_GRADES_ROUTING_KEY = settings.ENV_TOKENS.get( + 'POLICY_CHANGE_GRADES_ROUTING_KEY', settings.LOW_PRIORITY_QUEUE, + ) diff --git a/lms/djangoapps/grades/settings/common.py b/lms/djangoapps/grades/settings/common.py new file mode 100644 index 0000000000..fe198ed2c1 --- /dev/null +++ b/lms/djangoapps/grades/settings/common.py @@ -0,0 +1,6 @@ +def plugin_settings(settings): + # Queue to use for updating persistent grades + settings.RECALCULATE_GRADES_ROUTING_KEY = settings.LOW_PRIORITY_QUEUE + + # Queue to use for updating grades due to grading policy change + settings.POLICY_CHANGE_GRADES_ROUTING_KEY = settings.LOW_PRIORITY_QUEUE diff --git a/lms/djangoapps/grades/settings/test.py b/lms/djangoapps/grades/settings/test.py new file mode 100644 index 0000000000..7837e462fb --- /dev/null +++ b/lms/djangoapps/grades/settings/test.py @@ -0,0 +1,3 @@ +def plugin_settings(settings): + settings.FEATURES['PERSISTENT_GRADES_ENABLED_FOR_ALL_TESTS'] = True + settings.FEATURES['ASSUME_ZERO_GRADE_IF_ABSENT_FOR_ALL_TESTS'] = True diff --git a/lms/envs/aws.py b/lms/envs/aws.py index ec48edb938..391471490e 100644 --- a/lms/envs/aws.py +++ b/lms/envs/aws.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- """ -This is the default template for our main set of AWS servers. This does NOT -cover the content machines, which use content.py +This is the default template for our main set of AWS servers. Common traits: * Use memcached, and cache-backed sessions @@ -46,7 +45,6 @@ CONFIG_ROOT = path(os.environ.get('CONFIG_ROOT', ENV_ROOT)) # prefix. CONFIG_PREFIX = SERVICE_VARIANT + "." if SERVICE_VARIANT else "" - ################################ ALWAYS THE SAME ############################## DEBUG = False @@ -271,12 +269,6 @@ BULK_EMAIL_ROUTING_KEY = ENV_TOKENS.get('BULK_EMAIL_ROUTING_KEY', HIGH_PRIORITY_ # we have to reset the value here. BULK_EMAIL_ROUTING_KEY_SMALL_JOBS = ENV_TOKENS.get('BULK_EMAIL_ROUTING_KEY_SMALL_JOBS', LOW_PRIORITY_QUEUE) -# Queue to use for updating persistent grades -RECALCULATE_GRADES_ROUTING_KEY = ENV_TOKENS.get('RECALCULATE_GRADES_ROUTING_KEY', LOW_PRIORITY_QUEUE) - -# Queue to use for updating grades due to grading policy change -POLICY_CHANGE_GRADES_ROUTING_KEY = ENV_TOKENS.get('POLICY_CHANGE_GRADES_ROUTING_KEY', LOW_PRIORITY_QUEUE) - # Queue to use for expiring old entitlements ENTITLEMENTS_EXPIRATION_ROUTING_KEY = ENV_TOKENS.get('ENTITLEMENTS_EXPIRATION_ROUTING_KEY', LOW_PRIORITY_QUEUE) @@ -1077,15 +1069,6 @@ PARENTAL_CONSENT_AGE_LIMIT = ENV_TOKENS.get( PARENTAL_CONSENT_AGE_LIMIT ) -############## Settings for ACE #################################### -ACE_ENABLED_CHANNELS = ENV_TOKENS.get('ACE_ENABLED_CHANNELS', ACE_ENABLED_CHANNELS) -ACE_ENABLED_POLICIES = ENV_TOKENS.get('ACE_ENABLED_POLICIES', ACE_ENABLED_POLICIES) -ACE_CHANNEL_SAILTHRU_DEBUG = ENV_TOKENS.get('ACE_CHANNEL_SAILTHRU_DEBUG', ACE_CHANNEL_SAILTHRU_DEBUG) -ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME = ENV_TOKENS.get('ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME', ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME) -ACE_CHANNEL_SAILTHRU_API_KEY = AUTH_TOKENS.get('ACE_CHANNEL_SAILTHRU_API_KEY', ACE_CHANNEL_SAILTHRU_API_KEY) -ACE_CHANNEL_SAILTHRU_API_SECRET = AUTH_TOKENS.get('ACE_CHANNEL_SAILTHRU_API_SECRET', ACE_CHANNEL_SAILTHRU_API_SECRET) -ACE_ROUTING_KEY = ENV_TOKENS.get('ACE_ROUTING_KEY', ACE_ROUTING_KEY) - # Do NOT calculate this dynamically at startup with git because it's *slow*. EDX_PLATFORM_REVISION = ENV_TOKENS.get('EDX_PLATFORM_REVISION', EDX_PLATFORM_REVISION) @@ -1103,6 +1086,11 @@ COMPLETION_VIDEO_COMPLETE_PERCENTAGE = ENV_TOKENS.get( COMPLETION_VIDEO_COMPLETE_PERCENTAGE, ) +############################### Plugin Settings ############################### + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.LMS, SettingsType.AWS) + ########################## Derive Any Derived Settings ####################### derive_settings(__name__) diff --git a/lms/envs/common.py b/lms/envs/common.py index 3220992aa6..f0c29f3f84 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -2015,14 +2015,6 @@ BULK_EMAIL_LOG_SENT_EMAILS = False # parallel, and what the SES rate is. BULK_EMAIL_RETRY_DELAY_BETWEEN_SENDS = 0.02 -############################# Persistent Grades #################################### - -# Queue to use for updating persistent grades -RECALCULATE_GRADES_ROUTING_KEY = LOW_PRIORITY_QUEUE - -# Queue to use for updating grades due to grading policy change -POLICY_CHANGE_GRADES_ROUTING_KEY = LOW_PRIORITY_QUEUE - ############################# Email Opt In #################################### # Minimum age for organization-wide email opt in @@ -2105,9 +2097,6 @@ INSTALLED_APPS = [ # For content serving 'openedx.core.djangoapps.contentserver', - # Theming - 'openedx.core.djangoapps.theming.apps.ThemingConfig', - # Site configuration for theming and behavioral modification 'openedx.core.djangoapps.site_configuration', @@ -2136,7 +2125,6 @@ INSTALLED_APPS = [ 'openedx.core.djangoapps.course_groups', 'bulk_email', 'branding', - 'lms.djangoapps.grades.apps.GradesConfig', # Signals 'openedx.core.djangoapps.signals.apps.SignalConfig', @@ -2344,8 +2332,6 @@ INSTALLED_APPS = [ 'database_fixups', 'openedx.core.djangoapps.waffle_utils', - 'openedx.core.djangoapps.ace_common.apps.AceCommonConfig', - 'openedx.core.djangoapps.schedules.apps.SchedulesConfig', # Course Goals 'lms.djangoapps.course_goals', @@ -3443,20 +3429,6 @@ COURSES_API_CACHE_TIMEOUT = 3600 # Value is in seconds COURSEGRAPH_JOB_QUEUE = LOW_PRIORITY_QUEUE -############## Settings for ACE #################################### -ACE_ENABLED_CHANNELS = [ - 'file_email' -] -ACE_ENABLED_POLICIES = [ - 'bulk_email_optout' -] -ACE_CHANNEL_SAILTHRU_DEBUG = True -ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME = 'Automated Communication Engine Email' -ACE_CHANNEL_SAILTHRU_API_KEY = None -ACE_CHANNEL_SAILTHRU_API_SECRET = None - -ACE_ROUTING_KEY = LOW_PRIORITY_QUEUE - # Initialize to 'unknown', but read from JSON in aws.py EDX_PLATFORM_REVISION = 'unknown' @@ -3469,3 +3441,9 @@ COMPLETION_VIDEO_COMPLETE_PERCENTAGE = 0.95 ############### Settings for Django Rate limit ##################### RATELIMIT_ENABLE = True RATELIMIT_RATE = '30/m' + +############## Plugin Django Apps ######################### + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +INSTALLED_APPS.extend(DjangoAppRegistry.get_plugin_apps(ProjectType.LMS)) +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.LMS, SettingsType.COMMON) diff --git a/lms/envs/devstack.py b/lms/envs/devstack.py index 04e187a84b..c73423b122 100644 --- a/lms/envs/devstack.py +++ b/lms/envs/devstack.py @@ -269,21 +269,9 @@ JWT_AUTH.update({ 'JWT_AUDIENCE': 'lms-key', }) - -############## Settings for ACE #################################### -ACE_ENABLED_CHANNELS = [ - 'file_email' -] -ACE_ENABLED_POLICIES = [ - 'bulk_email_optout' -] -ACE_CHANNEL_SAILTHRU_DEBUG = True -ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME = 'Automated Communication Engine Email' -ACE_CHANNEL_SAILTHRU_API_KEY = None -ACE_CHANNEL_SAILTHRU_API_SECRET = None - -ACE_ROUTING_KEY = LOW_PRIORITY_QUEUE - +##################################################################### +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.LMS, SettingsType.DEVSTACK) ##################################################################### # See if the developer has any local overrides. diff --git a/lms/envs/docs/README.rst b/lms/envs/docs/README.rst new file mode 100644 index 0000000000..60948e9a92 --- /dev/null +++ b/lms/envs/docs/README.rst @@ -0,0 +1,101 @@ +LMS Configuration Settings +========================== + +The lms.envs module contains project-wide settings, defined in python modules +using the standard `Django Settings`_ mechanism. + +.. _Django Settings: https://docs.djangoproject.com/en/1.11/topics/settings/ + +Different python modules are used for different setting configuration options. +To prevent duplication of settings, modules import values from other modules, +as shown in the diagram below. + +.. image:: images/lms_settings.png + + +JSON Configuration Files +------------------------ + +In addition, there is a mechanism for reading and overriding configuration +settings from JSON files on-disk. The :file:`/lms/envs/aws.py` module loads +settings from ``lms.env.json`` and ``lms.auth.json`` files. All +security-sensitive settings and data belong in the ``lms.auth.json`` file, while +the rest are configured via the ``lms.env.json`` file. + +These JSON files allow open edX operators to configure the django runtime +without needing to make any changes to source-controlled python files in +edx-platform. Therefore, they are not checked into the edx-platform repo. +Rather, they are generated from the `edxapp playbook in the configuration +repo`_ and available in the ``/edx/app/edxapp/`` folder on edX servers. + +.. _edxapp playbook in the configuration repo: https://github.com/edx/configuration/tree/master/playbooks/roles/edxapp + + +Feature Flags and Settings Guidelines +------------------------------------- + +For guidelines on using Django settings and feature flag mechanisms in the edX +platform, please see `Feature Flags and Settings`_. + +.. _Feature Flags and Settings: https://openedx.atlassian.net/wiki/spaces/OpenDev/pages/40862688/Feature+Flags+and+Settings+on+edx-platform + + +Derived Settings +---------------- +In cases where you need to define one or more settings relative to the value of +another setting, you can explicitly designate them as derived calculations. +This can let you override one setting (such as a path or a feature toggle) and +have it automatically propagate to other settings which are defined in terms of +that value, without needing to track down all potentially impacted settings and +explicitly override them as well. This can be useful for test setting overrides +even if you don't anticipate end users customizing the value very often. + +For example:: + + def _make_locale_paths(settings): + locale_paths = [settings.REPO_ROOT + '/conf/locale'] # edx-platform/conf/locale/ + if settings.ENABLE_COMPREHENSIVE_THEMING: + # Add locale paths to settings for comprehensive theming. + for locale_path in settings.COMPREHENSIVE_THEME_LOCALE_PATHS: + locale_paths += (path(locale_path), ) + return locale_paths + LOCALE_PATHS = _make_locale_paths + derived('LOCALE_PATHS') + +In this case, ``LOCALE_PATHS`` will be defined correctly at the end of the +settings module parsing no matter what ``REPO_ROOT``, +`ENABLE_COMPREHENSIVE_THEMING`, and ``COMPREHENSIVE_THEME_LOCALE_PATHS`` are +currently set to. This is true even if the ``LOCALE_PATHS`` calculation was +defined in ``lms/envs/common.py`` and you're using ``lms/envs/aws.py`` which +includes overrides both from that module and the JSON configuration files. + +List entries and dictionary values can also be derived from other settings, even +when nested within each other:: + + def _make_mako_template_dirs(settings): + """ + Derives the final Mako template directories list from other settings. + """ + if settings.ENABLE_COMPREHENSIVE_THEMING: + themes_dirs = get_theme_base_dirs_from_settings(settings.COMPREHENSIVE_THEME_DIRS) + for theme in get_themes_unchecked(themes_dirs, settings.PROJECT_ROOT): + if theme.themes_base_dir not in settings.MAKO_TEMPLATE_DIRS_BASE: + settings.MAKO_TEMPLATE_DIRS_BASE.insert(0, theme.themes_base_dir) + if settings.FEATURES.get('USE_MICROSITES', False) and getattr(settings, "MICROSITE_CONFIGURATION", False): + settings.MAKO_TEMPLATE_DIRS_BASE.insert(0, settings.MICROSITE_ROOT_DIR) + return settings.MAKO_TEMPLATE_DIRS_BASE + + TEMPLATES = [ + { + 'NAME': 'django', + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + ... + }, + { + 'NAME': 'mako', + 'BACKEND': 'edxmako.backend.Mako', + 'APP_DIRS': False, + 'DIRS': _make_mako_template_dirs, + }, + ] + derived_collection_entry('TEMPLATES', 1, 'DIRS') diff --git a/lms/envs/docs/images/lms_settings.png b/lms/envs/docs/images/lms_settings.png new file mode 100644 index 0000000000..867f8a06bf Binary files /dev/null and b/lms/envs/docs/images/lms_settings.png differ diff --git a/lms/envs/test.py b/lms/envs/test.py index f42194289b..f32d50b6e7 100644 --- a/lms/envs/test.py +++ b/lms/envs/test.py @@ -297,10 +297,6 @@ OIDC_COURSE_HANDLER_CACHE_TIMEOUT = 0 FEATURES['ENABLE_MOBILE_REST_API'] = True FEATURES['ENABLE_VIDEO_ABSTRACTION_LAYER_API'] = True -########################### Grades ################################# -FEATURES['PERSISTENT_GRADES_ENABLED_FOR_ALL_TESTS'] = True -FEATURES['ASSUME_ZERO_GRADE_IF_ABSENT_FOR_ALL_TESTS'] = True - ###################### Payment ##############################3 # Enable fake payment processing page FEATURES['ENABLE_PAYMENT_FAKE'] = True @@ -532,9 +528,6 @@ NOTES_DISABLED_TABS = [] # Enable EdxNotes for tests. FEATURES['ENABLE_EDXNOTES'] = True -# Enable teams feature for tests. -FEATURES['ENABLE_TEAMS'] = True - # Enable courseware search for tests FEATURES['ENABLE_COURSEWARE_SEARCH'] = True @@ -601,6 +594,11 @@ ACTIVATION_EMAIL_FROM_ADDRESS = 'test_activate@edx.org' TEMPLATES[0]['OPTIONS']['debug'] = True +####################### Plugin Settings ########################## + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType, SettingsType +DjangoAppRegistry.add_plugin_settings(__name__, ProjectType.LMS, SettingsType.TEST) + ########################## Derive Any Derived Settings ####################### derive_settings(__name__) diff --git a/lms/urls.py b/lms/urls.py index 7a3f6fd78f..1098e6fa0b 100644 --- a/lms/urls.py +++ b/lms/urls.py @@ -133,9 +133,6 @@ urlpatterns = [ # URLs for managing dark launches of languages url(r'^update_lang/', include('openedx.core.djangoapps.dark_lang.urls', namespace='dark_lang')), - # URLs for managing theming - url(r'^theming/', include('openedx.core.djangoapps.theming.urls', namespace='theming')), - # For redirecting to help pages. url(r'^help_token/', include('help_tokens.urls')), @@ -478,13 +475,6 @@ urlpatterns += [ name='program_marketing_view', ), - # rest api for grades - url( - r'^api/grades/', - include('lms.djangoapps.grades.api.urls', namespace='grades_api') - ), - - # For the instructor url( r'^courses/{}/instructor$'.format( @@ -1084,3 +1074,7 @@ if settings.BRANCH_IO_KEY: urlpatterns += [ url(r'^text-me-the-app', 'student.views.text_me_the_app', name='text_me_the_app'), ] + + +from openedx.core.djangolib.django_plugins import DjangoAppRegistry, ProjectType +urlpatterns.extend(DjangoAppRegistry.get_plugin_url_patterns(ProjectType.LMS)) diff --git a/openedx/core/djangoapps/ace_common/apps.py b/openedx/core/djangoapps/ace_common/apps.py index 0527b125ea..33b1effbcb 100644 --- a/openedx/core/djangoapps/ace_common/apps.py +++ b/openedx/core/djangoapps/ace_common/apps.py @@ -4,6 +4,8 @@ Configuration for the ace_common Django app. from django.apps import AppConfig from django.utils.translation import ugettext_lazy as _ +from openedx.core.djangolib.django_plugins import ProjectType, PluginSettings, SettingsType + class AceCommonConfig(AppConfig): """ @@ -11,3 +13,13 @@ class AceCommonConfig(AppConfig): """ name = 'openedx.core.djangoapps.ace_common' verbose_name = _('ACE Common') + + plugin_app = { + PluginSettings.CONFIG: { + ProjectType.LMS: { + SettingsType.AWS: {PluginSettings.RELATIVE_PATH: u'settings.aws'}, + SettingsType.COMMON: {PluginSettings.RELATIVE_PATH: u'settings.common'}, + SettingsType.DEVSTACK: {PluginSettings.RELATIVE_PATH: u'settings.common'}, + } + } + } diff --git a/openedx/core/djangoapps/ace_common/settings/__init__.py b/openedx/core/djangoapps/ace_common/settings/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openedx/core/djangoapps/ace_common/settings/aws.py b/openedx/core/djangoapps/ace_common/settings/aws.py new file mode 100644 index 0000000000..56f149f247 --- /dev/null +++ b/openedx/core/djangoapps/ace_common/settings/aws.py @@ -0,0 +1,16 @@ +def plugin_settings(settings): + settings.ACE_ENABLED_CHANNELS = settings.ENV_TOKENS.get('ACE_ENABLED_CHANNELS', settings.ACE_ENABLED_CHANNELS) + settings.ACE_ENABLED_POLICIES = settings.ENV_TOKENS.get('ACE_ENABLED_POLICIES', settings.ACE_ENABLED_POLICIES) + settings.ACE_CHANNEL_SAILTHRU_DEBUG = settings.ENV_TOKENS.get( + 'ACE_CHANNEL_SAILTHRU_DEBUG', settings.ACE_CHANNEL_SAILTHRU_DEBUG, + ) + settings.ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME = settings.ENV_TOKENS.get( + 'ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME', settings.ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME, + ) + settings.ACE_CHANNEL_SAILTHRU_API_KEY = settings.AUTH_TOKENS.get( + 'ACE_CHANNEL_SAILTHRU_API_KEY', settings.ACE_CHANNEL_SAILTHRU_API_KEY, + ) + settings.ACE_CHANNEL_SAILTHRU_API_SECRET = settings.AUTH_TOKENS.get( + 'ACE_CHANNEL_SAILTHRU_API_SECRET', settings.ACE_CHANNEL_SAILTHRU_API_SECRET, + ) + settings.ACE_ROUTING_KEY = settings.ENV_TOKENS.get('ACE_ROUTING_KEY', settings.ACE_ROUTING_KEY) diff --git a/openedx/core/djangoapps/ace_common/settings/common.py b/openedx/core/djangoapps/ace_common/settings/common.py new file mode 100644 index 0000000000..c825505c2a --- /dev/null +++ b/openedx/core/djangoapps/ace_common/settings/common.py @@ -0,0 +1,15 @@ +def plugin_settings(settings): + settings.ACE_ENABLED_CHANNELS = [ + 'file_email' + ] + settings.ACE_ENABLED_POLICIES = [ + 'bulk_email_optout' + ] + settings.ACE_CHANNEL_SAILTHRU_DEBUG = True + settings.ACE_CHANNEL_SAILTHRU_TEMPLATE_NAME = 'Automated Communication Engine Email' + settings.ACE_CHANNEL_SAILTHRU_API_KEY = None + settings.ACE_CHANNEL_SAILTHRU_API_SECRET = None + + settings.ACE_ROUTING_KEY = 'edx.core.low' + + settings.FEATURES['test_django_plugin'] = True diff --git a/openedx/core/djangoapps/content/block_structure/transformer_registry.py b/openedx/core/djangoapps/content/block_structure/transformer_registry.py index 9c7b141f2c..2a57d64c9b 100644 --- a/openedx/core/djangoapps/content/block_structure/transformer_registry.py +++ b/openedx/core/djangoapps/content/block_structure/transformer_registry.py @@ -5,7 +5,7 @@ PluginManager. from base64 import b64encode from hashlib import sha1 -from openedx.core.lib.api.plugins import PluginManager +from openedx.core.lib.plugins import PluginManager from openedx.core.lib.cache_utils import memoized diff --git a/openedx/core/djangoapps/schedules/apps.py b/openedx/core/djangoapps/schedules/apps.py index a05be5447f..de788a11da 100644 --- a/openedx/core/djangoapps/schedules/apps.py +++ b/openedx/core/djangoapps/schedules/apps.py @@ -6,6 +6,8 @@ class SchedulesConfig(AppConfig): name = 'openedx.core.djangoapps.schedules' verbose_name = _('Schedules') + plugin_app = {} + def ready(self): # noinspection PyUnresolvedReferences from . import signals, tasks # pylint: disable=unused-variable diff --git a/openedx/core/djangoapps/schedules/docs/README.rst b/openedx/core/djangoapps/schedules/docs/README.rst index 34155ba12a..ef24ffe533 100644 --- a/openedx/core/djangoapps/schedules/docs/README.rst +++ b/openedx/core/djangoapps/schedules/docs/README.rst @@ -108,7 +108,7 @@ Glossary An Overview of edX's Dynamic Pacing System ------------------------------------------ -.. image:: img/system_diagram.png +.. image:: images/system_diagram.png Running the Management Commands diff --git a/openedx/core/djangoapps/schedules/docs/img/system_diagram.png b/openedx/core/djangoapps/schedules/docs/images/system_diagram.png similarity index 100% rename from openedx/core/djangoapps/schedules/docs/img/system_diagram.png rename to openedx/core/djangoapps/schedules/docs/images/system_diagram.png diff --git a/openedx/core/djangoapps/theming/apps.py b/openedx/core/djangoapps/theming/apps.py index 0b9fcf1ecd..9ea4f5c2d7 100644 --- a/openedx/core/djangoapps/theming/apps.py +++ b/openedx/core/djangoapps/theming/apps.py @@ -1,9 +1,19 @@ from django.apps import AppConfig +from openedx.core.djangolib.django_plugins import ProjectType, PluginURLs + + +plugin_urls_config = {PluginURLs.NAMESPACE: u'theming', PluginURLs.REGEX: u'theming/'} class ThemingConfig(AppConfig): name = 'openedx.core.djangoapps.theming' + plugin_app = { + PluginURLs.CONFIG: { + ProjectType.CMS: plugin_urls_config, + ProjectType.LMS: plugin_urls_config, + } + } verbose_name = "Theming" def ready(self): diff --git a/openedx/core/djangolib/django_plugins.py b/openedx/core/djangolib/django_plugins.py new file mode 100644 index 0000000000..9c18fa78e0 --- /dev/null +++ b/openedx/core/djangolib/django_plugins.py @@ -0,0 +1,347 @@ +""" +Provides functionality to enable improved plugin support of Django apps. + +Once a Django project is enhanced with this functionality, any participating +Django app (a.k.a. Plugin App) that is PIP-installed on the system is +automatically included in the Django project's INSTALLED_APPS list. In addition, +the participating Django app's URLs and Settings are automatically recognized by +the Django project. + +While Django+Python already support dynamic installation of components/apps, +they do not have out-of-the-box support for plugin apps that auto-install +into a containing Django project. + +This Django App Plugin functionality allows for Django-framework code to be +encapsulated within each Django app, rather than having a monolith Project that +is aware of the details of its Django apps. It is motivated by the following +design principles: + +* Single Responsibility Principle, which says "a class or module should have +one, and only one, reason to change." When code related to a single Django app +changes, there's no reason for its containing project to also change. The +encapsulation and modularity resulting from code being co-located with its +owning Django app helps prevent "God objects" that have too much responsibility +and knowledge of the details. + +* Open Closed Principle, which says "software entities should be open for +extension, but closed for modification." The edx-platform is extensible via +installation of Django apps. Having automatic Django App Plugin support allows +for this extensibility without modification to the edx-platform. Going forward, +we expect this capability to be widely used by external repos that depend on and +enhance the edx-platform without the need to modify the core platform. + +* Dependency Inversion Principle, which says "high level modules should not +depend upon low level modules." The high-level module here is the Django +project, while the participating Django app is the low-level module. For +long-term maintenance of a system, dependencies should go from low-level +modules/details to higher level ones. + + +== Django Projects == +In order to enable this functionality in a Django project, the project needs to +update: + +1. its settings to extend its INSTALLED_APPS to include the Plugin Apps: + INSTALLED_APPS.extend(DjangoAppRegistry.get_plugin_apps(...)) + +2. its settings to add all Plugin Settings: + DjangoAppRegistry.add_plugin_settings(__name__, ...) + +3. its urls to add all Plugin URLs: + urlpatterns.extend(DjangoAppRegistry.get_plugin_url_patterns(...)) + + +== Plugin Apps == +In order to make use of this functionality, plugin apps need to: + +1. create an AppConfig class in their apps module, as described in +https://docs.djangoproject.com/en/2.0/ref/applications/#django.apps.AppConfig. + +2. add their AppConfig class to the appropriate entry point in their setup.py +file: + + from setuptools import setup + setup( + ... + entry_points={ + "lms.djangoapp": [ + "my_app = full_python_path.my_app.apps:MyAppConfig", + ], + "cms.djangoapp": [ + ], + } + ) + +3. configure the Plugin App in their AppConfig class: + + from django.apps import AppConfig + from openedx.core.djangolib.django_plugins import ( + ProjectType, SettingsType, PluginURLs, PluginSettings + ) + class MyAppConfig(AppConfig): + name = u'full_python_path.my_app' + + # Class attribute that configures and enables this app as a Plugin App. + plugin_app = { + + # Configuration setting for Plugin URLs for this app. + PluginURLs.CONFIG: { + + # Configure the Plugin URLs for each project type, as needed. + ProjectType.LMS: { + + # The namespace to provide to django's urls.include, per + # https://docs.djangoproject.com/en/2.0/topics/http/urls/#url-namespaces + PluginURLs.NAMESPACE: u'my_app', + + # The regex to provide to django's urls.url. + PluginURLs.REGEX: u'api/my_app/', + + # The python path (relative to this app) to the URLs module + # to be plugged into the project. + PluginURLs.RELATIVE_PATH: u'api.urls', + } + }, + + + # Configuration setting for Plugin Settings for this app. + PluginSettings.CONFIG: { + + # Configure the Plugin Settings for each Project Type, as + # needed. + ProjectType.LMS: { + + # Configure each Settings Type, as needed. + SettingsType.AWS: { + # The python path (relative to this app) to the settings + # module for the relevant Project Type and Settings + # Type. + PluginSettings.RELATIVE_PATH: u'settings.aws', + }, + SettingsType.COMMON: { + PluginSettings.RELATIVE_PATH: u'settings.common', + }, + } + } + } + +OR use string constants when you cannot import from django_plugins. + + from django.apps import AppConfig + class MyAppConfig(AppConfig): + name = u'full_python_path.my_app' + + plugin_app = { + u'url_config': { + u'lms.djangoapp': { + u'namespace': u'my_app', + u'regex': u'api/my_app/', + u'relative_path': u'api.urls', + } + }, + u'settings_config': { + u'lms.djangoapp': { + u'aws': { relative_path: u'settings.aws' }, + u'common': { relative_path: u'settings.common'}, + } + } + } + +4. For Plugin Settings, insert the following function into each of the plugin +settings modules: + def plugin_settings(settings): + # Update the provided settings module with any app-specific settings. + # For example: + # settings.FEATURES['ENABLE_MY_APP'] = True + # settings.MY_APP_POLICY = 'foo' + +""" +from importlib import import_module +from django.conf.urls import include, url +from logging import getLogger +from openedx.core.lib.plugins import PluginManager + + +log = getLogger(__name__) + + +# Name of the class attribute to put in the AppConfig class of the Plugin App. +PLUGIN_APP_CLASS_ATTRIBUTE_NAME = u'plugin_app' + + +# Name of the function that belongs in the plugin Django app's settings file. +# The function should be defined as: +# def plugin_settings(settings): +# # enter code that should be injected into the given settings module. +PLUGIN_APP_SETTINGS_FUNC_NAME = u'plugin_settings' + + +class ProjectType(object): + """ + The ProjectType enum defines the possible values for the Django Projects + that are available in the edx-platform. Plugin apps use these values to + declare explicitly which projects they are extending. + """ + LMS = u'lms.djangoapp' + CMS = u'cms.djangoapp' + + +class SettingsType(object): + """ + The SettingsType enum defines the possible values for the settings files + that are available for extension in the edx-platform. Plugin apps use these + values (in addition to ProjectType) to declare explicitly which settings + (in the specified project) they are extending. + + See https://github.com/edx/edx-platform/master/lms/envs/docs/README.rst for + further information on each Settings Type. + """ + AWS = u'aws' + COMMON = u'common' + DEVSTACK = u'devstack' + TEST = u'test' + + +class PluginSettings(object): + """ + The PluginSettings enum defines dictionary field names (and defaults) + that can be specified by a Plugin App in order to configure the settings + that are injected into the project. + """ + CONFIG = u'settings_config' + RELATIVE_PATH = u'relative_path' + DEFAULT_RELATIVE_PATH = u'settings' + + +class PluginURLs(object): + """ + The PluginURLs enum defines dictionary field names (and defaults) that can + be specified by a Plugin App in order to configure the URLs that are + injected into the project. + """ + CONFIG = u'url_config' + APP_NAME = u'app_name' + NAMESPACE = u'namespace' + REGEX = u'regex' + RELATIVE_PATH = u'relative_path' + DEFAULT_RELATIVE_PATH = u'urls' + + +class DjangoAppRegistry(PluginManager): + """ + The DjangoAppRegistry class encapsulates the functionality to enable + improved plugin support of Django apps. + """ + + @classmethod + def get_plugin_apps(cls, project_type): + """ + Returns a list of all registered Plugin Apps, expected to be added to + the INSTALLED_APPS list for the given project_type. + """ + plugin_apps = [ + u'{module_name}.{class_name}'.format( + module_name=app_config.__module__, + class_name=app_config.__name__, + ) + for app_config in cls._get_app_configs(project_type) + if getattr(app_config, PLUGIN_APP_CLASS_ATTRIBUTE_NAME, None) is not None + ] + log.info(u'Plugin Apps: Found %s', plugin_apps) + return plugin_apps + + @classmethod + def add_plugin_settings(cls, settings_path, project_type, settings_type): + """ + Updates the module at the given ``settings_path`` with all Plugin + Settings appropriate for the given project_type and settings_type. + """ + settings_module = import_module(settings_path) + for plugin_settings in cls._iter_plugin_settings(project_type, settings_type): + settings_func = getattr(plugin_settings, PLUGIN_APP_SETTINGS_FUNC_NAME) + settings_func(settings_module) + + @classmethod + def get_plugin_url_patterns(cls, project_type): + """ + Returns a list of all registered Plugin URLs, expected to be added to + the URL patterns for the given project_type. + """ + return [ + url( + url_config.get(PluginURLs.REGEX, r''), + include( + url_module_path, + app_name=url_config.get(PluginURLs.APP_NAME), + namespace=url_config[PluginURLs.NAMESPACE], + ), + ) + for url_module_path, url_config in cls._iter_installable_urls(project_type) + ] + + @classmethod + def _iter_plugin_settings(cls, project_type, settings_type): + """ + Yields Plugin Settings modules that are registered for the given + project_type and settings_type. + """ + for app_config in cls._get_app_configs(project_type): + settings_config = _get_settings_config(app_config, project_type, settings_type) + if settings_config is None: + log.info( + u'Plugin Apps [Settings]: Did NOT find %s for %s and %s', + app_config.name, + project_type, + settings_type, + ) + continue + + plugin_settings_path = _get_module_path(app_config, settings_config, PluginSettings) + log.info(u'Plugin Apps [Settings]: Found %s for %s and %s', app_config.name, project_type, settings_type) + yield import_module(plugin_settings_path) + + @classmethod + def _iter_installable_urls(cls, project_type): + """ + Yields the module path and configuration for Plugin URLs registered for + the given project_type. + """ + for app_config in cls._get_app_configs(project_type): + url_config = _get_url_config(app_config, project_type) + if url_config is None: + log.info(u'Plugin Apps [URLs]: Did NOT find %s for %s', app_config.name, project_type) + continue + + urls_module_path = _get_module_path(app_config, url_config, PluginURLs) + url_config[PluginURLs.NAMESPACE] = url_config.get(PluginURLs.NAMESPACE, app_config.name) + log.info( + u'Plugin Apps [URLs]: Found %s with namespace %s for %s', + app_config.name, + url_config[PluginURLs.NAMESPACE], + project_type, + ) + yield urls_module_path, url_config + + @classmethod + def _get_app_configs(cls, project_type): + return cls.get_available_plugins(project_type).itervalues() + + +def _get_module_path(app_config, plugin_config, plugin_cls): + return u'{package_path}.{module_path}'.format( + package_path=app_config.name, + module_path=plugin_config.get(plugin_cls.RELATIVE_PATH, plugin_cls.DEFAULT_RELATIVE_PATH), + ) + + +def _get_settings_config(app_config, project_type, settings_type): + plugin_config = getattr(app_config, PLUGIN_APP_CLASS_ATTRIBUTE_NAME, {}) + settings_config = plugin_config.get(PluginSettings.CONFIG, {}) + project_type_settings = settings_config.get(project_type, {}) + return project_type_settings.get(settings_type) + + +def _get_url_config(app_config, project_type): + plugin_config = getattr(app_config, PLUGIN_APP_CLASS_ATTRIBUTE_NAME, {}) + url_config = plugin_config.get(PluginURLs.CONFIG, {}) + return url_config.get(project_type) diff --git a/openedx/core/lib/api/plugins.py b/openedx/core/lib/api/plugins.py deleted file mode 100644 index 576dc9cec5..0000000000 --- a/openedx/core/lib/api/plugins.py +++ /dev/null @@ -1,45 +0,0 @@ -""" -Adds support for first class features that can be added to the edX platform. -""" - -from stevedore.extension import ExtensionManager - - -class PluginError(Exception): - """ - Base Exception for when an error was found regarding features. - """ - pass - - -class PluginManager(object): - """ - Base class that manages plugins to the edX platform. - """ - @classmethod - def get_available_plugins(cls): - """ - Returns a dict of all the plugins that have been made available through the platform. - """ - # Note: we're creating the extension manager lazily to ensure that the Python path - # has been correctly set up. Trying to create this statically will fail, unfortunately. - if not hasattr(cls, "_plugins"): - plugins = {} - extension_manager = ExtensionManager(namespace=cls.NAMESPACE) # pylint: disable=no-member - for plugin_name in extension_manager.names(): - plugins[plugin_name] = extension_manager[plugin_name].plugin - cls._plugins = plugins - return cls._plugins - - @classmethod - def get_plugin(cls, name): - """ - Returns the plugin with the given name. - """ - plugins = cls.get_available_plugins() - if name not in plugins: - raise PluginError("No such plugin {name} for entry point {namespace}".format( - name=name, - namespace=cls.NAMESPACE # pylint: disable=no-member - )) - return plugins[name] diff --git a/openedx/core/lib/course_tabs.py b/openedx/core/lib/course_tabs.py index 4ce547c8fc..56d4ef1f2e 100644 --- a/openedx/core/lib/course_tabs.py +++ b/openedx/core/lib/course_tabs.py @@ -1,7 +1,7 @@ """ Tabs for courseware. """ -from openedx.core.lib.api.plugins import PluginManager +from openedx.core.lib.plugins import PluginManager # Stevedore extension point namespaces COURSE_TAB_NAMESPACE = 'openedx.course_tab' diff --git a/openedx/core/lib/plugins.py b/openedx/core/lib/plugins.py new file mode 100644 index 0000000000..6447cb4f96 --- /dev/null +++ b/openedx/core/lib/plugins.py @@ -0,0 +1,46 @@ +""" +Adds support for first class plugins that can be added to the edX platform. +""" +from collections import OrderedDict + +from stevedore.extension import ExtensionManager +from openedx.core.lib.cache_utils import memoized + + +class PluginError(Exception): + """ + Base Exception for when an error was found regarding plugins. + """ + pass + + +class PluginManager(object): + """ + Base class that manages plugins for the edX platform. + """ + @classmethod + @memoized + def get_available_plugins(cls, namespace=None): + """ + Returns a dict of all the plugins that have been made available through the platform. + """ + # Note: we're creating the extension manager lazily to ensure that the Python path + # has been correctly set up. Trying to create this statically will fail, unfortunately. + plugins = OrderedDict() + extension_manager = ExtensionManager(namespace=namespace or cls.NAMESPACE) # pylint: disable=no-member + for plugin_name in extension_manager.names(): + plugins[plugin_name] = extension_manager[plugin_name].plugin + return plugins + + @classmethod + def get_plugin(cls, name, namespace=None): + """ + Returns the plugin with the given name. + """ + plugins = cls.get_available_plugins(namespace) + if name not in plugins: + raise PluginError("No such plugin {name} for entry point {namespace}".format( + name=name, + namespace=namespace or cls.NAMESPACE, # pylint: disable=no-member + )) + return plugins[name] diff --git a/openedx/core/lib/tests/test_course_tab_api.py b/openedx/core/lib/tests/test_course_tab_api.py index 44537e295b..18607aec71 100644 --- a/openedx/core/lib/tests/test_course_tab_api.py +++ b/openedx/core/lib/tests/test_course_tab_api.py @@ -5,7 +5,7 @@ Tests for the plugin API from django.test import TestCase from nose.plugins.attrib import attr -from openedx.core.lib.api.plugins import PluginError +from openedx.core.lib.plugins import PluginError from openedx.core.lib.course_tabs import CourseTabPluginManager diff --git a/openedx/features/course_experience/course_tools.py b/openedx/features/course_experience/course_tools.py index 3be140ecc3..fe1f45f261 100644 --- a/openedx/features/course_experience/course_tools.py +++ b/openedx/features/course_experience/course_tools.py @@ -1,7 +1,7 @@ """ Support for course tool plugins. """ -from openedx.core.lib.api.plugins import PluginManager +from openedx.core.lib.plugins import PluginManager # Stevedore extension point namespace COURSE_TOOLS_NAMESPACE = 'openedx.course_tool' diff --git a/setup.py b/setup.py index c625903af8..cf594d82d5 100644 --- a/setup.py +++ b/setup.py @@ -63,5 +63,16 @@ setup( "openedx.ace.policy": [ "bulk_email_optout = lms.djangoapps.bulk_email.policies:CourseEmailOptout" ], + "lms.djangoapp": [ + "grades = lms.djangoapps.grades.apps:GradesConfig", + "ace_common = openedx.core.djangoapps.ace_common.apps:AceCommonConfig", + "schedules = openedx.core.djangoapps.schedules.apps:SchedulesConfig", + "theming = openedx.core.djangoapps.theming.apps:ThemingConfig", + ], + "cms.djangoapp": [ + "ace_common = openedx.core.djangoapps.ace_common.apps:AceCommonConfig", + "schedules = openedx.core.djangoapps.schedules.apps:SchedulesConfig", + "theming = openedx.core.djangoapps.theming.apps:ThemingConfig", + ], } )