From 353b4c4f670a53bbb9a0e6de949005ff16df8d2b Mon Sep 17 00:00:00 2001 From: Jeremy Bowman Date: Fri, 30 Jun 2017 18:28:40 -0400 Subject: [PATCH] PLAT-1499 Use appropriate default settings in Docker devstack --- cms/envs/devstack_optimized.py | 7 ++++++- cms/envs/devstack_with_worker.py | 7 ++++++- .../modulestore/tests/mongo_connection.py | 3 ++- common/lib/xmodule/xmodule/tests/test_video.py | 3 ++- lms/envs/devstack_optimized.py | 7 ++++++- lms/envs/devstack_with_worker.py | 7 ++++++- pavelib/assets.py | 6 +++--- pavelib/i18n.py | 3 ++- pavelib/paver_tests/test_assets.py | 10 ++++++---- pavelib/paver_tests/test_i18n.py | 7 ++++--- pavelib/paver_tests/test_servers.py | 18 +++++++++--------- pavelib/servers.py | 3 ++- pavelib/utils/envs.py | 2 ++ pavelib/utils/test/suites/nose_suite.py | 2 +- 14 files changed, 57 insertions(+), 28 deletions(-) diff --git a/cms/envs/devstack_optimized.py b/cms/envs/devstack_optimized.py index d5796406be..dc20d4ce75 100644 --- a/cms/envs/devstack_optimized.py +++ b/cms/envs/devstack_optimized.py @@ -18,9 +18,14 @@ as they are for non-optimized devstack. Instead, update_assets must be invoked each time that changes have been made. """ +import os + ########################## Devstack settings ################################### -from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import +if 'BOK_CHOY_HOSTNAME' in os.environ: + from .devstack_docker import * # pylint: disable=wildcard-import, unused-wildcard-import +else: + from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import TEST_ROOT = REPO_ROOT / "test_root" diff --git a/cms/envs/devstack_with_worker.py b/cms/envs/devstack_with_worker.py index eb9914d5e7..671f09191d 100644 --- a/cms/envs/devstack_with_worker.py +++ b/cms/envs/devstack_with_worker.py @@ -11,10 +11,15 @@ In two separate processes on devstack: ./manage.py cms celery worker --settings=devstack_with_worker """ +import os + # We intentionally define lots of variables that aren't used, and # want to import all variables from base settings files # pylint: disable=wildcard-import, unused-wildcard-import -from cms.envs.devstack import * +if 'BOK_CHOY_HOSTNAME' in os.environ: + from cms.envs.devstack_docker import * +else: + from cms.envs.devstack import * # Require a separate celery worker CELERY_ALWAYS_EAGER = False diff --git a/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py b/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py index 1b18d485d6..6469997453 100644 --- a/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py +++ b/common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py @@ -7,4 +7,5 @@ overrides don't exist import os MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) -MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost') +MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', + 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost') diff --git a/common/lib/xmodule/xmodule/tests/test_video.py b/common/lib/xmodule/xmodule/tests/test_video.py index 7e3d9c67e5..e38f11bd04 100644 --- a/common/lib/xmodule/xmodule/tests/test_video.py +++ b/common/lib/xmodule/xmodule/tests/test_video.py @@ -12,6 +12,7 @@ You can then use the CourseFactory and XModuleItemFactory as defined in common/lib/xmodule/xmodule/modulestore/tests/factories.py to create the course, section, subsection, unit, etc. """ +import os import unittest import datetime from uuid import uuid4 @@ -821,7 +822,7 @@ class VideoDescriptorIndexingTestCase(unittest.TestCase): settings.CONTENTSTORE = { 'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore', 'DOC_STORE_CONFIG': { - 'host': 'localhost', + 'host': 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost', 'db': 'test_xcontent_%s' % uuid4().hex, }, # allow for additional options that can be keyed on a name, e.g. 'trashcan' diff --git a/lms/envs/devstack_optimized.py b/lms/envs/devstack_optimized.py index cea9b31b49..51fe7a0c9d 100644 --- a/lms/envs/devstack_optimized.py +++ b/lms/envs/devstack_optimized.py @@ -18,9 +18,14 @@ as they are for non-optimized devstack. Instead, update_assets must be invoked each time that changes have been made. """ +import os + ########################## Devstack settings ################################### -from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import +if 'BOK_CHOY_HOSTNAME' in os.environ: + from .devstack_docker import * # pylint: disable=wildcard-import, unused-wildcard-import +else: + from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import TEST_ROOT = REPO_ROOT / "test_root" diff --git a/lms/envs/devstack_with_worker.py b/lms/envs/devstack_with_worker.py index 7a3377a584..0f4b610054 100644 --- a/lms/envs/devstack_with_worker.py +++ b/lms/envs/devstack_with_worker.py @@ -11,10 +11,15 @@ In two separate processes on devstack: ./manage.py lms celery worker --settings=devstack_with_worker """ +import os + # We intentionally define lots of variables that aren't used, and # want to import all variables from base settings files # pylint: disable=wildcard-import, unused-wildcard-import -from lms.envs.devstack import * +if 'BOK_CHOY_HOSTNAME' in os.environ: + from lms.envs.devstack_docker import * +else: + from lms.envs.devstack import * # Require a separate celery worker CELERY_ALWAYS_EAGER = False diff --git a/pavelib/assets.py b/pavelib/assets.py index 1d5e1f3efe..80081299f8 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -797,7 +797,7 @@ def watch_assets(options): # We only want Webpack to re-run on changes to its own entry points, not all JS files, so we use its own watcher # instead of subclassing from Watchdog like the other watchers do - execute_webpack_watch(settings='devstack') + execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS) if not getattr(options, 'background', False): # when running as a separate process, the main thread needs to loop @@ -826,7 +826,7 @@ def update_assets(args): help="lms or studio", ) parser.add_argument( - '--settings', type=str, default="devstack", + '--settings', type=str, default=Env.DEVSTACK_SETTINGS, help="Django settings module", ) parser.add_argument( @@ -859,7 +859,7 @@ def update_assets(args): process_xmodule_assets() process_npm_assets() compile_coffeescript() - execute_webpack(prod=(args.settings != "devstack"), settings=args.settings) + execute_webpack(prod=(args.settings != Env.DEVSTACK_SETTINGS), settings=args.settings) # Compile sass for themes and system execute_compile_sass(args) diff --git a/pavelib/i18n.py b/pavelib/i18n.py index 2e0f2035c2..0ba968c5f7 100644 --- a/pavelib/i18n.py +++ b/pavelib/i18n.py @@ -10,6 +10,7 @@ from path import Path as path from paver.easy import cmdopts, needs, sh, task from .utils.cmd import django_cmd +from .utils.envs import Env from .utils.timer import timed try: @@ -17,7 +18,7 @@ try: except ImportError: colorize = lambda color, text: text -DEFAULT_SETTINGS = 'devstack' +DEFAULT_SETTINGS = Env.DEVSTACK_SETTINGS @task diff --git a/pavelib/paver_tests/test_assets.py b/pavelib/paver_tests/test_assets.py index 548f6e5286..aa710e0f52 100644 --- a/pavelib/paver_tests/test_assets.py +++ b/pavelib/paver_tests/test_assets.py @@ -10,6 +10,7 @@ from watchdog.observers.polling import PollingObserver from pavelib.assets import COLLECTSTATIC_LOG_DIR_ARG, collect_assets +from ..utils.envs import Env from .utils import PaverTestCase ROOT_PATH = path(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) @@ -252,12 +253,12 @@ class TestCollectAssets(PaverTestCase): if specified_log_loc is None: collect_assets( systems, - "devstack" + Env.DEVSTACK_SETTINGS ) else: collect_assets( systems, - "devstack", + Env.DEVSTACK_SETTINGS, **specified_log_dict ) self.assertEqual(self.task_messages, expected_messages) @@ -271,7 +272,7 @@ class TestCollectAssets(PaverTestCase): systems = ["lms"] kwargs = {COLLECTSTATIC_LOG_DIR_ARG: None} expected_messages = self._set_expected_messages(log_location=expected_log_loc, systems=systems) - collect_assets(systems, "devstack", **kwargs) + collect_assets(systems, Env.DEVSTACK_SETTINGS, **kwargs) self.assertEqual(self.task_messages, expected_messages) def _set_expected_messages(self, log_location, systems): @@ -284,8 +285,9 @@ class TestCollectAssets(PaverTestCase): expected_messages = [] for sys in systems: expected_messages.append( - 'python manage.py {system} --settings=devstack collectstatic --noinput {log_loc}'.format( + 'python manage.py {system} --settings={settings} collectstatic --noinput {log_loc}'.format( system=sys, + settings=Env.DEVSTACK_SETTINGS, log_loc=log_location ) ) diff --git a/pavelib/paver_tests/test_i18n.py b/pavelib/paver_tests/test_i18n.py index cbacb17f57..d5991c1a77 100644 --- a/pavelib/paver_tests/test_i18n.py +++ b/pavelib/paver_tests/test_i18n.py @@ -11,6 +11,7 @@ from paver.easy import call_task, task import pavelib.i18n from pavelib.paver_tests.utils import PaverTestCase +from pavelib.utils.envs import Env TX_CONFIG_SIMPLE = """\ @@ -155,14 +156,14 @@ class TestI18nDummy(PaverTestCase): """ self.reset_task_messages() os.environ['NO_PREREQ_INSTALL'] = "true" - call_task('pavelib.i18n.i18n_dummy', options={"settings": 'test'}) + call_task('pavelib.i18n.i18n_dummy', options={"settings": Env.TEST_SETTINGS}) self.assertEquals( self.task_messages, [ u'i18n_tool extract', u'i18n_tool dummy', u'i18n_tool generate', - u'python manage.py lms --settings=test compilejsi18n', - u'python manage.py cms --settings=test compilejsi18n', + u'python manage.py lms --settings={} compilejsi18n'.format(Env.TEST_SETTINGS), + u'python manage.py cms --settings={} compilejsi18n'.format(Env.TEST_SETTINGS), ] ) diff --git a/pavelib/paver_tests/test_servers.py b/pavelib/paver_tests/test_servers.py index 7c517f8a71..4048af4611 100644 --- a/pavelib/paver_tests/test_servers.py +++ b/pavelib/paver_tests/test_servers.py @@ -102,7 +102,7 @@ class TestPaverServerTasks(PaverTestCase): """ options = server_options.copy() is_optimized = options.get("optimized", False) - expected_settings = "devstack_optimized" if is_optimized else options.get("settings", "devstack") + expected_settings = "devstack_optimized" if is_optimized else options.get("settings", Env.DEVSTACK_SETTINGS) # First test with LMS options["system"] = "lms" @@ -162,7 +162,7 @@ class TestPaverServerTasks(PaverTestCase): """ Test the "update_db" task. """ - settings = options.get("settings", "devstack") + settings = options.get("settings", Env.DEVSTACK_SETTINGS) call_task("pavelib.servers.update_db", options=options) # pylint: disable=line-too-long db_command = "NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{server} --traceback --pythonpath=. " @@ -185,7 +185,7 @@ class TestPaverServerTasks(PaverTestCase): """ Test the "check_settings" task. """ - settings = options.get("settings", "devstack") + settings = options.get("settings", Env.DEVSTACK_SETTINGS) call_task("pavelib.servers.check_settings", args=[system, settings]) self.assertEquals( self.task_messages, @@ -231,19 +231,19 @@ class TestPaverServerTasks(PaverTestCase): else: call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options) expected_messages = options.get("expected_messages", []) - expected_settings = settings if settings else "devstack" + expected_settings = settings if settings else Env.DEVSTACK_SETTINGS expected_asset_settings = asset_settings if asset_settings else expected_settings if is_optimized: expected_settings = "devstack_optimized" expected_asset_settings = "test_static_optimized" - expected_collect_static = not is_fast and expected_settings != "devstack" + expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS if not is_fast: expected_messages.append(u"xmodule_assets common/static/xmodule") expected_messages.append(u"install npm_assets") expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root)) expected_messages.extend([c.format(settings=expected_asset_settings) for c in EXPECTED_PRINT_SETTINGS_COMMAND]) expected_messages.append(EXPECTED_WEBPACK_COMMAND.format( - node_env="production" if expected_asset_settings != "devstack" else "development", + node_env="production" if expected_asset_settings != Env.DEVSTACK_SETTINGS else "development", static_root_lms=None, static_root_cms=None )) @@ -273,12 +273,12 @@ class TestPaverServerTasks(PaverTestCase): is_fast = options.get("fast", False) self.reset_task_messages() call_task("pavelib.servers.run_all_servers", options=options) - expected_settings = settings if settings else "devstack" + expected_settings = settings if settings else Env.DEVSTACK_SETTINGS expected_asset_settings = asset_settings if asset_settings else expected_settings if is_optimized: expected_settings = "devstack_optimized" expected_asset_settings = "test_static_optimized" - expected_collect_static = not is_fast and expected_settings != "devstack" + expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS expected_messages = [] if not is_fast: expected_messages.append(u"xmodule_assets common/static/xmodule") @@ -286,7 +286,7 @@ class TestPaverServerTasks(PaverTestCase): expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root)) expected_messages.extend([c.format(settings=expected_asset_settings) for c in EXPECTED_PRINT_SETTINGS_COMMAND]) expected_messages.append(EXPECTED_WEBPACK_COMMAND.format( - node_env="production" if expected_asset_settings != "devstack" else "development", + node_env="production" if expected_asset_settings != Env.DEVSTACK_SETTINGS else "development", static_root_lms=None, static_root_cms=None )) diff --git a/pavelib/servers.py b/pavelib/servers.py index 81579b1c51..2ba99a3acf 100644 --- a/pavelib/servers.py +++ b/pavelib/servers.py @@ -10,11 +10,12 @@ from paver.easy import call_task, cmdopts, consume_args, needs, sh, task from .assets import collect_assets from .utils.cmd import django_cmd +from .utils.envs import Env from .utils.process import run_multi_processes, run_process from .utils.timer import timed DEFAULT_PORT = {"lms": 8000, "studio": 8001} -DEFAULT_SETTINGS = 'devstack' +DEFAULT_SETTINGS = Env.DEVSTACK_SETTINGS OPTIMIZED_SETTINGS = "devstack_optimized" OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized" diff --git a/pavelib/utils/envs.py b/pavelib/utils/envs.py index 5172e57f13..c7fff8ac71 100644 --- a/pavelib/utils/envs.py +++ b/pavelib/utils/envs.py @@ -69,6 +69,8 @@ class Env(object): SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0') USING_DOCKER = SERVER_HOST != '0.0.0.0' SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy' + DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack' + TEST_SETTINGS = 'test_docker' if USING_DOCKER else 'test' BOK_CHOY_SERVERS = { 'lms': { diff --git a/pavelib/utils/test/suites/nose_suite.py b/pavelib/utils/test/suites/nose_suite.py index fce208367b..13f34b3600 100644 --- a/pavelib/utils/test/suites/nose_suite.py +++ b/pavelib/utils/test/suites/nose_suite.py @@ -119,7 +119,7 @@ class SystemTestSuite(NoseTestSuite): self.processes = kwargs.get('processes', None) self.randomize = kwargs.get('randomize', None) - self.settings = kwargs.get('settings', 'test') + self.settings = kwargs.get('settings', Env.TEST_SETTINGS) if self.processes is None: # Don't use multiprocessing by default