Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #15445 from edx/jmbowman/PLAT-1499

Browse Source 
PLAT-1499 Use appropriate default settings in Docker devstack
This commit is contained in:
Jeremy Bowman
2017-07-05 13:22:14 -04:00
committed by GitHub
parent eff621b7ba 353b4c4f67
commit e4c8ff3aa4
14 changed files with 57 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
cms/envs/devstack_optimized.py
View File

@@ -18,9 +18,14 @@ as they are for non-optimized devstack. Instead, update_assets must be
invoked each time that changes have been made.
"""
import os
########################## Devstack settings ###################################
from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import
if 'BOK_CHOY_HOSTNAME' in os.environ:
from .devstack_docker import * # pylint: disable=wildcard-import, unused-wildcard-import
else:
from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import
TEST_ROOT = REPO_ROOT / "test_root"

7
cms/envs/devstack_with_worker.py
View File

@@ -11,10 +11,15 @@ In two separate processes on devstack:
./manage.py cms celery worker --settings=devstack_with_worker
"""
import os
# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=wildcard-import, unused-wildcard-import
from cms.envs.devstack import *
if 'BOK_CHOY_HOSTNAME' in os.environ:
from cms.envs.devstack_docker import *
else:
from cms.envs.devstack import *
# Require a separate celery worker
CELERY_ALWAYS_EAGER = False

3
common/lib/xmodule/xmodule/modulestore/tests/mongo_connection.py
View File

@@ -7,4 +7,5 @@ overrides don't exist
import os
MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017'))
MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST', 'localhost')
MONGO_HOST = os.environ.get('EDXAPP_TEST_MONGO_HOST',
'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost')

3
common/lib/xmodule/xmodule/tests/test_video.py
View File

@@ -12,6 +12,7 @@ You can then use the CourseFactory and XModuleItemFactory as defined
in common/lib/xmodule/xmodule/modulestore/tests/factories.py to create
the course, section, subsection, unit, etc.
"""
import os
import unittest
import datetime
from uuid import uuid4
@@ -821,7 +822,7 @@ class VideoDescriptorIndexingTestCase(unittest.TestCase):
settings.CONTENTSTORE = {
'ENGINE': 'xmodule.contentstore.mongo.MongoContentStore',
'DOC_STORE_CONFIG': {
'host': 'localhost',
'host': 'edx.devstack.mongo' if 'BOK_CHOY_HOSTNAME' in os.environ else 'localhost',
'db': 'test_xcontent_%s' % uuid4().hex,
},
# allow for additional options that can be keyed on a name, e.g. 'trashcan'

7
lms/envs/devstack_optimized.py
View File

@@ -18,9 +18,14 @@ as they are for non-optimized devstack. Instead, update_assets must be
invoked each time that changes have been made.
"""
import os
########################## Devstack settings ###################################
from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import
if 'BOK_CHOY_HOSTNAME' in os.environ:
from .devstack_docker import * # pylint: disable=wildcard-import, unused-wildcard-import
else:
from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import
TEST_ROOT = REPO_ROOT / "test_root"

7
lms/envs/devstack_with_worker.py
View File

@@ -11,10 +11,15 @@ In two separate processes on devstack:
./manage.py lms celery worker --settings=devstack_with_worker
"""
import os
# We intentionally define lots of variables that aren't used, and
# want to import all variables from base settings files
# pylint: disable=wildcard-import, unused-wildcard-import
from lms.envs.devstack import *
if 'BOK_CHOY_HOSTNAME' in os.environ:
from lms.envs.devstack_docker import *
else:
from lms.envs.devstack import *
# Require a separate celery worker
CELERY_ALWAYS_EAGER = False

6
pavelib/assets.py
View File

@@ -799,7 +799,7 @@ def watch_assets(options):
# We only want Webpack to re-run on changes to its own entry points, not all JS files, so we use its own watcher
# instead of subclassing from Watchdog like the other watchers do
execute_webpack_watch(settings='devstack')
execute_webpack_watch(settings=Env.DEVSTACK_SETTINGS)
if not getattr(options, 'background', False):
# when running as a separate process, the main thread needs to loop
@@ -828,7 +828,7 @@ def update_assets(args):
help="lms or studio",
)
parser.add_argument(
'--settings', type=str, default="devstack",
'--settings', type=str, default=Env.DEVSTACK_SETTINGS,
help="Django settings module",
)
parser.add_argument(
@@ -861,7 +861,7 @@ def update_assets(args):
process_xmodule_assets()
process_npm_assets()
compile_coffeescript()
execute_webpack(prod=(args.settings != "devstack"), settings=args.settings)
execute_webpack(prod=(args.settings != Env.DEVSTACK_SETTINGS), settings=args.settings)
# Compile sass for themes and system
execute_compile_sass(args)

3
pavelib/i18n.py
View File

@@ -10,6 +10,7 @@ from path import Path as path
from paver.easy import cmdopts, needs, sh, task
from .utils.cmd import django_cmd
from .utils.envs import Env
from .utils.timer import timed
try:
@@ -17,7 +18,7 @@ try:
except ImportError:
colorize = lambda color, text: text
DEFAULT_SETTINGS = 'devstack'
DEFAULT_SETTINGS = Env.DEVSTACK_SETTINGS
@task

10
pavelib/paver_tests/test_assets.py
View File

@@ -10,6 +10,7 @@ from watchdog.observers.polling import PollingObserver
from pavelib.assets import COLLECTSTATIC_LOG_DIR_ARG, collect_assets
from ..utils.envs import Env
from .utils import PaverTestCase
ROOT_PATH = path(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
@@ -252,12 +253,12 @@ class TestCollectAssets(PaverTestCase):
if specified_log_loc is None:
collect_assets(
systems,
"devstack"
Env.DEVSTACK_SETTINGS
)
else:
collect_assets(
systems,
"devstack",
Env.DEVSTACK_SETTINGS,
**specified_log_dict
)
self.assertEqual(self.task_messages, expected_messages)
@@ -271,7 +272,7 @@ class TestCollectAssets(PaverTestCase):
systems = ["lms"]
kwargs = {COLLECTSTATIC_LOG_DIR_ARG: None}
expected_messages = self._set_expected_messages(log_location=expected_log_loc, systems=systems)
collect_assets(systems, "devstack", **kwargs)
collect_assets(systems, Env.DEVSTACK_SETTINGS, **kwargs)
self.assertEqual(self.task_messages, expected_messages)
def _set_expected_messages(self, log_location, systems):
@@ -284,8 +285,9 @@ class TestCollectAssets(PaverTestCase):
expected_messages = []
for sys in systems:
expected_messages.append(
'python manage.py {system} --settings=devstack collectstatic --noinput {log_loc}'.format(
'python manage.py {system} --settings={settings} collectstatic --noinput {log_loc}'.format(
system=sys,
settings=Env.DEVSTACK_SETTINGS,
log_loc=log_location
)
)

7
pavelib/paver_tests/test_i18n.py
View File

@@ -11,6 +11,7 @@ from paver.easy import call_task, task
import pavelib.i18n
from pavelib.paver_tests.utils import PaverTestCase
from pavelib.utils.envs import Env
TX_CONFIG_SIMPLE = """\
@@ -155,14 +156,14 @@ class TestI18nDummy(PaverTestCase):
"""
self.reset_task_messages()
os.environ['NO_PREREQ_INSTALL'] = "true"
call_task('pavelib.i18n.i18n_dummy', options={"settings": 'test'})
call_task('pavelib.i18n.i18n_dummy', options={"settings": Env.TEST_SETTINGS})
self.assertEquals(
self.task_messages,
[
u'i18n_tool extract',
u'i18n_tool dummy',
u'i18n_tool generate',
u'python manage.py lms --settings=test compilejsi18n',
u'python manage.py cms --settings=test compilejsi18n',
u'python manage.py lms --settings={} compilejsi18n'.format(Env.TEST_SETTINGS),
u'python manage.py cms --settings={} compilejsi18n'.format(Env.TEST_SETTINGS),
]
)

18
pavelib/paver_tests/test_servers.py
View File

@@ -102,7 +102,7 @@ class TestPaverServerTasks(PaverTestCase):
"""
options = server_options.copy()
is_optimized = options.get("optimized", False)
expected_settings = "devstack_optimized" if is_optimized else options.get("settings", "devstack")
expected_settings = "devstack_optimized" if is_optimized else options.get("settings", Env.DEVSTACK_SETTINGS)
# First test with LMS
options["system"] = "lms"
@@ -162,7 +162,7 @@ class TestPaverServerTasks(PaverTestCase):
"""
Test the "update_db" task.
"""
settings = options.get("settings", "devstack")
settings = options.get("settings", Env.DEVSTACK_SETTINGS)
call_task("pavelib.servers.update_db", options=options)
# pylint: disable=line-too-long
db_command = "NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{server} --traceback --pythonpath=. "
@@ -185,7 +185,7 @@ class TestPaverServerTasks(PaverTestCase):
"""
Test the "check_settings" task.
"""
settings = options.get("settings", "devstack")
settings = options.get("settings", Env.DEVSTACK_SETTINGS)
call_task("pavelib.servers.check_settings", args=[system, settings])
self.assertEquals(
self.task_messages,
@@ -231,19 +231,19 @@ class TestPaverServerTasks(PaverTestCase):
else:
call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options)
expected_messages = options.get("expected_messages", [])
expected_settings = settings if settings else "devstack"
expected_settings = settings if settings else Env.DEVSTACK_SETTINGS
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS
if not is_fast:
expected_messages.append(u"xmodule_assets common/static/xmodule")
expected_messages.append(u"install npm_assets")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
expected_messages.extend([c.format(settings=expected_asset_settings) for c in EXPECTED_PRINT_SETTINGS_COMMAND])
expected_messages.append(EXPECTED_WEBPACK_COMMAND.format(
node_env="production" if expected_asset_settings != "devstack" else "development",
node_env="production" if expected_asset_settings != Env.DEVSTACK_SETTINGS else "development",
static_root_lms=None,
static_root_cms=None
))
@@ -273,12 +273,12 @@ class TestPaverServerTasks(PaverTestCase):
is_fast = options.get("fast", False)
self.reset_task_messages()
call_task("pavelib.servers.run_all_servers", options=options)
expected_settings = settings if settings else "devstack"
expected_settings = settings if settings else Env.DEVSTACK_SETTINGS
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS
expected_messages = []
if not is_fast:
expected_messages.append(u"xmodule_assets common/static/xmodule")
@@ -286,7 +286,7 @@ class TestPaverServerTasks(PaverTestCase):
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=self.platform_root))
expected_messages.extend([c.format(settings=expected_asset_settings) for c in EXPECTED_PRINT_SETTINGS_COMMAND])
expected_messages.append(EXPECTED_WEBPACK_COMMAND.format(
node_env="production" if expected_asset_settings != "devstack" else "development",
node_env="production" if expected_asset_settings != Env.DEVSTACK_SETTINGS else "development",
static_root_lms=None,
static_root_cms=None
))

3
pavelib/servers.py
View File

@@ -10,11 +10,12 @@ from paver.easy import call_task, cmdopts, consume_args, needs, sh, task
from .assets import collect_assets
from .utils.cmd import django_cmd
from .utils.envs import Env
from .utils.process import run_multi_processes, run_process
from .utils.timer import timed
DEFAULT_PORT = {"lms": 8000, "studio": 8001}
DEFAULT_SETTINGS = 'devstack'
DEFAULT_SETTINGS = Env.DEVSTACK_SETTINGS
OPTIMIZED_SETTINGS = "devstack_optimized"
OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized"

2
pavelib/utils/envs.py
View File

@@ -69,6 +69,8 @@ class Env(object):
SERVER_HOST = os.environ.get('BOK_CHOY_HOSTNAME', '0.0.0.0')
USING_DOCKER = SERVER_HOST != '0.0.0.0'
SETTINGS = 'bok_choy_docker' if USING_DOCKER else 'bok_choy'
DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack'
TEST_SETTINGS = 'test_docker' if USING_DOCKER else 'test'
BOK_CHOY_SERVERS = {
'lms': {

2
pavelib/utils/test/suites/nose_suite.py
View File

@@ -119,7 +119,7 @@ class SystemTestSuite(NoseTestSuite):
self.processes = kwargs.get('processes', None)
self.randomize = kwargs.get('randomize', None)
self.settings = kwargs.get('settings', 'test')
self.settings = kwargs.get('settings', Env.TEST_SETTINGS)
if self.processes is None:
# Don't use multiprocessing by default