From 85814f0bbf75974f22264c84047b3a8984e49dd0 Mon Sep 17 00:00:00 2001 From: Andy Armstrong Date: Thu, 25 Jun 2015 18:03:01 -0400 Subject: [PATCH] Support running Studio with optimized assets --- cms/envs/bok_choy.py | 4 +- cms/envs/devstack_optimized.py | 40 ++++ cms/envs/test_static_optimized.py | 38 ++-- lms/envs/bok_choy.py | 2 +- lms/envs/devstack_optimized.py | 40 ++++ lms/envs/test_static_optimized.py | 39 +++- pavelib/assets.py | 10 +- pavelib/paver_tests/test_servers.py | 278 ++++++++++++++++++++++++++++ pavelib/paver_tests/utils.py | 55 ++++++ pavelib/servers.py | 134 +++++++++++--- pavelib/utils/process.py | 10 + 11 files changed, 590 insertions(+), 60 deletions(-) create mode 100644 cms/envs/devstack_optimized.py create mode 100644 lms/envs/devstack_optimized.py create mode 100644 pavelib/paver_tests/test_servers.py create mode 100644 pavelib/paver_tests/utils.py diff --git a/cms/envs/bok_choy.py b/cms/envs/bok_choy.py index 29f9accd97..8c7e7f8585 100644 --- a/cms/envs/bok_choy.py +++ b/cms/envs/bok_choy.py @@ -36,7 +36,7 @@ from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import INSTALLED_APPS += ('django_extensions',) # Redirect to the test_root folder within the repo -TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" # pylint: disable=no-value-for-parameter +TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath() LOG_DIR = (TEST_ROOT / "log").abspath() @@ -64,7 +64,7 @@ STATICFILES_FINDERS = ( 'staticfiles.finders.FileSystemFinder', ) STATICFILES_DIRS = ( - (TEST_ROOT / "staticfiles").abspath(), + (TEST_ROOT / "staticfiles" / "cms").abspath(), ) # Silence noisy logs diff --git a/cms/envs/devstack_optimized.py b/cms/envs/devstack_optimized.py new file mode 100644 index 0000000000..5694a96ff1 --- /dev/null +++ b/cms/envs/devstack_optimized.py @@ -0,0 +1,40 @@ +""" +Settings to run Studio in devstack using optimized static assets. + +This configuration changes Studio to use the optimized static assets generated for testing, +rather than picking up the files directly from the source tree. + +The following Paver command can be used to run Studio in optimized mode: + + paver devstack studio --optimized + +You can also generate the assets explicitly and then run Studio: + + paver update_assets cms --settings=test_static_optimized + paver devstack studio --settings=devstack_optimized --fast + +Note that changes to JavaScript assets will not be picked up automatically +as they are for non-optimized devstack. Instead, update_assets must be +invoked each time that changes have been made. +""" + +########################## Devstack settings ################################### + +from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import + +TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter + +############################ STATIC FILES ############################# + +# Enable debug so that static assets are served by Django +DEBUG = True + +# Serve static files at /static directly from the staticfiles directory under test root. +# Note: optimized files for testing are generated with settings from test_static_optimized +STATIC_URL = "/static/" +STATICFILES_FINDERS = ( + 'staticfiles.finders.FileSystemFinder', +) +STATICFILES_DIRS = ( + (TEST_ROOT / "staticfiles" / "cms").abspath(), +) diff --git a/cms/envs/test_static_optimized.py b/cms/envs/test_static_optimized.py index c2b333b547..ae5d91f521 100644 --- a/cms/envs/test_static_optimized.py +++ b/cms/envs/test_static_optimized.py @@ -10,36 +10,26 @@ support both generating static assets to a directory and also serving static from the same directory. """ -import os -from path import path # pylint: disable=no-name-in-module +# Start with the common settings +from .common import * # pylint: disable=wildcard-import, unused-wildcard-import -# Pylint gets confused by path.py instances, which report themselves as class -# objects. As a result, pylint applies the wrong regex in validating names, -# and throws spurious errors. Therefore, we disable invalid-name checking. -# pylint: disable=invalid-name +# Use an in-memory database since this settings file is only used for updating assets +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + }, +} +######################### Static file overrides #################################### -########################## Prod-like settings ################################### -# These should be as close as possible to the settings we use in production. -# As in prod, we read in environment and auth variables from JSON files. -# Unlike in prod, we use the JSON files stored in this repo. -# This is a convenience for ensuring (a) that we can consistently find the files -# and (b) that the files are the same in Jenkins as in local dev. -os.environ['SERVICE_VARIANT'] = 'bok_choy' -os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname() # pylint: disable=no-value-for-parameter - -from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import - -######################### Testing overrides #################################### - -# Redirects to the test_root folder within the repo -TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" # pylint: disable=no-value-for-parameter +# Redirect to the test_root folder within the repo +TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter LOG_DIR = (TEST_ROOT / "log").abspath() -# Stores the static files under test root so that they don't overwrite existing static assets -STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath() +# Store the static files under test root so that they don't overwrite existing static assets +STATIC_ROOT = (TEST_ROOT / "staticfiles" / "cms").abspath() -# Disables uglify when tests are running (used by build.js). +# Disable uglify when tests are running (used by build.js). # 1. Uglify is by far the slowest part of the build process # 2. Having full source code makes debugging tests easier for developers os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none' diff --git a/lms/envs/bok_choy.py b/lms/envs/bok_choy.py index 26e0852826..897029a772 100644 --- a/lms/envs/bok_choy.py +++ b/lms/envs/bok_choy.py @@ -75,7 +75,7 @@ EDXNOTES_PUBLIC_API = 'http://localhost:8042/api/v1' EDXNOTES_INTERNAL_API = 'http://localhost:8042/api/v1' # Enable django-pipeline and staticfiles -STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath() +STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath() # Silence noisy logs import logging diff --git a/lms/envs/devstack_optimized.py b/lms/envs/devstack_optimized.py new file mode 100644 index 0000000000..26b8cc3e21 --- /dev/null +++ b/lms/envs/devstack_optimized.py @@ -0,0 +1,40 @@ +""" +Settings to run LMS in devstack using optimized static assets. + +This configuration changes LMS to use the optimized static assets generated for testing, +rather than picking up the files directly from the source tree. + +The following Paver command can be used to run LMS in optimized mode: + + paver devstack lms --optimized + +You can also generate the assets explicitly and then run Studio: + + paver update_assets lms --settings=test_static_optimized + paver devstack lms --settings=devstack_optimized --fast + +Note that changes to JavaScript assets will not be picked up automatically +as they are for non-optimized devstack. Instead, update_assets must be +invoked each time that changes have been made. +""" + +########################## Devstack settings ################################### + +from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import + +TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter + +############################ STATIC FILES ############################# + +# Enable debug so that static assets are served by Django +DEBUG = True + +# Serve static files at /static directly from the staticfiles directory under test root. +# Note: optimized files for testing are generated with settings from test_static_optimized +STATIC_URL = "/static/" +STATICFILES_FINDERS = ( + 'staticfiles.finders.FileSystemFinder', +) +STATICFILES_DIRS = ( + (TEST_ROOT / "staticfiles" / "lms").abspath(), +) diff --git a/lms/envs/test_static_optimized.py b/lms/envs/test_static_optimized.py index 61c99b6590..8ee4091d47 100644 --- a/lms/envs/test_static_optimized.py +++ b/lms/envs/test_static_optimized.py @@ -1,16 +1,47 @@ """ Settings used when generating static assets for use in tests. -Bok Choy uses two different settings files: +For example, Bok Choy uses two different settings files: 1. test_static_optimized is used when invoking collectstatic 2. bok_choy is used when running CMS and LMS Note: it isn't possible to have a single settings file, because Django doesn't support both generating static assets to a directory and also serving static from the same directory. - """ -# TODO: update the Bok Choy tests to run with optimized static assets (as is done in Studio) +# Start with the common settings +from .common import * # pylint: disable=wildcard-import, unused-wildcard-import -from .bok_choy import * # pylint: disable=wildcard-import, unused-wildcard-import +# Use an in-memory database since this settings file is only used for updating assets +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + }, + +} + +# Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up +XQUEUE_INTERFACE = { + "url": "https://sandbox-xqueue.edx.org", + "django_auth": { + "username": "lms", + "password": "***REMOVED***" + }, + "basic_auth": ('anant', 'agarwal'), +} + + +######################### Static file overrides #################################### + +# Redirect to the test_root folder within the repo +TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter +LOG_DIR = (TEST_ROOT / "log").abspath() + +# Store the static files under test root so that they don't overwrite existing static assets +STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath() + +# Disable uglify when tests are running (used by build.js). +# 1. Uglify is by far the slowest part of the build process +# 2. Having full source code makes debugging tests easier for developers +os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none' diff --git a/pavelib/assets.py b/pavelib/assets.py index a49a586fd3..c159986c35 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -3,12 +3,12 @@ Asset compilation and collection. """ from __future__ import print_function import argparse +from paver import tasks from paver.easy import sh, path, task, cmdopts, needs, consume_args, call_task, no_help from watchdog.observers import Observer from watchdog.events import PatternMatchingEventHandler import glob import traceback -import os from .utils.envs import Env from .utils.cmd import cmd, django_cmd @@ -191,6 +191,10 @@ def watch_assets(options): """ Watch for changes to asset files, and regenerate js/css """ + # Don't watch assets when performing a dry run + if tasks.environment.dry_run: + return + observer = Observer() CoffeeScriptWatcher().register(observer) @@ -246,10 +250,10 @@ def update_assets(args): compile_templated_sass(args.system, args.settings) process_xmodule_assets() compile_coffeescript() - call_task('compile_sass', options={'debug': args.debug}) + call_task('pavelib.assets.compile_sass', options={'debug': args.debug}) if args.collect: collect_assets(args.system, args.settings) if args.watch: - call_task('watch_assets', options={'background': not args.debug}) + call_task('pavelib.assets.watch_assets', options={'background': not args.debug}) diff --git a/pavelib/paver_tests/test_servers.py b/pavelib/paver_tests/test_servers.py new file mode 100644 index 0000000000..6c3267dd23 --- /dev/null +++ b/pavelib/paver_tests/test_servers.py @@ -0,0 +1,278 @@ +"""Unit tests for the Paver server tasks.""" + +import ddt +import os +from paver.easy import call_task + +from .utils import PaverTestCase + +EXPECTED_COFFEE_COMMAND = ( + "node_modules/.bin/coffee --compile `find {platform_root}/lms " + "{platform_root}/cms {platform_root}/common -type f -name \"*.coffee\"`" +) +EXPECTED_SASS_COMMAND = ( + "sass --update --cache-location /tmp/sass-cache --default-encoding utf-8 --style compressed" + " --quiet --load-path common/static --load-path common/static/sass" + " --load-path lms/static/sass --load-path lms/static/certificates/sass" + " --load-path cms/static/sass --load-path common/static/sass" + " lms/static/sass:lms/static/css lms/static/certificates/sass:lms/static/certificates/css" + " cms/static/sass:cms/static/css common/static/sass:common/static/css" +) +EXPECTED_PREPROCESS_ASSETS_COMMAND = ( + "python manage.py {system} --settings={asset_settings} preprocess_assets" +) +EXPECTED_COLLECT_STATIC_COMMAND = ( + "python manage.py {system} --settings={asset_settings} collectstatic --noinput > /dev/null" +) +EXPECTED_CELERY_COMMAND = ( + "python manage.py lms --settings={settings} celery worker --beat --loglevel=INFO --pythonpath=." +) +EXPECTED_RUN_SERVER_COMMAND = ( + "python manage.py {system} --settings={settings} runserver --traceback --pythonpath=. 0.0.0.0:{port}" +) + + +@ddt.ddt +class TestPaverServerTasks(PaverTestCase): + """ + Test the Paver server tasks. + """ + @ddt.data( + [{}], + [{"settings": "aws"}], + [{"asset-settings": "test_static_optimized"}], + [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}], + [{"fast": True}], + [{"port": 8030}], + ) + @ddt.unpack + def test_lms(self, options): + """ + Test the "devstack" task. + """ + self.verify_server_task("lms", options) + + @ddt.data( + [{}], + [{"settings": "aws"}], + [{"asset-settings": "test_static_optimized"}], + [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}], + [{"fast": True}], + [{"port": 8031}], + ) + @ddt.unpack + def test_studio(self, options): + """ + Test the "devstack" task. + """ + self.verify_server_task("studio", options) + + @ddt.data( + [{}], + [{"settings": "aws"}], + [{"asset-settings": "test_static_optimized"}], + [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}], + [{"fast": True}], + [{"optimized": True}], + [{"optimized": True, "fast": True}], + [{"no-contracts": True}], + ) + @ddt.unpack + def test_devstack(self, server_options): + """ + Test the "devstack" task. + """ + options = server_options.copy() + + # First test with LMS + options["system"] = "lms" + self.verify_server_task("devstack", options, contracts_default=True) + + # Then test with Studio + options["system"] = "cms" + self.verify_server_task("devstack", options, contracts_default=True) + + @ddt.data( + [{}], + [{"settings": "aws"}], + [{"asset_settings": "test_static_optimized"}], + [{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}], + [{"fast": True}], + [{"optimized": True}], + [{"optimized": True, "fast": True}], + ) + @ddt.unpack + def test_run_all_servers(self, options): + """ + Test the "run_all_servers" task. + """ + self.verify_run_all_servers_task(options) + + @ddt.data( + [{}], + [{"settings": "aws"}], + ) + @ddt.unpack + def test_celery(self, options): + """ + Test the "celery" task. + """ + settings = options.get("settings", "dev_with_worker") + call_task("pavelib.servers.celery", options=options) + self.assertEquals(self.task_messages, [EXPECTED_CELERY_COMMAND.format(settings=settings)]) + + @ddt.data( + [{}], + [{"settings": "aws"}], + ) + @ddt.unpack + def test_update_db(self, options): + """ + Test the "update_db" task. + """ + settings = options.get("settings", "devstack") + call_task("pavelib.servers.update_db", options=options) + db_command = "python manage.py {server} --settings={settings} syncdb --migrate --traceback --pythonpath=." + self.assertEquals( + self.task_messages, + [ + db_command.format(server="lms", settings=settings), + db_command.format(server="cms", settings=settings), + ] + ) + + @ddt.data( + ["lms", {}], + ["lms", {"settings": "aws"}], + ["cms", {}], + ["cms", {"settings": "aws"}], + ) + @ddt.unpack + def test_check_settings(self, system, options): + """ + Test the "check_settings" task. + """ + settings = options.get("settings", "devstack") + call_task("pavelib.servers.check_settings", args=[system, settings]) + self.assertEquals( + self.task_messages, + [ + "echo 'import {system}.envs.{settings}' " + "| python manage.py {system} --settings={settings} shell --plain --pythonpath=.".format( + system=system, settings=settings + ), + ] + ) + + def verify_server_task(self, task_name, options, contracts_default=False): + """ + Verify the output of a server task. + """ + settings = options.get("settings", None) + asset_settings = options.get("asset-settings", None) + is_optimized = options.get("optimized", False) + is_fast = options.get("fast", False) + no_contracts = options.get("no-contracts", not contracts_default) + if task_name == "devstack": + system = options.get("system") + elif task_name == "studio": + system = "cms" + else: + system = "lms" + port = options.get("port", "8000" if system == "lms" else "8001") + self.reset_task_messages() + if task_name == "devstack": + args = ["studio" if system == "cms" else system] + if settings: + args.append("--settings={settings}".format(settings=settings)) + if asset_settings: + args.append("--asset-settings={asset_settings}".format(asset_settings=asset_settings)) + if is_optimized: + args.append("--optimized") + if is_fast: + args.append("--fast") + if no_contracts: + args.append("--no-contracts") + call_task("pavelib.servers.devstack", args=args) + else: + call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options) + expected_messages = [] + expected_settings = settings if settings else "devstack" + expected_asset_settings = asset_settings if asset_settings else expected_settings + if is_optimized: + expected_settings = "devstack_optimized" + expected_asset_settings = "test_static_optimized" + expected_collect_static = not is_fast and expected_settings != "devstack" + platform_root = os.getcwd() + if not is_fast: + expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format( + system=system, asset_settings=expected_asset_settings + )) + expected_messages.append("xmodule_assets common/static/xmodule") + expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root)) + expected_messages.append(EXPECTED_SASS_COMMAND) + if expected_collect_static: + expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format( + system=system, asset_settings=expected_asset_settings + )) + expected_run_server_command = EXPECTED_RUN_SERVER_COMMAND.format( + system=system, + settings=expected_settings, + port=port, + ) + if not no_contracts: + expected_run_server_command += " --contracts" + expected_messages.append(expected_run_server_command) + self.assertEquals(self.task_messages, expected_messages) + + def verify_run_all_servers_task(self, options): + """ + Verify the output of a server task. + """ + settings = options.get("settings", None) + asset_settings = options.get("asset_settings", None) + is_optimized = options.get("optimized", False) + is_fast = options.get("fast", False) + self.reset_task_messages() + call_task("pavelib.servers.run_all_servers", options=options) + expected_settings = settings if settings else "devstack" + expected_asset_settings = asset_settings if asset_settings else expected_settings + if is_optimized: + expected_settings = "devstack_optimized" + expected_asset_settings = "test_static_optimized" + expected_collect_static = not is_fast and expected_settings != "devstack" + platform_root = os.getcwd() + expected_messages = [] + if not is_fast: + expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format( + system="lms", asset_settings=expected_asset_settings + )) + expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format( + system="cms", asset_settings=expected_asset_settings + )) + expected_messages.append("xmodule_assets common/static/xmodule") + expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root)) + expected_messages.append(EXPECTED_SASS_COMMAND) + if expected_collect_static: + expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format( + system="lms", asset_settings=expected_asset_settings + )) + expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format( + system="cms", asset_settings=expected_asset_settings + )) + expected_messages.append( + EXPECTED_RUN_SERVER_COMMAND.format( + system="lms", + settings=expected_settings, + port=8000, + ) + ) + expected_messages.append( + EXPECTED_RUN_SERVER_COMMAND.format( + system="cms", + settings=expected_settings, + port=8001, + ) + ) + expected_messages.append(EXPECTED_CELERY_COMMAND.format(settings="dev_with_worker")) + self.assertEquals(self.task_messages, expected_messages) diff --git a/pavelib/paver_tests/utils.py b/pavelib/paver_tests/utils.py new file mode 100644 index 0000000000..76c7185ba5 --- /dev/null +++ b/pavelib/paver_tests/utils.py @@ -0,0 +1,55 @@ +"""Unit tests for the Paver server tasks.""" + +import os +from paver import tasks +from unittest import TestCase + + +class PaverTestCase(TestCase): + """ + Base class for Paver test cases. + """ + def setUp(self): + super(PaverTestCase, self).setUp() + + # Show full length diffs upon test failure + self.maxDiff = None # pylint: disable=invalid-name + + # Create a mock Paver environment + tasks.environment = MockEnvironment() + + # Don't run pre-reqs + os.environ['NO_PREREQ_INSTALL'] = 'true' + + def tearDown(self): + super(PaverTestCase, self).tearDown() + tasks.environment = tasks.Environment() + del os.environ['NO_PREREQ_INSTALL'] + + @property + def task_messages(self): + """Returns the messages output by the Paver task.""" + return tasks.environment.messages + + def reset_task_messages(self): + """Clear the recorded message""" + tasks.environment.messages = [] + + +class MockEnvironment(tasks.Environment): + """ + Mock environment that collects information about Paver commands. + """ + def __init__(self): + super(MockEnvironment, self).__init__() + self.dry_run = True + self.messages = [] + + def info(self, message, *args): + """Capture any messages that have been recorded""" + if args: + output = message % args + else: + output = message + if not output.startswith("--->"): + self.messages.append(output) diff --git a/pavelib/servers.py b/pavelib/servers.py index 8076a0e46e..a790ef83dd 100644 --- a/pavelib/servers.py +++ b/pavelib/servers.py @@ -2,24 +2,36 @@ Run and manage servers for local development. """ from __future__ import print_function -import sys import argparse from paver.easy import * + +from .assets import collect_assets from .utils.cmd import django_cmd from .utils.process import run_process, run_multi_processes DEFAULT_PORT = {"lms": 8000, "studio": 8001} DEFAULT_SETTINGS = 'devstack' +OPTIMIZED_SETTINGS = "devstack_optimized" +OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized" + +ASSET_SETTINGS_HELP = ( + "Settings file used for updating assets. Defaults to the value of the settings variable if not provided." +) -def run_server(system, settings=None, port=None, skip_assets=False, contracts=False): - """ - Start the server for the specified `system` (lms or studio). - `settings` is the Django settings module to use; if not provided, use the default. - `port` is the port to run the server on; if not provided, use the default port for the system. +def run_server( + system, fast=False, settings=None, asset_settings=None, port=None, contracts=False +): + """Start the server for LMS or Studio. - If `skip_assets` is True, skip the asset compilation step. + Args: + system (str): The system to be run (lms or studio). + fast (bool): If true, then start the server immediately without updating assets (defaults to False). + settings (str): The Django settings module to use; if not provided, use the default. + asset_settings (str) The settings to use when generating assets. If not provided, assets are not generated. + port (str): The port number to run the server on. If not provided, uses the default port for the system. + contracts (bool) If true then PyContracts is enabled (defaults to False). """ if system not in ['lms', 'studio']: print("System must be either lms or studio", file=sys.stderr) @@ -28,9 +40,13 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa if not settings: settings = DEFAULT_SETTINGS - if not skip_assets: - # Local dev settings use staticfiles to serve assets, so we can skip the collecstatic step - args = [system, '--settings={}'.format(settings), '--skip-collect', '--watch'] + if not fast and asset_settings: + args = [system, '--settings={}'.format(asset_settings), '--watch'] + # The default settings use DEBUG mode for running the server which means that + # the optimized assets are ignored, so we skip collectstatic in that case + # to save time. + if settings == DEFAULT_SETTINGS: + args.append('--skip-collect') call_task('pavelib.assets.update_assets', args=args) if port is None: @@ -48,40 +64,55 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa @needs('pavelib.prereqs.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings"), + ("asset-settings=", "a", ASSET_SETTINGS_HELP), ("port=", "p", "Port"), - ("fast", "f", "Skip updating assets") + ("fast", "f", "Skip updating assets"), ]) def lms(options): """ Run the LMS server. """ - settings = getattr(options, 'settings', None) + settings = getattr(options, 'settings', DEFAULT_SETTINGS) + asset_settings = getattr(options, 'asset-settings', settings) port = getattr(options, 'port', None) fast = getattr(options, 'fast', False) - run_server('lms', settings=settings, port=port, skip_assets=fast) + run_server( + 'lms', + fast=fast, + settings=settings, + asset_settings=asset_settings, + port=port, + ) @task @needs('pavelib.prereqs.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings"), + ("asset-settings=", "a", ASSET_SETTINGS_HELP), ("port=", "p", "Port"), - ("fast", "f", "Skip updating assets") + ("fast", "f", "Skip updating assets"), ]) def studio(options): """ Run the Studio server. """ - settings = getattr(options, 'settings', None) + settings = getattr(options, 'settings', DEFAULT_SETTINGS) + asset_settings = getattr(options, 'asset-settings', settings) port = getattr(options, 'port', None) fast = getattr(options, 'fast', False) - run_server('studio', settings=settings, port=port, skip_assets=fast) + run_server( + 'studio', + fast=fast, + settings=settings, + asset_settings=asset_settings, + port=port, + ) @task @needs('pavelib.prereqs.install_prereqs') @consume_args -@no_help def devstack(args): """ Start the devstack lms or studio server @@ -89,6 +120,9 @@ def devstack(args): parser = argparse.ArgumentParser(prog='paver devstack') parser.add_argument('system', type=str, nargs=1, help="lms or studio") parser.add_argument('--fast', action='store_true', default=False, help="Skip updating assets") + parser.add_argument('--optimized', action='store_true', default=False, help="Run with optimized assets") + parser.add_argument('--settings', type=str, default=DEFAULT_SETTINGS, help="Settings file") + parser.add_argument('--asset-settings', type=str, default=None, help=ASSET_SETTINGS_HELP) parser.add_argument( '--no-contracts', action='store_true', @@ -96,7 +130,18 @@ def devstack(args): help="Disable contracts. By default, they're enabled in devstack." ) args = parser.parse_args(args) - run_server(args.system[0], settings='devstack', skip_assets=args.fast, contracts=(not args.no_contracts)) + settings = args.settings + asset_settings = args.asset_settings if args.asset_settings else settings + if args.optimized: + settings = OPTIMIZED_SETTINGS + asset_settings = OPTIMIZED_ASSETS_SETTINGS + run_server( + args.system[0], + fast=args.fast, + settings=settings, + asset_settings=asset_settings, + contracts=not args.no_contracts, + ) @task @@ -116,33 +161,70 @@ def celery(options): @needs('pavelib.prereqs.install_prereqs') @cmdopts([ ("settings=", "s", "Django settings for both LMS and Studio"), + ("asset_settings=", "a", "Django settings for updating assets for both LMS and Studio (defaults to settings)"), ("worker_settings=", "w", "Celery worker Django settings"), ("fast", "f", "Skip updating assets"), + ("optimized", "o", "Run with optimized assets"), ("settings_lms=", "l", "Set LMS only, overriding the value from --settings (if provided)"), + ("asset_settings_lms=", "al", "Set LMS only, overriding the value from --asset_settings (if provided)"), ("settings_cms=", "c", "Set Studio only, overriding the value from --settings (if provided)"), + ("asset_settings_cms=", "ac", "Set Studio only, overriding the value from --asset_settings (if provided)"), ]) def run_all_servers(options): """ Runs Celery workers, Studio, and LMS. """ settings = getattr(options, 'settings', DEFAULT_SETTINGS) - settings_lms = getattr(options, 'settings_lms', settings) - settings_cms = getattr(options, 'settings_cms', settings) + asset_settings = getattr(options, 'asset_settings', settings) worker_settings = getattr(options, 'worker_settings', 'dev_with_worker') fast = getattr(options, 'fast', False) + optimized = getattr(options, 'optimized', False) + + if optimized: + settings = OPTIMIZED_SETTINGS + asset_settings = OPTIMIZED_ASSETS_SETTINGS + + settings_lms = getattr(options, 'settings_lms', settings) + settings_cms = getattr(options, 'settings_cms', settings) + asset_settings_lms = getattr(options, 'asset_settings_lms', asset_settings) + asset_settings_cms = getattr(options, 'asset_settings_cms', asset_settings) if not fast: - args = ['lms', '--settings={}'.format(settings_lms), '--skip-collect'] + # First update assets for both LMS and Studio but don't collect static yet + args = [ + 'lms', 'studio', + '--settings={}'.format(asset_settings), + '--skip-collect' + ] call_task('pavelib.assets.update_assets', args=args) - args = ['studio', '--settings={}'.format(settings_cms), '--skip-collect'] - call_task('pavelib.assets.update_assets', args=args) + # Now collect static for each system separately with the appropriate settings. + # Note that the default settings use DEBUG mode for running the server which + # means that the optimized assets are ignored, so we skip collectstatic in that + # case to save time. + if settings != DEFAULT_SETTINGS: + collect_assets(['lms'], asset_settings_lms) + collect_assets(['studio'], asset_settings_cms) + # Install an asset watcher to regenerate files that change call_task('pavelib.assets.watch_assets', options={'background': True}) + + # Start up LMS, CMS and Celery + lms_port = DEFAULT_PORT['lms'] + cms_port = DEFAULT_PORT['studio'] + lms_runserver_args = ["0.0.0.0:{}".format(lms_port)] + cms_runserver_args = ["0.0.0.0:{}".format(cms_port)] + run_multi_processes([ - django_cmd('lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['lms'])), - django_cmd('studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['studio'])), - django_cmd('lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.') + django_cmd( + 'lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', *lms_runserver_args + ), + django_cmd( + 'studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', *cms_runserver_args + ), + django_cmd( + 'lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.' + ) ]) diff --git a/pavelib/utils/process.py b/pavelib/utils/process.py index a687218723..4d4cb9e3e9 100644 --- a/pavelib/utils/process.py +++ b/pavelib/utils/process.py @@ -9,6 +9,8 @@ import signal import psutil import atexit +from paver import tasks + def kill_process(proc): """ @@ -41,6 +43,14 @@ def run_multi_processes(cmd_list, out_log=None, err_log=None): err_log_file = open(err_log, 'w') kwargs['stderr'] = err_log_file + # If the user is performing a dry run of a task, then just log + # the command strings and return so that no destructive operations + # are performed. + if tasks.environment.dry_run: + for cmd in cmd_list: + tasks.environment.info(cmd) + return + try: for cmd in cmd_list: pids.extend([subprocess.Popen(cmd, **kwargs)])