Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Support running Studio with optimized assets

Browse Source
This commit is contained in:
Andy Armstrong
2015-06-25 18:03:01 -04:00
parent 072ae99d12
commit 85814f0bbf
11 changed files with 590 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
cms/envs/bok_choy.py
View File

@@ -36,7 +36,7 @@ from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import
INSTALLED_APPS += ('django_extensions',)
# Redirect to the test_root folder within the repo
TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" # pylint: disable=no-value-for-parameter
TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter
GITHUB_REPO_ROOT = (TEST_ROOT / "data").abspath()
LOG_DIR = (TEST_ROOT / "log").abspath()
@@ -64,7 +64,7 @@ STATICFILES_FINDERS = (
'staticfiles.finders.FileSystemFinder',
)
STATICFILES_DIRS = (
(TEST_ROOT / "staticfiles").abspath(),
(TEST_ROOT / "staticfiles" / "cms").abspath(),
)
# Silence noisy logs

40
cms/envs/devstack_optimized.py Normal file
View File

@@ -0,0 +1,40 @@
"""
Settings to run Studio in devstack using optimized static assets.
This configuration changes Studio to use the optimized static assets generated for testing,
rather than picking up the files directly from the source tree.
The following Paver command can be used to run Studio in optimized mode:
paver devstack studio --optimized
You can also generate the assets explicitly and then run Studio:
paver update_assets cms --settings=test_static_optimized
paver devstack studio --settings=devstack_optimized --fast
Note that changes to JavaScript assets will not be picked up automatically
as they are for non-optimized devstack. Instead, update_assets must be
invoked each time that changes have been made.
"""
########################## Devstack settings ###################################
from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import
TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter
############################ STATIC FILES #############################
# Enable debug so that static assets are served by Django
DEBUG = True
# Serve static files at /static directly from the staticfiles directory under test root.
# Note: optimized files for testing are generated with settings from test_static_optimized
STATIC_URL = "/static/"
STATICFILES_FINDERS = (
'staticfiles.finders.FileSystemFinder',
)
STATICFILES_DIRS = (
(TEST_ROOT / "staticfiles" / "cms").abspath(),
)

38
cms/envs/test_static_optimized.py
View File

@@ -10,36 +10,26 @@ support both generating static assets to a directory and also serving static
from the same directory.
"""
import os
from path import path # pylint: disable=no-name-in-module
# Start with the common settings
from .common import * # pylint: disable=wildcard-import, unused-wildcard-import
# Pylint gets confused by path.py instances, which report themselves as class
# objects. As a result, pylint applies the wrong regex in validating names,
# and throws spurious errors. Therefore, we disable invalid-name checking.
# pylint: disable=invalid-name
# Use an in-memory database since this settings file is only used for updating assets
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
},
}
######################### Static file overrides ####################################
########################## Prod-like settings ###################################
# These should be as close as possible to the settings we use in production.
# As in prod, we read in environment and auth variables from JSON files.
# Unlike in prod, we use the JSON files stored in this repo.
# This is a convenience for ensuring (a) that we can consistently find the files
# and (b) that the files are the same in Jenkins as in local dev.
os.environ['SERVICE_VARIANT'] = 'bok_choy'
os.environ['CONFIG_ROOT'] = path(__file__).abspath().dirname() # pylint: disable=no-value-for-parameter
from .aws import * # pylint: disable=wildcard-import, unused-wildcard-import
######################### Testing overrides ####################################
# Redirects to the test_root folder within the repo
TEST_ROOT = CONFIG_ROOT.dirname().dirname() / "test_root" # pylint: disable=no-value-for-parameter
# Redirect to the test_root folder within the repo
TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter
LOG_DIR = (TEST_ROOT / "log").abspath()
# Stores the static files under test root so that they don't overwrite existing static assets
STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
# Store the static files under test root so that they don't overwrite existing static assets
STATIC_ROOT = (TEST_ROOT / "staticfiles" / "cms").abspath()
# Disables uglify when tests are running (used by build.js).
# Disable uglify when tests are running (used by build.js).
# 1. Uglify is by far the slowest part of the build process
# 2. Having full source code makes debugging tests easier for developers
os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none'

2
lms/envs/bok_choy.py
View File

@@ -75,7 +75,7 @@ EDXNOTES_PUBLIC_API = 'http://localhost:8042/api/v1'
EDXNOTES_INTERNAL_API = 'http://localhost:8042/api/v1'
# Enable django-pipeline and staticfiles
STATIC_ROOT = (TEST_ROOT / "staticfiles").abspath()
STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath()
# Silence noisy logs
import logging

40
lms/envs/devstack_optimized.py Normal file
View File

@@ -0,0 +1,40 @@
"""
Settings to run LMS in devstack using optimized static assets.
This configuration changes LMS to use the optimized static assets generated for testing,
rather than picking up the files directly from the source tree.
The following Paver command can be used to run LMS in optimized mode:
paver devstack lms --optimized
You can also generate the assets explicitly and then run Studio:
paver update_assets lms --settings=test_static_optimized
paver devstack lms --settings=devstack_optimized --fast
Note that changes to JavaScript assets will not be picked up automatically
as they are for non-optimized devstack. Instead, update_assets must be
invoked each time that changes have been made.
"""
########################## Devstack settings ###################################
from .devstack import * # pylint: disable=wildcard-import, unused-wildcard-import
TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter
############################ STATIC FILES #############################
# Enable debug so that static assets are served by Django
DEBUG = True
# Serve static files at /static directly from the staticfiles directory under test root.
# Note: optimized files for testing are generated with settings from test_static_optimized
STATIC_URL = "/static/"
STATICFILES_FINDERS = (
'staticfiles.finders.FileSystemFinder',
)
STATICFILES_DIRS = (
(TEST_ROOT / "staticfiles" / "lms").abspath(),
)

39
lms/envs/test_static_optimized.py
View File

@@ -1,16 +1,47 @@
"""
Settings used when generating static assets for use in tests.
Bok Choy uses two different settings files:
For example, Bok Choy uses two different settings files:
1. test_static_optimized is used when invoking collectstatic
2. bok_choy is used when running CMS and LMS
Note: it isn't possible to have a single settings file, because Django doesn't
support both generating static assets to a directory and also serving static
from the same directory.
"""
# TODO: update the Bok Choy tests to run with optimized static assets (as is done in Studio)
# Start with the common settings
from .common import * # pylint: disable=wildcard-import, unused-wildcard-import
from .bok_choy import * # pylint: disable=wildcard-import, unused-wildcard-import
# Use an in-memory database since this settings file is only used for updating assets
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
},
}
# Provide a dummy XQUEUE_INTERFACE setting as LMS expects it to exist on start up
XQUEUE_INTERFACE = {
"url": "https://sandbox-xqueue.edx.org",
"django_auth": {
"username": "lms",
"password": "***REMOVED***"
},
"basic_auth": ('anant', 'agarwal'),
}
######################### Static file overrides ####################################
# Redirect to the test_root folder within the repo
TEST_ROOT = REPO_ROOT / "test_root" # pylint: disable=no-value-for-parameter
LOG_DIR = (TEST_ROOT / "log").abspath()
# Store the static files under test root so that they don't overwrite existing static assets
STATIC_ROOT = (TEST_ROOT / "staticfiles" / "lms").abspath()
# Disable uglify when tests are running (used by build.js).
# 1. Uglify is by far the slowest part of the build process
# 2. Having full source code makes debugging tests easier for developers
os.environ['REQUIRE_BUILD_PROFILE_OPTIMIZE'] = 'none'

10
pavelib/assets.py
View File

@@ -3,12 +3,12 @@ Asset compilation and collection.
"""
from __future__ import print_function
import argparse
from paver import tasks
from paver.easy import sh, path, task, cmdopts, needs, consume_args, call_task, no_help
from watchdog.observers import Observer
from watchdog.events import PatternMatchingEventHandler
import glob
import traceback
import os
from .utils.envs import Env
from .utils.cmd import cmd, django_cmd
@@ -191,6 +191,10 @@ def watch_assets(options):
"""
Watch for changes to asset files, and regenerate js/css
"""
# Don't watch assets when performing a dry run
if tasks.environment.dry_run:
return
observer = Observer()
CoffeeScriptWatcher().register(observer)
@@ -246,10 +250,10 @@ def update_assets(args):
compile_templated_sass(args.system, args.settings)
process_xmodule_assets()
compile_coffeescript()
call_task('compile_sass', options={'debug': args.debug})
call_task('pavelib.assets.compile_sass', options={'debug': args.debug})
if args.collect:
collect_assets(args.system, args.settings)
if args.watch:
call_task('watch_assets', options={'background': not args.debug})
call_task('pavelib.assets.watch_assets', options={'background': not args.debug})

278
pavelib/paver_tests/test_servers.py Normal file
View File

@@ -0,0 +1,278 @@
"""Unit tests for the Paver server tasks."""
import ddt
import os
from paver.easy import call_task
from .utils import PaverTestCase
EXPECTED_COFFEE_COMMAND = (
"node_modules/.bin/coffee --compile `find {platform_root}/lms "
"{platform_root}/cms {platform_root}/common -type f -name \"*.coffee\"`"
)
EXPECTED_SASS_COMMAND = (
"sass --update --cache-location /tmp/sass-cache --default-encoding utf-8 --style compressed"
" --quiet --load-path common/static --load-path common/static/sass"
" --load-path lms/static/sass --load-path lms/static/certificates/sass"
" --load-path cms/static/sass --load-path common/static/sass"
" lms/static/sass:lms/static/css lms/static/certificates/sass:lms/static/certificates/css"
" cms/static/sass:cms/static/css common/static/sass:common/static/css"
)
EXPECTED_PREPROCESS_ASSETS_COMMAND = (
"python manage.py {system} --settings={asset_settings} preprocess_assets"
)
EXPECTED_COLLECT_STATIC_COMMAND = (
"python manage.py {system} --settings={asset_settings} collectstatic --noinput > /dev/null"
)
EXPECTED_CELERY_COMMAND = (
"python manage.py lms --settings={settings} celery worker --beat --loglevel=INFO --pythonpath=."
)
EXPECTED_RUN_SERVER_COMMAND = (
"python manage.py {system} --settings={settings} runserver --traceback --pythonpath=. 0.0.0.0:{port}"
)
@ddt.ddt
class TestPaverServerTasks(PaverTestCase):
"""
Test the Paver server tasks.
"""
@ddt.data(
[{}],
[{"settings": "aws"}],
[{"asset-settings": "test_static_optimized"}],
[{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
[{"fast": True}],
[{"port": 8030}],
)
@ddt.unpack
def test_lms(self, options):
"""
Test the "devstack" task.
"""
self.verify_server_task("lms", options)
@ddt.data(
[{}],
[{"settings": "aws"}],
[{"asset-settings": "test_static_optimized"}],
[{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
[{"fast": True}],
[{"port": 8031}],
)
@ddt.unpack
def test_studio(self, options):
"""
Test the "devstack" task.
"""
self.verify_server_task("studio", options)
@ddt.data(
[{}],
[{"settings": "aws"}],
[{"asset-settings": "test_static_optimized"}],
[{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
[{"fast": True}],
[{"optimized": True}],
[{"optimized": True, "fast": True}],
[{"no-contracts": True}],
)
@ddt.unpack
def test_devstack(self, server_options):
"""
Test the "devstack" task.
"""
options = server_options.copy()
# First test with LMS
options["system"] = "lms"
self.verify_server_task("devstack", options, contracts_default=True)
# Then test with Studio
options["system"] = "cms"
self.verify_server_task("devstack", options, contracts_default=True)
@ddt.data(
[{}],
[{"settings": "aws"}],
[{"asset_settings": "test_static_optimized"}],
[{"settings": "devstack_optimized", "asset-settings": "test_static_optimized"}],
[{"fast": True}],
[{"optimized": True}],
[{"optimized": True, "fast": True}],
)
@ddt.unpack
def test_run_all_servers(self, options):
"""
Test the "run_all_servers" task.
"""
self.verify_run_all_servers_task(options)
@ddt.data(
[{}],
[{"settings": "aws"}],
)
@ddt.unpack
def test_celery(self, options):
"""
Test the "celery" task.
"""
settings = options.get("settings", "dev_with_worker")
call_task("pavelib.servers.celery", options=options)
self.assertEquals(self.task_messages, [EXPECTED_CELERY_COMMAND.format(settings=settings)])
@ddt.data(
[{}],
[{"settings": "aws"}],
)
@ddt.unpack
def test_update_db(self, options):
"""
Test the "update_db" task.
"""
settings = options.get("settings", "devstack")
call_task("pavelib.servers.update_db", options=options)
db_command = "python manage.py {server} --settings={settings} syncdb --migrate --traceback --pythonpath=."
self.assertEquals(
self.task_messages,
[
db_command.format(server="lms", settings=settings),
db_command.format(server="cms", settings=settings),
]
)
@ddt.data(
["lms", {}],
["lms", {"settings": "aws"}],
["cms", {}],
["cms", {"settings": "aws"}],
)
@ddt.unpack
def test_check_settings(self, system, options):
"""
Test the "check_settings" task.
"""
settings = options.get("settings", "devstack")
call_task("pavelib.servers.check_settings", args=[system, settings])
self.assertEquals(
self.task_messages,
[
"echo 'import {system}.envs.{settings}' "
"| python manage.py {system} --settings={settings} shell --plain --pythonpath=.".format(
system=system, settings=settings
),
]
)
def verify_server_task(self, task_name, options, contracts_default=False):
"""
Verify the output of a server task.
"""
settings = options.get("settings", None)
asset_settings = options.get("asset-settings", None)
is_optimized = options.get("optimized", False)
is_fast = options.get("fast", False)
no_contracts = options.get("no-contracts", not contracts_default)
if task_name == "devstack":
system = options.get("system")
elif task_name == "studio":
system = "cms"
else:
system = "lms"
port = options.get("port", "8000" if system == "lms" else "8001")
self.reset_task_messages()
if task_name == "devstack":
args = ["studio" if system == "cms" else system]
if settings:
args.append("--settings={settings}".format(settings=settings))
if asset_settings:
args.append("--asset-settings={asset_settings}".format(asset_settings=asset_settings))
if is_optimized:
args.append("--optimized")
if is_fast:
args.append("--fast")
if no_contracts:
args.append("--no-contracts")
call_task("pavelib.servers.devstack", args=args)
else:
call_task("pavelib.servers.{task_name}".format(task_name=task_name), options=options)
expected_messages = []
expected_settings = settings if settings else "devstack"
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
platform_root = os.getcwd()
if not is_fast:
expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
system=system, asset_settings=expected_asset_settings
))
expected_messages.append("xmodule_assets common/static/xmodule")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
expected_messages.append(EXPECTED_SASS_COMMAND)
if expected_collect_static:
expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
system=system, asset_settings=expected_asset_settings
))
expected_run_server_command = EXPECTED_RUN_SERVER_COMMAND.format(
system=system,
settings=expected_settings,
port=port,
)
if not no_contracts:
expected_run_server_command += " --contracts"
expected_messages.append(expected_run_server_command)
self.assertEquals(self.task_messages, expected_messages)
def verify_run_all_servers_task(self, options):
"""
Verify the output of a server task.
"""
settings = options.get("settings", None)
asset_settings = options.get("asset_settings", None)
is_optimized = options.get("optimized", False)
is_fast = options.get("fast", False)
self.reset_task_messages()
call_task("pavelib.servers.run_all_servers", options=options)
expected_settings = settings if settings else "devstack"
expected_asset_settings = asset_settings if asset_settings else expected_settings
if is_optimized:
expected_settings = "devstack_optimized"
expected_asset_settings = "test_static_optimized"
expected_collect_static = not is_fast and expected_settings != "devstack"
platform_root = os.getcwd()
expected_messages = []
if not is_fast:
expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
system="lms", asset_settings=expected_asset_settings
))
expected_messages.append(EXPECTED_PREPROCESS_ASSETS_COMMAND.format(
system="cms", asset_settings=expected_asset_settings
))
expected_messages.append("xmodule_assets common/static/xmodule")
expected_messages.append(EXPECTED_COFFEE_COMMAND.format(platform_root=platform_root))
expected_messages.append(EXPECTED_SASS_COMMAND)
if expected_collect_static:
expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
system="lms", asset_settings=expected_asset_settings
))
expected_messages.append(EXPECTED_COLLECT_STATIC_COMMAND.format(
system="cms", asset_settings=expected_asset_settings
))
expected_messages.append(
EXPECTED_RUN_SERVER_COMMAND.format(
system="lms",
settings=expected_settings,
port=8000,
)
)
expected_messages.append(
EXPECTED_RUN_SERVER_COMMAND.format(
system="cms",
settings=expected_settings,
port=8001,
)
)
expected_messages.append(EXPECTED_CELERY_COMMAND.format(settings="dev_with_worker"))
self.assertEquals(self.task_messages, expected_messages)

55
pavelib/paver_tests/utils.py Normal file
View File

@@ -0,0 +1,55 @@
"""Unit tests for the Paver server tasks."""
import os
from paver import tasks
from unittest import TestCase
class PaverTestCase(TestCase):
"""
Base class for Paver test cases.
"""
def setUp(self):
super(PaverTestCase, self).setUp()
# Show full length diffs upon test failure
self.maxDiff = None # pylint: disable=invalid-name
# Create a mock Paver environment
tasks.environment = MockEnvironment()
# Don't run pre-reqs
os.environ['NO_PREREQ_INSTALL'] = 'true'
def tearDown(self):
super(PaverTestCase, self).tearDown()
tasks.environment = tasks.Environment()
del os.environ['NO_PREREQ_INSTALL']
@property
def task_messages(self):
"""Returns the messages output by the Paver task."""
return tasks.environment.messages
def reset_task_messages(self):
"""Clear the recorded message"""
tasks.environment.messages = []
class MockEnvironment(tasks.Environment):
"""
Mock environment that collects information about Paver commands.
"""
def __init__(self):
super(MockEnvironment, self).__init__()
self.dry_run = True
self.messages = []
def info(self, message, *args):
"""Capture any messages that have been recorded"""
if args:
output = message % args
else:
output = message
if not output.startswith("--->"):
self.messages.append(output)

134
pavelib/servers.py
View File

@@ -2,24 +2,36 @@
Run and manage servers for local development.
"""
from __future__ import print_function
import sys
import argparse
from paver.easy import *
from .assets import collect_assets
from .utils.cmd import django_cmd
from .utils.process import run_process, run_multi_processes
DEFAULT_PORT = {"lms": 8000, "studio": 8001}
DEFAULT_SETTINGS = 'devstack'
OPTIMIZED_SETTINGS = "devstack_optimized"
OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized"
ASSET_SETTINGS_HELP = (
"Settings file used for updating assets. Defaults to the value of the settings variable if not provided."
)
def run_server(system, settings=None, port=None, skip_assets=False, contracts=False):
"""
Start the server for the specified `system` (lms or studio).
`settings` is the Django settings module to use; if not provided, use the default.
`port` is the port to run the server on; if not provided, use the default port for the system.
def run_server(
system, fast=False, settings=None, asset_settings=None, port=None, contracts=False
):
"""Start the server for LMS or Studio.
If `skip_assets` is True, skip the asset compilation step.
Args:
system (str): The system to be run (lms or studio).
fast (bool): If true, then start the server immediately without updating assets (defaults to False).
settings (str): The Django settings module to use; if not provided, use the default.
asset_settings (str) The settings to use when generating assets. If not provided, assets are not generated.
port (str): The port number to run the server on. If not provided, uses the default port for the system.
contracts (bool) If true then PyContracts is enabled (defaults to False).
"""
if system not in ['lms', 'studio']:
print("System must be either lms or studio", file=sys.stderr)
@@ -28,9 +40,13 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa
if not settings:
settings = DEFAULT_SETTINGS
if not skip_assets:
# Local dev settings use staticfiles to serve assets, so we can skip the collecstatic step
args = [system, '--settings={}'.format(settings), '--skip-collect', '--watch']
if not fast and asset_settings:
args = [system, '--settings={}'.format(asset_settings), '--watch']
# The default settings use DEBUG mode for running the server which means that
# the optimized assets are ignored, so we skip collectstatic in that case
# to save time.
if settings == DEFAULT_SETTINGS:
args.append('--skip-collect')
call_task('pavelib.assets.update_assets', args=args)
if port is None:
@@ -48,40 +64,55 @@ def run_server(system, settings=None, port=None, skip_assets=False, contracts=Fa
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
("settings=", "s", "Django settings"),
("asset-settings=", "a", ASSET_SETTINGS_HELP),
("port=", "p", "Port"),
("fast", "f", "Skip updating assets")
("fast", "f", "Skip updating assets"),
])
def lms(options):
"""
Run the LMS server.
"""
settings = getattr(options, 'settings', None)
settings = getattr(options, 'settings', DEFAULT_SETTINGS)
asset_settings = getattr(options, 'asset-settings', settings)
port = getattr(options, 'port', None)
fast = getattr(options, 'fast', False)
run_server('lms', settings=settings, port=port, skip_assets=fast)
run_server(
'lms',
fast=fast,
settings=settings,
asset_settings=asset_settings,
port=port,
)
@task
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
("settings=", "s", "Django settings"),
("asset-settings=", "a", ASSET_SETTINGS_HELP),
("port=", "p", "Port"),
("fast", "f", "Skip updating assets")
("fast", "f", "Skip updating assets"),
])
def studio(options):
"""
Run the Studio server.
"""
settings = getattr(options, 'settings', None)
settings = getattr(options, 'settings', DEFAULT_SETTINGS)
asset_settings = getattr(options, 'asset-settings', settings)
port = getattr(options, 'port', None)
fast = getattr(options, 'fast', False)
run_server('studio', settings=settings, port=port, skip_assets=fast)
run_server(
'studio',
fast=fast,
settings=settings,
asset_settings=asset_settings,
port=port,
)
@task
@needs('pavelib.prereqs.install_prereqs')
@consume_args
@no_help
def devstack(args):
"""
Start the devstack lms or studio server
@@ -89,6 +120,9 @@ def devstack(args):
parser = argparse.ArgumentParser(prog='paver devstack')
parser.add_argument('system', type=str, nargs=1, help="lms or studio")
parser.add_argument('--fast', action='store_true', default=False, help="Skip updating assets")
parser.add_argument('--optimized', action='store_true', default=False, help="Run with optimized assets")
parser.add_argument('--settings', type=str, default=DEFAULT_SETTINGS, help="Settings file")
parser.add_argument('--asset-settings', type=str, default=None, help=ASSET_SETTINGS_HELP)
parser.add_argument(
'--no-contracts',
action='store_true',
@@ -96,7 +130,18 @@ def devstack(args):
help="Disable contracts. By default, they're enabled in devstack."
)
args = parser.parse_args(args)
run_server(args.system[0], settings='devstack', skip_assets=args.fast, contracts=(not args.no_contracts))
settings = args.settings
asset_settings = args.asset_settings if args.asset_settings else settings
if args.optimized:
settings = OPTIMIZED_SETTINGS
asset_settings = OPTIMIZED_ASSETS_SETTINGS
run_server(
args.system[0],
fast=args.fast,
settings=settings,
asset_settings=asset_settings,
contracts=not args.no_contracts,
)
@task
@@ -116,33 +161,70 @@ def celery(options):
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
("settings=", "s", "Django settings for both LMS and Studio"),
("asset_settings=", "a", "Django settings for updating assets for both LMS and Studio (defaults to settings)"),
("worker_settings=", "w", "Celery worker Django settings"),
("fast", "f", "Skip updating assets"),
("optimized", "o", "Run with optimized assets"),
("settings_lms=", "l", "Set LMS only, overriding the value from --settings (if provided)"),
("asset_settings_lms=", "al", "Set LMS only, overriding the value from --asset_settings (if provided)"),
("settings_cms=", "c", "Set Studio only, overriding the value from --settings (if provided)"),
("asset_settings_cms=", "ac", "Set Studio only, overriding the value from --asset_settings (if provided)"),
])
def run_all_servers(options):
"""
Runs Celery workers, Studio, and LMS.
"""
settings = getattr(options, 'settings', DEFAULT_SETTINGS)
settings_lms = getattr(options, 'settings_lms', settings)
settings_cms = getattr(options, 'settings_cms', settings)
asset_settings = getattr(options, 'asset_settings', settings)
worker_settings = getattr(options, 'worker_settings', 'dev_with_worker')
fast = getattr(options, 'fast', False)
optimized = getattr(options, 'optimized', False)
if optimized:
settings = OPTIMIZED_SETTINGS
asset_settings = OPTIMIZED_ASSETS_SETTINGS
settings_lms = getattr(options, 'settings_lms', settings)
settings_cms = getattr(options, 'settings_cms', settings)
asset_settings_lms = getattr(options, 'asset_settings_lms', asset_settings)
asset_settings_cms = getattr(options, 'asset_settings_cms', asset_settings)
if not fast:
args = ['lms', '--settings={}'.format(settings_lms), '--skip-collect']
# First update assets for both LMS and Studio but don't collect static yet
args = [
'lms', 'studio',
'--settings={}'.format(asset_settings),
'--skip-collect'
]
call_task('pavelib.assets.update_assets', args=args)
args = ['studio', '--settings={}'.format(settings_cms), '--skip-collect']
call_task('pavelib.assets.update_assets', args=args)
# Now collect static for each system separately with the appropriate settings.
# Note that the default settings use DEBUG mode for running the server which
# means that the optimized assets are ignored, so we skip collectstatic in that
# case to save time.
if settings != DEFAULT_SETTINGS:
collect_assets(['lms'], asset_settings_lms)
collect_assets(['studio'], asset_settings_cms)
# Install an asset watcher to regenerate files that change
call_task('pavelib.assets.watch_assets', options={'background': True})
# Start up LMS, CMS and Celery
lms_port = DEFAULT_PORT['lms']
cms_port = DEFAULT_PORT['studio']
lms_runserver_args = ["0.0.0.0:{}".format(lms_port)]
cms_runserver_args = ["0.0.0.0:{}".format(cms_port)]
run_multi_processes([
django_cmd('lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['lms'])),
django_cmd('studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', "0.0.0.0:{}".format(DEFAULT_PORT['studio'])),
django_cmd('lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.')
django_cmd(
'lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', *lms_runserver_args
),
django_cmd(
'studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', *cms_runserver_args
),
django_cmd(
'lms', worker_settings, 'celery', 'worker', '--beat', '--loglevel=INFO', '--pythonpath=.'
)
])

10
pavelib/utils/process.py
View File

@@ -9,6 +9,8 @@ import signal
import psutil
import atexit
from paver import tasks
def kill_process(proc):
"""
@@ -41,6 +43,14 @@ def run_multi_processes(cmd_list, out_log=None, err_log=None):
err_log_file = open(err_log, 'w')
kwargs['stderr'] = err_log_file
# If the user is performing a dry run of a task, then just log
# the command strings and return so that no destructive operations
# are performed.
if tasks.environment.dry_run:
for cmd in cmd_list:
tasks.environment.info(cmd)
return
try:
for cmd in cmd_list:
pids.extend([subprocess.Popen(cmd, **kwargs)])