edx-platform/pavelib/servers.py

"""
Run and manage servers for local development.
"""


import argparse
import sys

from paver.easy import call_task, cmdopts, consume_args, needs, sh, task

from .assets import collect_assets
from .utils.cmd import cmd, django_cmd
from .utils.envs import Env
from .utils.process import run_multi_processes, run_process
from .utils.timer import timed

DEFAULT_PORT = {"lms": 8000, "studio": 8001}
DEFAULT_SETTINGS = Env.DEVSTACK_SETTINGS
OPTIMIZED_SETTINGS = "devstack_optimized"
OPTIMIZED_ASSETS_SETTINGS = "test_static_optimized"

ASSET_SETTINGS_HELP = (
    "Settings file used for updating assets. Defaults to the value of the settings variable if not provided."
)


def run_server(
        system, fast=False, settings=None, asset_settings=None, port=None
):
    """Start the server for LMS or Studio.

    Args:
        system (str): The system to be run (lms or studio).
        fast (bool): If true, then start the server immediately without updating assets (defaults to False).
        settings (str): The Django settings module to use; if not provided, use the default.
        asset_settings (str) The settings to use when generating assets. If not provided, assets are not generated.
        port (str): The port number to run the server on. If not provided, uses the default port for the system.
    """
    if system not in ['lms', 'studio']:
        print("System must be either lms or studio", file=sys.stderr)
        exit(1)  # lint-amnesty, pylint: disable=consider-using-sys-exit

    if not settings:
        settings = DEFAULT_SETTINGS

    if not fast and asset_settings:
        args = [system, f'--settings={asset_settings}', '--watch']
        # The default settings use DEBUG mode for running the server which means that
        # the optimized assets are ignored, so we skip collectstatic in that case
        # to save time.
        if settings == DEFAULT_SETTINGS:
            args.append('--skip-collect')
        call_task('pavelib.assets.update_assets', args=args)

    if port is None:
        port = DEFAULT_PORT[system]

    args = [settings, 'runserver', '--traceback', '--pythonpath=.', f'0.0.0.0:{port}']

    run_process(django_cmd(system, *args))


@task
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
    ("settings=", "s", "Django settings"),
    ("asset-settings=", "a", ASSET_SETTINGS_HELP),
    ("port=", "p", "Port"),
    ("fast", "f", "Skip updating assets"),
])
def lms(options):
    """
    Run the LMS server.
    """
    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
    asset_settings = getattr(options, 'asset-settings', settings)
    port = getattr(options, 'port', None)
    fast = getattr(options, 'fast', False)
    run_server(
        'lms',
        fast=fast,
        settings=settings,
        asset_settings=asset_settings,
        port=port,
    )


@task
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
    ("settings=", "s", "Django settings"),
    ("asset-settings=", "a", ASSET_SETTINGS_HELP),
    ("port=", "p", "Port"),
    ("fast", "f", "Skip updating assets"),
])
def studio(options):
    """
    Run the Studio server.
    """
    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
    asset_settings = getattr(options, 'asset-settings', settings)
    port = getattr(options, 'port', None)
    fast = getattr(options, 'fast', False)
    run_server(
        'studio',
        fast=fast,
        settings=settings,
        asset_settings=asset_settings,
        port=port,
    )


@task
@needs('pavelib.prereqs.install_prereqs')
@consume_args
def devstack(args):
    """
    Start the devstack lms or studio server
    """
    parser = argparse.ArgumentParser(prog='paver devstack')
    parser.add_argument('system', type=str, nargs=1, help="lms or studio")
    parser.add_argument('--fast', action='store_true', default=False, help="Skip updating assets")
    parser.add_argument('--optimized', action='store_true', default=False, help="Run with optimized assets")
    parser.add_argument('--settings', type=str, default=DEFAULT_SETTINGS, help="Settings file")
    parser.add_argument('--asset-settings', type=str, default=None, help=ASSET_SETTINGS_HELP)
    args = parser.parse_args(args)
    settings = args.settings
    asset_settings = args.asset_settings if args.asset_settings else settings
    if args.optimized:
        settings = OPTIMIZED_SETTINGS
        asset_settings = OPTIMIZED_ASSETS_SETTINGS
    sh(django_cmd('cms', settings, 'reindex_course', '--setup'))
    run_server(
        args.system[0],
        fast=args.fast,
        settings=settings,
        asset_settings=asset_settings,
    )


@task
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
    ("settings=", "s", "Django settings"),
])
def celery(options):
    """
    Runs Celery workers.
    """
    settings = getattr(options, 'settings', 'devstack_with_worker')
    run_process(cmd(f'DJANGO_SETTINGS_MODULE=lms.envs.{settings}',
                    'celery', 'worker', '--app=lms.celery:APP',
                    '--beat', '--loglevel=INFO', '--pythonpath=.'))


@task
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
    ("settings=", "s", "Django settings for both LMS and Studio"),
    ("asset-settings=", "a", "Django settings for updating assets for both LMS and Studio (defaults to settings)"),
    ("worker-settings=", "w", "Celery worker Django settings"),
    ("fast", "f", "Skip updating assets"),
    ("optimized", "o", "Run with optimized assets"),
    ("settings-lms=", "l", "Set LMS only, overriding the value from --settings (if provided)"),
    ("asset-settings-lms=", None, "Set LMS only, overriding the value from --asset-settings (if provided)"),
    ("settings-cms=", "c", "Set Studio only, overriding the value from --settings (if provided)"),
    ("asset-settings-cms=", None, "Set Studio only, overriding the value from --asset-settings (if provided)"),

    ("asset_settings=", None, "deprecated in favor of asset-settings"),
    ("asset_settings_cms=", None, "deprecated in favor of asset-settings-cms"),
    ("asset_settings_lms=", None, "deprecated in favor of asset-settings-lms"),
    ("settings_cms=", None, "deprecated in favor of settings-cms"),
    ("settings_lms=", None, "deprecated in favor of settings-lms"),
    ("worker_settings=", None, "deprecated in favor of worker-settings"),
])
def run_all_servers(options):
    """
    Runs Celery workers, Studio, and LMS.
    """
    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
    asset_settings = getattr(options, 'asset_settings', settings)
    worker_settings = getattr(options, 'worker_settings', 'devstack_with_worker')
    fast = getattr(options, 'fast', False)
    optimized = getattr(options, 'optimized', False)

    if optimized:
        settings = OPTIMIZED_SETTINGS
        asset_settings = OPTIMIZED_ASSETS_SETTINGS

    settings_lms = getattr(options, 'settings_lms', settings)
    settings_cms = getattr(options, 'settings_cms', settings)
    asset_settings_lms = getattr(options, 'asset_settings_lms', asset_settings)
    asset_settings_cms = getattr(options, 'asset_settings_cms', asset_settings)

    if not fast:
        # First update assets for both LMS and Studio but don't collect static yet
        args = [
            'lms', 'studio',
            f'--settings={asset_settings}',
            '--skip-collect'
        ]
        call_task('pavelib.assets.update_assets', args=args)

        # Now collect static for each system separately with the appropriate settings.
        # Note that the default settings use DEBUG mode for running the server which
        # means that the optimized assets are ignored, so we skip collectstatic in that
        # case to save time.
        if settings != DEFAULT_SETTINGS:
            collect_assets(['lms'], asset_settings_lms)
            collect_assets(['studio'], asset_settings_cms)

        # Install an asset watcher to regenerate files that change
        call_task('pavelib.assets.watch_assets', options={'background': True})

    # Start up LMS, CMS and Celery
    lms_port = DEFAULT_PORT['lms']
    cms_port = DEFAULT_PORT['studio']
    lms_runserver_args = [f"0.0.0.0:{lms_port}"]
    cms_runserver_args = [f"0.0.0.0:{cms_port}"]

    run_multi_processes([
        django_cmd(
            'lms', settings_lms, 'runserver', '--traceback', '--pythonpath=.', *lms_runserver_args
        ),
        django_cmd(
            'studio', settings_cms, 'runserver', '--traceback', '--pythonpath=.', *cms_runserver_args
        ),
        cmd(
            f'DJANGO_SETTINGS_MODULE=lms.envs.{worker_settings}',
            'celery', 'worker', '--app=lms.celery:APP',
            '--beat', '--loglevel=INFO', '--pythonpath=.'
        )
    ])


@task
@needs('pavelib.prereqs.install_prereqs')
@cmdopts([
    ("settings=", "s", "Django settings"),
    ("fake-initial", None, "Fake the initial migrations"),
])
@timed
def update_db(options):
    """
    Migrates the lms and cms across all databases
    """
    settings = getattr(options, 'settings', DEFAULT_SETTINGS)
    fake = "--fake-initial" if getattr(options, 'fake_initial', False) else ""
    for system in ('lms', 'cms'):
        # pylint: disable=line-too-long
        sh("NO_EDXAPP_SUDO=1 EDX_PLATFORM_SETTINGS_OVERRIDE={settings} /edx/bin/edxapp-migrate-{system} --traceback --pythonpath=. {fake}".format(
            settings=settings,
            system=system,
            fake=fake))


@task
@needs('pavelib.prereqs.install_prereqs')
@consume_args
@timed
def check_settings(args):
    """
    Checks settings files.
    """
    parser = argparse.ArgumentParser(prog='paver check_settings')
    parser.add_argument('system', type=str, nargs=1, help="lms or studio")
    parser.add_argument('settings', type=str, nargs=1, help='Django settings')
    args = parser.parse_args(args)

    system = args.system[0]
    settings = args.settings[0]

    try:
        import_cmd = f"echo 'import {system}.envs.{settings}'"
        django_shell_cmd = django_cmd(system, settings, 'shell', '--plain', '--pythonpath=.')
        sh(f"{import_cmd} | {django_shell_cmd}")

    except:  # pylint: disable=bare-except
        print("Failed to import settings", file=sys.stderr)