feat!: Remove Paver
BREAKING CHANGE: Removes all remaining Paver commands including `pavelib/prereqs.py:*` and `pavelib/assets.py:*`. BREAKING CHANGE: Removes `./manage.py [lms|cms] compile_sass`, which was just a wrapper around Paver commands. BREAKING CHANGE: Removes paver.txt. Operators should install testing.txt instead. Part of: https://github.com/openedx/edx-platform/issues/34467
This commit is contained in:
Kyle D. McCormick
committed by
Kyle McCormick
Kyle McCormick
parent
38dc4eab5d
commit
2d5543a9ae
2
.github/workflows/pylint-checks.yml
vendored
2
.github/workflows/pylint-checks.yml
vendored
@@ -22,7 +22,7 @@ jobs:
|
||||
- module-name: openedx-2
|
||||
path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/"
|
||||
- module-name: common
|
||||
path: "common pavelib"
|
||||
path: "common"
|
||||
- module-name: cms
|
||||
path: "cms"
|
||||
- module-name: xmodule
|
||||
|
||||
6
.github/workflows/unit-test-shards.json
vendored
6
.github/workflows/unit-test-shards.json
vendored
@@ -255,15 +255,13 @@
|
||||
"common-with-lms": {
|
||||
"settings": "lms.envs.test",
|
||||
"paths": [
|
||||
"common/djangoapps/",
|
||||
"pavelib/"
|
||||
"common/djangoapps/"
|
||||
]
|
||||
},
|
||||
"common-with-cms": {
|
||||
"settings": "cms.envs.test",
|
||||
"paths": [
|
||||
"common/djangoapps/",
|
||||
"pavelib/"
|
||||
"common/djangoapps/"
|
||||
]
|
||||
},
|
||||
"xmodule-with-lms": {
|
||||
|
||||
2
.github/workflows/unit-tests.yml
vendored
2
.github/workflows/unit-tests.yml
vendored
@@ -158,7 +158,7 @@ jobs:
|
||||
shell: bash
|
||||
run: |
|
||||
echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
|
||||
echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ pavelib/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
|
||||
echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV
|
||||
|
||||
- name: get GHA unit test paths
|
||||
shell: bash
|
||||
|
||||
7
Makefile
7
Makefile
@@ -93,7 +93,6 @@ requirements: dev-requirements ## install development environment requirements
|
||||
# Order is very important in this list: files must appear after everything they include!
|
||||
REQ_FILES = \
|
||||
requirements/edx/coverage \
|
||||
requirements/edx/paver \
|
||||
requirements/edx-sandbox/base \
|
||||
requirements/edx/base \
|
||||
requirements/edx/doc \
|
||||
@@ -179,7 +178,7 @@ xsslint: ## check xss for quality issuest
|
||||
--config=scripts.xsslint_config \
|
||||
--thresholds=scripts/xsslint_thresholds.json
|
||||
|
||||
pycodestyle: ## check python files for quality issues
|
||||
pycodestyle: ## check python files for quality issues
|
||||
pycodestyle .
|
||||
|
||||
## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved
|
||||
@@ -190,13 +189,13 @@ pii_check: ## check django models for pii annotations
|
||||
--app_name cms \
|
||||
--coverage \
|
||||
--lint
|
||||
|
||||
|
||||
DJANGO_SETTINGS_MODULE=lms.envs.test \
|
||||
code_annotations django_find_annotations \
|
||||
--config_file .pii_annotations.yml \
|
||||
--app_name lms \
|
||||
--coverage \
|
||||
--lint
|
||||
--lint
|
||||
|
||||
check_keywords: ## check django models for reserve keywords
|
||||
DJANGO_SETTINGS_MODULE=cms.envs.test \
|
||||
|
||||
@@ -1,112 +0,0 @@
|
||||
"""
|
||||
Management command for compiling sass.
|
||||
|
||||
DEPRECATED in favor of `npm run compile-sass`.
|
||||
"""
|
||||
import shlex
|
||||
|
||||
from django.core.management import BaseCommand
|
||||
from django.conf import settings
|
||||
|
||||
from pavelib.assets import run_deprecated_command_wrapper
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
"""
|
||||
Compile theme sass and collect theme assets.
|
||||
"""
|
||||
|
||||
help = "DEPRECATED. Use 'npm run compile-sass' instead."
|
||||
|
||||
# NOTE (CCB): This allows us to compile static assets in Docker containers without database access.
|
||||
requires_system_checks = []
|
||||
|
||||
def add_arguments(self, parser):
|
||||
"""
|
||||
Add arguments for compile_sass command.
|
||||
|
||||
Args:
|
||||
parser (django.core.management.base.CommandParser): parsed for parsing command line arguments.
|
||||
"""
|
||||
parser.add_argument(
|
||||
'system', type=str, nargs='*', default=["lms", "cms"],
|
||||
help="lms or studio",
|
||||
)
|
||||
|
||||
# Named (optional) arguments
|
||||
parser.add_argument(
|
||||
'--theme-dirs',
|
||||
dest='theme_dirs',
|
||||
type=str,
|
||||
nargs='+',
|
||||
default=None,
|
||||
help="List of dirs where given themes would be looked.",
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--themes',
|
||||
type=str,
|
||||
nargs='+',
|
||||
default=["all"],
|
||||
help="List of themes whose sass need to compiled. Or 'no'/'all' to compile for no/all themes.",
|
||||
)
|
||||
|
||||
# Named (optional) arguments
|
||||
parser.add_argument(
|
||||
'--force',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="DEPRECATED. Full recompilation is now always forced.",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--debug',
|
||||
action='store_true',
|
||||
default=False,
|
||||
help="Disable Sass compression",
|
||||
)
|
||||
|
||||
def handle(self, *args, **options):
|
||||
"""
|
||||
Handle compile_sass command.
|
||||
"""
|
||||
systems = set(
|
||||
{"lms": "lms", "cms": "cms", "studio": "cms"}[sys]
|
||||
for sys in options.get("system", ["lms", "cms"])
|
||||
)
|
||||
theme_dirs = options.get("theme_dirs") or settings.COMPREHENSIVE_THEME_DIRS or []
|
||||
themes_option = options.get("themes") or [] # '[]' means 'all'
|
||||
if not settings.ENABLE_COMPREHENSIVE_THEMING:
|
||||
compile_themes = False
|
||||
themes = []
|
||||
elif "no" in themes_option:
|
||||
compile_themes = False
|
||||
themes = []
|
||||
elif "all" in themes_option:
|
||||
compile_themes = True
|
||||
themes = []
|
||||
else:
|
||||
compile_themes = True
|
||||
themes = themes_option
|
||||
run_deprecated_command_wrapper(
|
||||
old_command="./manage.py [lms|cms] compile_sass",
|
||||
ignored_old_flags=list(set(["force"]) & set(options)),
|
||||
new_command=shlex.join([
|
||||
"npm",
|
||||
"run",
|
||||
("compile-sass-dev" if options.get("debug") else "compile-sass"),
|
||||
"--",
|
||||
*(["--skip-lms"] if "lms" not in systems else []),
|
||||
*(["--skip-cms"] if "cms" not in systems else []),
|
||||
*(["--skip-themes"] if not compile_themes else []),
|
||||
*(
|
||||
arg
|
||||
for theme_dir in theme_dirs
|
||||
for arg in ["--theme-dir", str(theme_dir)]
|
||||
),
|
||||
*(
|
||||
arg
|
||||
for theme in themes
|
||||
for arg in ["--theme", theme]
|
||||
),
|
||||
]),
|
||||
)
|
||||
@@ -3,7 +3,11 @@ print_setting
|
||||
=============
|
||||
|
||||
Django command to output a single Django setting.
|
||||
Useful when paver or a shell script needs such a value.
|
||||
Originally used by "paver" scripts before we removed them.
|
||||
Still useful when a shell script needs such a value.
|
||||
Keep in mind that the LMS/CMS startup time is slow, so if you invoke this
|
||||
Django management multiple times in a command that gets run often, you are
|
||||
going to be sad.
|
||||
|
||||
This handles the one specific use case of the "print_settings" command from
|
||||
django-extensions that we were actually using.
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
""" # lint-amnesty, pylint: disable=django-not-configured
|
||||
paver commands
|
||||
"""
|
||||
|
||||
|
||||
from . import assets
|
||||
@@ -1,506 +0,0 @@
|
||||
"""
|
||||
Asset compilation and collection.
|
||||
|
||||
This entire module is DEPRECATED. In Redwood, it exists just as a collection of temporary compatibility wrappers.
|
||||
In Sumac, this module will be deleted. To migrate, follow the advice in the printed warnings and/or read the
|
||||
instructions on the DEPR ticket: https://github.com/openedx/edx-platform/issues/31895
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import glob
|
||||
import json
|
||||
import shlex
|
||||
import traceback
|
||||
from functools import wraps
|
||||
from threading import Timer
|
||||
|
||||
from paver import tasks
|
||||
from paver.easy import call_task, cmdopts, consume_args, needs, no_help, sh, task
|
||||
from watchdog.events import PatternMatchingEventHandler
|
||||
from watchdog.observers import Observer # pylint: disable=unused-import # Used by Tutor. Remove after Sumac cut.
|
||||
|
||||
from .utils.cmd import django_cmd
|
||||
from .utils.envs import Env
|
||||
from .utils.timer import timed
|
||||
|
||||
|
||||
SYSTEMS = {
|
||||
'lms': 'lms',
|
||||
'cms': 'cms',
|
||||
'studio': 'cms',
|
||||
}
|
||||
|
||||
WARNING_SYMBOLS = "⚠️ " * 50 # A row of 'warning' emoji to catch CLI users' attention
|
||||
|
||||
|
||||
def run_deprecated_command_wrapper(*, old_command, ignored_old_flags, new_command):
|
||||
"""
|
||||
Run the new version of shell command, plus a warning that the old version is deprecated.
|
||||
"""
|
||||
depr_warning = (
|
||||
"\n" +
|
||||
f"{WARNING_SYMBOLS}\n" +
|
||||
"\n" +
|
||||
f"WARNING: '{old_command}' is DEPRECATED! It will be removed before Sumac.\n" +
|
||||
"The command you ran is now just a temporary wrapper around a new,\n" +
|
||||
"supported command, which you should use instead:\n" +
|
||||
"\n" +
|
||||
f"\t{new_command}\n" +
|
||||
"\n" +
|
||||
"Details: https://github.com/openedx/edx-platform/issues/31895\n" +
|
||||
"".join(
|
||||
f" WARNING: ignored deprecated paver flag '{flag}'\n"
|
||||
for flag in ignored_old_flags
|
||||
) +
|
||||
f"{WARNING_SYMBOLS}\n" +
|
||||
"\n"
|
||||
)
|
||||
# Print deprecation warning twice so that it's more likely to be seen in the logs.
|
||||
print(depr_warning)
|
||||
sh(new_command)
|
||||
print(depr_warning)
|
||||
|
||||
|
||||
def debounce(seconds=1):
|
||||
"""
|
||||
Prevents the decorated function from being called more than every `seconds`
|
||||
seconds. Waits until calls stop coming in before calling the decorated
|
||||
function.
|
||||
|
||||
This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor.
|
||||
"""
|
||||
def decorator(func):
|
||||
func.timer = None
|
||||
|
||||
@wraps(func)
|
||||
def wrapper(*args, **kwargs):
|
||||
def call():
|
||||
func(*args, **kwargs)
|
||||
func.timer = None
|
||||
if func.timer:
|
||||
func.timer.cancel()
|
||||
func.timer = Timer(seconds, call)
|
||||
func.timer.start()
|
||||
|
||||
return wrapper
|
||||
return decorator
|
||||
|
||||
|
||||
class SassWatcher(PatternMatchingEventHandler):
|
||||
"""
|
||||
Watches for sass file changes
|
||||
|
||||
This is DEPRECATED. It exists in Redwood just to ease the transition for Tutor.
|
||||
"""
|
||||
ignore_directories = True
|
||||
patterns = ['*.scss']
|
||||
|
||||
def register(self, observer, directories):
|
||||
"""
|
||||
register files with observer
|
||||
Arguments:
|
||||
observer (watchdog.observers.Observer): sass file observer
|
||||
directories (list): list of directories to be register for sass watcher.
|
||||
"""
|
||||
for dirname in directories:
|
||||
paths = []
|
||||
if '*' in dirname:
|
||||
paths.extend(glob.glob(dirname))
|
||||
else:
|
||||
paths.append(dirname)
|
||||
|
||||
for obs_dirname in paths:
|
||||
observer.schedule(self, obs_dirname, recursive=True)
|
||||
|
||||
@debounce()
|
||||
def on_any_event(self, event):
|
||||
print('\tCHANGED:', event.src_path)
|
||||
try:
|
||||
compile_sass() # pylint: disable=no-value-for-parameter
|
||||
except Exception: # pylint: disable=broad-except
|
||||
traceback.print_exc()
|
||||
|
||||
|
||||
@task
|
||||
@no_help
|
||||
@cmdopts([
|
||||
('system=', 's', 'The system to compile sass for (defaults to all)'),
|
||||
('theme-dirs=', '-td', 'Theme dirs containing all themes (defaults to None)'),
|
||||
('themes=', '-t', 'The theme to compile sass for (defaults to None)'),
|
||||
('debug', 'd', 'Whether to use development settings'),
|
||||
('force', '', 'DEPRECATED. Full recompilation is now always forced.'),
|
||||
])
|
||||
@timed
|
||||
def compile_sass(options):
|
||||
"""
|
||||
Compile Sass to CSS. If command is called without any arguments, it will
|
||||
only compile lms, cms sass for the open source theme. And none of the comprehensive theme's sass would be compiled.
|
||||
|
||||
If you want to compile sass for all comprehensive themes you will have to run compile_sass
|
||||
specifying all the themes that need to be compiled..
|
||||
|
||||
The following is a list of some possible ways to use this command.
|
||||
|
||||
Command:
|
||||
paver compile_sass
|
||||
Description:
|
||||
compile sass files for both lms and cms. If command is called like above (i.e. without any arguments) it will
|
||||
only compile lms, cms sass for the open source theme. None of the theme's sass will be compiled.
|
||||
|
||||
Command:
|
||||
paver compile_sass --theme-dirs /edx/app/edxapp/edx-platform/themes --themes=red-theme
|
||||
Description:
|
||||
compile sass files for both lms and cms for 'red-theme' present in '/edx/app/edxapp/edx-platform/themes'
|
||||
|
||||
Command:
|
||||
paver compile_sass --theme-dirs=/edx/app/edxapp/edx-platform/themes --themes red-theme stanford-style
|
||||
Description:
|
||||
compile sass files for both lms and cms for 'red-theme' and 'stanford-style' present in
|
||||
'/edx/app/edxapp/edx-platform/themes'.
|
||||
|
||||
Command:
|
||||
paver compile_sass --system=cms
|
||||
--theme-dirs /edx/app/edxapp/edx-platform/themes /edx/app/edxapp/edx-platform/common/test/
|
||||
--themes red-theme stanford-style test-theme
|
||||
Description:
|
||||
compile sass files for cms only for 'red-theme', 'stanford-style' and 'test-theme' present in
|
||||
'/edx/app/edxapp/edx-platform/themes' and '/edx/app/edxapp/edx-platform/common/test/'.
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead.
|
||||
"""
|
||||
systems = [SYSTEMS[sys] for sys in get_parsed_option(options, 'system', ['lms', 'cms'])] # normalize studio->cms
|
||||
run_deprecated_command_wrapper(
|
||||
old_command="paver compile_sass",
|
||||
ignored_old_flags=(set(["--force"]) & set(options)),
|
||||
new_command=shlex.join([
|
||||
"npm",
|
||||
"run",
|
||||
("compile-sass-dev" if options.get("debug") else "compile-sass"),
|
||||
"--",
|
||||
*(["--dry"] if tasks.environment.dry_run else []),
|
||||
*(["--skip-lms"] if "lms" not in systems else []),
|
||||
*(["--skip-cms"] if "cms" not in systems else []),
|
||||
*(
|
||||
arg
|
||||
for theme_dir in get_parsed_option(options, 'theme_dirs', [])
|
||||
for arg in ["--theme-dir", str(theme_dir)]
|
||||
),
|
||||
*(
|
||||
arg
|
||||
for theme in get_parsed_option(options, "themes", [])
|
||||
for arg in ["--theme", theme]
|
||||
),
|
||||
]),
|
||||
)
|
||||
|
||||
|
||||
def _compile_sass(system, theme, debug, force, _timing_info):
|
||||
"""
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER
|
||||
|
||||
It exists to ease the transition for Tutor in Redwood, which directly imported and used this function.
|
||||
"""
|
||||
run_deprecated_command_wrapper(
|
||||
old_command="pavelib.assets:_compile_sass",
|
||||
ignored_old_flags=(set(["--force"]) if force else set()),
|
||||
new_command=[
|
||||
"npm",
|
||||
"run",
|
||||
("compile-sass-dev" if debug else "compile-sass"),
|
||||
"--",
|
||||
*(["--dry"] if tasks.environment.dry_run else []),
|
||||
*(
|
||||
["--skip-default", "--theme-dir", str(theme.parent), "--theme", str(theme.name)]
|
||||
if theme
|
||||
else []
|
||||
),
|
||||
("--skip-cms" if system == "lms" else "--skip-lms"),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def process_npm_assets():
|
||||
"""
|
||||
Process vendor libraries installed via NPM.
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER. It is now handled as part of `npm clean-install`.
|
||||
If you need to invoke it explicitly, you can run `npm run postinstall`.
|
||||
"""
|
||||
run_deprecated_command_wrapper(
|
||||
old_command="pavelib.assets:process_npm_assets",
|
||||
ignored_old_flags=[],
|
||||
new_command=shlex.join(["npm", "run", "postinstall"]),
|
||||
)
|
||||
|
||||
|
||||
@task
|
||||
@no_help
|
||||
def process_xmodule_assets():
|
||||
"""
|
||||
Process XModule static assets.
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY STUB. Refrences to it should be deleted.
|
||||
"""
|
||||
print(
|
||||
"\n" +
|
||||
f"{WARNING_SYMBOLS}",
|
||||
"\n" +
|
||||
"WARNING: 'paver process_xmodule_assets' is DEPRECATED! It will be removed before Sumac.\n" +
|
||||
"\n" +
|
||||
"Starting with Quince, it is no longer necessary to post-process XModule assets, so \n" +
|
||||
"'paver process_xmodule_assets' is a no-op. Please simply remove it from your build scripts.\n" +
|
||||
"\n" +
|
||||
"Details: https://github.com/openedx/edx-platform/issues/31895\n" +
|
||||
f"{WARNING_SYMBOLS}",
|
||||
)
|
||||
|
||||
|
||||
def collect_assets(systems, settings, **kwargs):
|
||||
"""
|
||||
Collect static assets, including Django pipeline processing.
|
||||
`systems` is a list of systems (e.g. 'lms' or 'studio' or both)
|
||||
`settings` is the Django settings module to use.
|
||||
`**kwargs` include arguments for using a log directory for collectstatic output. Defaults to /dev/null.
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER
|
||||
|
||||
It exists to ease the transition for Tutor in Redwood, which directly imported and used this function.
|
||||
"""
|
||||
run_deprecated_command_wrapper(
|
||||
old_command="pavelib.asset:collect_assets",
|
||||
ignored_old_flags=[],
|
||||
new_command=" && ".join(
|
||||
"( " +
|
||||
shlex.join(
|
||||
["./manage.py", SYSTEMS[sys], f"--settings={settings}", "collectstatic", "--noinput"]
|
||||
) + (
|
||||
""
|
||||
if "collect_log_dir" not in kwargs else
|
||||
" > /dev/null"
|
||||
if kwargs["collect_log_dir"] is None else
|
||||
f"> {kwargs['collect_log_dir']}/{SYSTEMS[sys]}-collectstatic.out"
|
||||
) +
|
||||
" )"
|
||||
for sys in systems
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def execute_compile_sass(args):
|
||||
"""
|
||||
Construct django management command compile_sass (defined in theming app) and execute it.
|
||||
Args:
|
||||
args: command line argument passed via update_assets command
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run compile-sass` instead.
|
||||
"""
|
||||
for sys in args.system:
|
||||
options = ""
|
||||
options += " --theme-dirs " + " ".join(args.theme_dirs) if args.theme_dirs else ""
|
||||
options += " --themes " + " ".join(args.themes) if args.themes else ""
|
||||
options += " --debug" if args.debug else ""
|
||||
|
||||
sh(
|
||||
django_cmd(
|
||||
sys,
|
||||
args.settings,
|
||||
"compile_sass {system} {options}".format(
|
||||
system='cms' if sys == 'studio' else sys,
|
||||
options=options,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@task
|
||||
@cmdopts([
|
||||
('settings=', 's', "Django settings (defaults to devstack)"),
|
||||
('watch', 'w', "DEPRECATED. This flag never did anything anyway."),
|
||||
])
|
||||
@timed
|
||||
def webpack(options):
|
||||
"""
|
||||
Run a Webpack build.
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run webpack` instead.
|
||||
"""
|
||||
settings = getattr(options, 'settings', Env.DEVSTACK_SETTINGS)
|
||||
result = Env.get_django_settings(['STATIC_ROOT', 'WEBPACK_CONFIG_PATH'], "lms", settings=settings)
|
||||
static_root_lms, config_path = result
|
||||
static_root_cms, = Env.get_django_settings(["STATIC_ROOT"], "cms", settings=settings)
|
||||
js_env_extra_config_setting, = Env.get_django_json_settings(["JS_ENV_EXTRA_CONFIG"], "cms", settings=settings)
|
||||
js_env_extra_config = json.dumps(js_env_extra_config_setting or "{}")
|
||||
node_env = "development" if config_path == 'webpack.dev.config.js' else "production"
|
||||
run_deprecated_command_wrapper(
|
||||
old_command="paver webpack",
|
||||
ignored_old_flags=(set(["watch"]) & set(options)),
|
||||
new_command=' '.join([
|
||||
f"WEBPACK_CONFIG_PATH={config_path}",
|
||||
f"NODE_ENV={node_env}",
|
||||
f"STATIC_ROOT_LMS={static_root_lms}",
|
||||
f"STATIC_ROOT_CMS={static_root_cms}",
|
||||
f"JS_ENV_EXTRA_CONFIG={js_env_extra_config}",
|
||||
"npm",
|
||||
"run",
|
||||
"webpack",
|
||||
]),
|
||||
)
|
||||
|
||||
|
||||
def get_parsed_option(command_opts, opt_key, default=None):
|
||||
"""
|
||||
Extract user command option and parse it.
|
||||
Arguments:
|
||||
command_opts: Command line arguments passed via paver command.
|
||||
opt_key: name of option to get and parse
|
||||
default: if `command_opt_value` not in `command_opts`, `command_opt_value` will be set to default.
|
||||
Returns:
|
||||
list or None
|
||||
"""
|
||||
command_opt_value = getattr(command_opts, opt_key, default)
|
||||
if command_opt_value:
|
||||
command_opt_value = listfy(command_opt_value)
|
||||
|
||||
return command_opt_value
|
||||
|
||||
|
||||
def listfy(data):
|
||||
"""
|
||||
Check and convert data to list.
|
||||
Arguments:
|
||||
data: data structure to be converted.
|
||||
"""
|
||||
|
||||
if isinstance(data, str):
|
||||
data = data.split(',')
|
||||
elif not isinstance(data, list):
|
||||
data = [data]
|
||||
|
||||
return data
|
||||
|
||||
|
||||
@task
|
||||
@cmdopts([
|
||||
('background', 'b', 'DEPRECATED. Use shell tools like & to run in background if needed.'),
|
||||
('settings=', 's', "DEPRECATED. Django is not longer invoked to compile JS/Sass."),
|
||||
('theme-dirs=', '-td', 'The themes dir containing all themes (defaults to None)'),
|
||||
('themes=', '-t', 'DEPRECATED. All themes in --theme-dirs are now watched.'),
|
||||
('wait=', '-w', 'DEPRECATED. Watchdog\'s default wait time is now used.'),
|
||||
])
|
||||
@timed
|
||||
def watch_assets(options):
|
||||
"""
|
||||
Watch for changes to asset files, and regenerate js/css
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER. Use `npm run watch` instead.
|
||||
"""
|
||||
# Don't watch assets when performing a dry run
|
||||
if tasks.environment.dry_run:
|
||||
return
|
||||
|
||||
theme_dirs = ':'.join(get_parsed_option(options, 'theme_dirs', []))
|
||||
run_deprecated_command_wrapper(
|
||||
old_command="paver watch_assets",
|
||||
ignored_old_flags=(set(["debug", "themes", "settings", "background"]) & set(options)),
|
||||
new_command=shlex.join([
|
||||
*(
|
||||
["env", f"COMPREHENSIVE_THEME_DIRS={theme_dirs}"]
|
||||
if theme_dirs else []
|
||||
),
|
||||
"npm",
|
||||
"run",
|
||||
"watch",
|
||||
]),
|
||||
)
|
||||
|
||||
|
||||
@task
|
||||
@needs(
|
||||
'pavelib.prereqs.install_node_prereqs',
|
||||
'pavelib.prereqs.install_python_prereqs',
|
||||
)
|
||||
@consume_args
|
||||
@timed
|
||||
def update_assets(args):
|
||||
"""
|
||||
Compile Sass, then collect static assets.
|
||||
|
||||
This is a DEPRECATED COMPATIBILITY WRAPPER around other DEPRECATED COMPATIBILITY WRAPPERS.
|
||||
The aggregate affect of this command can be achieved with this sequence of commands instead:
|
||||
|
||||
* pip install -r requirements/edx/assets.txt # replaces install_python_prereqs
|
||||
* npm clean-install # replaces install_node_prereqs
|
||||
* npm run build # replaces execute_compile_sass and webpack
|
||||
* ./manage.py lms collectstatic --noinput # replaces collect_assets (for LMS)
|
||||
* ./manage.py cms collectstatic --noinput # replaces collect_assets (for CMS)
|
||||
"""
|
||||
parser = argparse.ArgumentParser(prog='paver update_assets')
|
||||
parser.add_argument(
|
||||
'system', type=str, nargs='*', default=["lms", "studio"],
|
||||
help="lms or studio",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--settings', type=str, default=Env.DEVSTACK_SETTINGS,
|
||||
help="Django settings module",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--debug', action='store_true', default=False,
|
||||
help="Enable all debugging",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--debug-collect', action='store_true', default=False,
|
||||
help="Disable collect static",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--skip-collect', dest='collect', action='store_false', default=True,
|
||||
help="Skip collection of static assets",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--watch', action='store_true', default=False,
|
||||
help="Watch files for changes",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--theme-dirs', dest='theme_dirs', type=str, nargs='+', default=None,
|
||||
help="base directories where themes are placed",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--themes', type=str, nargs='+', default=None,
|
||||
help="list of themes to compile sass for. ignored when --watch is used; all themes are watched.",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--collect-log', dest="collect_log_dir", default=None,
|
||||
help="When running collectstatic, direct output to specified log directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
'--wait', type=float, default=0.0,
|
||||
help="DEPRECATED. Watchdog's default wait time is now used.",
|
||||
)
|
||||
args = parser.parse_args(args)
|
||||
|
||||
# Build Webpack
|
||||
call_task('pavelib.assets.webpack', options={'settings': args.settings})
|
||||
|
||||
# Compile sass for themes and system
|
||||
execute_compile_sass(args)
|
||||
|
||||
if args.collect:
|
||||
if args.collect_log_dir:
|
||||
collect_log_args = {"collect_log_dir": args.collect_log_dir}
|
||||
elif args.debug or args.debug_collect:
|
||||
collect_log_args = {"collect_log_dir": None}
|
||||
else:
|
||||
collect_log_args = {}
|
||||
|
||||
collect_assets(args.system, args.settings, **collect_log_args)
|
||||
|
||||
if args.watch:
|
||||
call_task(
|
||||
'pavelib.assets.watch_assets',
|
||||
options={
|
||||
'background': not args.debug,
|
||||
'settings': args.settings,
|
||||
'theme_dirs': args.theme_dirs,
|
||||
'themes': args.themes,
|
||||
'wait': [float(args.wait)]
|
||||
},
|
||||
)
|
||||
@@ -1,6 +0,0 @@
|
||||
[
|
||||
"foo/bar.py:192: [C0111(missing-docstring), Bliptv] Missing docstring",
|
||||
"foo/bar/test.py:74: [C0322(no-space-before-operator)] Operator not preceded by a space",
|
||||
"ugly/string/test.py:16: [C0103(invalid-name)] Invalid name \"whats up\" for type constant (should match (([A-Z_][A-Z0-9_]*)|(__.*__)|log|urlpatterns)$)",
|
||||
"multiple/lines/test.py:72: [C0322(no-space-before-operator)] Operator not preceded by a space\nFOO_BAR='pipeline.storage.NonPackagingPipelineStorage'\n ^"
|
||||
]
|
||||
@@ -1,130 +0,0 @@
|
||||
"""Unit tests for the Paver asset tasks."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from unittest import TestCase
|
||||
from unittest.mock import patch
|
||||
|
||||
import ddt
|
||||
import paver.easy
|
||||
from paver import tasks
|
||||
|
||||
import pavelib.assets
|
||||
from pavelib.assets import Env
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).parent.parent.parent
|
||||
|
||||
LMS_SETTINGS = {
|
||||
"WEBPACK_CONFIG_PATH": "webpack.fake.config.js",
|
||||
"STATIC_ROOT": "/fake/lms/staticfiles",
|
||||
|
||||
}
|
||||
CMS_SETTINGS = {
|
||||
"WEBPACK_CONFIG_PATH": "webpack.fake.config",
|
||||
"STATIC_ROOT": "/fake/cms/staticfiles",
|
||||
"JS_ENV_EXTRA_CONFIG": json.dumps({"key1": [True, False], "key2": {"key2.1": 1369, "key2.2": "1369"}}),
|
||||
}
|
||||
|
||||
|
||||
def _mock_get_django_settings(django_settings, system, settings=None): # pylint: disable=unused-argument
|
||||
return [(LMS_SETTINGS if system == "lms" else CMS_SETTINGS)[s] for s in django_settings]
|
||||
|
||||
|
||||
@ddt.ddt
|
||||
@patch.object(Env, 'get_django_settings', _mock_get_django_settings)
|
||||
@patch.object(Env, 'get_django_json_settings', _mock_get_django_settings)
|
||||
class TestDeprecatedPaverAssets(TestCase):
|
||||
"""
|
||||
Simple test to ensure that the soon-to-be-removed Paver commands are correctly translated into the new npm-run
|
||||
commands.
|
||||
"""
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
self.maxDiff = None
|
||||
os.environ['NO_PREREQ_INSTALL'] = 'true'
|
||||
tasks.environment = tasks.Environment()
|
||||
|
||||
def tearDown(self):
|
||||
super().tearDown()
|
||||
del os.environ['NO_PREREQ_INSTALL']
|
||||
|
||||
@ddt.data(
|
||||
dict(
|
||||
task_name='pavelib.assets.compile_sass',
|
||||
args=[],
|
||||
kwargs={},
|
||||
expected=["npm run compile-sass --"],
|
||||
),
|
||||
dict(
|
||||
task_name='pavelib.assets.compile_sass',
|
||||
args=[],
|
||||
kwargs={"system": "lms,studio"},
|
||||
expected=["npm run compile-sass --"],
|
||||
),
|
||||
dict(
|
||||
task_name='pavelib.assets.compile_sass',
|
||||
args=[],
|
||||
kwargs={"debug": True},
|
||||
expected=["npm run compile-sass-dev --"],
|
||||
),
|
||||
dict(
|
||||
task_name='pavelib.assets.compile_sass',
|
||||
args=[],
|
||||
kwargs={"system": "lms"},
|
||||
expected=["npm run compile-sass -- --skip-cms"],
|
||||
),
|
||||
dict(
|
||||
task_name='pavelib.assets.compile_sass',
|
||||
args=[],
|
||||
kwargs={"system": "studio"},
|
||||
expected=["npm run compile-sass -- --skip-lms"],
|
||||
),
|
||||
dict(
|
||||
task_name='pavelib.assets.compile_sass',
|
||||
args=[],
|
||||
kwargs={"system": "cms", "theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes"},
|
||||
expected=[
|
||||
"npm run compile-sass -- --skip-lms " +
|
||||
f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes"
|
||||
],
|
||||
),
|
||||
dict(
|
||||
task_name='pavelib.assets.compile_sass',
|
||||
args=[],
|
||||
kwargs={"theme_dirs": f"{REPO_ROOT}/common/test,{REPO_ROOT}/themes", "themes": "red-theme,test-theme"},
|
||||
expected=[
|
||||
"npm run compile-sass -- " +
|
||||
f"--theme-dir {REPO_ROOT}/common/test --theme-dir {REPO_ROOT}/themes " +
|
||||
"--theme red-theme --theme test-theme"
|
||||
],
|
||||
),
|
||||
dict(
|
||||
task_name='pavelib.assets.update_assets',
|
||||
args=["lms", "studio", "--settings=fake.settings"],
|
||||
kwargs={},
|
||||
expected=[
|
||||
(
|
||||
"WEBPACK_CONFIG_PATH=webpack.fake.config.js " +
|
||||
"NODE_ENV=production " +
|
||||
"STATIC_ROOT_LMS=/fake/lms/staticfiles " +
|
||||
"STATIC_ROOT_CMS=/fake/cms/staticfiles " +
|
||||
'JS_ENV_EXTRA_CONFIG=' +
|
||||
'"{\\"key1\\": [true, false], \\"key2\\": {\\"key2.1\\": 1369, \\"key2.2\\": \\"1369\\"}}" ' +
|
||||
"npm run webpack"
|
||||
),
|
||||
"python manage.py lms --settings=fake.settings compile_sass lms ",
|
||||
"python manage.py cms --settings=fake.settings compile_sass cms ",
|
||||
(
|
||||
"( ./manage.py lms --settings=fake.settings collectstatic --noinput ) && " +
|
||||
"( ./manage.py cms --settings=fake.settings collectstatic --noinput )"
|
||||
),
|
||||
],
|
||||
),
|
||||
)
|
||||
@ddt.unpack
|
||||
@patch.object(pavelib.assets, 'sh')
|
||||
def test_paver_assets_wrapper_invokes_new_commands(self, mock_sh, task_name, args, kwargs, expected):
|
||||
paver.easy.call_task(task_name, args=args, options=kwargs)
|
||||
assert [call_args[0] for (call_args, call_kwargs) in mock_sh.call_args_list] == expected
|
||||
@@ -1,97 +0,0 @@
|
||||
"""Unit tests for the Paver server tasks."""
|
||||
|
||||
|
||||
import os
|
||||
from unittest import TestCase
|
||||
from uuid import uuid4
|
||||
|
||||
from paver import tasks
|
||||
from paver.easy import BuildFailure
|
||||
|
||||
|
||||
class PaverTestCase(TestCase):
|
||||
"""
|
||||
Base class for Paver test cases.
|
||||
"""
|
||||
def setUp(self):
|
||||
super().setUp()
|
||||
|
||||
# Show full length diffs upon test failure
|
||||
self.maxDiff = None # pylint: disable=invalid-name
|
||||
|
||||
# Create a mock Paver environment
|
||||
tasks.environment = MockEnvironment()
|
||||
|
||||
# Don't run pre-reqs
|
||||
os.environ['NO_PREREQ_INSTALL'] = 'true'
|
||||
|
||||
def tearDown(self):
|
||||
super().tearDown()
|
||||
tasks.environment = tasks.Environment()
|
||||
del os.environ['NO_PREREQ_INSTALL']
|
||||
|
||||
@property
|
||||
def task_messages(self):
|
||||
"""Returns the messages output by the Paver task."""
|
||||
return tasks.environment.messages
|
||||
|
||||
@property
|
||||
def platform_root(self):
|
||||
"""Returns the current platform's root directory."""
|
||||
return os.getcwd()
|
||||
|
||||
def reset_task_messages(self):
|
||||
"""Clear the recorded message"""
|
||||
tasks.environment.messages = []
|
||||
|
||||
|
||||
class MockEnvironment(tasks.Environment):
|
||||
"""
|
||||
Mock environment that collects information about Paver commands.
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.dry_run = True
|
||||
self.messages = []
|
||||
|
||||
def info(self, message, *args):
|
||||
"""Capture any messages that have been recorded"""
|
||||
if args:
|
||||
output = message % args
|
||||
else:
|
||||
output = message
|
||||
if not output.startswith("--->"):
|
||||
self.messages.append(str(output))
|
||||
|
||||
|
||||
def fail_on_eslint(*args, **kwargs):
|
||||
"""
|
||||
For our tests, we need the call for diff-quality running eslint reports
|
||||
to fail, since that is what is going to fail when we pass in a
|
||||
percentage ("p") requirement.
|
||||
"""
|
||||
if "eslint" in args[0]: # lint-amnesty, pylint: disable=no-else-raise
|
||||
raise BuildFailure('Subprocess return code: 1')
|
||||
else:
|
||||
if kwargs.get('capture', False):
|
||||
return uuid4().hex
|
||||
else:
|
||||
return
|
||||
|
||||
|
||||
def fail_on_npm_install():
|
||||
"""
|
||||
Used to simulate an error when executing "npm install"
|
||||
"""
|
||||
return 1
|
||||
|
||||
|
||||
def unexpected_fail_on_npm_install(*args, **kwargs): # pylint: disable=unused-argument
|
||||
"""
|
||||
For our tests, we need the call for diff-quality running pycodestyle reports to fail, since that is what
|
||||
is going to fail when we pass in a percentage ("p") requirement.
|
||||
"""
|
||||
if ["npm", "install", "--verbose"] == args[0]: # lint-amnesty, pylint: disable=no-else-raise
|
||||
raise BuildFailure('Subprocess return code: 50')
|
||||
else:
|
||||
return
|
||||
@@ -1,351 +0,0 @@
|
||||
"""
|
||||
Install Python and Node prerequisites.
|
||||
"""
|
||||
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from distutils import sysconfig # pylint: disable=deprecated-module
|
||||
|
||||
from paver.easy import sh, task # lint-amnesty, pylint: disable=unused-import
|
||||
|
||||
from .utils.envs import Env
|
||||
from .utils.timer import timed
|
||||
|
||||
PREREQS_STATE_DIR = os.getenv('PREREQ_CACHE_DIR', Env.REPO_ROOT / '.prereqs_cache')
|
||||
NO_PREREQ_MESSAGE = "NO_PREREQ_INSTALL is set, not installing prereqs"
|
||||
NO_PYTHON_UNINSTALL_MESSAGE = 'NO_PYTHON_UNINSTALL is set. No attempts will be made to uninstall old Python libs.'
|
||||
COVERAGE_REQ_FILE = 'requirements/edx/coverage.txt'
|
||||
|
||||
# If you make any changes to this list you also need to make
|
||||
# a corresponding change to circle.yml, which is how the python
|
||||
# prerequisites are installed for builds on circleci.com
|
||||
toxenv = os.environ.get('TOXENV')
|
||||
if toxenv and toxenv != 'quality':
|
||||
PYTHON_REQ_FILES = ['requirements/edx/testing.txt']
|
||||
else:
|
||||
PYTHON_REQ_FILES = ['requirements/edx/development.txt']
|
||||
|
||||
# Developers can have private requirements, for local copies of github repos,
|
||||
# or favorite debugging tools, etc.
|
||||
PRIVATE_REQS = 'requirements/edx/private.txt'
|
||||
if os.path.exists(PRIVATE_REQS):
|
||||
PYTHON_REQ_FILES.append(PRIVATE_REQS)
|
||||
|
||||
|
||||
def str2bool(s):
|
||||
s = str(s)
|
||||
return s.lower() in ('yes', 'true', 't', '1')
|
||||
|
||||
|
||||
def no_prereq_install():
|
||||
"""
|
||||
Determine if NO_PREREQ_INSTALL should be truthy or falsy.
|
||||
"""
|
||||
return str2bool(os.environ.get('NO_PREREQ_INSTALL', 'False'))
|
||||
|
||||
|
||||
def no_python_uninstall():
|
||||
""" Determine if we should run the uninstall_python_packages task. """
|
||||
return str2bool(os.environ.get('NO_PYTHON_UNINSTALL', 'False'))
|
||||
|
||||
|
||||
def create_prereqs_cache_dir():
|
||||
"""Create the directory for storing the hashes, if it doesn't exist already."""
|
||||
try:
|
||||
os.makedirs(PREREQS_STATE_DIR)
|
||||
except OSError:
|
||||
if not os.path.isdir(PREREQS_STATE_DIR):
|
||||
raise
|
||||
|
||||
|
||||
def compute_fingerprint(path_list):
|
||||
"""
|
||||
Hash the contents of all the files and directories in `path_list`.
|
||||
Returns the hex digest.
|
||||
"""
|
||||
|
||||
hasher = hashlib.sha1()
|
||||
|
||||
for path_item in path_list:
|
||||
|
||||
# For directories, create a hash based on the modification times
|
||||
# of first-level subdirectories
|
||||
if os.path.isdir(path_item):
|
||||
for dirname in sorted(os.listdir(path_item)):
|
||||
path_name = os.path.join(path_item, dirname)
|
||||
if os.path.isdir(path_name):
|
||||
hasher.update(str(os.stat(path_name).st_mtime).encode('utf-8'))
|
||||
|
||||
# For files, hash the contents of the file
|
||||
if os.path.isfile(path_item):
|
||||
with open(path_item, "rb") as file_handle:
|
||||
hasher.update(file_handle.read())
|
||||
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def prereq_cache(cache_name, paths, install_func):
|
||||
"""
|
||||
Conditionally execute `install_func()` only if the files/directories
|
||||
specified by `paths` have changed.
|
||||
|
||||
If the code executes successfully (no exceptions are thrown), the cache
|
||||
is updated with the new hash.
|
||||
"""
|
||||
# Retrieve the old hash
|
||||
cache_filename = cache_name.replace(" ", "_")
|
||||
cache_file_path = os.path.join(PREREQS_STATE_DIR, f"{cache_filename}.sha1")
|
||||
old_hash = None
|
||||
if os.path.isfile(cache_file_path):
|
||||
with open(cache_file_path) as cache_file:
|
||||
old_hash = cache_file.read()
|
||||
|
||||
# Compare the old hash to the new hash
|
||||
# If they do not match (either the cache hasn't been created, or the files have changed),
|
||||
# then execute the code within the block.
|
||||
new_hash = compute_fingerprint(paths)
|
||||
if new_hash != old_hash:
|
||||
install_func()
|
||||
|
||||
# Update the cache with the new hash
|
||||
# If the code executed within the context fails (throws an exception),
|
||||
# then this step won't get executed.
|
||||
create_prereqs_cache_dir()
|
||||
with open(cache_file_path, "wb") as cache_file:
|
||||
# Since the pip requirement files are modified during the install
|
||||
# process, we need to store the hash generated AFTER the installation
|
||||
post_install_hash = compute_fingerprint(paths)
|
||||
cache_file.write(post_install_hash.encode('utf-8'))
|
||||
else:
|
||||
print(f'{cache_name} unchanged, skipping...')
|
||||
|
||||
|
||||
def node_prereqs_installation():
|
||||
"""
|
||||
Configures npm and installs Node prerequisites
|
||||
"""
|
||||
# Before July 2023, these directories were created and written to
|
||||
# as root. Afterwards, they are created as being owned by the
|
||||
# `app` user -- but also need to be deleted by that user (due to
|
||||
# how npm runs post-install scripts.) Developers with an older
|
||||
# devstack installation who are reprovisioning will see errors
|
||||
# here if the files are still owned by root. Deleting the files in
|
||||
# advance prevents this error.
|
||||
#
|
||||
# This hack should probably be left in place for at least a year.
|
||||
# See ADR 17 for more background on the transition.
|
||||
sh("rm -rf common/static/common/js/vendor/ common/static/common/css/vendor/")
|
||||
# At the time of this writing, the js dir has git-versioned files
|
||||
# but the css dir does not, so the latter would have been created
|
||||
# as root-owned (in the process of creating the vendor
|
||||
# subdirectory). Delete it only if empty, just in case
|
||||
# git-versioned files are added later.
|
||||
sh("rmdir common/static/common/css || true")
|
||||
|
||||
# NPM installs hang sporadically. Log the installation process so that we
|
||||
# determine if any packages are chronic offenders.
|
||||
npm_log_file_path = f'{Env.GEN_LOG_DIR}/npm-install.log'
|
||||
npm_log_file = open(npm_log_file_path, 'wb') # lint-amnesty, pylint: disable=consider-using-with
|
||||
npm_command = 'npm ci --verbose'.split()
|
||||
|
||||
# The implementation of Paver's `sh` function returns before the forked
|
||||
# actually returns. Using a Popen object so that we can ensure that
|
||||
# the forked process has returned
|
||||
proc = subprocess.Popen(npm_command, stderr=npm_log_file) # lint-amnesty, pylint: disable=consider-using-with
|
||||
retcode = proc.wait()
|
||||
if retcode == 1:
|
||||
raise Exception(f"npm install failed: See {npm_log_file_path}")
|
||||
print("Successfully clean-installed NPM packages. Log found at {}".format(
|
||||
npm_log_file_path
|
||||
))
|
||||
|
||||
|
||||
def python_prereqs_installation():
|
||||
"""
|
||||
Installs Python prerequisites
|
||||
"""
|
||||
# edx-platform installs some Python projects from within the edx-platform repo itself.
|
||||
sh("pip install -e .")
|
||||
for req_file in PYTHON_REQ_FILES:
|
||||
pip_install_req_file(req_file)
|
||||
|
||||
|
||||
def pip_install_req_file(req_file):
|
||||
"""Pip install the requirements file."""
|
||||
pip_cmd = 'pip install -q --disable-pip-version-check --exists-action w'
|
||||
sh(f"{pip_cmd} -r {req_file}")
|
||||
|
||||
|
||||
@task
|
||||
@timed
|
||||
def install_node_prereqs():
|
||||
"""
|
||||
Installs Node prerequisites
|
||||
"""
|
||||
if no_prereq_install():
|
||||
print(NO_PREREQ_MESSAGE)
|
||||
return
|
||||
|
||||
prereq_cache("Node prereqs", ["package.json", "package-lock.json"], node_prereqs_installation)
|
||||
|
||||
|
||||
# To add a package to the uninstall list, just add it to this list! No need
|
||||
# to touch any other part of this file.
|
||||
PACKAGES_TO_UNINSTALL = [
|
||||
"MySQL-python", # Because mysqlclient shares the same directory name
|
||||
"South", # Because it interferes with Django 1.8 migrations.
|
||||
"edxval", # Because it was bork-installed somehow.
|
||||
"django-storages",
|
||||
"django-oauth2-provider", # Because now it's called edx-django-oauth2-provider.
|
||||
"edx-oauth2-provider", # Because it moved from github to pypi
|
||||
"enum34", # Because enum34 is not needed in python>3.4
|
||||
"i18n-tools", # Because now it's called edx-i18n-tools
|
||||
"moto", # Because we no longer use it and it conflicts with recent jsondiff versions
|
||||
"python-saml", # Because python3-saml shares the same directory name
|
||||
"pytest-faulthandler", # Because it was bundled into pytest
|
||||
"djangorestframework-jwt", # Because now its called drf-jwt.
|
||||
]
|
||||
|
||||
|
||||
@task
|
||||
@timed
|
||||
def uninstall_python_packages():
|
||||
"""
|
||||
Uninstall Python packages that need explicit uninstallation.
|
||||
|
||||
Some Python packages that we no longer want need to be explicitly
|
||||
uninstalled, notably, South. Some other packages were once installed in
|
||||
ways that were resistant to being upgraded, like edxval. Also uninstall
|
||||
them.
|
||||
"""
|
||||
|
||||
if no_python_uninstall():
|
||||
print(NO_PYTHON_UNINSTALL_MESSAGE)
|
||||
return
|
||||
|
||||
# So that we don't constantly uninstall things, use a hash of the packages
|
||||
# to be uninstalled. Check it, and skip this if we're up to date.
|
||||
hasher = hashlib.sha1()
|
||||
hasher.update(repr(PACKAGES_TO_UNINSTALL).encode('utf-8'))
|
||||
expected_version = hasher.hexdigest()
|
||||
state_file_path = os.path.join(PREREQS_STATE_DIR, "Python_uninstall.sha1")
|
||||
create_prereqs_cache_dir()
|
||||
|
||||
if os.path.isfile(state_file_path):
|
||||
with open(state_file_path) as state_file:
|
||||
version = state_file.read()
|
||||
if version == expected_version:
|
||||
print('Python uninstalls unchanged, skipping...')
|
||||
return
|
||||
|
||||
# Run pip to find the packages we need to get rid of. Believe it or not,
|
||||
# edx-val is installed in a way that it is present twice, so we have a loop
|
||||
# to really really get rid of it.
|
||||
for _ in range(3):
|
||||
uninstalled = False
|
||||
frozen = sh("pip freeze", capture=True)
|
||||
|
||||
for package_name in PACKAGES_TO_UNINSTALL:
|
||||
if package_in_frozen(package_name, frozen):
|
||||
# Uninstall the pacakge
|
||||
sh(f"pip uninstall --disable-pip-version-check -y {package_name}")
|
||||
uninstalled = True
|
||||
if not uninstalled:
|
||||
break
|
||||
else:
|
||||
# We tried three times and didn't manage to get rid of the pests.
|
||||
print("Couldn't uninstall unwanted Python packages!")
|
||||
return
|
||||
|
||||
# Write our version.
|
||||
with open(state_file_path, "wb") as state_file:
|
||||
state_file.write(expected_version.encode('utf-8'))
|
||||
|
||||
|
||||
def package_in_frozen(package_name, frozen_output):
|
||||
"""Is this package in the output of 'pip freeze'?"""
|
||||
# Look for either:
|
||||
#
|
||||
# PACKAGE-NAME==
|
||||
#
|
||||
# or:
|
||||
#
|
||||
# blah_blah#egg=package_name-version
|
||||
#
|
||||
pattern = r"(?mi)^{pkg}==|#egg={pkg_under}-".format(
|
||||
pkg=re.escape(package_name),
|
||||
pkg_under=re.escape(package_name.replace("-", "_")),
|
||||
)
|
||||
return bool(re.search(pattern, frozen_output))
|
||||
|
||||
|
||||
@task
|
||||
@timed
|
||||
def install_coverage_prereqs():
|
||||
""" Install python prereqs for measuring coverage. """
|
||||
if no_prereq_install():
|
||||
print(NO_PREREQ_MESSAGE)
|
||||
return
|
||||
pip_install_req_file(COVERAGE_REQ_FILE)
|
||||
|
||||
|
||||
@task
|
||||
@timed
|
||||
def install_python_prereqs():
|
||||
"""
|
||||
Installs Python prerequisites.
|
||||
"""
|
||||
if no_prereq_install():
|
||||
print(NO_PREREQ_MESSAGE)
|
||||
return
|
||||
|
||||
uninstall_python_packages()
|
||||
|
||||
# Include all of the requirements files in the fingerprint.
|
||||
files_to_fingerprint = list(PYTHON_REQ_FILES)
|
||||
|
||||
# Also fingerprint the directories where packages get installed:
|
||||
# ("/edx/app/edxapp/venvs/edxapp/lib/python2.7/site-packages")
|
||||
files_to_fingerprint.append(sysconfig.get_python_lib())
|
||||
|
||||
# In a virtualenv, "-e installs" get put in a src directory.
|
||||
if Env.PIP_SRC:
|
||||
src_dir = Env.PIP_SRC
|
||||
else:
|
||||
src_dir = os.path.join(sys.prefix, "src")
|
||||
if os.path.isdir(src_dir):
|
||||
files_to_fingerprint.append(src_dir)
|
||||
|
||||
# Also fingerprint this source file, so that if the logic for installations
|
||||
# changes, we will redo the installation.
|
||||
this_file = __file__
|
||||
if this_file.endswith(".pyc"):
|
||||
this_file = this_file[:-1] # use the .py file instead of the .pyc
|
||||
files_to_fingerprint.append(this_file)
|
||||
|
||||
prereq_cache("Python prereqs", files_to_fingerprint, python_prereqs_installation)
|
||||
|
||||
|
||||
@task
|
||||
@timed
|
||||
def install_prereqs():
|
||||
"""
|
||||
Installs Node and Python prerequisites
|
||||
"""
|
||||
if no_prereq_install():
|
||||
print(NO_PREREQ_MESSAGE)
|
||||
return
|
||||
|
||||
if not str2bool(os.environ.get('SKIP_NPM_INSTALL', 'False')):
|
||||
install_node_prereqs()
|
||||
install_python_prereqs()
|
||||
log_installed_python_prereqs()
|
||||
|
||||
|
||||
def log_installed_python_prereqs():
|
||||
""" Logs output of pip freeze for debugging. """
|
||||
sh("pip freeze > {}".format(Env.GEN_LOG_DIR + "/pip_freeze.log"))
|
||||
@@ -1,24 +0,0 @@
|
||||
"""
|
||||
Helper functions for constructing shell commands.
|
||||
"""
|
||||
|
||||
|
||||
def cmd(*args):
|
||||
"""
|
||||
Concatenate the arguments into a space-separated shell command.
|
||||
"""
|
||||
return " ".join(str(arg) for arg in args if arg)
|
||||
|
||||
|
||||
def django_cmd(sys, settings, *args):
|
||||
"""
|
||||
Construct a Django management command.
|
||||
|
||||
`sys` is either 'lms' or 'studio'.
|
||||
`settings` is the Django settings module (such as "dev" or "test")
|
||||
`args` are concatenated to form the rest of the command.
|
||||
"""
|
||||
# Maintain backwards compatibility with manage.py,
|
||||
# which calls "studio" "cms"
|
||||
sys = 'cms' if sys == 'studio' else sys
|
||||
return cmd("python manage.py", sys, f"--settings={settings}", *args)
|
||||
@@ -1,271 +0,0 @@
|
||||
"""
|
||||
Helper functions for loading environment settings.
|
||||
"""
|
||||
import configparser
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from time import sleep
|
||||
|
||||
from lazy import lazy
|
||||
from path import Path as path
|
||||
from paver.easy import BuildFailure, sh
|
||||
|
||||
from pavelib.utils.cmd import django_cmd
|
||||
|
||||
|
||||
def repo_root():
|
||||
"""
|
||||
Get the root of the git repository (edx-platform).
|
||||
|
||||
This sometimes fails on Docker Devstack, so it's been broken
|
||||
down with some additional error handling. It usually starts
|
||||
working within 30 seconds or so; for more details, see
|
||||
https://openedx.atlassian.net/browse/PLAT-1629 and
|
||||
https://github.com/docker/for-mac/issues/1509
|
||||
"""
|
||||
file_path = path(__file__)
|
||||
attempt = 1
|
||||
while True:
|
||||
try:
|
||||
absolute_path = file_path.abspath()
|
||||
break
|
||||
except OSError:
|
||||
print(f'Attempt {attempt}/180 to get an absolute path failed')
|
||||
if attempt < 180:
|
||||
attempt += 1
|
||||
sleep(1)
|
||||
else:
|
||||
print('Unable to determine the absolute path of the edx-platform repo, aborting')
|
||||
raise
|
||||
return absolute_path.parent.parent.parent
|
||||
|
||||
|
||||
class Env:
|
||||
"""
|
||||
Load information about the execution environment.
|
||||
"""
|
||||
|
||||
# Root of the git repository (edx-platform)
|
||||
REPO_ROOT = repo_root()
|
||||
|
||||
# Reports Directory
|
||||
REPORT_DIR = REPO_ROOT / 'reports'
|
||||
METRICS_DIR = REPORT_DIR / 'metrics'
|
||||
QUALITY_DIR = REPORT_DIR / 'quality_junitxml'
|
||||
|
||||
# Generic log dir
|
||||
GEN_LOG_DIR = REPO_ROOT / "test_root" / "log"
|
||||
|
||||
# Python unittest dirs
|
||||
PYTHON_COVERAGERC = REPO_ROOT / ".coveragerc"
|
||||
|
||||
# Which Python version should be used in xdist workers?
|
||||
PYTHON_VERSION = os.environ.get("PYTHON_VERSION", "2.7")
|
||||
|
||||
# Directory that videos are served from
|
||||
VIDEO_SOURCE_DIR = REPO_ROOT / "test_root" / "data" / "video"
|
||||
|
||||
PRINT_SETTINGS_LOG_FILE = GEN_LOG_DIR / "print_settings.log"
|
||||
|
||||
# Detect if in a Docker container, and if so which one
|
||||
FRONTEND_TEST_SERVER_HOST = os.environ.get('FRONTEND_TEST_SERVER_HOSTNAME', '0.0.0.0')
|
||||
USING_DOCKER = FRONTEND_TEST_SERVER_HOST != '0.0.0.0'
|
||||
DEVSTACK_SETTINGS = 'devstack_docker' if USING_DOCKER else 'devstack'
|
||||
TEST_SETTINGS = 'test'
|
||||
|
||||
# Mongo databases that will be dropped before/after the tests run
|
||||
MONGO_HOST = 'localhost'
|
||||
|
||||
# Test Ids Directory
|
||||
TEST_DIR = REPO_ROOT / ".testids"
|
||||
|
||||
# Configured browser to use for the js test suites
|
||||
SELENIUM_BROWSER = os.environ.get('SELENIUM_BROWSER', 'firefox')
|
||||
if USING_DOCKER:
|
||||
KARMA_BROWSER = 'ChromeDocker' if SELENIUM_BROWSER == 'chrome' else 'FirefoxDocker'
|
||||
else:
|
||||
KARMA_BROWSER = 'FirefoxNoUpdates'
|
||||
|
||||
# Files used to run each of the js test suites
|
||||
# TODO: We have [temporarily disabled] the three Webpack-based tests suites. They have been silently
|
||||
# broken for a long time; after noticing they were broken, we added the DieHardPlugin to
|
||||
# webpack.common.config.js to prevent future silent breakage, but have not yet been able to
|
||||
# fix and re-enable the suites. Note that the LMS suite is all Webpack-based even though it's
|
||||
# not in the name.
|
||||
# Issue: https://github.com/openedx/edx-platform/issues/35956
|
||||
KARMA_CONFIG_FILES = [
|
||||
REPO_ROOT / 'cms/static/karma_cms.conf.js',
|
||||
REPO_ROOT / 'cms/static/karma_cms_squire.conf.js',
|
||||
## [temporarily disabled] REPO_ROOT / 'cms/static/karma_cms_webpack.conf.js',
|
||||
## [temporarily disabled] REPO_ROOT / 'lms/static/karma_lms.conf.js',
|
||||
REPO_ROOT / 'xmodule/js/karma_xmodule.conf.js',
|
||||
## [temporarily disabled] REPO_ROOT / 'xmodule/js/karma_xmodule_webpack.conf.js',
|
||||
REPO_ROOT / 'common/static/karma_common.conf.js',
|
||||
REPO_ROOT / 'common/static/karma_common_requirejs.conf.js',
|
||||
]
|
||||
|
||||
JS_TEST_ID_KEYS = [
|
||||
'cms',
|
||||
'cms-squire',
|
||||
## [temporarily-disabled] 'cms-webpack',
|
||||
## [temporarily-disabled] 'lms',
|
||||
'xmodule',
|
||||
## [temporarily-disabled] 'xmodule-webpack',
|
||||
'common',
|
||||
'common-requirejs',
|
||||
'jest-snapshot'
|
||||
]
|
||||
|
||||
JS_REPORT_DIR = REPORT_DIR / 'javascript'
|
||||
|
||||
# Directories used for pavelib/ tests
|
||||
IGNORED_TEST_DIRS = ('__pycache__', '.cache', '.pytest_cache')
|
||||
LIB_TEST_DIRS = [path("pavelib/paver_tests"), path("scripts/xsslint/tests")]
|
||||
|
||||
# Directory for i18n test reports
|
||||
I18N_REPORT_DIR = REPORT_DIR / 'i18n'
|
||||
|
||||
# Directory for keeping src folder that comes with pip installation.
|
||||
# Setting this is equivalent to passing `--src <dir>` to pip directly.
|
||||
PIP_SRC = os.environ.get("PIP_SRC")
|
||||
|
||||
# Service variant (lms, cms, etc.) configured with an environment variable
|
||||
# We use this to determine which envs.json file to load.
|
||||
SERVICE_VARIANT = os.environ.get('SERVICE_VARIANT', None)
|
||||
|
||||
# If service variant not configured in env, then pass the correct
|
||||
# environment for lms / cms
|
||||
if not SERVICE_VARIANT: # this will intentionally catch "";
|
||||
if any(i in sys.argv[1:] for i in ('cms', 'studio')):
|
||||
SERVICE_VARIANT = 'cms'
|
||||
else:
|
||||
SERVICE_VARIANT = 'lms'
|
||||
|
||||
@classmethod
|
||||
def get_django_settings(cls, django_settings, system, settings=None, print_setting_args=None):
|
||||
"""
|
||||
Interrogate Django environment for specific settings values
|
||||
:param django_settings: list of django settings values to get
|
||||
:param system: the django app to use when asking for the setting (lms | cms)
|
||||
:param settings: the settings file to use when asking for the value
|
||||
:param print_setting_args: the additional arguments to send to print_settings
|
||||
:return: unicode value of the django setting
|
||||
"""
|
||||
if not settings:
|
||||
settings = os.environ.get("EDX_PLATFORM_SETTINGS", "aws")
|
||||
log_dir = os.path.dirname(cls.PRINT_SETTINGS_LOG_FILE)
|
||||
if not os.path.exists(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
settings_length = len(django_settings)
|
||||
django_settings = ' '.join(django_settings) # parse_known_args makes a list again
|
||||
print_setting_args = ' '.join(print_setting_args or [])
|
||||
try:
|
||||
value = sh(
|
||||
django_cmd(
|
||||
system,
|
||||
settings,
|
||||
"print_setting {django_settings} 2>{log_file} {print_setting_args}".format(
|
||||
django_settings=django_settings,
|
||||
print_setting_args=print_setting_args,
|
||||
log_file=cls.PRINT_SETTINGS_LOG_FILE
|
||||
).strip()
|
||||
),
|
||||
capture=True
|
||||
)
|
||||
# else for cases where values are not found & sh returns one None value
|
||||
return tuple(str(value).splitlines()) if value else tuple(None for _ in range(settings_length))
|
||||
except BuildFailure:
|
||||
print(f"Unable to print the value of the {django_settings} setting:")
|
||||
with open(cls.PRINT_SETTINGS_LOG_FILE) as f:
|
||||
print(f.read())
|
||||
sys.exit(1)
|
||||
|
||||
@classmethod
|
||||
def get_django_json_settings(cls, django_settings, system, settings=None):
|
||||
"""
|
||||
Interrogate Django environment for specific settings value
|
||||
:param django_settings: list of django settings values to get
|
||||
:param system: the django app to use when asking for the setting (lms | cms)
|
||||
:param settings: the settings file to use when asking for the value
|
||||
:return: json string value of the django setting
|
||||
"""
|
||||
return cls.get_django_settings(
|
||||
django_settings,
|
||||
system,
|
||||
settings=settings,
|
||||
print_setting_args=["--json"],
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def covered_modules(cls):
|
||||
"""
|
||||
List the source modules listed in .coveragerc for which coverage
|
||||
will be measured.
|
||||
"""
|
||||
coveragerc = configparser.RawConfigParser()
|
||||
coveragerc.read(cls.PYTHON_COVERAGERC)
|
||||
modules = coveragerc.get('run', 'source')
|
||||
result = []
|
||||
for module in modules.split('\n'):
|
||||
module = module.strip()
|
||||
if module:
|
||||
result.append(module)
|
||||
return result
|
||||
|
||||
@lazy
|
||||
def env_tokens(self):
|
||||
"""
|
||||
Return a dict of environment settings.
|
||||
If we couldn't find the JSON file, issue a warning and return an empty dict.
|
||||
"""
|
||||
|
||||
# Find the env JSON file
|
||||
if self.SERVICE_VARIANT:
|
||||
env_path = self.REPO_ROOT.parent / f"{self.SERVICE_VARIANT}.env.json"
|
||||
else:
|
||||
env_path = path("env.json").abspath()
|
||||
|
||||
# If the file does not exist, here or one level up,
|
||||
# issue a warning and return an empty dict
|
||||
if not env_path.isfile():
|
||||
env_path = env_path.parent.parent / env_path.basename()
|
||||
if not env_path.isfile():
|
||||
print(
|
||||
"Warning: could not find environment JSON file "
|
||||
"at '{path}'".format(path=env_path),
|
||||
file=sys.stderr,
|
||||
)
|
||||
return {}
|
||||
|
||||
# Otherwise, load the file as JSON and return the resulting dict
|
||||
try:
|
||||
with open(env_path) as env_file:
|
||||
return json.load(env_file)
|
||||
|
||||
except ValueError:
|
||||
print(
|
||||
"Error: Could not parse JSON "
|
||||
"in {path}".format(path=env_path),
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
@lazy
|
||||
def feature_flags(self):
|
||||
"""
|
||||
Return a dictionary of feature flags configured by the environment.
|
||||
"""
|
||||
return self.env_tokens.get('FEATURES', {})
|
||||
|
||||
@classmethod
|
||||
def rsync_dirs(cls):
|
||||
"""
|
||||
List the directories that should be synced during pytest-xdist
|
||||
execution. Needs to include all modules for which coverage is
|
||||
measured, not just the tests being run.
|
||||
"""
|
||||
result = set()
|
||||
for module in cls.covered_modules():
|
||||
result.add(module.split('/')[0])
|
||||
return result
|
||||
@@ -1,121 +0,0 @@
|
||||
"""
|
||||
Helper functions for managing processes.
|
||||
"""
|
||||
|
||||
|
||||
import atexit
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import psutil
|
||||
from paver import tasks
|
||||
|
||||
|
||||
def kill_process(proc):
|
||||
"""
|
||||
Kill the process `proc` created with `subprocess`.
|
||||
"""
|
||||
p1_group = psutil.Process(proc.pid)
|
||||
child_pids = p1_group.children(recursive=True)
|
||||
|
||||
for child_pid in child_pids:
|
||||
os.kill(child_pid.pid, signal.SIGKILL)
|
||||
|
||||
|
||||
def run_multi_processes(cmd_list, out_log=None, err_log=None):
|
||||
"""
|
||||
Run each shell command in `cmd_list` in a separate process,
|
||||
piping stdout to `out_log` (a path) and stderr to `err_log` (also a path).
|
||||
|
||||
Terminates the processes on CTRL-C and ensures the processes are killed
|
||||
if an error occurs.
|
||||
"""
|
||||
kwargs = {'shell': True, 'cwd': None}
|
||||
pids = []
|
||||
|
||||
if out_log:
|
||||
out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with
|
||||
kwargs['stdout'] = out_log_file
|
||||
|
||||
if err_log:
|
||||
err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with
|
||||
kwargs['stderr'] = err_log_file
|
||||
|
||||
# If the user is performing a dry run of a task, then just log
|
||||
# the command strings and return so that no destructive operations
|
||||
# are performed.
|
||||
if tasks.environment.dry_run:
|
||||
for cmd in cmd_list:
|
||||
tasks.environment.info(cmd)
|
||||
return
|
||||
|
||||
try:
|
||||
for cmd in cmd_list:
|
||||
pids.extend([subprocess.Popen(cmd, **kwargs)])
|
||||
|
||||
# pylint: disable=unused-argument
|
||||
def _signal_handler(*args):
|
||||
"""
|
||||
What to do when process is ended
|
||||
"""
|
||||
print("\nEnding...")
|
||||
|
||||
signal.signal(signal.SIGINT, _signal_handler)
|
||||
print("Enter CTL-C to end")
|
||||
signal.pause()
|
||||
print("Processes ending")
|
||||
|
||||
# pylint: disable=broad-except
|
||||
except Exception as err:
|
||||
print(f"Error running process {err}", file=sys.stderr)
|
||||
|
||||
finally:
|
||||
for pid in pids:
|
||||
kill_process(pid)
|
||||
|
||||
|
||||
def run_process(cmd, out_log=None, err_log=None):
|
||||
"""
|
||||
Run the shell command `cmd` in a separate process,
|
||||
piping stdout to `out_log` (a path) and stderr to `err_log` (also a path).
|
||||
|
||||
Terminates the process on CTRL-C or if an error occurs.
|
||||
"""
|
||||
return run_multi_processes([cmd], out_log=out_log, err_log=err_log)
|
||||
|
||||
|
||||
def run_background_process(cmd, out_log=None, err_log=None, cwd=None):
|
||||
"""
|
||||
Runs a command as a background process. Sends SIGINT at exit.
|
||||
"""
|
||||
|
||||
kwargs = {'shell': True, 'cwd': cwd}
|
||||
if out_log:
|
||||
out_log_file = open(out_log, 'w') # lint-amnesty, pylint: disable=consider-using-with
|
||||
kwargs['stdout'] = out_log_file
|
||||
|
||||
if err_log:
|
||||
err_log_file = open(err_log, 'w') # lint-amnesty, pylint: disable=consider-using-with
|
||||
kwargs['stderr'] = err_log_file
|
||||
|
||||
proc = subprocess.Popen(cmd, **kwargs) # lint-amnesty, pylint: disable=consider-using-with
|
||||
|
||||
def exit_handler():
|
||||
"""
|
||||
Send SIGINT to the process's children. This is important
|
||||
for running commands under coverage, as coverage will not
|
||||
produce the correct artifacts if the child process isn't
|
||||
killed properly.
|
||||
"""
|
||||
p1_group = psutil.Process(proc.pid)
|
||||
child_pids = p1_group.children(recursive=True)
|
||||
|
||||
for child_pid in child_pids:
|
||||
os.kill(child_pid.pid, signal.SIGINT)
|
||||
|
||||
# Wait for process to actually finish
|
||||
proc.wait()
|
||||
|
||||
atexit.register(exit_handler)
|
||||
@@ -1,83 +0,0 @@
|
||||
"""
|
||||
Tools for timing paver tasks
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
from datetime import datetime
|
||||
from os.path import dirname, exists
|
||||
|
||||
import wrapt
|
||||
|
||||
LOGGER = logging.getLogger(__file__)
|
||||
PAVER_TIMER_LOG = os.environ.get('PAVER_TIMER_LOG')
|
||||
|
||||
|
||||
@wrapt.decorator
|
||||
def timed(wrapped, instance, args, kwargs): # pylint: disable=unused-argument
|
||||
"""
|
||||
Log execution time for a function to a log file.
|
||||
|
||||
Logging is only actually executed if the PAVER_TIMER_LOG environment variable
|
||||
is set. That variable is expanded for the current user and current
|
||||
environment variables. It also can have :meth:`~Datetime.strftime` format
|
||||
identifiers which are substituted using the time when the task started.
|
||||
|
||||
For example, ``PAVER_TIMER_LOG='~/.paver.logs/%Y-%d-%m.log'`` will create a new
|
||||
log file every day containing reconds for paver tasks run that day, and
|
||||
will put those log files in the ``.paver.logs`` directory inside the users
|
||||
home.
|
||||
|
||||
Must be earlier in the decorator stack than the paver task declaration.
|
||||
"""
|
||||
start = datetime.utcnow()
|
||||
exception_info = {}
|
||||
try:
|
||||
return wrapped(*args, **kwargs)
|
||||
except Exception as exc:
|
||||
exception_info = {
|
||||
'exception': "".join(traceback.format_exception_only(type(exc), exc)).strip()
|
||||
}
|
||||
raise
|
||||
finally:
|
||||
end = datetime.utcnow()
|
||||
|
||||
# N.B. This is intended to provide a consistent interface and message format
|
||||
# across all of Open edX tooling, so it deliberately eschews standard
|
||||
# python logging infrastructure.
|
||||
if PAVER_TIMER_LOG is not None:
|
||||
|
||||
log_path = start.strftime(PAVER_TIMER_LOG)
|
||||
|
||||
log_message = {
|
||||
'python_version': sys.version,
|
||||
'task': f"{wrapped.__module__}.{wrapped.__name__}",
|
||||
'args': [repr(arg) for arg in args],
|
||||
'kwargs': {key: repr(value) for key, value in kwargs.items()},
|
||||
'started_at': start.isoformat(' '),
|
||||
'ended_at': end.isoformat(' '),
|
||||
'duration': (end - start).total_seconds(),
|
||||
}
|
||||
log_message.update(exception_info)
|
||||
|
||||
try:
|
||||
log_dir = dirname(log_path)
|
||||
if log_dir and not exists(log_dir):
|
||||
os.makedirs(log_dir)
|
||||
|
||||
with open(log_path, 'a') as outfile:
|
||||
json.dump(
|
||||
log_message,
|
||||
outfile,
|
||||
separators=(',', ':'),
|
||||
sort_keys=True,
|
||||
)
|
||||
outfile.write('\n')
|
||||
except OSError:
|
||||
# Squelch OSErrors, because we expect them and they shouldn't
|
||||
# interrupt the rest of the process.
|
||||
LOGGER.exception("Unable to write timing logs")
|
||||
12
pavement.py
12
pavement.py
@@ -1,12 +0,0 @@
|
||||
import sys # lint-amnesty, pylint: disable=django-not-configured, missing-module-docstring
|
||||
import os
|
||||
|
||||
# Ensure that we can import pavelib, and that our copy of pavelib
|
||||
# takes precedence over anything else installed in the virtualenv.
|
||||
# In local dev, we usually don't need to do this, because Python
|
||||
# automatically puts the current working directory on the system path.
|
||||
# Until we re-run pip install, the other copies of edx-platform could
|
||||
# take precedence, leading to some strange results.
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
from pavelib import * # lint-amnesty, pylint: disable=wildcard-import, wrong-import-position
|
||||
@@ -25,7 +25,6 @@
|
||||
# Follow up issue to remove this fork: https://github.com/openedx/edx-platform/issues/33456
|
||||
https://github.com/overhangio/py2neo/releases/download/2021.2.3/py2neo-2021.2.3.tar.gz
|
||||
|
||||
pygments # Used to support colors in paver command output
|
||||
# i18n_tool is needed at build time for pulling translations
|
||||
edx-i18n-tools>=0.4.6 # Commands for developers and translators to extract, compile and validate translations
|
||||
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
-c ../constraints.txt
|
||||
|
||||
-r github.in # Forks and other dependencies not yet on PyPI
|
||||
-r paver.txt # Requirements for running paver commands
|
||||
|
||||
# DON'T JUST ADD NEW DEPENDENCIES!!!
|
||||
# Please follow these guidelines whenever you change this file:
|
||||
|
||||
@@ -1,27 +0,0 @@
|
||||
# Requirements to run and test Paver
|
||||
#
|
||||
# DON'T JUST ADD NEW DEPENDENCIES!!!
|
||||
#
|
||||
# If you open a pull request that adds a new dependency, you should:
|
||||
# * verify that the dependency has a license compatible with AGPLv3
|
||||
# * confirm that it has no system requirements beyond what we already install
|
||||
# * run "make upgrade" to update the detailed requirements files
|
||||
#
|
||||
|
||||
-c ../constraints.txt
|
||||
|
||||
edx-opaque-keys # Create and introspect course and xblock identities
|
||||
lazy # Lazily-evaluated attributes for Python objects
|
||||
libsass # Python bindings for the LibSass CSS compiler
|
||||
markupsafe # XML/HTML/XHTML Markup safe strings
|
||||
mock # Stub out code with mock objects and make assertions about how they have been used
|
||||
path # Easier manipulation of filesystem paths
|
||||
paver # Build, distribution and deployment scripting tool
|
||||
psutil # Library for retrieving information on running processes and system utilization
|
||||
pymongo # via edx-opaque-keys
|
||||
python-memcached # Python interface to the memcached memory cache daemon
|
||||
pymemcache # Python interface to the memcached memory cache daemon
|
||||
requests # Simple interface for making HTTP requests
|
||||
stevedore # Support for runtime plugins, used for XBlocks and edx-platform Django app plugins
|
||||
watchdog # Used in paver watch_assets
|
||||
wrapt # Decorator utilities used in the @timed paver task decorator
|
||||
@@ -1,65 +0,0 @@
|
||||
#
|
||||
# This file is autogenerated by pip-compile with Python 3.11
|
||||
# by the following command:
|
||||
#
|
||||
# make upgrade
|
||||
#
|
||||
certifi==2024.8.30
|
||||
# via requests
|
||||
charset-normalizer==2.0.12
|
||||
# via
|
||||
# -c requirements/edx/../constraints.txt
|
||||
# requests
|
||||
dnspython==2.7.0
|
||||
# via pymongo
|
||||
edx-opaque-keys==2.11.0
|
||||
# via -r requirements/edx/paver.in
|
||||
idna==3.10
|
||||
# via requests
|
||||
lazy==1.6
|
||||
# via -r requirements/edx/paver.in
|
||||
libsass==0.10.0
|
||||
# via
|
||||
# -c requirements/edx/../constraints.txt
|
||||
# -r requirements/edx/paver.in
|
||||
markupsafe==3.0.2
|
||||
# via -r requirements/edx/paver.in
|
||||
mock==5.1.0
|
||||
# via -r requirements/edx/paver.in
|
||||
path==16.11.0
|
||||
# via
|
||||
# -c requirements/edx/../constraints.txt
|
||||
# -r requirements/edx/paver.in
|
||||
paver==1.3.4
|
||||
# via -r requirements/edx/paver.in
|
||||
pbr==6.1.0
|
||||
# via stevedore
|
||||
psutil==6.1.0
|
||||
# via -r requirements/edx/paver.in
|
||||
pymemcache==4.0.0
|
||||
# via -r requirements/edx/paver.in
|
||||
pymongo==4.4.0
|
||||
# via
|
||||
# -c requirements/edx/../constraints.txt
|
||||
# -r requirements/edx/paver.in
|
||||
# edx-opaque-keys
|
||||
python-memcached==1.62
|
||||
# via -r requirements/edx/paver.in
|
||||
requests==2.32.3
|
||||
# via -r requirements/edx/paver.in
|
||||
six==1.17.0
|
||||
# via
|
||||
# libsass
|
||||
# paver
|
||||
stevedore==5.4.0
|
||||
# via
|
||||
# -r requirements/edx/paver.in
|
||||
# edx-opaque-keys
|
||||
typing-extensions==4.12.2
|
||||
# via edx-opaque-keys
|
||||
urllib3==2.2.3
|
||||
# via requests
|
||||
watchdog==6.0.0
|
||||
# via -r requirements/edx/paver.in
|
||||
wrapt==1.17.0
|
||||
# via -r requirements/edx/paver.in
|
||||
@@ -1,89 +0,0 @@
|
||||
# shellcheck disable=all
|
||||
# ^ Paver in edx-platform is on the way out
|
||||
# (https://github.com/openedx/edx-platform/issues/31798)
|
||||
# so we're not going to bother fixing these shellcheck
|
||||
# violations.
|
||||
|
||||
# Courtesy of Gregory Nicholas
|
||||
|
||||
_subcommand_opts()
|
||||
{
|
||||
local awkfile command cur usage
|
||||
command=$1
|
||||
cur=${COMP_WORDS[COMP_CWORD]}
|
||||
awkfile=/tmp/paver-option-awkscript-$$.awk
|
||||
echo '
|
||||
BEGIN {
|
||||
opts = "";
|
||||
}
|
||||
|
||||
{
|
||||
for (i = 1; i <= NF; i = i + 1) {
|
||||
# Match short options (-a, -S, -3)
|
||||
# or long options (--long-option, --another_option)
|
||||
# in output from paver help [subcommand]
|
||||
if ($i ~ /^(-[A-Za-z0-9]|--[A-Za-z][A-Za-z0-9_-]*)/) {
|
||||
opt = $i;
|
||||
# remove trailing , and = characters.
|
||||
match(opt, "[,=]");
|
||||
if (RSTART > 0) {
|
||||
opt = substr(opt, 0, RSTART);
|
||||
}
|
||||
opts = opts " " opt;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
END {
|
||||
print opts
|
||||
}' > $awkfile
|
||||
|
||||
usage=`paver help $command`
|
||||
options=`echo "$usage"|awk -f $awkfile`
|
||||
|
||||
COMPREPLY=( $(compgen -W "$options" -- "$cur") )
|
||||
}
|
||||
|
||||
|
||||
_paver()
|
||||
{
|
||||
local cur prev
|
||||
COMPREPLY=()
|
||||
# Variable to hold the current word
|
||||
cur="${COMP_WORDS[COMP_CWORD]}"
|
||||
prev="${COMP_WORDS[COMP_CWORD - 1]}"
|
||||
|
||||
# Build a list of the available tasks from: `paver --help --quiet`
|
||||
local cmds=$(paver -hq | awk '/^ ([a-zA-Z][a-zA-Z0-9_]+)/ {print $1}')
|
||||
|
||||
subcmd="${COMP_WORDS[1]}"
|
||||
# Generate possible matches and store them in the
|
||||
# array variable COMPREPLY
|
||||
|
||||
if [[ -n $subcmd ]]
|
||||
then
|
||||
|
||||
if [[ ${#COMP_WORDS[*]} == 3 ]]
|
||||
then
|
||||
_subcommand_opts $subcmd
|
||||
return 0
|
||||
else
|
||||
if [[ "$cur" == -* ]]
|
||||
then
|
||||
_subcommand_opts $subcmd
|
||||
return 0
|
||||
else
|
||||
COMPREPLY=( $(compgen -o nospace -- "$cur") )
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ ${#COMP_WORDS[*]} == 2 ]]
|
||||
then
|
||||
COMPREPLY=( $(compgen -W "${cmds}" -- "$cur") )
|
||||
fi
|
||||
}
|
||||
|
||||
# Assign the auto-completion function for our command.
|
||||
|
||||
complete -F _paver -o default paver
|
||||
Reference in New Issue
Block a user