build: npm run watch (experimental) (#32866)
Implements the `npm run watch` section of the assets ADR [1], plus some modifications since I decided to switch from pywatchman to watchdog (see ADR changes for justification). This will replace `paver watch_assets` (edx-platform) and `openedx-assets watch-themes` (Tutor). Specifically, this PR adds three experimental commands: * `npm run watch-sass` : Watch for Sass changes with watchdog. * `npm run watch-webpack` : Invoke Webpack-watch for JS changes. * `npm run watch` : Invoke both `watch-sass` and `watch-webpack` simultaneously. These commands are only intended to work in development mode. They have been tested both on bare-metal edx-platform and through `tutor dev run` on on Linux. Before removing the "experimental" label, we need to: * Test through Devstack on Linux. * Test through Devstack and `tutor dev run` on macOS. * Test on bare-metal macOS. Might not work, which is OK, but we should document that. * Document the commands in edx-platform's README. * Confirm that this not only works through `tutor dev run`, but also as a suitable replacement in the `watchthemes` service that Tutor runs automatically as part of `tutor dev start`. Tweak if necessary. References: 1. https://github.com/openedx/edx-platform/blob/master/docs/decisions/0017-reimplement-asset-processing.rst Part of: https://github.com/openedx/edx-platform/issues/31612
This commit is contained in:
Kyle McCormick
@@ -218,7 +218,9 @@ The three top-level edx-platform asset processing actions are *build*, *collect*
|
||||
|
||||
- ``npm run watch``
|
||||
|
||||
Bash wrappers around invocation(s) of `watchman <https://facebook.github.io/watchman/>`_, a popular file-watching library maintained by Meta. Watchman is already installed into edx-platform (and other services) via the pywatchman pip wrapper package.
|
||||
Bash wrappers around invocations of the `watchdog library <https://pypi.org/project/watchdog/>`_ for themable/themed assets, and `webpack --watch <https://webpack.js.org/configuration/watch/>`_ for Webpack-managed assets. Both of these tools are available via dependencies that are already installed into edx-platform.
|
||||
|
||||
We considered using `watchman <https://facebook.github.io/watchman/>`_, a popular file-watching library maintained by Meta, but found that the Python release of the library is poorly maintained (latest release 2017) and the documentation is difficult to follow. `Django uses pywatchman but is planning to migrate off of it <https://code.djangoproject.com/ticket/34479>`_ and onto `watchfiles <https://pypi.org/project/watchfiles/>`_. We considered watchfiles, but decided against adding another developer dependency to edx-platform. Future developers could consider migrating to watchfiles if it seemed worthwile.
|
||||
|
||||
|
||||
Build Configuration
|
||||
|
||||
@@ -9,7 +9,10 @@
|
||||
"webpack": "NODE_ENV=${NODE_ENV:-production} \"$(npm bin)/webpack\" --config=${WEBPACK_CONFIG_PATH:-webpack.prod.config.js}",
|
||||
"webpack-dev": "NODE_ENV=development \"$(npm bin)/webpack\" --config=webpack.dev.config.js",
|
||||
"compile-sass": "scripts/compile_sass.py --env=${NODE_ENV:-production}",
|
||||
"compile-sass-dev": "scripts/compile_sass.py --env=development"
|
||||
"compile-sass-dev": "scripts/compile_sass.py --env=development",
|
||||
"watch": "echo 'WARNING: `npm run watch` in edx-platform is experimental. Use at your own risk.' && { npm run watch-webpack& npm run watch-sass& } && sleep infinity",
|
||||
"watch-webpack": "npm run webpack-dev -- --watch",
|
||||
"watch-sass": "scripts/watch_sass.sh"
|
||||
},
|
||||
"dependencies": {
|
||||
"@babel/core": "7.19.0",
|
||||
|
||||
@@ -22,3 +22,4 @@ djangorestframework-stubs # Typing stubs for DRF
|
||||
mypy # static type checking
|
||||
pywatchman # More efficient checking for runserver reload trigger events
|
||||
vulture # Detects possible dead/unused code, used in scripts/find-dead-code.sh
|
||||
watchdog # Used by `npm run watch` to auto-recompile when assets are changed
|
||||
|
||||
170
scripts/watch_sass.sh
Executable file
170
scripts/watch_sass.sh
Executable file
@@ -0,0 +1,170 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Wait for changes to Sass, and recompile.
|
||||
# Invoke from repo root as `npm run watch-sass`.
|
||||
# This script tries to recompile the minimal set of Sass for any given change.
|
||||
|
||||
# By default, only watches default Sass.
|
||||
# To watch themes too, provide colon-separated paths in the EDX_PLATFORM_THEME_DIRS environment variable.
|
||||
# Each path will be treated as a "theme dir", which means that every immediate child directory is watchable as a theme.
|
||||
# For example:
|
||||
#
|
||||
# EDX_PLATFORM_THEME_DIRS=/openedx/themes:./themes npm run watch-sass
|
||||
#
|
||||
# would watch default Sass as well as /openedx/themes/indigo, /openedx/themes/mytheme, ./themes/red-theme, etc.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
COL_SECTION="\e[1;36m" # Section header color (bold cyan)
|
||||
COL_LOG="\e[36m" # Log color (cyan)
|
||||
COL_WARNING="\e[1;33m" # Warning (bold yellow)
|
||||
COL_ERROR="\e[1;31m" # Error (bold red)
|
||||
COL_CMD="\e[35m" # Command echoing (magenta)
|
||||
COL_OFF="\e[0m" # Normal color
|
||||
|
||||
section() {
|
||||
# Print a header in bold cyan to indicate sections of output.
|
||||
echo -e "${COL_SECTION}$*${COL_OFF}"
|
||||
}
|
||||
log() {
|
||||
# Info line. Indented by one space so that it appears as nested under section headers.
|
||||
echo -e " ${COL_LOG}$*${COL_OFF}"
|
||||
}
|
||||
warning() {
|
||||
# Bright yellow warning message.
|
||||
echo -e "${COL_WARNING}WARNING: $*${COL_OFF}"
|
||||
}
|
||||
error() {
|
||||
# Bright red error message.
|
||||
echo -e "${COL_ERROR}ERROR: $*${COL_OFF}"
|
||||
}
|
||||
echo_quoted_cmd() {
|
||||
# Echo args, each single-quoted, so that the user could copy-paste and run them as a command.
|
||||
# Indented by two spaces so it appears as nested under log lines.
|
||||
echo -e " ${COL_CMD}$(printf "'%s' " "$@")${COL_OFF}"
|
||||
}
|
||||
|
||||
start_sass_watch() {
|
||||
# Start a watch for .scss files in a particular dir. Run in the background.
|
||||
# start_sass_watch NAME_FOR_LOGGING WATCH_ROOT_PATH HANDLER_COMMAND
|
||||
local name="$1"
|
||||
local path="$2"
|
||||
local handler="$3"
|
||||
log "Starting watcher for $name:"
|
||||
# Note: --drop means that we should ignore any change events that happen during recompilation.
|
||||
# This is good because it means that if you edit 3 files, we won't fire off three simultaneous compiles.
|
||||
# It's not perfect, though, because if you change 3 files, only the first one will trigger a recompile,
|
||||
# so depending on the timing, your latest changes may or may not be picked up. We accept this as a reasonable
|
||||
# tradeoff. Some watcher libraries are smarter than watchdog, in that they wait until the filesystem "settles"
|
||||
# before firing off a the recompilation. This sounds nice but did not seem worth the effort for legacy assets.
|
||||
local watch_cmd=(watchmedo shell-command -v --patterns '*.scss' --recursive --drop --command "$handler" "$path")
|
||||
echo_quoted_cmd "${watch_cmd[@]}"
|
||||
"${watch_cmd[@]}" &
|
||||
}
|
||||
|
||||
clean_up() {
|
||||
# Kill all background processes we started.
|
||||
# Since they're all 'watchmedo' instances, we can just use killall.
|
||||
log "Stopping all watchers:"
|
||||
local stop_cmd=(killall watchmedo)
|
||||
echo_quoted_cmd "${stop_cmd[@]}"
|
||||
"${stop_cmd[@]}" || true
|
||||
log "Watchers stopped."
|
||||
}
|
||||
|
||||
warning "'npm run watch-sass' in edx-platform is experimental. Use at your own risk."
|
||||
|
||||
if [[ ! -d common/static/sass ]] ; then
|
||||
error 'This command must be run from the root of edx-platform!'
|
||||
exit 1
|
||||
fi
|
||||
if ! type watchmedo 1>/dev/null 2>&1 ; then
|
||||
error "command not found: watchmedo"
|
||||
log "The \`watchdog\` Python package is probably not installed. You can install it with:"
|
||||
log " pip install -r requirements/edx/development.txt"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
trap clean_up EXIT
|
||||
|
||||
# Start by compiling all watched Sass right away, mirroring the behavior of webpack --watch.
|
||||
section "COMPILING SASS:"
|
||||
npm run compile-sass
|
||||
echo
|
||||
echo
|
||||
|
||||
section "STARTING DEFAULT SASS WATCHERS:"
|
||||
|
||||
# Changes to LMS Sass require a full recompilation, since LMS Sass can be used in CMS and in themes.
|
||||
start_sass_watch "default LMS Sass" \
|
||||
lms/static/sass \
|
||||
'npm run compile-sass-dev'
|
||||
|
||||
# Changes to default cert Sass only require recompilation of default cert Sass, since cert Sass
|
||||
# cannot be included into LMS, CMS, or themes.
|
||||
start_sass_watch "default certificate Sass" \
|
||||
lms/static/certificates/sass \
|
||||
'npm run compile-sass-dev -- --skip-cms --skip-themes'
|
||||
|
||||
# Changes to CMS Sass require recompilation of default & themed CMS Sass, but not LMS Sass.
|
||||
start_sass_watch "default CMS Sass" \
|
||||
cms/static/sass \
|
||||
'npm run compile-sass-dev -- --skip-lms'
|
||||
|
||||
# Sass changes in common, node_modules, and xmodule all require full recompilations.
|
||||
start_sass_watch "default common Sass" \
|
||||
common/static \
|
||||
'npm run compile-sass-dev'
|
||||
start_sass_watch "node_modules Sass" \
|
||||
node_modules \
|
||||
'npm run compile-sass-dev'
|
||||
start_sass_watch "builtin XBlock Sass" \
|
||||
xmodule/assets \
|
||||
'npm run compile-sass-dev'
|
||||
|
||||
export IFS=";"
|
||||
for theme_dir in ${EDX_PLATFORM_THEME_DIRS:-} ; do
|
||||
for theme_path in "$theme_dir"/* ; do
|
||||
|
||||
theme_name="${theme_path#"$theme_dir/"}"
|
||||
lms_sass="$theme_path/lms/static/sass"
|
||||
cert_sass="$theme_path/lms/static/certificates/sass"
|
||||
cms_sass="$theme_path/cms/static/sass"
|
||||
|
||||
if [[ -d "$lms_sass" ]] || [[ -d "$cert_sass" ]] || [[ -d "$cms_sass" ]] ; then
|
||||
section "STARTING WATCHERS FOR THEME '$theme_name':"
|
||||
fi
|
||||
|
||||
# Changes to a theme's LMS Sass require that the full theme is recompiled, since LMS
|
||||
# Sass is used in certs and CMS.
|
||||
if [[ -d "$lms_sass" ]] ; then
|
||||
start_sass_watch "$theme_name LMS Sass" \
|
||||
"$lms_sass" \
|
||||
"npm run compile-sass-dev -- -T $theme_dir -t $theme_name --skip-default"
|
||||
fi
|
||||
|
||||
# Changes to a theme's certs Sass only requires that its certs Sass be recompiled.
|
||||
if [[ -d "$cert_sass" ]] ; then
|
||||
start_sass_watch "$theme_name certificate Sass" \
|
||||
"$cert_sass" \
|
||||
"npm run compile-sass-dev -- -T $theme_dir -t $theme_name --skip-default --skip-cms"
|
||||
fi
|
||||
|
||||
# Changes to a theme's CMS Sass only requires that its CMS Sass be recompiled.
|
||||
if [[ -d "$cms_sass" ]] ; then
|
||||
start_sass_watch "$theme_name CMS Sass" \
|
||||
"$cms_sass" \
|
||||
"npm run compile-sass-dev -- -T $theme_dir -t $theme_name --skip-default --skip-lms"
|
||||
fi
|
||||
|
||||
done
|
||||
done
|
||||
|
||||
sleep infinity &
|
||||
echo
|
||||
echo "Watching Open edX LMS & CMS Sass for changes."
|
||||
echo "Use Ctrl+c to exit."
|
||||
echo
|
||||
echo
|
||||
wait $!
|
||||
|
||||
@@ -10,7 +10,7 @@ The ``xmodule`` folder contains a variety of old-yet-important functionality cor
|
||||
* the implementations of several different built-in content-level XBlocks, such as ``problem`` and ``html``; and
|
||||
* `assets for those built-in XBlocks`_.
|
||||
|
||||
.. _assets for those built-in XBlocks: https://github.com/openedx/edx-platform/tree/master/assets
|
||||
.. _assets for those built-in XBlocks: https://github.com/openedx/edx-platform/tree/master/xmodule/assets#readme
|
||||
|
||||
Historical Context
|
||||
******************
|
||||
|
||||
Reference in New Issue
Block a user