Andal.LND/edx-platform
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

build: npm run build (experimental) (#32823)

Browse Source 
This PR implements much of the static assets rework ADR [1], including:

* `npm run build[-dev]`, and its subcommands,
* `npm run webpack[-dev]` and
* `npm run compile-sass[-dev]`.

This is backwards-compatible. `paver update_assets` should not be affected.
The new command warns that it is "experimental" for a few reasons:

* `npm run build` will fail in the webpack phase unless you first
run  `xmodule_assets`. This will be changed soon [2].

* We have tested the new build, but not quite so thoroughly that we'd
recommend it as the production default yet. Once the xmodule_assets
work lands, we'll share this on the forums so early adopters can try it
out.

* The commands lack some top-level documentation. Once they stabilize more,
we'll add a section to the README that explains how and when to use `npm run
build` and its subcommands and its env vars.

* `npm run watch` is not yet implemented.

References:
1. https://github.com/openedx/edx-platform/blob/master/docs/decisions/0017-reimplement-asset-processing.rst
2. https://github.com/openedx/edx-platform/pull/32685

Part of: https://github.com/openedx/edx-platform/issues/31604
This commit is contained in:
Kyle McCormick
2023-07-26 08:04:45 -04:00
committed by GitHub
parent 23d696675f
commit 3e67719a24
15 changed files with 511 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
package.json
View File

@@ -3,7 +3,13 @@
"version": "0.1.0",
"repository": "https://github.com/openedx/edx-platform",
"scripts": {
"postinstall": "scripts/copy-node-modules.sh"
"postinstall": "scripts/copy-node-modules.sh",
"build": "echo 'WARNING: `npm run build` in edx-platform is experimental. Use at your own risk.' && npm run webpack && npm run compile-sass",
"build-dev": "echo 'WARNING: `npm run build-dev` in edx-platform is experimental. Use at your own risk.' && npm run webpack-dev && npm run compile-sass-dev",
"webpack": "NODE_ENV=${NODE_ENV:-production} \"$(npm bin)/webpack\" --config=${WEBPACK_CONFIG_PATH:-webpack.prod.config.js}",
"webpack-dev": "NODE_ENV=development \"$(npm bin)/webpack\" --config=webpack.dev.config.js",
"compile-sass": "scripts/compile_sass.py --env=${NODE_ENV:-production}",
"compile-sass-dev": "scripts/compile_sass.py --env=development"
},
"dependencies": {
"@babel/core": "7.19.0",

7
requirements/constraints.txt
View File

@@ -110,7 +110,6 @@ drf-yasg<1.21.6
# Adding pin to avoid any major upgrade
djangorestframework<3.15.0
# tests failing with greater version. Fix this in separate ticket.
pillow<10.0.0
@@ -118,3 +117,9 @@ pillow<10.0.0
# 1.16.0 works with Django 3.2 through 4.1
django-stubs==1.16.0
djangorestframework-stubs==3.14.0 # Pinned to match django-stubs. Remove this when we can remove the above pin.
# Our legacy Sass code is incompatible with anything except this ancient libsass version.
# Here is a ticket to upgrade, but it's of debatable importance given that we are rapidly moving
# away from legacy LMS/CMS frontends:
# https://github.com/openedx/edx-platform/issues/31616
libsass==0.10.0

11
requirements/edx/assets.in Normal file
View File

@@ -0,0 +1,11 @@
# Requirements for running `npm run build`.
# This file is NOT included into base.txt, because we do not want to
# superfluously install frontend build requirements into the production
# backend requirements.
-c ../constraints.txt
click
libsass
nodeenv

21
requirements/edx/assets.txt Normal file
View File

@@ -0,0 +1,21 @@
#
# This file is autogenerated by pip-compile with Python 3.8
# by the following command:
#
# make upgrade
#
click==8.1.5
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/assets.in
libsass==0.10.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/assets.in
nodeenv==1.8.0
# via -r requirements/edx/assets.in
six==1.16.0
# via libsass
# The following packages are considered to be unsafe in a requirements file:
# setuptools

4
requirements/edx/base.txt
View File

@@ -661,7 +661,9 @@ lazy==1.5
# ora2
# xblock
libsass==0.10.0
# via -r requirements/edx/paver.txt
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/paver.txt
loremipsum==1.0.5
# via ora2
lti-consumer-xblock==9.5.5

1
requirements/edx/development.in
View File

@@ -13,6 +13,7 @@
-r ../pip-tools.txt # pip-tools and its dependencies, for managing requirements files
-r testing.txt # Dependencies for running the various test suites
-r doc.txt # Dependencies for building the documentation locally.
-r assets.txt # Allow developers to rebuild assets with `npm run build`.
click # Used for perf_tests utilities in modulestore
django-debug-toolbar # A set of panels that display debug information about the current request/response

5
requirements/edx/development.txt
View File

@@ -214,6 +214,7 @@ click==8.1.6
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/../pip-tools.txt
# -r requirements/edx/assets.txt
# -r requirements/edx/development.in
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
@@ -1123,6 +1124,8 @@ lazy-object-proxy==1.9.0
# astroid
libsass==0.10.0
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/assets.txt
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
loremipsum==1.0.5
@@ -1243,6 +1246,7 @@ nltk==3.8.1
# chem
nodeenv==1.8.0
# via
# -r requirements/edx/assets.txt
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
numpy==1.22.4
@@ -1803,6 +1807,7 @@ singledispatch==4.0.0
# via -r requirements/edx/testing.txt
six==1.16.0
# via
# -r requirements/edx/assets.txt
# -r requirements/edx/doc.txt
# -r requirements/edx/testing.txt
# analytics-python

4
requirements/edx/doc.txt
View File

@@ -787,7 +787,9 @@ lazy==1.5
# ora2
# xblock
libsass==0.10.0
# via -r requirements/edx/base.txt
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
loremipsum==1.0.5
# via
# -r requirements/edx/base.txt

2
requirements/edx/paver.in
View File

@@ -12,7 +12,7 @@
edx-opaque-keys # Create and introspect course and xblock identities
lazy # Lazily-evaluated attributes for Python objects
libsass==0.10.0 # Python bindings for the LibSass CSS compiler
libsass # Python bindings for the LibSass CSS compiler
markupsafe # XML/HTML/XHTML Markup safe strings
mock # Stub out code with mock objects and make assertions about how they have been used
path # Easier manipulation of filesystem paths

4
requirements/edx/paver.txt
View File

@@ -17,7 +17,9 @@ idna==3.4
lazy==1.5
# via -r requirements/edx/paver.in
libsass==0.10.0
# via -r requirements/edx/paver.in
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/paver.in
markupsafe==2.1.3
# via -r requirements/edx/paver.in
mock==5.1.0

4
requirements/edx/testing.txt
View File

@@ -854,7 +854,9 @@ lazy==1.5
lazy-object-proxy==1.9.0
# via astroid
libsass==0.10.0
# via -r requirements/edx/base.txt
# via
# -c requirements/edx/../constraints.txt
# -r requirements/edx/base.txt
loremipsum==1.0.5
# via
# -r requirements/edx/base.txt

438
scripts/compile_sass.py Executable file
View File

@@ -0,0 +1,438 @@
#!/usr/bin/env python
"""
Defines a CLI for compiling Sass (both default and themed) into CSS.
Should be run from the root of edx-platform using `npm run` wrapper.
Requirements for this scripts are stored in requirements/edx/assets.in.
Get more details:
npm run compile-sass -- --help
npm run compile-sass -- --dry
Setup (Tutor and Devstack will do this for you):
python -m venv venv
. venv/bin/activate
pip install -r requirements/edx/assets.txt
Usage:
npm run compile-sass # prod, no args
npm run compile-sass -- ARGS # prod, with args
npm run compile-sass-dev # dev, no args
npm run compile-sass-dev -- ARGS # dev, with args
This script is intentionally implemented in a very simplistic way. It prefers repetition
over abstraction, and its dependencies are minimal (just click and libsass-python, ideally).
We do this because:
* If and when we migrate from libsass-python to something less ancient like node-sass or dart-sass,
we will want to re-write this script in Bash or JavaScript so that it can work without any
backend tooling. By keeping the script dead simple, that will be easier.
* The features this script supports (legacy frontends & comprehensive theming) are on the way out,
in favor of micro-frontends, branding, and Paragon design tokens. We're not sure how XBlock
view styling will fit into that, but it probably can be much simpler than comprehensive theming.
So, we don't need this script to be modular and extensible. We just need it to be obvious, robust,
and easy to maintain until we can delete it.
See docs/decisions/0017-reimplement-asset-processing.rst for more details.
"""
from __future__ import annotations
import glob
import subprocess
from pathlib import Path
import click
# Accept both long- and short-forms of these words, but normalize to long form.
# We accept both because edx-platform asset build scripts historically use the short form,
# but NODE_ENV uses the long form, so to make them integrate more seamlessly we accept both.
NORMALIZED_ENVS = {
"prod": "production",
"dev": "development",
"production": "production",
"development": "development",
}
@click.option(
"-T",
"--theme-dir",
"theme_dirs",
metavar="PATH",
multiple=True,
envvar="EDX_PLATFORM_THEME_DIRS",
type=click.Path(
exists=True, file_okay=False, readable=True, writable=True, path_type=Path
),
help=(
"Consider sub-dirs of PATH as themes. "
"Multiple theme dirs are accepted. "
"If none are provided, we look at colon-separated paths on the EDX_PLATFORM_THEME_DIRS env var."
),
)
@click.option(
"-t",
"--theme",
"themes",
metavar="NAME",
multiple=True,
type=str,
help=(
"A theme to compile. "
"NAME should be a sub-dir of a PATH provided by --theme-dir. "
"Multiple themes are accepted. "
"If none are provided, all available themes are compiled."
),
)
@click.option(
"--skip-default",
is_flag=True,
help="Don't compile default Sass.",
)
@click.option(
"--skip-themes",
is_flag=True,
help="Don't compile any themes (overrides --theme* options).",
)
@click.option(
"--skip-lms",
is_flag=True,
help="Don't compile any LMS Sass.",
)
@click.option(
"--skip-cms",
is_flag=True,
help="Don't compile any CMS Sass.",
)
@click.option(
"--env",
type=click.Choice(["dev", "development", "prod", "production"]),
default="prod",
help="Optimize CSS for this environment. Defaults to 'prod'.",
)
@click.option(
"--dry",
is_flag=True,
help="Print what would be compiled, but don't compile it.",
)
@click.option(
"-h",
"--help",
"show_help",
is_flag=True,
help="Print this help.",
)
@click.command()
@click.pass_context
def main(
context: click.Context,
theme_dirs: list[Path],
themes: list[str],
skip_default: bool,
skip_themes: bool,
skip_lms: bool,
skip_cms: bool,
env: str,
dry: bool,
show_help: bool,
) -> None:
"""
Compile Sass for edx-platform and its themes.
Default Sass is compiled unless explicitly skipped.
Additionally, any number of themes may be specified using --theme-dir and --theme.
Default CSS is compiled to css/ directories in edx-platform.
Themed CSS is compiled to css/ directories in their source themes.
"""
def compile_sass_dir(
message: str,
source: Path,
dest: Path,
includes: list[Path],
tolerate_missing: bool = False,
) -> None:
"""
Compile a directory of Sass into a target CSS directory, and generate any missing RTL CSS.
Structure of source dir is mirrored in target dir.
"""
use_dev_settings = NORMALIZED_ENVS[env] == "development"
click.secho(f" {message}...", fg="cyan")
click.secho(f" Source: {source}")
click.secho(f" Target: {dest}")
if not source.is_dir():
if tolerate_missing:
click.secho(f" Skipped because source directory does not exist.", fg="yellow")
return
else:
raise FileNotFoundError(f"missing Sass source dir: {source}")
click.echo(f" Include paths:")
for include in includes:
click.echo(f" {include}")
if not dry:
# Import sass late so that this script can be dry-run without installing
# libsass, which takes a while as it must be compiled from its C source.
import sass
dest.mkdir(parents=True, exist_ok=True)
sass.compile(
dirname=(str(source), str(dest)),
include_paths=[str(include_path) for include_path in includes],
source_comments=use_dev_settings,
output_style=("nested" if use_dev_settings else "compressed"),
)
click.secho(f" Compiled.", fg="green")
# For Sass files without explicit RTL versions, generate
# an RTL version of the CSS using the rtlcss library.
for sass_path in glob.glob(str(source) + "/**/*.scss"):
if Path(sass_path).name.startswith("_"):
# Don't generate RTL CSS for partials
continue
if sass_path.endswith("-rtl.scss"):
# Don't generate RTL CSS if the file is itself an RTL version
continue
if Path(sass_path.replace(".scss", "-rtl.scss")).exists():
# Don't generate RTL CSS if there is an explicit Sass version for RTL
continue
click.echo(" Generating missing right-to-left CSS:")
source_css_file = sass_path.replace(str(source), str(dest)).replace(
".scss", ".css"
)
target_css_file = source_css_file.replace(".css", "-rtl.css")
click.echo(f" Source: {source_css_file}")
click.echo(f" Target: {target_css_file}")
if not dry:
subprocess.run(["rtlcss", source_css_file, target_css_file])
click.secho(" Generated.", fg="green")
# Information
click.secho(f"USING ENV: {NORMALIZED_ENVS[env]}", fg="blue")
if dry:
click.secho(f"DRY RUN: Will print compile steps, but will not compile anything.", fg="blue")
click.echo()
# Warnings
click.secho("WARNING: `npm run compile-sass` is experimental. Use at your own risk.", fg="yellow", bold=True)
if show_help:
click.echo(context.get_help())
return
if skip_lms and skip_cms:
click.secho("WARNING: You are skipping both LMS and CMS... nothing will be compiled!", fg="yellow")
if skip_default and skip_themes:
click.secho("WARNING: You are skipped both default Sass and themed Sass... nothing will be compiled!", fg="yellow")
click.echo()
# Build a list of theme paths:
if skip_themes:
theme_paths = []
else:
theme_paths = [
theme_dir / theme
# For every theme dir,
for theme_dir in theme_dirs
for theme in (
# for every theme name (if theme names provided),
themes
or
# or for every subdir of theme dirs (if no theme name provided),
(theme_dir_entry.name for theme_dir_entry in theme_dir.iterdir())
)
# consider the path a theme if it has a lms/ or cms/ subdirectory.
if (theme_dir / theme / "lms").is_dir() or (theme_dir / theme / "cms").is_dir()
]
# We expect this script to be run from the edx-platform root.
repo = Path(".")
if not (repo / "xmodule").is_dir():
# Sanity check: If the xmodule/ folder is missing, we're definitely not at the root
# of edx-platform, so save the user some headache by exiting early.
raise Exception(f"{__file__} must be run from the root of edx-platform")
# Every Sass compilation will use have these directories as lookup paths,
# regardless of theme.
common_includes = [
repo / "common" / "static",
repo / "common" / "static" / "sass",
repo / "node_modules" / "@edx",
repo / "node_modules",
]
if not skip_default:
click.secho(f"Compiling default Sass...", fg="cyan", bold=True)
if not skip_lms:
compile_sass_dir(
"Compiling default LMS Sass",
repo / "lms" / "static" / "sass",
repo / "lms" / "static" / "css",
includes=[
*common_includes,
repo / "lms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
],
)
compile_sass_dir(
"Compiling default certificate Sass",
repo / "lms" / "static" / "certificates" / "sass",
repo / "lms" / "static" / "certificates" / "css",
includes=[
*common_includes,
repo / "lms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
],
)
compile_sass_dir(
"Compiling built-in XBlock Sass for default LMS",
repo / "xmodule" / "assets",
repo / "lms" / "static" / "css",
includes=[
*common_includes,
repo / "lms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
repo / "cms" / "static" / "sass",
],
)
if not skip_cms:
compile_sass_dir(
"Compiling default CMS Sass",
repo / "cms" / "static" / "sass",
repo / "cms" / "static" / "css",
includes=[
*common_includes,
repo / "lms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass",
],
)
compile_sass_dir(
"Compiling built-in XBlock Sass for default CMS",
repo / "xmodule" / "assets",
repo / "cms" / "static" / "css",
includes=[
*common_includes,
repo / "lms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
repo / "cms" / "static" / "sass",
],
)
click.secho(f"Done compiling default Sass!", fg="cyan", bold=True)
click.echo()
for theme in theme_paths:
click.secho(f"Compiling Sass for theme at {theme}...", fg="cyan", bold=True)
if not skip_lms:
compile_sass_dir(
"Compiling default LMS Sass with themed partials",
repo / "lms" / "static" / "sass",
theme / "lms" / "static" / "css",
includes=[
*common_includes,
theme / "lms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
],
tolerate_missing=True,
)
compile_sass_dir(
"Compiling themed LMS Sass as overrides to CSS from previous step",
theme / "lms" / "static" / "sass",
theme / "lms" / "static" / "css",
includes=[
*common_includes,
theme / "lms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
],
tolerate_missing=True,
)
compile_sass_dir(
"Compiling themed certificate Sass",
theme / "lms" / "static" / "certificates" / "sass",
theme / "lms" / "static" / "certificates" / "css",
includes=[
*common_includes,
theme / "lms" / "static" / "sass" / "partials",
theme / "lms" / "static" / "sass",
],
tolerate_missing=True,
)
compile_sass_dir(
"Compiling built-in XBlock Sass for themed LMS",
repo / "xmodule" / "assets",
theme / "lms" / "static" / "css",
includes=[
*common_includes,
theme / "lms" / "static" / "sass" / "partials",
theme / "cms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
repo / "cms" / "static" / "sass",
],
)
if not skip_cms:
compile_sass_dir(
"Compiling default CMS Sass with themed partials",
repo / "cms" / "static" / "sass",
theme / "cms" / "static" / "css",
includes=[
*common_includes,
repo / "lms" / "static" / "sass" / "partials",
theme / "cms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass",
],
tolerate_missing=True,
)
compile_sass_dir(
"Compiling themed CMS Sass as overrides to CSS from previous step",
theme / "cms" / "static" / "sass",
theme / "cms" / "static" / "css",
includes=[
*common_includes,
repo / "lms" / "static" / "sass" / "partials",
theme / "cms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass",
],
tolerate_missing=True,
)
compile_sass_dir(
"Compiling built-in XBlock Sass for themed CMS",
repo / "xmodule" / "assets",
theme / "cms" / "static" / "css",
includes=[
*common_includes,
theme / "lms" / "static" / "sass" / "partials",
theme / "cms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass" / "partials",
repo / "cms" / "static" / "sass" / "partials",
repo / "lms" / "static" / "sass",
repo / "cms" / "static" / "sass",
],
)
click.secho(f"Done compiling Sass for theme at {theme}!", fg="cyan", bold=True)
click.echo()
# Report what we did.
click.secho("Successfully compiled:", fg="green", bold=True)
if not skip_default:
click.secho(f" - {repo.absolute()} (default Sass)", fg="green")
for theme in theme_paths:
click.secho(f" - {theme}", fg="green")
if skip_lms:
click.secho(f"(skipped LMS)", fg="yellow")
if skip_cms:
click.secho(f"(skipped CMS)", fg="yellow")
if __name__ == "__main__":
main(prog_name="npm run compile-sass --")

10
webpack.common.config.js
View File

@@ -26,6 +26,9 @@ var defineFooter = new RegExp('(' + defineCallFooter.source + ')|('
+ defineDirectFooter.source + ')|('
+ defineFancyFooter.source + ')', 'm');
var staticRootLms = process.env.STATIC_ROOT_LMS || "./test_root/staticfiles";
var staticRootCms = process.env.STATIC_ROOT_CMS || (staticRootLms + "/studio");
var workerConfig = function() {
try {
return {
@@ -39,7 +42,7 @@ var workerConfig = function() {
},
plugins: [
new BundleTracker({
path: process.env.STATIC_ROOT_LMS,
path: staticRootLms,
filename: 'webpack-worker-stats.json'
}),
new webpack.DefinePlugin({
@@ -131,14 +134,15 @@ module.exports = Merge.smart({
},
plugins: [
new webpack.ProgressPlugin(), // report progress during compilation
new webpack.NoEmitOnErrorsPlugin(),
new webpack.NamedModulesPlugin(),
new BundleTracker({
path: process.env.STATIC_ROOT_CMS,
path: staticRootCms,
filename: 'webpack-stats.json'
}),
new BundleTracker({
path: process.env.STATIC_ROOT_LMS,
path: staticRootLms,
filename: 'webpack-stats.json'
}),
new webpack.ProvidePlugin({

2
webpack.dev.config.js
View File

@@ -21,7 +21,7 @@ module.exports = _.values(Merge.smart(commonConfig, {
}),
new webpack.DefinePlugin({
'process.env.NODE_ENV': JSON.stringify('development'),
'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG
'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG || "{}"
})
],
module: {

2
webpack.prod.config.js
View File

@@ -18,7 +18,7 @@ var optimizedConfig = Merge.smart(commonConfig, {
plugins: [
new webpack.DefinePlugin({
'process.env.NODE_ENV': JSON.stringify('production'),
'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG
'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG || "{}"
}),
new webpack.LoaderOptionsPlugin({ // This may not be needed; legacy option for loaders written for webpack 1
minimize: true