diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst index f77b4add1ba1..1048edf4c075 100644 --- a/docs/decisions/0017-reimplement-asset-processing.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -218,7 +218,9 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``npm run watch`` - Bash wrappers around invocation(s) of `watchman `_, a popular file-watching library maintained by Meta. Watchman is already installed into edx-platform (and other services) via the pywatchman pip wrapper package. + Bash wrappers around invocations of the `watchdog library `_ for themable/themed assets, and `webpack --watch `_ for Webpack-managed assets. Both of these tools are available via dependencies that are already installed into edx-platform. + + We considered using `watchman `_, a popular file-watching library maintained by Meta, but found that the Python release of the library is poorly maintained (latest release 2017) and the documentation is difficult to follow. `Django uses pywatchman but is planning to migrate off of it `_ and onto `watchfiles `_. We considered watchfiles, but decided against adding another developer dependency to edx-platform. Future developers could consider migrating to watchfiles if it seemed worthwile. Build Configuration diff --git a/package.json b/package.json index 21c80c872c44..a589b3da57e6 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,10 @@ "webpack": "NODE_ENV=${NODE_ENV:-production} \"$(npm bin)/webpack\" --config=${WEBPACK_CONFIG_PATH:-webpack.prod.config.js}", "webpack-dev": "NODE_ENV=development \"$(npm bin)/webpack\" --config=webpack.dev.config.js", "compile-sass": "scripts/compile_sass.py --env=${NODE_ENV:-production}", - "compile-sass-dev": "scripts/compile_sass.py --env=development" + "compile-sass-dev": "scripts/compile_sass.py --env=development", + "watch": "echo 'WARNING: `npm run watch` in edx-platform is experimental. Use at your own risk.' && { npm run watch-webpack& npm run watch-sass& } && sleep infinity", + "watch-webpack": "npm run webpack-dev -- --watch", + "watch-sass": "scripts/watch_sass.sh" }, "dependencies": { "@babel/core": "7.19.0", diff --git a/requirements/edx/development.in b/requirements/edx/development.in index fb3d9d5497b2..00c9e533b126 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -22,3 +22,4 @@ djangorestframework-stubs # Typing stubs for DRF mypy # static type checking pywatchman # More efficient checking for runserver reload trigger events vulture # Detects possible dead/unused code, used in scripts/find-dead-code.sh +watchdog # Used by `npm run watch` to auto-recompile when assets are changed diff --git a/scripts/watch_sass.sh b/scripts/watch_sass.sh new file mode 100755 index 000000000000..dee0d88f6215 --- /dev/null +++ b/scripts/watch_sass.sh @@ -0,0 +1,170 @@ +#!/usr/bin/env bash + +# Wait for changes to Sass, and recompile. +# Invoke from repo root as `npm run watch-sass`. +# This script tries to recompile the minimal set of Sass for any given change. + +# By default, only watches default Sass. +# To watch themes too, provide colon-separated paths in the EDX_PLATFORM_THEME_DIRS environment variable. +# Each path will be treated as a "theme dir", which means that every immediate child directory is watchable as a theme. +# For example: +# +# EDX_PLATFORM_THEME_DIRS=/openedx/themes:./themes npm run watch-sass +# +# would watch default Sass as well as /openedx/themes/indigo, /openedx/themes/mytheme, ./themes/red-theme, etc. + +set -euo pipefail + +COL_SECTION="\e[1;36m" # Section header color (bold cyan) +COL_LOG="\e[36m" # Log color (cyan) +COL_WARNING="\e[1;33m" # Warning (bold yellow) +COL_ERROR="\e[1;31m" # Error (bold red) +COL_CMD="\e[35m" # Command echoing (magenta) +COL_OFF="\e[0m" # Normal color + +section() { + # Print a header in bold cyan to indicate sections of output. + echo -e "${COL_SECTION}$*${COL_OFF}" +} +log() { + # Info line. Indented by one space so that it appears as nested under section headers. + echo -e " ${COL_LOG}$*${COL_OFF}" +} +warning() { + # Bright yellow warning message. + echo -e "${COL_WARNING}WARNING: $*${COL_OFF}" +} +error() { + # Bright red error message. + echo -e "${COL_ERROR}ERROR: $*${COL_OFF}" +} +echo_quoted_cmd() { + # Echo args, each single-quoted, so that the user could copy-paste and run them as a command. + # Indented by two spaces so it appears as nested under log lines. + echo -e " ${COL_CMD}$(printf "'%s' " "$@")${COL_OFF}" +} + +start_sass_watch() { + # Start a watch for .scss files in a particular dir. Run in the background. + # start_sass_watch NAME_FOR_LOGGING WATCH_ROOT_PATH HANDLER_COMMAND + local name="$1" + local path="$2" + local handler="$3" + log "Starting watcher for $name:" + # Note: --drop means that we should ignore any change events that happen during recompilation. + # This is good because it means that if you edit 3 files, we won't fire off three simultaneous compiles. + # It's not perfect, though, because if you change 3 files, only the first one will trigger a recompile, + # so depending on the timing, your latest changes may or may not be picked up. We accept this as a reasonable + # tradeoff. Some watcher libraries are smarter than watchdog, in that they wait until the filesystem "settles" + # before firing off a the recompilation. This sounds nice but did not seem worth the effort for legacy assets. + local watch_cmd=(watchmedo shell-command -v --patterns '*.scss' --recursive --drop --command "$handler" "$path") + echo_quoted_cmd "${watch_cmd[@]}" + "${watch_cmd[@]}" & +} + +clean_up() { + # Kill all background processes we started. + # Since they're all 'watchmedo' instances, we can just use killall. + log "Stopping all watchers:" + local stop_cmd=(killall watchmedo) + echo_quoted_cmd "${stop_cmd[@]}" + "${stop_cmd[@]}" || true + log "Watchers stopped." +} + +warning "'npm run watch-sass' in edx-platform is experimental. Use at your own risk." + +if [[ ! -d common/static/sass ]] ; then + error 'This command must be run from the root of edx-platform!' + exit 1 +fi +if ! type watchmedo 1>/dev/null 2>&1 ; then + error "command not found: watchmedo" + log "The \`watchdog\` Python package is probably not installed. You can install it with:" + log " pip install -r requirements/edx/development.txt" + exit 1 +fi + +trap clean_up EXIT + +# Start by compiling all watched Sass right away, mirroring the behavior of webpack --watch. +section "COMPILING SASS:" +npm run compile-sass +echo +echo + +section "STARTING DEFAULT SASS WATCHERS:" + +# Changes to LMS Sass require a full recompilation, since LMS Sass can be used in CMS and in themes. +start_sass_watch "default LMS Sass" \ + lms/static/sass \ + 'npm run compile-sass-dev' + +# Changes to default cert Sass only require recompilation of default cert Sass, since cert Sass +# cannot be included into LMS, CMS, or themes. +start_sass_watch "default certificate Sass" \ + lms/static/certificates/sass \ + 'npm run compile-sass-dev -- --skip-cms --skip-themes' + +# Changes to CMS Sass require recompilation of default & themed CMS Sass, but not LMS Sass. +start_sass_watch "default CMS Sass" \ + cms/static/sass \ + 'npm run compile-sass-dev -- --skip-lms' + +# Sass changes in common, node_modules, and xmodule all require full recompilations. +start_sass_watch "default common Sass" \ + common/static \ + 'npm run compile-sass-dev' +start_sass_watch "node_modules Sass" \ + node_modules \ + 'npm run compile-sass-dev' +start_sass_watch "builtin XBlock Sass" \ + xmodule/assets \ + 'npm run compile-sass-dev' + +export IFS=";" +for theme_dir in ${EDX_PLATFORM_THEME_DIRS:-} ; do + for theme_path in "$theme_dir"/* ; do + + theme_name="${theme_path#"$theme_dir/"}" + lms_sass="$theme_path/lms/static/sass" + cert_sass="$theme_path/lms/static/certificates/sass" + cms_sass="$theme_path/cms/static/sass" + + if [[ -d "$lms_sass" ]] || [[ -d "$cert_sass" ]] || [[ -d "$cms_sass" ]] ; then + section "STARTING WATCHERS FOR THEME '$theme_name':" + fi + + # Changes to a theme's LMS Sass require that the full theme is recompiled, since LMS + # Sass is used in certs and CMS. + if [[ -d "$lms_sass" ]] ; then + start_sass_watch "$theme_name LMS Sass" \ + "$lms_sass" \ + "npm run compile-sass-dev -- -T $theme_dir -t $theme_name --skip-default" + fi + + # Changes to a theme's certs Sass only requires that its certs Sass be recompiled. + if [[ -d "$cert_sass" ]] ; then + start_sass_watch "$theme_name certificate Sass" \ + "$cert_sass" \ + "npm run compile-sass-dev -- -T $theme_dir -t $theme_name --skip-default --skip-cms" + fi + + # Changes to a theme's CMS Sass only requires that its CMS Sass be recompiled. + if [[ -d "$cms_sass" ]] ; then + start_sass_watch "$theme_name CMS Sass" \ + "$cms_sass" \ + "npm run compile-sass-dev -- -T $theme_dir -t $theme_name --skip-default --skip-lms" + fi + + done +done + +sleep infinity & +echo +echo "Watching Open edX LMS & CMS Sass for changes." +echo "Use Ctrl+c to exit." +echo +echo +wait $! + diff --git a/xmodule/README.rst b/xmodule/README.rst index 73aeca96cc7c..5096c78c2abe 100644 --- a/xmodule/README.rst +++ b/xmodule/README.rst @@ -10,7 +10,7 @@ The ``xmodule`` folder contains a variety of old-yet-important functionality cor * the implementations of several different built-in content-level XBlocks, such as ``problem`` and ``html``; and * `assets for those built-in XBlocks`_. -.. _assets for those built-in XBlocks: https://github.com/openedx/edx-platform/tree/master/assets +.. _assets for those built-in XBlocks: https://github.com/openedx/edx-platform/tree/master/xmodule/assets#readme Historical Context ******************