From 554176e0524ccb16bb715764dded8355e5228b2a Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Tue, 17 Oct 2023 13:48:19 -0400 Subject: [PATCH] feat!: remove dependency on Paver scripts BREAKING CHANGE: `openedx-assets` is replaed with `npm run` subcommands. For details, see the changelog entry. For further details and rationale, see the upstream DEPR ticket: https://github.com/openedx/edx-platform/issues/31895 --- .../20240318_154804_kyle_assets_build.md | 21 ++ docs/__init__.py | 0 docs/dev.rst | 4 +- .../templates/apps/openedx/config/cms.env.yml | 2 - .../templates/apps/openedx/config/lms.env.yml | 2 - .../openedx/settings/partials/common_cms.py | 4 +- .../openedx/settings/partials/common_lms.py | 4 +- tutor/templates/build/openedx/Dockerfile | 41 ++-- .../build/openedx/bin/openedx-assets | 218 ------------------ .../build/openedx/settings/cms/assets.py | 3 +- .../build/openedx/settings/lms/assets.py | 3 +- .../build/openedx/settings/partials/assets.py | 3 - .../build/openedx/settings/partials/i18n.py | 2 - tutor/templates/dev/docker-compose.yml | 2 +- .../jobs/init/mounted-directories.sh | 2 +- 15 files changed, 51 insertions(+), 260 deletions(-) create mode 100644 changelog.d/20240318_154804_kyle_assets_build.md create mode 100644 docs/__init__.py delete mode 100755 tutor/templates/build/openedx/bin/openedx-assets diff --git a/changelog.d/20240318_154804_kyle_assets_build.md b/changelog.d/20240318_154804_kyle_assets_build.md new file mode 100644 index 0000000000..67f0a143da --- /dev/null +++ b/changelog.d/20240318_154804_kyle_assets_build.md @@ -0,0 +1,21 @@ +- 💥[Feature] The `openedx-assets` command is replaced with `npm run` subcommands. + This will slightly reduce the build time for edx-platform assets and comprehensive themes. + It will also open up the door for more significant build time reductions in the future. + Here is a migration guide, where each command is to be run in the `lms` or `cms` container. + + | **Before** | **After** | + |------------------------------------------|-------------------------------------------------------------------------------------| + | `openedx-assets build --env=prod ARGS` | `npm run build -- ARGS` | + | `openedx-assets build --env=dev ARGS` | `npm run build-dev -- ARGS` | + | `openedx-assets common --env=prod ARGS` | `npm run compile-sass -- --skip-themes ARGS` | + | `openedx-assets common --env=dev ARGS` | `npm run compile-sass-dev -- --skip-themes ARGS` | + | `openedx-assets webpack --env=prod ARGS` | `npm run webpack -- ARGS` | + | `openedx-assets webpack --env=dev ARGS` | `npm run webpack-dev -- ARGS` | + | `openedx-assets npm` | `npm run postinstall` (`npm clean-install` runs this automatically) | + | `openedx-assets xmodule` | (no longer necessary) | + | `openedx-assets collect ARGS` | `./manage.py lms collectstatic --noinput ARGS && ./manage.py cms collectstatic ARGS` | + | `openedx-assets watch-themes ARGS` | `npm run watch-themes -- ARGS` | + + For more details, see the [deprecation notice for paver](https://github.com/openedx/edx-platform/issues/34467) + and the [static assets reference](https://github.com/openedx/edx-platform/tree/open-release/redwood.master/docs/references/static-assets.rst) + in edx-platform. diff --git a/docs/__init__.py b/docs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs/dev.rst b/docs/dev.rst index 6b1f1975bf..216dfb3819 100644 --- a/docs/dev.rst +++ b/docs/dev.rst @@ -99,9 +99,9 @@ To open a python shell in the LMS or CMS, run:: You can then import edx-platform and django modules and execute python code. -To rebuild assets, you can use the ``openedx-assets`` command that ships with Tutor:: +To rebuild assets, you can run the ``build-dev`` NPM script that comes with edx-plaform:: - tutor dev run lms openedx-assets build --env=dev + tutor dev run lms npm run build-dev .. _specialized for developer usage: diff --git a/tutor/templates/apps/openedx/config/cms.env.yml b/tutor/templates/apps/openedx/config/cms.env.yml index c062b2c230..53e99b0c2d 100644 --- a/tutor/templates/apps/openedx/config/cms.env.yml +++ b/tutor/templates/apps/openedx/config/cms.env.yml @@ -27,8 +27,6 @@ CELERY_BROKER_USER: "{{ REDIS_USERNAME }}" CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}" ALTERNATE_WORKER_QUEUES: "lms" ENABLE_COMPREHENSIVE_THEMING: true -COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"] -STATIC_ROOT_BASE: "/openedx/staticfiles" EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" EMAIL_HOST: "{{ SMTP_HOST }}" EMAIL_PORT: {{ SMTP_PORT }} diff --git a/tutor/templates/apps/openedx/config/lms.env.yml b/tutor/templates/apps/openedx/config/lms.env.yml index a1f93eb9a5..565820c81b 100644 --- a/tutor/templates/apps/openedx/config/lms.env.yml +++ b/tutor/templates/apps/openedx/config/lms.env.yml @@ -33,8 +33,6 @@ CELERY_BROKER_USER: "{{ REDIS_USERNAME }}" CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}" ALTERNATE_WORKER_QUEUES: "cms" ENABLE_COMPREHENSIVE_THEMING: true -COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"] -STATIC_ROOT_BASE: "/openedx/staticfiles" EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend" EMAIL_HOST: "{{ SMTP_HOST }}" EMAIL_PORT: {{ SMTP_PORT }} diff --git a/tutor/templates/apps/openedx/settings/partials/common_cms.py b/tutor/templates/apps/openedx/settings/partials/common_cms.py index 1b2f48a2f9..ecc7c46b57 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_cms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_cms.py @@ -21,9 +21,9 @@ FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register' # Create folders if necessary -for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: +for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]: if not os.path.exists(folder): - os.makedirs(folder, exist_ok=True) + os.makedirs(folder, exist_ok=True, parents=True) {{ patch("openedx-cms-common-settings") }} diff --git a/tutor/templates/apps/openedx/settings/partials/common_lms.py b/tutor/templates/apps/openedx/settings/partials/common_lms.py index e3b86492e0..3adb38da5e 100644 --- a/tutor/templates/apps/openedx/settings/partials/common_lms.py +++ b/tutor/templates/apps/openedx/settings/partials/common_lms.py @@ -38,9 +38,9 @@ } # Create folders if necessary -for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]: +for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]: if not os.path.exists(folder): - os.makedirs(folder, exist_ok=True) + os.makedirs(folder, exist_ok=True, parents=True) {{ patch("openedx-lms-common-settings") }} diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index bb66c511a9..7be169eb9d 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -163,6 +163,10 @@ RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV VIRTUAL_ENV /openedx/venv/ +ENV COMPREHENSIVE_THEME_DIRS /openedx/themes +ENV STATIC_ROOT_LMS /openedx/staticfiles +ENV STATIC_ROOT_CMS /openedx/staticfiles/studio + WORKDIR /openedx/edx-platform {# Install auto-mounted directories as Python packages. #} @@ -205,27 +209,22 @@ ENV PATH /openedx/bin:${PATH} {{ patch("openedx-dockerfile-pre-assets") }} -# Collect production assets. By default, only assets from the default theme +# Build & collect production assets. By default, only assets from the default theme # will be processed. This makes the docker image lighter and faster to build. -# Only the custom themes added to /openedx/themes will be compiled. -# Here, we don't run "paver update_assets" which is slow, compiles all themes -# and requires a complex settings file. Instead, we decompose the commands -# and run each one individually to collect the production static assets to -# /openedx/staticfiles. -ENV NO_PYTHON_UNINSTALL 1 -ENV NO_PREREQ_INSTALL 1 -# We need to rely on a separate openedx-assets command to accelerate asset processing. -# For instance, we don't want to run all steps of asset collection every time the theme -# is modified. -RUN openedx-assets xmodule \ - && openedx-assets npm \ - && openedx-assets webpack --env=prod \ - && openedx-assets common -COPY --chown=app:app ./themes/ /openedx/themes/ -RUN openedx-assets themes \ - && openedx-assets collect --settings=tutor.assets \ - # De-duplicate static assets with symlinks - && rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ +RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. +RUN npm run compile-sass -- --skip-themes +RUN npm run webpack + +# Now that the default theme is built, build any custom themes +COPY --chown=app:app ./themes/ /openedx/themes +RUN npm run compile-sass -- --skip-default + +# and finally, collect assets for the production image, +# de-duping assets with symlinks. +RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \ + ./manage.py cms collectstatic --noinput --settings=tutor.assets && \ + # De-duplicate static assets with symlinks \ + rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ # Create a data directory, which might be used (or not) RUN mkdir /openedx/data @@ -278,7 +277,7 @@ ENV PYTHONBREAKPOINT=ipdb.set_trace # static assets, then production assets will be served instead. RUN rm -r /openedx/staticfiles && \ mkdir /openedx/staticfiles && \ - openedx-assets webpack --env=dev + npm run build-dev {{ patch("openedx-dev-dockerfile-post-python-requirements") }} diff --git a/tutor/templates/build/openedx/bin/openedx-assets b/tutor/templates/build/openedx/bin/openedx-assets deleted file mode 100755 index 1b89434d67..0000000000 --- a/tutor/templates/build/openedx/bin/openedx-assets +++ /dev/null @@ -1,218 +0,0 @@ -#! /usr/bin/env python -from __future__ import print_function -import argparse -import os -import subprocess -import sys -import traceback - -from path import Path - -from pavelib import assets - - -DEFAULT_STATIC_ROOT = "/openedx/staticfiles" -DEFAULT_THEMES_DIR = "/openedx/themes" - - -def main(): - parser = argparse.ArgumentParser( - description="Various assets processing/building/collection utility for Open edX" - ) - subparsers = parser.add_subparsers() - - npm = subparsers.add_parser("npm", help="Copy static assets from node_modules") - npm.set_defaults(func=run_npm) - - build = subparsers.add_parser("build", help="Build all assets") - build.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") - build.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR]) - build.add_argument("--themes", nargs="+", default=["all"]) - build.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - build.add_argument("--systems", nargs="+", default=["lms", "cms"]) - build.set_defaults(func=run_build) - - xmodule = subparsers.add_parser("xmodule", help="Process assets from xmodule") - xmodule.set_defaults(func=run_xmodule) - - webpack = subparsers.add_parser("webpack", help="Run webpack") - webpack.add_argument("-r", "--static-root", default=DEFAULT_STATIC_ROOT) - webpack.add_argument("-e", "--env", choices=["prod", "dev"], default="prod") - webpack.set_defaults(func=run_webpack) - - common = subparsers.add_parser( - "common", help="Compile static assets for common theme" - ) - common.add_argument("--systems", nargs="+", default=["lms", "cms"]) - common.set_defaults(func=run_common) - - themes = subparsers.add_parser( - "themes", help="Compile static assets for custom themes" - ) - themes.add_argument("--theme-dirs", nargs="+", default=[DEFAULT_THEMES_DIR]) - themes.add_argument("--themes", nargs="+", default=["all"]) - themes.add_argument("--systems", nargs="+", default=["lms", "cms"]) - themes.set_defaults(func=run_themes) - - collect = subparsers.add_parser( - "collect", help="Collect static assets to be served by webserver" - ) - collect.add_argument( - "-s", - "--settings", - default="tutor.assets", - help="Django settings module", - ) - collect.add_argument( - "--systems", - nargs="+", - choices=["lms", "cms"], - default=["lms", "cms"], - help="Limit collection to lms or cms", - ) - collect.set_defaults(func=run_collect) - - watch_themes = subparsers.add_parser( - "watch-themes", help="Watch theme assets for changes and recompile on-the-fly" - ) - watch_themes.add_argument( - "-e", - "--env", - choices=["prod", "dev"], - default="prod", - help="Webpack target to run", - ) - watch_themes.add_argument("--theme-dirs", default=[DEFAULT_THEMES_DIR]) - watch_themes.set_defaults(func=run_watch_themes) - - args = parser.parse_args() - args.func(args) - - -def run_build(args): - run_xmodule(args) - run_npm(args) - run_webpack(args) - run_common(args) - run_themes(args) - - -def run_xmodule(_args): - # Collecting xmodule assets is incompatible with setting the django path, because - # of an unfortunate call to settings.configure() - django_settings_module = os.environ.get("DJANGO_SETTINGS_MODULE") - if django_settings_module: - os.environ.pop("DJANGO_SETTINGS_MODULE") - - sys.argv[1:] = ["common/static/xmodule"] - import xmodule.static_content - - xmodule.static_content.main() - - if django_settings_module: - os.environ["DJANGO_SETTINGS_MODULE"] = django_settings_module - - -def run_npm(_args): - assets.process_npm_assets() - - -def run_webpack(args): - os.environ["STATIC_ROOT_LMS"] = args.static_root - os.environ["STATIC_ROOT_CMS"] = os.path.join(args.static_root, "studio") - os.environ["NODE_ENV"] = {"prod": "production", "dev": "development"}[args.env] - subprocess.check_call( - [ - "webpack", - "--progress", - "--config=webpack.{env}.config.js".format(env=args.env), - ] - ) - - -def run_common(args): - for system in args.systems: - print("Compiling {} sass assets from common theme...".format(system)) - assets._compile_sass(system, None, False, False, []) - - -def run_themes(args): - for theme_dir in args.theme_dirs: - local_themes = ( - list_subdirectories(theme_dir) if "all" in args.themes else args.themes - ) - for theme in local_themes: - theme_path = os.path.join(theme_dir, theme) - if os.path.exists(theme_path): - for system in args.systems: - print( - "Compiling {} sass assets from theme {}...".format( - system, theme_path - ) - ) - assets._compile_sass(system, Path(theme_path), False, False, []) - - -def run_collect(args): - assets.collect_assets(args.systems, args.settings) - - -def run_watch_themes(args): - """ - Watch static assets for changes and re-compile those changes when - necessary. This piece of code is heavily inspired from the - edx-platform/pavelib/assets.py:watch_assets function, which could not be - used directly because it does not properly read the platform settings - environment variable. - - Note that this function will only work for watching assets in development - mode. In production, watching changes does not make much sense anyway. - """ - observer = assets.Observer() - for theme_dir in args.theme_dirs: - print("Watching changes in {}...".format(theme_dir)) - ThemeWatcher(theme_dir).register(observer) - observer.start() - try: - while True: - observer.join(2) - except KeyboardInterrupt: - observer.stop() - - -def list_subdirectories(path): - return [ - subpath - for subpath in os.listdir(path) - if os.path.isdir(os.path.join(path, subpath)) - ] - - -class ThemeWatcher(assets.SassWatcher): - def __init__(self, theme_dir): - super(ThemeWatcher, self).__init__() - self.theme_dir = theme_dir - - # pylint: disable=arguments-differ - def register(self, observer): - return super(ThemeWatcher, self).register(observer, [self.theme_dir]) - - @assets.debounce() - def on_any_event(self, event): - components = os.path.relpath(event.src_path, self.theme_dir).split("/") - try: - theme = components[0] - system = components[1] - except IndexError: - return - try: - print("Detected change:", event.src_path) - print("\tRecompiling {} theme for {}".format(theme, system)) - assets._compile_sass(system, Path(self.theme_dir) / theme, False, False, []) - print("\tDone recompiling {} theme for {}".format(theme, system)) - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - -if __name__ == "__main__": - main() diff --git a/tutor/templates/build/openedx/settings/cms/assets.py b/tutor/templates/build/openedx/settings/cms/assets.py index 914453dc4c..010df2b44a 100644 --- a/tutor/templates/build/openedx/settings/cms/assets.py +++ b/tutor/templates/build/openedx/settings/cms/assets.py @@ -1,6 +1,5 @@ {% include "build/openedx/settings/partials/assets.py" %} -STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio' -WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" +WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = path(STATIC_ROOT) / "webpack-stats.json" derive_settings(__name__) diff --git a/tutor/templates/build/openedx/settings/lms/assets.py b/tutor/templates/build/openedx/settings/lms/assets.py index b3f85deec4..010df2b44a 100644 --- a/tutor/templates/build/openedx/settings/lms/assets.py +++ b/tutor/templates/build/openedx/settings/lms/assets.py @@ -1,6 +1,5 @@ {% include "build/openedx/settings/partials/assets.py" %} -STATIC_ROOT = path(STATIC_ROOT_BASE) -WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json" +WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = path(STATIC_ROOT) / "webpack-stats.json" derive_settings(__name__) diff --git a/tutor/templates/build/openedx/settings/partials/assets.py b/tutor/templates/build/openedx/settings/partials/assets.py index fce97cdb85..86ef9110ab 100644 --- a/tutor/templates/build/openedx/settings/partials/assets.py +++ b/tutor/templates/build/openedx/settings/partials/assets.py @@ -5,9 +5,6 @@ from openedx.core.lib.derived import derive_settings ENABLE_COMPREHENSIVE_THEMING = True -COMPREHENSIVE_THEME_DIRS.append('/openedx/themes') - -STATIC_ROOT_BASE = '/openedx/staticfiles' SECRET_KEY = 'secret' XQUEUE_INTERFACE = { diff --git a/tutor/templates/build/openedx/settings/partials/i18n.py b/tutor/templates/build/openedx/settings/partials/i18n.py index 3ca5259847..4c5e98a77f 100644 --- a/tutor/templates/build/openedx/settings/partials/i18n.py +++ b/tutor/templates/build/openedx/settings/partials/i18n.py @@ -1,8 +1,6 @@ from ..common import * from openedx.core.lib.derived import derive_settings -STATIC_ROOT_BASE = '/openedx/staticfiles' - SECRET_KEY = 'secret' XQUEUE_INTERFACE = { 'django_auth': None, diff --git a/tutor/templates/dev/docker-compose.yml b/tutor/templates/dev/docker-compose.yml index 4aa01ee56c..1575c9165a 100644 --- a/tutor/templates/dev/docker-compose.yml +++ b/tutor/templates/dev/docker-compose.yml @@ -43,7 +43,7 @@ services: # Additional service for watching theme changes watchthemes: <<: *openedx-service - command: openedx-assets watch-themes --env dev + command: npm run watch-sass restart: unless-stopped {% if RUN_ELASTICSEARCH and is_docker_rootless() %} diff --git a/tutor/templates/jobs/init/mounted-directories.sh b/tutor/templates/jobs/init/mounted-directories.sh index e10dc7b7c9..0f7c615b8b 100644 --- a/tutor/templates/jobs/init/mounted-directories.sh +++ b/tutor/templates/jobs/init/mounted-directories.sh @@ -37,7 +37,7 @@ pip install -e . npm clean-install # Regenerate static assets. -openedx-assets build --env=dev +npm run build-dev set -x echo "Done setting up bind-mounted edx-platform."