feat!: remove dependency on Paver scripts (overhangio#1042)

BREAKING CHANGE: `openedx-assets` is replaed with `npm run` subcommands.
For details, see the changelog entry.
For further details and rationale, see the upstream DEPR ticket:
openedx/edx-platform#31895

changelog.d/20240318_154804_kyle_assets_build.md

@@ -0,0 +1,21 @@
+- 💥[Feature] The `openedx-assets` command is replaced with `npm run` subcommands.
+  This will slightly reduce the build time for edx-platform assets and comprehensive themes.
+  It will also open up the door for more significant build time reductions in the future.
+  Here is a migration guide, where each command is to be run in the `lms` or `cms` container.
+
+  **Before**                               | **After**                                                                           
+  -----------------------------------------|-------------------------------------------------------------------------------------
+  `openedx-assets build --env=prod ARGS`   | `npm run build -- ARGS`                                                             
+  `openedx-assets build --env=dev ARGS`    | `npm run build-dev -- ARGS`                                                         
+  `openedx-assets common --env=prod ARGS`  | `npm run compile-sass -- --skip-themes ARGS`                                    
+  `openedx-assets common --env=dev ARGS`   | `npm run compile-sass-dev -- --skip-themes ARGS`                                    
+  `openedx-assets webpack --env=prod ARGS` | `npm run webpack -- ARGS`                                                           
+  `openedx-assets webpack --env=dev ARGS`  | `npm run webpack-dev -- ARGS`                                                       
+  `openedx-assets npm`                     | `npm run postinstall` (`npm clean-install` runs this automatically)                 
+  `openedx-assets xmodule`                 | (no longer necessary)                                                               
+  `openedx-assets collect ARGS`            | `./manage.py lms collectstatic --noinput ARGS && ./manage.py cms collectstatic ARGS` 
+  `openedx-assets watch-themes ARGS`       | `npm run watch-themes -- ARGS`                                                      
+
+  For more details, see the [deprecation notice for paver](https://github.com/openedx/edx-platform/issues/34467)
+  and the [static assets reference](https://github.com/openedx/edx-platform/tree/open-release/redwood.master/docs/references/static-assets.rst)
+  in edx-platform.

docs/dev.rst

@@ -99,9 +99,9 @@ To open a python shell in the LMS or CMS, run::
 
 You can then import edx-platform and django modules and execute python code.
 
-To rebuild assets, you can use the ``openedx-assets`` command that ships with Tutor::
+To rebuild assets, you can run the ``build-dev`` NPM script that comes with edx-plaform::
 
-    tutor dev run lms openedx-assets build --env=dev
+    tutor dev run lms npm run build-dev
 
 
 .. _specialized for developer usage:

tutor/templates/apps/openedx/config/cms.env.yml

@@ -27,8 +27,6 @@ CELERY_BROKER_USER: "{{ REDIS_USERNAME }}"
 CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}"
 ALTERNATE_WORKER_QUEUES: "lms"
 ENABLE_COMPREHENSIVE_THEMING: true
-COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"]
-STATIC_ROOT_BASE: "/openedx/staticfiles"
 EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
 EMAIL_HOST: "{{ SMTP_HOST }}"
 EMAIL_PORT: {{ SMTP_PORT }}

tutor/templates/apps/openedx/config/lms.env.yml

@@ -33,8 +33,6 @@ CELERY_BROKER_USER: "{{ REDIS_USERNAME }}"
 CELERY_BROKER_PASSWORD: "{{ REDIS_PASSWORD }}"
 ALTERNATE_WORKER_QUEUES: "cms"
 ENABLE_COMPREHENSIVE_THEMING: true
-COMPREHENSIVE_THEME_DIRS: ["/openedx/themes"]
-STATIC_ROOT_BASE: "/openedx/staticfiles"
 EMAIL_BACKEND: "django.core.mail.backends.smtp.EmailBackend"
 EMAIL_HOST: "{{ SMTP_HOST }}"
 EMAIL_PORT: {{ SMTP_PORT }}

tutor/templates/apps/openedx/settings/partials/common_cms.py

@@ -21,7 +21,7 @@
 FRONTEND_REGISTER_URL = LMS_ROOT_URL + '/register'
 
 # Create folders if necessary
-for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]:
+for folder in [LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]:
     if not os.path.exists(folder):
         os.makedirs(folder, exist_ok=True)
 

tutor/templates/apps/openedx/settings/partials/common_lms.py

@@ -38,7 +38,7 @@
 }
 
 # Create folders if necessary
-for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT_BASE, ORA2_FILEUPLOAD_ROOT]:
+for folder in [DATA_DIR, LOG_DIR, MEDIA_ROOT, STATIC_ROOT, ORA2_FILEUPLOAD_ROOT]:
     if not os.path.exists(folder):
         os.makedirs(folder, exist_ok=True)
 

tutor/templates/build/openedx/Dockerfile

@@ -162,6 +162,10 @@ RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules
 
 ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH}
 ENV VIRTUAL_ENV /openedx/venv/
+ENV COMPREHENSIVE_THEME_DIRS /openedx/themes
+ENV STATIC_ROOT_LMS /openedx/staticfiles
+ENV STATIC_ROOT_CMS /openedx/staticfiles/studio
+
 WORKDIR /openedx/edx-platform
 
 {# Install auto-mounted directories as Python packages. #}
@@ -204,27 +208,22 @@ ENV PATH /openedx/bin:${PATH}
 
 {{ patch("openedx-dockerfile-pre-assets") }}
 
-# Collect production assets. By default, only assets from the default theme
+# Build & collect production assets. By default, only assets from the default theme
 # will be processed. This makes the docker image lighter and faster to build.
-# Only the custom themes added to /openedx/themes will be compiled.
-# Here, we don't run "paver update_assets" which is slow, compiles all themes
-# and requires a complex settings file. Instead, we decompose the commands
-# and run each one individually to collect the production static assets to
-# /openedx/staticfiles.
-ENV NO_PYTHON_UNINSTALL 1
-ENV NO_PREREQ_INSTALL 1
-# We need to rely on a separate openedx-assets command to accelerate asset processing.
-# For instance, we don't want to run all steps of asset collection every time the theme
-# is modified.
-RUN openedx-assets xmodule \
-    && openedx-assets npm \
-    && openedx-assets webpack --env=prod \
-    && openedx-assets common
-COPY --chown=app:app ./themes/ /openedx/themes/
-RUN openedx-assets themes \
-    && openedx-assets collect --settings=tutor.assets \
-    # De-duplicate static assets with symlinks
-    && rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/
+RUN npm run postinstall  # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too.
+RUN npm run compile-sass -- --skip-themes
+RUN npm run webpack
+
+# Now that the default theme is built, build any custom themes
+COPY --chown=app:app ./themes/ /openedx/themes
+RUN npm run compile-sass -- --skip-default
+
+# and finally, collect assets for the production image,
+# de-duping assets with symlinks.
+RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \
+    ./manage.py cms collectstatic --noinput --settings=tutor.assets && \
+    # De-duplicate static assets with symlinks \
+    rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/
 
 # Create a data directory, which might be used (or not)
 RUN mkdir /openedx/data
@@ -277,7 +276,7 @@ ENV PYTHONBREAKPOINT=ipdb.set_trace
 # static assets, then production assets will be served instead.
 RUN rm -r /openedx/staticfiles && \
     mkdir /openedx/staticfiles && \
-    openedx-assets webpack --env=dev
+    npm run build-dev
 
 {{ patch("openedx-dev-dockerfile-post-python-requirements") }}
 

tutor/templates/build/openedx/settings/cms/assets.py

@@ -1,6 +1,5 @@
 {% include "build/openedx/settings/partials/assets.py" %}
 
-STATIC_ROOT = path(STATIC_ROOT_BASE) / 'studio'
-WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
+WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = path(STATIC_ROOT) / "webpack-stats.json"
 
 derive_settings(__name__)

tutor/templates/build/openedx/settings/lms/assets.py

@@ -1,6 +1,5 @@
 {% include "build/openedx/settings/partials/assets.py" %}
 
-STATIC_ROOT = path(STATIC_ROOT_BASE)
-WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = STATIC_ROOT / "webpack-stats.json"
+WEBPACK_LOADER['DEFAULT']['STATS_FILE'] = path(STATIC_ROOT) / "webpack-stats.json"
 
 derive_settings(__name__)

tutor/templates/build/openedx/settings/partials/assets.py

@@ -5,9 +5,6 @@
 from openedx.core.lib.derived import derive_settings
 
 ENABLE_COMPREHENSIVE_THEMING = True
-COMPREHENSIVE_THEME_DIRS.append('/openedx/themes')
-
-STATIC_ROOT_BASE = '/openedx/staticfiles'
 
 SECRET_KEY = 'secret'
 XQUEUE_INTERFACE = {

tutor/templates/build/openedx/settings/partials/i18n.py

@@ -1,8 +1,6 @@
 from ..common import *
 from openedx.core.lib.derived import derive_settings
 
-STATIC_ROOT_BASE = '/openedx/staticfiles'
-
 SECRET_KEY = 'secret'
 XQUEUE_INTERFACE = {
     'django_auth': None,

tutor/templates/dev/docker-compose.yml

@@ -43,7 +43,7 @@ services:
   # Additional service for watching theme changes
   watchthemes:
     <<: *openedx-service
-    command: openedx-assets watch-themes --env dev
+    command: npm run watch-sass
     restart: unless-stopped
 
   {% if RUN_ELASTICSEARCH and is_docker_rootless() %}