From 012faeff2d1f74359a8d0d5c1487942976dc20ee Mon Sep 17 00:00:00 2001 From: "Kyle D. McCormick" Date: Mon, 3 Jun 2024 08:53:41 -0400 Subject: [PATCH] feat: symlink to prebuilt artifacts from bindmount --- tutor/commands/jobs.py | 9 -- tutor/templates/build/openedx/Dockerfile | 101 +++++++++--------- .../build/openedx/bin/create-artifact-links | 54 ++++++++++ tutor/templates/jobs/init/lms.sh | 2 + .../jobs/init/mounted-directories.sh | 43 -------- 5 files changed, 109 insertions(+), 100 deletions(-) create mode 100644 tutor/templates/build/openedx/bin/create-artifact-links delete mode 100644 tutor/templates/jobs/init/mounted-directories.sh diff --git a/tutor/commands/jobs.py b/tutor/commands/jobs.py index 3d8e845ae5..264f1c7db6 100644 --- a/tutor/commands/jobs.py +++ b/tutor/commands/jobs.py @@ -42,15 +42,6 @@ def _add_core_init_tasks() -> None: ("mysql", env.read_core_template_file("jobs", "init", "mysql.sh")) ) with hooks.Contexts.app("lms").enter(): - hooks.Filters.CLI_DO_INIT_TASKS.add_item( - ( - "lms", - env.read_core_template_file("jobs", "init", "mounted-directories.sh"), - ), - # If edx-platform is mounted, then we may need to perform some setup - # before other initialization scripts can be run. - priority=priorities.HIGH, - ) hooks.Filters.CLI_DO_INIT_TASKS.add_item( ("lms", env.read_core_template_file("jobs", "init", "lms.sh")) ) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 844bd8e413..86d9f5bc88 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -77,6 +77,7 @@ RUN curl -fsSL https://github.com/openedx/edx-platform/commit/3ff69fd5813256f935 # docker build --build-context edx-platform=/path/to/edx-platform FROM scratch as edx-platform COPY --from=code /openedx/edx-platform / +RUN make clean # Avoid spurious cache misses by ignoring generated files {# Create empty layers for all bind-mounted directories #} {% for name in iter_mounted_directories(MOUNTS, "openedx") %} @@ -100,8 +101,8 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ setuptools==69.1.1 pip==24.0 wheel==0.43.0 # Install base requirements -RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ - --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ +COPY --link --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install -r /openedx/edx-platform/requirements/edx/base.txt # Install extra requirements @@ -122,7 +123,7 @@ RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install '{{ extra_requirements }}' {% endfor %} -###### Install nodejs with nodeenv in /openedx/nodeenv +###### nodejs with nodeenv in /openedx/nodeenv FROM python as nodejs # Install nodeenv with the version provided by edx-platform @@ -130,7 +131,7 @@ FROM python as nodejs RUN pip install nodeenv==1.8.0 RUN nodeenv /openedx/nodeenv --node=18.20.1 --prebuilt -# Install nodejs requirements +###### nodejs + node requirements FROM nodejs as nodejs-requirements ARG NPM_REGISTRY={{ NPM_REGISTRY }} WORKDIR /openedx/edx-platform @@ -143,8 +144,8 @@ RUN --mount=type=cache,target=/root/.npm,sharing=shared \ FROM nodejs-requirements as pre-assets {{ patch("openedx-dockerfile-pre-assets") }} -# Gather minimal sources for webpacking JS into bundles -# Copy the entire source tree, and then delete everything that isn't relevant. +###### Minimal set of source files for webpacking JS into bundles +# We copy the entire source tree, and then delete everything that isn't relevant. FROM pre-assets as js-sources COPY --link --from=edx-platform / . RUN find . -type f -a \! \ @@ -159,17 +160,17 @@ RUN find . -type f -a \! \ -o -name '.babelrc' \ \) -delete -# Intermediate image to capture prod JS build +###### Intermediate image to capture prod JS build FROM pre-assets as js-production COPY --link --from=js-sources /openedx/edx-platform /openedx/edx-platform RUN npm run webpack -# Intermediate image to capture dev JS build +###### Intermediate image to capture dev JS build FROM pre-assets as js-development COPY --link --from=js-sources /openedx/edx-platform /openedx/edx-platform RUN npm run webpack-dev -# Gather minimal requirements and sources for compiling Sass into CSS +####### Minimal set of requirements and source files for compiling Sass into CSS FROM pre-assets as css-sources COPY --link --from=edx-platform /requirements/edx/assets.txt requirements/edx/assets.txt RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ @@ -182,21 +183,20 @@ COPY --link --from=edx-platform /lms/static/certificates/sass lms/static/certifi COPY --link --from=edx-platform /cms/static/sass cms/static/sass COPY --link --from=edx-platform /cms/static/sass/partials cms/static/sass/partials COPY --link --from=edx-platform /xmodule/assets xmodule/assets -COPY --link --from=edx-platform /lms/static/css/vendor lms/static/css/vendor -# Intermediate image to capture compressed CSS build +###### Intermediate image to capture prod (compressed) CSS build FROM css-sources as css-production RUN npm run compile-sass -- --skip-themes COPY --link ./themes/ /openedx/themes RUN npm run compile-sass -- --skip-default -# Intermediate image to capture uncompressed CSS build +###### Intermediate image to capture dev (uncompressed) CSS build FROM css-sources as css-development RUN npm run compile-sass-dev -- --skip-themes COPY --link ./themes/ /openedx/themes RUN npm run compile-sass-dev -- --skip-default -###### Intermediate image shared between production-final and development +###### Intermediate image shared between final dev and prod images FROM minimal as production # Install system requirements @@ -228,11 +228,19 @@ COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=python-requirements /opened COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=python-requirements /mnt /mnt COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs /openedx/nodeenv /openedx/nodeenv +# @@TODO describe +RUN mkdir /openedx/artifacts + +# Link JS requirements into /openedx/artifacts +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/common/static/common/js/vendor /openedx/artifacts/common/static/common/js/vendor +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/common/static/common/css/vendor /openedx/artifacts/common/static/common/css/vendor +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/artifacts/node_modules + WORKDIR /openedx/edx-platform # We install edx-platform here because it creates an egg-info folder in the current # repo. We need both the source code and the virtualenv to run this command. -RUN pip install -e . +RUN pip install -e . && mv Open_edX.egg-info /openedx/artifacts # Create folder that will store lms/cms.env.yml files, as well as # the tutor-specific settings files. @@ -249,16 +257,12 @@ COPY --chown=app:app ./bin /openedx/bin RUN chmod a+x /openedx/bin/* ENV PATH /openedx/bin:${PATH} +# Create symlinks from /openedx/edx-platform to /openedx/artifacts. See script for details. +RUN create-artifact-links + # Create a data directory, which might be used (or not) RUN mkdir /openedx/data -# If this "canary" file is missing from a container, then that indicates that a -# local edx-platform was bind-mounted into that container, thus overwriting the -# canary. This information is useful during edx-platform initialisation. -RUN echo \ - "This copy of edx-platform was built into a Docker image." \ - > bindmount-canary - # service variant is "lms" or "cms" ENV SERVICE_VARIANT lms ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.production @@ -303,36 +307,16 @@ ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.development CMD ./manage.py $SERVICE_VARIANT runserver 0.0.0.0:8000 -# Link in dev JS requirements and dev assets from intermediate dev images -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/edx-platform/common/static/bundles common/static/bundles -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules node_modules +# Link dev assets into dev image, placing edx-platform assets in /openedx/artifacts COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/staticfiles /openedx/staticfiles -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/cms/static/css cms/static/css -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/css lms/static/css -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/certificates/css lms/static/certificates/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-development /openedx/edx-platform/common/static/bundles /openedx/artifacts/common/static/bundles +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/cms/static/css /openedx/artifacts/cms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/css /openedx/artifacts/lms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/edx-platform/lms/static/certificates/css /openedx/artifacts/lms/static/certificates/css COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/themes /openedx/themes -RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. - -###### Final image with production assets and command -FROM production as final - -{# Install auto-mounted directories as Python packages. #} -{% for name in iter_mounted_directories(MOUNTS, "openedx") %} -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=mnt-{{ name }} / /mnt/{{ name }} -RUN pip install -e "/mnt/{{ name }}" -{% endfor %} -# Link in JS requirements and static assets from intermediate images -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=nodejs-requirements /openedx/edx-platform/node_modules node_modules -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/staticfiles /openedx/staticfiles -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/edx-platform/common/static/bundles common/static/bundles -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/cms/static/css cms/static/css -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/css lms/static/css -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/certificates/css lms/static/certificates/css -COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/themes /openedx/themes -RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. - -# Pull latest translations via atlas +###### Intermediate image with translations pulled from atlas +FROM production as translations RUN make clean_translations RUN ./manage.py lms --settings=tutor.i18n pull_plugin_translations --verbose --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} RUN ./manage.py lms --settings=tutor.i18n pull_xblock_translations --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} @@ -346,6 +330,27 @@ RUN ./manage.py lms --settings=tutor.i18n compilemessages -v1 RUN ./manage.py lms --settings=tutor.i18n compilejsi18n RUN ./manage.py cms --settings=tutor.i18n compilejsi18n +###### Final image with production assets and command +FROM production as final + +{# Install auto-mounted directories as Python packages. #} +{% for name in iter_mounted_directories(MOUNTS, "openedx") %} +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=mnt-{{ name }} / /mnt/{{ name }} +RUN pip install -e "/mnt/{{ name }}" +{% endfor %} + +# Link prod assets into prod image, placing edx-platform assets in /openedx/artifacts +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/staticfiles /openedx/staticfiles +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=js-production /openedx/edx-platform/common/static/bundles /openedx/artifacts/common/static/bundles +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/cms/static/css /openedx/artifacts/cms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/css /openedx/artifacts/lms/static/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-production /openedx/edx-platform/lms/static/certificates/css /openedx/artifacts/lms/static/certificates/css +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=css-development /openedx/themes /openedx/themes + +# Link translations into prod image +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=translations /openedx/edx-platform/conf/locale conf/locale +COPY --link --chown=$APP_USER_ID:$APP_USER_ID --from=translations /openedx/edx-platform/conf/plugins-locale conf/plugins-locale + # and finally, collect assets for the production image, # de-duping assets with symlinks. RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \ diff --git a/tutor/templates/build/openedx/bin/create-artifact-links b/tutor/templates/build/openedx/bin/create-artifact-links new file mode 100644 index 0000000000..14b7c7c95c --- /dev/null +++ b/tutor/templates/build/openedx/bin/create-artifact-links @@ -0,0 +1,54 @@ +#!/bin/sh +# +# Several important build artifacts need to be generated in the edx-platform repo. +# However, when a developer bind-mounts edx-platform, it completely overwrites the repo. +# So, the Dockerfile generates the artifacts outside of edx-platform (at /openedx/artifacts) +# where they will not be overwritten by a bind-mount. This script ensures that edx-platform +# contains symlinks into edx-platfor. This script is run both in the Dockerfile and in lms's +# init job; that way, the symlinks exist regardless of whether edx-platform is bind-mounted. +# +# ARTIFACT DIRECTORY | PURPOSE +# ----------------------------------+--------------------------------------------- +# Open_edX.egg-info | edx-platform metadata generated by setup.py +# node_modules | npm packages +# common/static/common/js/vendor | npm JS copies, for use by RequireJS +# common/static/common/css/vendor | npm CSS copies, for use by RequireJS +# common/static/bundles | JS bundles, generated by Webpack +# cms/static/css | Studio CSS, compiled from Sass +# lms/static/css | LMS CSS, compiled from Sass +# lms/static/certificates/css | Certificate CSS, compiled from Sass + +echo "Symlinking build artifacts in /openedx/edx-platform to /openedx/artifacts..." +set -x + +mkdir -p /openedx/artifacts + +for dir in \ + Open_edX.egg-info \ + node_modules \ + common/static/common/js/vendor \ + common/static/common/css/vendor \ + common/static/bundles \ + cms/static/css \ + lms/static/css \ + lms/static/certificates/css ; do + + # If there isn't a symlink or there's one to the wrong place, then fix it + if test `readlink -f $dir` != /openedx/artifacts/$dir ; then + + # If there's an existing symlink (to the wrong place), delete it + if [ -L $dir ] ; then + rm $dir + + # If there's a file or a dir already, back it up + elif [ -d $dir || -f $dir ] ; then + mv -f $dir $dir.bak + fi + + # Create the correct symlink + ln -s /openedx/artifacts/$dir $dir + fi +done + +set -x +echo "Done symlinking build artifacts." diff --git a/tutor/templates/jobs/init/lms.sh b/tutor/templates/jobs/init/lms.sh index 88c94625c5..904451c770 100644 --- a/tutor/templates/jobs/init/lms.sh +++ b/tutor/templates/jobs/init/lms.sh @@ -1,5 +1,7 @@ dockerize -wait tcp://{{ MYSQL_HOST }}:{{ MYSQL_PORT }} -timeout 20s +create-artifact-links + {%- if MONGODB_HOST.startswith("mongodb+srv://") %} echo "MongoDB is using SRV records, so we cannot wait for it to be ready" {%- else %} diff --git a/tutor/templates/jobs/init/mounted-directories.sh b/tutor/templates/jobs/init/mounted-directories.sh deleted file mode 100644 index 0f7c615b8b..0000000000 --- a/tutor/templates/jobs/init/mounted-directories.sh +++ /dev/null @@ -1,43 +0,0 @@ -# The initialization job contains various re-install operations needed to be done -# on mounted directories (edx-platform, /mnt/*xblock, /mnt/) -# 1. /mnt/* -# Whenever xblocks or other installable packages are mounted, during the image build, they are copied over to container -# and installed. This results in egg_info generation for the mounted directories. However, the egg_info is not carried -# over to host. When the containers are launched, the host directories without egg_info are mounted on runtime -# and disappear from pip list. -# -# 2. edx-platform -# When a new local copy of edx-platform is bind-mounted, certain build -# artifacts from the openedx image's edx-platform directory are lost. -# We regenerate them here. - - -for mounted_dir in /mnt/*; do - if [ -f $mounted_dir/setup.py ] && ! ls $mounted_dir/*.egg-info >/dev/null 2>&1 ; then - echo "Unable to locate egg-info in $mounted_dir" - pip install -e $mounted_dir - fi -done - -if [ -f /openedx/edx-platform/bindmount-canary ] ; then - # If this file exists, then edx-platform has not been bind-mounted, - # so no build artifacts need to be regenerated. - echo "Using edx-platform from image (not bind-mount)." - echo "No extra setup is required." - exit -fi - -echo "Performing additional setup for bind-mounted edx-platform." -set -x # Echo out executed lines - -# Regenerate Open_edX.egg-info -pip install -e . - -# Regenerate node_modules -npm clean-install - -# Regenerate static assets. -npm run build-dev - -set -x -echo "Done setting up bind-mounted edx-platform."