diff --git a/tutor/commands/compose.py b/tutor/commands/compose.py index 5724da4aeb..55ebd154d1 100644 --- a/tutor/commands/compose.py +++ b/tutor/commands/compose.py @@ -20,6 +20,7 @@ from tutor.exceptions import TutorError from tutor.tasks import BaseComposeTaskRunner from tutor.types import Config +from tutor.utils import execute as execute_shell class ComposeTaskRunner(BaseComposeTaskRunner): @@ -114,6 +115,10 @@ def launch( click.echo(fmt.title("Docker image updates")) context.invoke(dc_command, command="pull") + if bindmount.get_mounts(config): + click.echo(fmt.title("Copying build artifacts into bind-mounted directories")) + context.invoke(copyartifacts) + click.echo(fmt.title("Starting the platform in detached mode")) context.invoke(start, detach=True) @@ -365,6 +370,63 @@ def copyfrom( ) +@click.command( + help="TODO describe" +) +@click.argument( + "mount_paths", + metavar="mount_path", + nargs=-1, + type=click.Path(dir_okay=True, file_okay=False, resolve_path=True), +) +@click.pass_obj +def copyartifacts(context: BaseComposeContext, mount_paths: list[Path]) -> None: + """ + TODO write docstring + TODO move this to `tutor images build --populate` ?? + """ + config = tutor_config.load(context.root) + host_mount_paths: list[str] = [ + os.path.abspath(os.path.expanduser(mount_path)) + for mount_path + in mount_paths or bindmount.get_mounts(config) + ] + + # Sort out the (source, target) pairs by image name so that we can work one-at-a-time later. + copies_by_image: dict[str, tuple[str, str]] = {} + for host_mount_path in host_mount_paths: + mount_name = os.path.basename(host_mount_path) + for image, container_mount_path in hooks.Filters.COMPOSE_MOUNTS.iterate(mount_name): + if image != "openedx-dev": + # TODO just do openedx-dev for now -- will generalize this soon. + continue + for path_in_mount in hooks.Filters.IMAGES_BUILD_MOUNT_ARTIFACTS.iterate(mount_name, image): + source = f"{container_mount_path}/{path_in_mount}" + target = f"{host_mount_path}/{path_in_mount}" + copies_by_image.setdefault(image, []).append((source, target)) + + container_name = "tutor_mounts_populate_temp" # TODO: improve name? + runner = context.job_runner(config) + + # For each image: create a temporary container, do the copy operations, and then rm the container. + if not copies_by_image: + fmt.echo_alert("Nothing to copy.") + return + for image, copies in copies_by_image.items(): + execute_shell("docker", "rm", "-f", container_name) + for name, _, tag, __ in hooks.Filters.IMAGES_BUILD.iterate(config): + if name == image: + execute_shell("docker", "create", "--name", container_name, tag) + break + else: + raise ValueError(f"unknown image name: {name}") + for source, target in copies: + execute_shell("rm", "-rf", target) # Wipe any existing artifact. + execute_shell("sh", "-c", f'mkdir -p "$(dirname "{target}")"') # Ensure parent dirs exist. + execute_shell("docker", "cp", f"{container_name}:{source}", target) # Actually do the copy. + execute_shell("docker", "rm", "-f", container_name) + + @click.command( short_help="Run a command in a running container", help=( @@ -426,6 +488,17 @@ def dc_command( context.job_runner(config).docker_compose(command, *args) +@hooks.Filters.APP_PUBLIC_HOSTS.add() +def _edx_platform_public_hosts( + hosts: list[str], context_name: t.Literal["local", "dev"] +) -> list[str]: + if context_name == "dev": + hosts += ["{{ LMS_HOST }}:8000", "{{ CMS_HOST }}:8001"] + else: + hosts += ["{{ LMS_HOST }}", "{{ CMS_HOST }}"] + return hosts + + hooks.Filters.ENV_TEMPLATE_VARIABLES.add_item(("iter_mounts", bindmount.iter_mounts)) @@ -439,6 +512,7 @@ def add_commands(command_group: click.Group) -> None: command_group.add_command(dc_command) command_group.add_command(run) command_group.add_command(copyfrom) + command_group.add_command(copyartifacts) command_group.add_command(execute) command_group.add_command(logs) command_group.add_command(status) diff --git a/tutor/commands/images.py b/tutor/commands/images.py index ed3174af63..f8facff481 100644 --- a/tutor/commands/images.py +++ b/tutor/commands/images.py @@ -34,7 +34,7 @@ def _add_core_images_to_build( image, os.path.join("build", image), tutor_config.get_typed(config, tag, str), - (), + ("--target=production",), ) ) diff --git a/tutor/hooks/catalog.py b/tutor/hooks/catalog.py index 60ec819d26..8612790c5a 100644 --- a/tutor/hooks/catalog.py +++ b/tutor/hooks/catalog.py @@ -401,6 +401,34 @@ def your_filter_callback(some_data): #: ``os.path.basename(path)``) to conditionally add mounts. IMAGES_BUILD_MOUNTS: Filter[list[tuple[str, str]], [str]] = Filter() + #: Relative paths of build artifacts, to be copied into a host-mounted folder from a service image. + #: (TODO update this description since Quince changes) + #: + #: Docker images contain many build artifacts, such as generated assets and packaging metadata, which + #: must exist at runtime in order for services to run properly. When a user bind-mounts a directory + #: from their host machine, there is no guarantee that the host directory will contain those essential + #: artifacts, and regenerating them from source may be time consuming. To remedy this, Tutor provides + #: the ``copyartifacts`` command, which efficiently copies the necessary artifacts from the original image + #: into the host directory. This command is also automatically run as part of ``launch``. + #: + #: The ``COMPOSE_MOUNT_ARTIFACTS`` filter tells Tutor which artifacts must be copied from which + #: service for any given host-mounted directory. By default, for edx-platform, this includes + #: several directories such as ``node_modules`` and ``lms/static/css``, to be copied from the lms + #: service's image. + #: + #: Note that any given artifact should only be specified once in this filter. If an artifact + #: exists on an image used by multiple services, choose one of those service for it to be + #: copied from. + #: + #: :parameter artifacts list[str]: files or directories considered build artifacts for the given + #: host-mounted folder in the given service. Paths must be relative to the mount directory. + #: :parameter str mount_name: basename of a host-mounted folder. + #: :parameter image: internal name of a Docker image from which artifacts will be copied. The ``IMAGES_BUILD_MOUNTS`` + #: filter should map ``mount_name`` somewhere in this image; that mount location will be used + #: as the source for ``artifacts``. + IMAGES_BUILD_MOUNT_ARTIFACTS: Filter[list[str], [str, str]] = Filter() + + #: List of images to be pulled when we run ``tutor images pull ...``. #: #: :parameter list[tuple[str, str]] tasks: list of ``(name, tag)`` tuples. diff --git a/tutor/plugins/openedx.py b/tutor/plugins/openedx.py index 2984d60792..efc433d5fd 100644 --- a/tutor/plugins/openedx.py +++ b/tutor/plugins/openedx.py @@ -44,6 +44,27 @@ def _mount_edx_platform_build( return volumes +@hooks.Filters.IMAGES_BUILD_MOUNT_ARTIFACTS.add() +def _populate_edx_platform( + paths_to_copy: list[str], mount_name: str, image: str +) -> list[tuple[str, str]]: + """ + TODO describe + """ + if mount_name == "edx-platform" and image in ["openedx", "openedx-dev"]: + paths_to_copy += [ + "Open_edX.egg-info", + "node_modules", + "lms/static/css", + "lms/static/certificates/css", + "cms/static/css", + "common/static/bundles", + "common/static/common/js/vendor", + "common/static/common/css/vendor", + ] + return paths_to_copy + + @hooks.Filters.COMPOSE_MOUNTS.add() def _mount_edx_platform_compose( volumes: list[tuple[str, str]], name: str @@ -55,20 +76,22 @@ def _mount_edx_platform_compose( if name == "edx-platform": path = "/openedx/edx-platform" volumes.append(("openedx", path)) + volumes.append(("openedx-dev", path)) return volumes # Auto-magically bind-mount xblock directories and some common dependencies. -hooks.Filters.MOUNTED_DIRECTORIES.add_items( - [ - ("openedx", r".*[xX][bB]lock.*"), - ("openedx", "edx-enterprise"), - ("openedx", "edx-ora2"), - ("openedx", "edx-search"), - ("openedx", "openedx-learning"), - ("openedx", r"platform-plugin-.*"), - ] -) +for openedx_image in ["openedx", "openedx-dev"]: + hooks.Filters.MOUNTED_DIRECTORIES.add_items( + [ + (openedx_image, r".*[xX][bB]lock.*"), + (openedx_image, "edx-enterprise"), + (openedx_image, "edx-ora2"), + (openedx_image, "edx-search"), + (openedx_image, "openedx-learning"), + (openedx_image, r"platform-plugin-.*"), + ] + ) @hooks.Filters.MOUNTED_DIRECTORIES.add(priority=hooks.priorities.LOW) diff --git a/tutor/templates/build/openedx/Dockerfile b/tutor/templates/build/openedx/Dockerfile index 368268e812..6ce9b608d5 100644 --- a/tutor/templates/build/openedx/Dockerfile +++ b/tutor/templates/build/openedx/Dockerfile @@ -1,51 +1,36 @@ # syntax=docker/dockerfile:1 -###### Minimal image with base system requirements for most stages + FROM docker.io/ubuntu:20.04 as minimal LABEL maintainer="Overhang.io " - ENV DEBIAN_FRONTEND=noninteractive RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt update && \ - apt install -y build-essential curl git language-pack-en + apt install \ + build-essential \ + curl \ + git \ + language-pack-en \ + --yes ENV LC_ALL en_US.UTF-8 {{ patch("openedx-dockerfile-minimal") }} - -###### Install python with pyenv in /opt/pyenv and create virtualenv in /openedx/venv -FROM minimal as python -# https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites -RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ - --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt update && \ - apt install -y libssl-dev zlib1g-dev libbz2-dev \ - libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev \ - xz-utils tk-dev libffi-dev liblzma-dev python-openssl git - -# Install pyenv -# https://www.python.org/downloads/ -# https://github.com/pyenv/pyenv/releases ARG PYTHON_VERSION=3.11.8 +ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} ENV PYENV_ROOT /opt/pyenv -RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.36 --depth 1 - -# Install Python -RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION - -# Create virtualenv -RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv +ENV VIRTUAL_ENV /openedx/venv/ +ENV XDG_CACHE_HOME /openedx/.cache +ENV COMPREHENSIVE_THEME_DIRS /openedx/themes +ENV STATIC_ROOT_LMS /openedx/staticfiles +ENV STATIC_ROOT_CMS /openedx/staticfiles/studio -###### Checkout edx-platform code -FROM minimal as code +FROM minimal as git-clone-edx-platform ARG EDX_PLATFORM_REPOSITORY={{ EDX_PLATFORM_REPOSITORY }} ARG EDX_PLATFORM_VERSION={{ EDX_PLATFORM_VERSION }} RUN mkdir -p /openedx/edx-platform && \ git clone $EDX_PLATFORM_REPOSITORY --branch $EDX_PLATFORM_VERSION --depth 1 /openedx/edx-platform WORKDIR /openedx/edx-platform - -# Identify tutor user to apply patches using git RUN git config --global user.email "tutor@overhang.io" \ && git config --global user.name "Tutor" - {%- if patch("openedx-dockerfile-git-patches-default") %} # Custom edx-platform patches {{ patch("openedx-dockerfile-git-patches-default") }} @@ -61,138 +46,211 @@ RUN curl -fsSL https://github.com/openedx/edx-platform/commit/ad201cd664b6c722cb # https://discuss.openedx.org/t/upcoming-security-fix-for-edx-platform-on-2024-05-17/13004 RUN curl -fsSL https://github.com/openedx/edx-platform/commit/3ff69fd5813256f935f19c237ea0c42d4c16edbf.patch | git am {%- endif %} - {# Example: RUN curl -fsSL https://github.com/openedx/edx-platform/commit/.patch | git am #} {{ patch("openedx-dockerfile-post-git-checkout") }} -##### Empty layer with just the repo at the root. -# This is useful when overriding the build context with a host repo: -# docker build --build-context edx-platform=/path/to/edx-platform FROM scratch as edx-platform -COPY --from=code /openedx/edx-platform / +COPY --from=git-clone-edx-platform /openedx/edx-platform / -{# Create empty layers for all bind-mounted directories #} {% for name in iter_mounted_directories(MOUNTS, "openedx") %} FROM scratch as mnt-{{ name }} {% endfor %} -###### Install python requirements in virtualenv -FROM python as python-requirements -ENV PATH /openedx/venv/bin:${PATH} -ENV VIRTUAL_ENV /openedx/venv/ -ENV XDG_CACHE_HOME /openedx/.cache - +FROM minimal as pyenv RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt update \ - && apt install -y software-properties-common libmysqlclient-dev libxmlsec1-dev libgeos-dev - -# Install the right version of pip/setuptools + apt update && \ + apt install \ + curl \ + git \ + libbz2-dev \ + libffi-dev \ + liblzma-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + llvm \ + python-openssl \ + tk-dev \ + wget \ + xz-utils \ + zlib1g-dev \ + --yes +RUN git clone https://github.com/pyenv/pyenv $PYENV_ROOT --branch v2.3.36 --depth 1 +RUN $PYENV_ROOT/bin/pyenv install $PYTHON_VERSION +RUN $PYENV_ROOT/versions/$PYTHON_VERSION/bin/python -m venv /openedx/venv RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install \ - # https://pypi.org/project/setuptools/ - # https://pypi.org/project/pip/ - # https://pypi.org/project/wheel/ - setuptools==69.1.1 pip==24.0 wheel==0.43.0 + setuptools==69.1.1 \ + pip==24.0 \ + wheel==0.43.0 -# Install base requirements -RUN --mount=type=bind,from=edx-platform,source=/requirements/edx/base.txt,target=/openedx/edx-platform/requirements/edx/base.txt \ - --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install -r /openedx/edx-platform/requirements/edx/base.txt +FROM pyenv as nodeenv +WORKDIR /openedx/edx-platform +COPY --from=edx-platform /requirements/edx/assets.txt requirements/edx/assets.txt +RUN pip install -r requirements/edx/assets.txt +ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} +RUN nodeenv /openedx/nodeenv --node=18.20.1 --prebuilt +ARG NPM_REGISTRY={{ NPM_REGISTRY }} +COPY --from=edx-platform /package.json package.json +COPY --from=edx-platform /package-lock.json package-lock.json +COPY --from=edx-platform /scripts/copy-node-modules.sh scripts/copy-node-modules.sh +RUN --mount=type=cache,target=/root/.npm,sharing=shared \ + npm clean-install --no-audit --registry=$NPM_REGISTRY -# Install extra requirements -RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install \ - # Use redis as a django cache https://pypi.org/project/django-redis/ - django-redis==5.4.0 \ - # uwsgi server https://pypi.org/project/uWSGI/ - uwsgi==2.0.24 +FROM nodeenv as js-src +COPY --link --from=edx-platform /.babelrc .babelrc +COPY --link --from=edx-platform /webpack-config/file-lists.js webpack-config/file-lists.js +COPY --link --from=edx-platform /webpack.builtinblocks.config.js webpack.builtinblocks.config.js +COPY --link --from=edx-platform /webpack.common.config.js webpack.common.config.js +COPY --link --from=edx-platform /webpack.dev.config.js webpack.dev.config.js +COPY --link --from=edx-platform /webpack.prod.config.js webpack.prod.config.js +COPY --link --from=edx-platform /cms/djangoapps/pipeline_js/js cms/djangoapps/pipeline_js/js +COPY --link --from=edx-platform /cms/static cms/static +COPY --link --from=edx-platform /cms/templates/ cms/templates/ +COPY --link --from=edx-platform /common/static/common common/static/common +COPY --link --from=edx-platform /common/static/js/ common/static/js +COPY --link --from=edx-platform /lms/djangoapps/discussion/static lms/djangoapps/discussion/static +COPY --link --from=edx-platform /lms/djangoapps/instructor/static lms/djangoapps/instructor/static +COPY --link --from=edx-platform /lms/djangoapps/support/static/support lms/djangoapps/support/static/support +COPY --link --from=edx-platform /lms/djangoapps/teams/static lms/djangoapps/teams/static +COPY --link --from=edx-platform /lms/static/ lms/static/ +COPY --link --from=edx-platform /lms/templates/ lms/templates/ +COPY --link --from=edx-platform /openedx/features/announcements/static openedx/features/announcements/static +COPY --link --from=edx-platform /openedx/features/course_bookmarks/static openedx/features/course_bookmarks/static +COPY --link --from=edx-platform /openedx/features/course_experience/static openedx/features/course_experience/static +COPY --link --from=edx-platform /openedx/features/course_search/static openedx/features/course_search/static +COPY --link --from=edx-platform /openedx/features/learner_profile/static openedx/features/learner_profile/static +COPY --link --from=edx-platform /xmodule/assets xmodule/assets +COPY --link --from=edx-platform /xmodule/js xmodule/js +COPY --link --from=nodeenv /openedx/edx-platform/common/static/common/js/vendor common/static/common/js/vendor +COPY --link --from=nodeenv /openedx/edx-platform/common/static/common/css/vendor common/static/common/css/vendor + +FROM js-src as js-prod +RUN npm run webpack -{{ patch("openedx-dockerfile-post-python-requirements") }} +FROM js-src as js-dev +RUN npm run webpack-dev + +FROM nodeenv as sass +COPY --link --from=edx-platform /scripts/compile_sass.py scripts/compile_sass.py +COPY --link --from=edx-platform /common/static common/static +COPY --link --from=edx-platform /lms/static/sass lms/static/sass +COPY --link --from=edx-platform /lms/static/sass/partials lms/static/sass/partials +COPY --link --from=edx-platform /lms/static/certificates/sass lms/static/certificates/sass +COPY --link --from=edx-platform /cms/static/sass cms/static/sass +COPY --link --from=edx-platform /cms/static/sass/partials cms/static/sass/partials +COPY --link --from=edx-platform /xmodule/assets xmodule/assets +COPY --link --from=edx-platform /lms/static/css/vendor lms/static/css/vendor + +FROM sass as css-prod +RUN npm run compile-sass -- --skip-themes +COPY --link ./themes/ /openedx/themes/ +RUN npm run compile-sass -- --skip-default -# Install scorm xblock -RUN pip install "openedx-scorm-xblock>=17.0.0,<18.0.0" +FROM sass as css-dev +RUN npm run compile-sass -- --skip-themes --env=dev +COPY --link ./themes/ /openedx/themes/ +RUN npm run compile-sass -- --skip-default --env=dev +FROM nodeenv as fullenv +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt update && \ + apt install \ + libgeos-dev \ + libmysqlclient-dev \ + libxmlsec1-dev \ + software-properties-common \ + --yes +COPY --link --from=edx-platform /requirements/edx/base.txt /openedx/edx-platform/requirements/edx/base.txt +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install \ + -r /openedx/edx-platform/requirements/edx/base.txt \ + django-redis==5.4.0 \ + uwsgi==2.0.24 \ + 'openedx-scorm-xblock>=17.0.0,<18.0.0' +{{ patch("openedx-dockerfile-post-python-requirements") }} {% for extra_requirements in OPENEDX_EXTRA_PIP_REQUIREMENTS %} RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ pip install '{{ extra_requirements }}' {% endfor %} +{% for name in iter_mounted_directories(MOUNTS, "openedx") %} +COPY --link --from=mnt-{{ name }} / /mnt/{{ name }} +RUN pip install -e "/mnt/{{ name }}" +{% endfor %} -###### Install nodejs with nodeenv in /openedx/nodeenv -FROM python as nodejs-requirements -ENV PATH /openedx/nodeenv/bin:/openedx/venv/bin:${PATH} - -# Install nodeenv with the version provided by edx-platform -# https://github.com/openedx/edx-platform/blob/master/requirements/edx/base.txt -RUN pip install nodeenv==1.8.0 -RUN nodeenv /openedx/nodeenv --node=18.20.1 --prebuilt - -# Install nodejs requirements -ARG NPM_REGISTRY={{ NPM_REGISTRY }} -WORKDIR /openedx/edx-platform -RUN --mount=type=bind,from=edx-platform,source=/package.json,target=/openedx/edx-platform/package.json \ - --mount=type=bind,from=edx-platform,source=/package-lock.json,target=/openedx/edx-platform/package-lock.json \ - --mount=type=bind,from=edx-platform,source=/scripts/copy-node-modules.sh,target=/openedx/edx-platform/scripts/copy-node-modules.sh \ - --mount=type=cache,target=/root/.npm,sharing=shared \ - npm clean-install --no-audit --registry=$NPM_REGISTRY - -###### Production image with system and python requirements -FROM minimal as production +FROM fullenv as fullenv-dev +COPY --link --from=edx-platform /requirements/edx/development.txt /openedx/edx-platform/requirements/edx/development.txt +RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ + pip install \ + -r /openedx/edx-platform/requirements/edx/development.txt \ + ipdb==0.13.13 \ + ipython==8.24.0 -# Install system requirements +FROM minimal as app RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ - apt update \ - && apt install -y gettext gfortran graphviz graphviz-dev libffi-dev libfreetype6-dev libgeos-dev libjpeg8-dev liblapack-dev libmysqlclient-dev libpng-dev libsqlite3-dev libxmlsec1-dev lynx mysql-client ntp pkg-config rdfind - -# From then on, run as unprivileged "app" user -# Note that this must always be different from root (APP_USER_ID=0) + apt update && \ + apt install \ + gettext \ + gfortran \ + graphviz \ + graphviz-dev \ + libffi-dev \ + libfreetype6-dev \ + libgeos-dev \ + libjpeg8-dev \ + liblapack-dev \ + libmysqlclient-dev \ + libpng-dev \ + libsqlite3-dev \ + libxmlsec1-dev \ + lynx \ + mysql-client \ + ntp \ + pkg-config \ + rdfind \ + --yes ARG APP_USER_ID=1000 RUN if [ "$APP_USER_ID" = 0 ]; then echo "app user may not be root" && false; fi RUN useradd --no-log-init --home-dir /openedx --create-home --shell /bin/bash --uid ${APP_USER_ID} app USER ${APP_USER_ID} - -# https://hub.docker.com/r/powerman/dockerize/tags COPY --link --from=docker.io/powerman/dockerize:0.19.0 /usr/local/bin/dockerize /usr/local/bin/dockerize -COPY --chown=app:app --from=edx-platform / /openedx/edx-platform -COPY --chown=app:app --from=python /opt/pyenv /opt/pyenv -COPY --chown=app:app --from=python-requirements /openedx/venv /openedx/venv -COPY --chown=app:app --from=python-requirements /mnt /mnt -COPY --chown=app:app --from=nodejs-requirements /openedx/nodeenv /openedx/nodeenv -COPY --chown=app:app --from=nodejs-requirements /openedx/edx-platform/node_modules /openedx/node_modules - -# Symlink node_modules such that we can bind-mount the edx-platform repository -RUN ln -s /openedx/node_modules /openedx/edx-platform/node_modules - -ENV PATH /openedx/venv/bin:./node_modules/.bin:/openedx/nodeenv/bin:${PATH} -ENV VIRTUAL_ENV /openedx/venv/ -ENV COMPREHENSIVE_THEME_DIRS /openedx/themes -ENV STATIC_ROOT_LMS /openedx/staticfiles -ENV STATIC_ROOT_CMS /openedx/staticfiles/studio - -WORKDIR /openedx/edx-platform - -{# Install auto-mounted directories as Python packages. #} -{% for name in iter_mounted_directories(MOUNTS, "openedx") %} -COPY --from=mnt-{{ name }} --chown=app:app / /mnt/{{ name }} -RUN pip install -e "/mnt/{{ name }}" -{% endfor %} - -# We install edx-platform here because it creates an egg-info folder in the current -# repo. We need both the source code and the virtualenv to run this command. -RUN pip install -e . - -# Create folder that will store lms/cms.env.yml files, as well as -# the tutor-specific settings files. -RUN mkdir -p /openedx/config ./lms/envs/tutor ./cms/envs/tutor -COPY --chown=app:app revisions.yml /openedx/config/ +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv /opt/pyenv /opt/pyenv +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv /openedx/nodeenv /openedx/nodeenv +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv /openedx/venv /openedx/venv +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv /mnt /mnt +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=edx-platform / /openedx/edx-platform +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv \ + /openedx/edx-platform/node_modules \ + /openedx/edx-platform/node_modules +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv \ + /openedx/edx-platform/common/static/common/js/vendor \ + /openedx/edx-platform/common/static/common/js/vendor +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv \ + /openedx/edx-platform/common/static/common/css/vendor \ + /openedx/edx-platform/common/static/common/css/vendor +RUN mkdir -p /openedx/config /openedx/edx-platform/lms/envs/tutor /openedx/edx-platform/cms/envs/tutor +COPY --link revisions.yml /openedx/config/ +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} settings/lms/*.py /openedx/edx-platform/lms/envs/tutor/ +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} settings/cms/*.py /openedx/edx-platform/cms/envs/tutor/ +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} ./bin /openedx/bin +RUN chmod a+x /openedx/bin/* +ENV PATH /openedx/bin:${PATH} +ENV SERVICE_VARIANT lms ENV LMS_CFG /openedx/config/lms.env.yml ENV CMS_CFG /openedx/config/cms.env.yml ENV REVISION_CFG /openedx/config/revisions.yml -COPY --chown=app:app settings/lms/*.py ./lms/envs/tutor/ -COPY --chown=app:app settings/cms/*.py ./cms/envs/tutor/ +RUN mkdir /openedx/data +WORKDIR /openedx/edx-platform +RUN pip install -e . +{{ patch("openedx-dockerfile") }} +EXPOSE 8000 -# Pull latest translations via atlas +FROM app as locales RUN make clean_translations RUN ./manage.py lms --settings=tutor.i18n pull_plugin_translations --verbose --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} RUN ./manage.py lms --settings=tutor.i18n pull_xblock_translations --repository='{{ ATLAS_REPOSITORY }}' --revision='{{ ATLAS_REVISION }}' {{ ATLAS_OPTIONS }} @@ -206,101 +264,76 @@ RUN ./manage.py lms --settings=tutor.i18n compilemessages -v1 RUN ./manage.py lms --settings=tutor.i18n compilejsi18n RUN ./manage.py cms --settings=tutor.i18n compilejsi18n -# Copy scripts -COPY --chown=app:app ./bin /openedx/bin -RUN chmod a+x /openedx/bin/* -ENV PATH /openedx/bin:${PATH} - -{{ patch("openedx-dockerfile-pre-assets") }} - -# Build & collect production assets. By default, only assets from the default theme -# will be processed. This makes the docker image lighter and faster to build. -RUN npm run postinstall # Postinstall artifacts are stuck in nodejs-requirements layer. Create them here too. -RUN npm run compile-sass -- --skip-themes -RUN npm run webpack - -# Now that the default theme is built, build any custom themes -COPY --chown=app:app ./themes/ /openedx/themes -RUN npm run compile-sass -- --skip-default - -# and finally, collect assets for the production image, -# de-duping assets with symlinks. -RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets && \ - ./manage.py cms collectstatic --noinput --settings=tutor.assets && \ - # De-duplicate static assets with symlinks \ - rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ - -# Create a data directory, which might be used (or not) -RUN mkdir /openedx/data - -# If this "canary" file is missing from a container, then that indicates that a -# local edx-platform was bind-mounted into that container, thus overwriting the -# canary. This information is useful during edx-platform initialisation. -RUN echo \ - "This copy of edx-platform was built into a Docker image." \ - > bindmount-canary - -# service variant is "lms" or "cms" -ENV SERVICE_VARIANT lms +FROM app as production +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=locales \ + /openedx/edx-platform/conf/locale \ + /openedx/edx-platform/conf/locale +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=locales \ + /openedx/edx-platform/conf/plugins-locale \ + /openedx/edx-platform/conf/plugins-locale +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=js-prod \ + /openedx/staticfiles \ + /openedx/staticfiles +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=js-prod \ + /openedx/edx-platform/common/static/bundles \ + common/static/bundles +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-prod \ + /openedx/edx-platform/lms/static/css \ + lms/static/css +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-prod \ + /openedx/edx-platform/lms/static/certificates/css \ + lms/static/certificates/css +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-prod \ + /openedx/edx-platform/cms/static/css \ + cms/static/css +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-prod \ + /openedx/themes \ + /openedx/themes +RUN ./manage.py lms collectstatic --noinput --settings=tutor.assets +RUN ./manage.py cms collectstatic --noinput --settings=tutor.assets +RUN rdfind -makesymlinks true -followsymlinks true /openedx/staticfiles/ +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} settings/uwsgi.ini . ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.production +ENV UWSGI_WORKERS=2 +CMD uwsgi uwsgi.ini +{{ patch("openedx-dockerfile-final") }} -{{ patch("openedx-dockerfile") }} - -EXPOSE 8000 - -###### Intermediate image with dev/test dependencies -FROM production as development - -# Install useful system requirements (as root) +FROM app as development USER root RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt,sharing=locked \ apt update && \ - apt install -y vim iputils-ping dnsutils telnet + apt install \ + dnsutils \ + iputils-ping \ + telnet \ + vim \ + --yes USER app - -# Install dev python requirements -RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install -r requirements/edx/development.txt -# https://pypi.org/project/ipdb/ -# https://pypi.org/project/ipython (>=Python 3.10 started with 8.20) -RUN --mount=type=cache,target=/openedx/.cache/pip,sharing=shared \ - pip install ipdb==0.13.13 ipython==8.24.0 - -{# Re-install mounted requirements, otherwise they will be superseded by upstream reqs #} -{% for name in iter_mounted_directories(MOUNTS, "openedx") %} -COPY --from=mnt-{{ name }} --chown=app:app / /mnt/{{ name }} -RUN pip install -e "/mnt/{{ name }}" -{% endfor %} - -# Add ipdb as default PYTHONBREAKPOINT -ENV PYTHONBREAKPOINT=ipdb.set_trace - -# Recompile static assets: in development mode all static assets are stored in edx-platform, -# and the location of these files is stored in webpack-stats.json. If we don't recompile -# static assets, then production assets will be served instead. -RUN rm -r /openedx/staticfiles && \ - mkdir /openedx/staticfiles && \ - npm run build-dev - +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=fullenv-dev /openedx/venv /openedx/venv +RUN pip install -e . {{ patch("openedx-dev-dockerfile-post-python-requirements") }} - -# Default django settings +ENV PYTHONBREAKPOINT=ipdb.set_trace +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=js-dev \ + /openedx/staticfiles/webpack-stats.json \ + /openedx/staticfiles/webpack-stats.json +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=js-dev \ + /openedx/staticfiles/studio/webpack-stats.json \ + /openedx/staticfiles/studio/webpack-stats.json +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=js-dev \ + /openedx/edx-platform/common/static/bundles \ + common/static/bundles +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-dev \ + /openedx/edx-platform/lms/static/css \ + lms/static/css +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-dev \ + /openedx/edx-platform/lms/static/certificates/css \ + lms/static/certificates/css +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-dev \ + /openedx/edx-platform/cms/static/css \ + cms/static/css +COPY --link --chown=${APP_USER_ID}:${APP_USER_ID} --from=css-dev \ + /openedx/themes \ + /openedx/themes ENV DJANGO_SETTINGS_MODULE lms.envs.tutor.development - CMD ./manage.py $SERVICE_VARIANT runserver 0.0.0.0:8000 - -###### Final image with production cmd -FROM production as final - -# Default amount of uWSGI processes -ENV UWSGI_WORKERS=2 - -# Copy the default uWSGI configuration -COPY --chown=app:app settings/uwsgi.ini . - -# Run server -CMD uwsgi uwsgi.ini - -{{ patch("openedx-dockerfile-final") }} -