Dockerfy the Openverse development environment (#4343)

* Dockerfy Openverse dev env

* Switch to Fedora for ARM support

* Fix group ID reading for macOS

* Apply suggestions from code review

Co-authored-by: Dhruv Bhanushali <[email protected]>

* Document new just recipes

* Make Dockerfied development environment compatible with macOS (#4401)

* Update packages and their explanations

* Use `/opt` as the preserved volume target

* Remove user management that's incompatible with macOS

* Add missing hyphen for default value

* Remove redundant packages and update comment

* Fix permissions (hopefully) and get aliases working again

* Fix shellcheck lint

* Tidy up

* Fix argument string conversion location

* Remove use of associative arrays to support macOS bash

* Unuse Perl

* Remove potentially problematic link utility

* Add g++ for renovate pre-commit node-gyp

* Remove unnecessary gcc; make running ov in parallel possible; fix whitespace

* Add pre-push hook and pass arguments to `ov hook`

* Fix pdm venv location

* Silence errors in docker volume when checking for volume

* Use &> to clobber all output

* Update ov

Co-authored-by: Dhruv Bhanushali <[email protected]>

---------

Co-authored-by: Dhruv Bhanushali <[email protected]>
Co-authored-by: Dhruv Bhanushali <[email protected]>

.github/CODEOWNERS

@@ -46,3 +46,4 @@ README.md                   @WordPress/openverse-maintainers
 docker-compose.yml          @WordPress/openverse-maintainers
 env.template                @WordPress/openverse-maintainers
 justfile                    @WordPress/openverse-maintainers
+ov                          @WordPress/openverse-maintainers

.gitignore

@@ -13,6 +13,7 @@ logs/
 
 # Environment
 .env
+.ovprofile
 
 # Python environments
 env/

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
         # -i: edit in place (no argument means no backup)
         # -p: loop over all files provided, print error message if file cannot be opened
         # -e: use the code provided inline
-        entry: perl -i -pe 's/```console/```bash/g'
+        entry: python3 -c 'import fileinput; [print(line.replace("```console", "```bash"), end="") for line in fileinput.input(inplace=True)]'
         language: system
 
   - repo: https://github.com/pre-commit/pre-commit-hooks

docker/dev_env/Dockerfile

@@ -0,0 +1,67 @@
+FROM docker.io/library/fedora:latest
+
+# We want to keep all important things in `/opt` as we will preserve the
+# `/opt` directory as a volume.
+
+# Set HOME to /opt so XDG-respecting utilities automatically use it
+# without additional configuration
+# This gets chmodded with wide open permissions, so that
+# on Linux hosts, the explicitly passed user (with docker group)
+# is able to read/write from here, as well as the root user used by
+# macOS hosts (who don't have Docker permissions or host filesystem
+# permissions issues to contend with and as such run the container as root)
+ENV HOME="/opt"
+
+# location where PDM installs Python interpreters
+ENV PDM_PYTHONS="${HOME}/pdm/bin"
+# location where `n` installs Node.js versions
+ENV N_PREFIX="${HOME}/n"
+
+# Add tooling installed in custom locations to `PATH`.
+ENV PATH="${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}"
+
+# Dependency explanations:
+# - git: required by some python package; contributors should use git on their host
+# - perl: used in linting
+# - gcc, g++: required by some pre-commit hooks for node-gyp
+# - just: command runner
+# - which: locate a program file in `PATH`
+# - pipx: Python CLI app installer
+# - nodejs: language runtime (includes npm but not Corepack)
+# - docker*: used to interact with host Docker socket
+#
+# pipx dependencies:
+# - pdm, pipenv: Python package managers
+# - pre-commit: Git pre-commit and pre-push hook manager
+#
+# Node dependencies:
+# - n: Node.js distribution manager
+# - corepack: Node.js package-manager-manager
+RUN dnf -y install dnf-plugins-core \
+  && dnf -y config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \
+  && dnf -y install \
+    git \
+    g++ \
+    just \
+    which \
+    nodejs npm \
+    python3.12 pipx \
+    docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \
+  && pipx install \
+    pdm pipenv  \
+    pre-commit \
+  && npm install -g \
+    n \
+    corepack \
+  && corepack enable
+
+RUN bash -c 'chmod -Rv 0777 /opt'
+
+# Avoid overwriting `.venv`'s from the host
+ENV PDM_VENV_IN_PROJECT="False"
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]
+
+CMD ["/bin/sh"]

docker/dev_env/entrypoint.sh

@@ -0,0 +1,53 @@
+#! /usr/bin/env bash
+
+set -e
+
+if [ ! -d "$OPENVERSE_PROJECT"/.git ]; then
+  printf "Repository not mounted to container!\n"
+  exit 1
+fi
+
+cd "$OPENVERSE_PROJECT" || exit 1
+
+corepack install 1>/dev/null
+
+if [ -z "$(n ls 2>/dev/null)" ]; then
+  printf "Installing the specific Node JS version required by Openverse frontend; this is only necessary the first time the toolkit runs\n"
+  n install auto
+fi
+
+if [ -n "$PNPM_HOME" ]; then
+  export PATH="$PNPM_HOME:$PATH"
+fi
+
+pdm config python.install_root "/opt/pdm/python"
+pdm config venv.location "/opt/pdm/venvs"
+
+_python3s=(/opt/pdm/python/[email protected].*/bin/python3)
+
+if [ ! -x "${_python3s[0]}" ]; then
+  printf "Installing the specific Python version required for pipenv environments; this is only necessary the first time the toolkit runs\n"
+  pdm python install 3.11
+  _python3s=(/opt/pdm/python/[email protected].*/bin/python3)
+fi
+
+PYTHON_311=${_python3s[0]}
+export PYTHON_311
+
+mkdir -p "$PDM_PYTHONS"
+
+if [ ! -x "$PDM_PYTHONS"/python3.11 ]; then
+  ln -s "$PYTHON_311" "$PDM_PYTHONS"/python3.11
+fi
+
+if [ -z "$(command -v pipenv)" ]; then
+  # Install pipenv with the specific python version required by environments
+  # still using it, otherwise it gets confused about which dependencies to use
+  pipx install pipenv --python "$PYTHON_311"
+fi
+
+if [ -n "$PDM_CACHE_DIR" ]; then
+  pdm config install.cache on
+fi
+
+bash -c "$*"

docker/dev_env/hooks/pre-commit

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+
+export TERM=xterm
+
+ov hook pre-commit "$@"

docker/dev_env/hooks/pre-push

@@ -0,0 +1,5 @@
+#! /usr/bin/env bash
+
+export TERM=xterm
+
+ov hook pre-push "$@"

docker/dev_env/run.sh

@@ -0,0 +1,74 @@
+#! /usr/bin/env bash
+
+set -e
+
+volume_name="openverse-dev-env"
+
+if ! docker volume inspect openverse-dev-env &>/dev/null; then
+  docker volume create "$volume_name" 1>/dev/null
+fi
+
+run_args=(
+  -i
+  --rm
+  --env "OPENVERSE_PROJECT=$OPENVERSE_PROJECT"
+  --env "TERM=xterm-256color"
+  --network host
+  # Bind the repo to the same exact location inside the container so that pre-commit
+  # and others don't get confused about where files are supposed to be
+  -v "$OPENVERSE_PROJECT:$OPENVERSE_PROJECT:rw,z"
+  --workdir "$OPENVERSE_PROJECT"
+  # Save the /opt directory of the container so we can reuse it each time
+  --mount "type=volume,src=$volume_name,target=/opt"
+  # Expose the host's docker socket to the container so the container can run docker/compose etc
+  -v /var/run/docker.sock:/var/run/docker.sock
+)
+
+# When running `ov` directly, `-t 0` will show that stdin is available, so
+# we should provision a TTY in the docker container (making it possible to
+# interact with the container directly)
+# However, when running in pre-commit (for example), there is no TTY, and
+# docker run will complain if `-t` requests a TTY when the execution
+# environment doesn't have one to attach.
+# In other words, only tell Docker to attach a TTY to the container when
+# there's one to attach in the first place.
+if [ -t 0 ]; then
+  run_args+=(-t)
+fi
+
+case "$OSTYPE" in
+linux*)
+  run_args+=(--user "$UID:$(getent group docker | cut -d: -f3)")
+  ;;
+darwin*)
+  # noop, just catching them to avoid the fall-through error case
+  ;;
+*)
+  printf "Openverse development is only supported on Linux and macOS hosts. Please use WSL to run the Openverse development environment under Linux on Windows computers." >/dev/stderr
+  exit 1
+  ;;
+esac
+
+host_pnpm_store="$(pnpm store path 2>/dev/null || echo)"
+
+# Share the pnpm cache with the container, if it's available locally
+if [ "$host_pnpm_store" != "" ]; then
+  pnpm_home="$(dirname "$host_pnpm_store")"
+  run_args+=(
+    --env PNPM_HOME="$pnpm_home"
+    -v "$pnpm_home:$pnpm_home:rw,z"
+  )
+fi
+
+# Share the PDM cache with the container, if it's available locally
+# --quiet so PDM doesn't repeatedly fill the console with update messages
+# if they're enabled
+if [ "$(pdm config --quiet install.cache)" == "True" ]; then
+  host_pdm_cache="$(pdm config --quiet cache_dir)"
+  run_args+=(
+    --env "PDM_CACHE_DIR=$host_pdm_cache"
+    -v "$host_pdm_cache:$host_pdm_cache:rw,z"
+  )
+fi
+
+docker run "${run_args[@]}" openverse-dev-env:latest "$@"

justfile

@@ -62,6 +62,10 @@ install:
     just node-install
     just py-install
 
+# Install `ov`-based git hooks
+@install-hooks:
+    bash -c "cp ./docker/dev_env/hooks/* ./.git/hooks"
+
 # Setup pre-commit as a Git hook
 precommit:
     #!/usr/bin/env bash

ov

@@ -0,0 +1,85 @@
+#! /usr/bin/env bash
+
+set -e
+
+# https://stackoverflow.com/a/1482133
+OPENVERSE_PROJECT="$(dirname "$(readlink -f -- "$0")")"
+export OPENVERSE_PROJECT
+
+_self="$OPENVERSE_PROJECT/ov"
+
+_cmd="$1"
+
+dev_env="$OPENVERSE_PROJECT"/docker/dev_env
+
+if [[ $_cmd == "help" || -z $_cmd ]]; then
+  cat <<-'EOF'
+Openverse development toolkit
+
+USAGE
+  ov <subcommand> [args]
+
+COMMANDS
+  ov init
+    Initialise the Openverse development toolkit for the first time
+    Alias for:
+
+        ov build && ov just install-hooks
+
+  ov build
+    Build the Openverse development toolkit Docker image
+
+  ov clean
+    Remove the Openverse development toolkit Docker image and volume
+    Use in conjunction with `ov init` to recreate the environment from
+    scratch:
+
+        ov clean && ov init
+
+  ov hook HOOK
+    Run Git hooks through pre-commit inside the development toolkit container
+
+  ov COMMAND
+    Run COMMAND inside the development toolkit container
+    Hints: The toolkit comes loaded with many tools for working with Openverse! Try
+    some of the following:
+
+      - ov just
+      - ov pdm
+      - ov pnpm
+      - ov python
+      - ov bash
+EOF
+
+  exit 0
+fi
+
+case "$_cmd" in
+init)
+  "$_self" build
+  "$_self" just install-hooks
+  ;;
+
+build)
+  docker build -t openverse-dev-env:latest "$dev_env"
+  ;;
+
+clean)
+  docker volume rm openverse-dev-env
+  ;;
+
+hook)
+  # Arguments match the implementation of hooks installed by pre-commit
+  "$_self" pre-commit hook-impl \
+    --config=.pre-commit-config.yaml \
+    --hook-type="$2" \
+    --hook-dir "$OPENVERSE_PROJECT"/.git/hooks \
+    --color=always \
+    -- "${@:3}"
+  ;;
+
+*)
+  "$dev_env"/run.sh "$@"
+  ;;
+
+esac