diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index cd631547ec6..2d5ecee3412 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -46,3 +46,4 @@ README.md @WordPress/openverse-maintainers docker-compose.yml @WordPress/openverse-maintainers env.template @WordPress/openverse-maintainers justfile @WordPress/openverse-maintainers +ov @WordPress/openverse-maintainers diff --git a/.gitignore b/.gitignore index 531cf353b30..2a5c1d36961 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ logs/ # Environment .env +.ovprofile # Python environments env/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 30ee7283088..47c4e9e1a2c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: # -i: edit in place (no argument means no backup) # -p: loop over all files provided, print error message if file cannot be opened # -e: use the code provided inline - entry: perl -i -pe 's/```console/```bash/g' + entry: python3 -c 'import fileinput; [print(line.replace("```console", "```bash"), end="") for line in fileinput.input(inplace=True)]' language: system - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/docker/dev_env/Dockerfile b/docker/dev_env/Dockerfile new file mode 100644 index 00000000000..ea6e18ab7a0 --- /dev/null +++ b/docker/dev_env/Dockerfile @@ -0,0 +1,67 @@ +FROM docker.io/library/fedora:latest + +# We want to keep all important things in `/opt` as we will preserve the +# `/opt` directory as a volume. + +# Set HOME to /opt so XDG-respecting utilities automatically use it +# without additional configuration +# This gets chmodded with wide open permissions, so that +# on Linux hosts, the explicitly passed user (with docker group) +# is able to read/write from here, as well as the root user used by +# macOS hosts (who don't have Docker permissions or host filesystem +# permissions issues to contend with and as such run the container as root) +ENV HOME="/opt" + +# location where PDM installs Python interpreters +ENV PDM_PYTHONS="${HOME}/pdm/bin" +# location where `n` installs Node.js versions +ENV N_PREFIX="${HOME}/n" + +# Add tooling installed in custom locations to `PATH`. +ENV PATH="${N_PREFIX}/bin:${PDM_PYTHONS}:${HOME}/.local/bin:${PATH}" + +# Dependency explanations: +# - git: required by some python package; contributors should use git on their host +# - perl: used in linting +# - gcc, g++: required by some pre-commit hooks for node-gyp +# - just: command runner +# - which: locate a program file in `PATH` +# - pipx: Python CLI app installer +# - nodejs: language runtime (includes npm but not Corepack) +# - docker*: used to interact with host Docker socket +# +# pipx dependencies: +# - pdm, pipenv: Python package managers +# - pre-commit: Git pre-commit and pre-push hook manager +# +# Node dependencies: +# - n: Node.js distribution manager +# - corepack: Node.js package-manager-manager +RUN dnf -y install dnf-plugins-core \ + && dnf -y config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo \ + && dnf -y install \ + git \ + g++ \ + just \ + which \ + nodejs npm \ + python3.12 pipx \ + docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin \ + && pipx install \ + pdm pipenv \ + pre-commit \ + && npm install -g \ + n \ + corepack \ + && corepack enable + +RUN bash -c 'chmod -Rv 0777 /opt' + +# Avoid overwriting `.venv`'s from the host +ENV PDM_VENV_IN_PROJECT="False" + +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] + +CMD ["/bin/sh"] diff --git a/docker/dev_env/entrypoint.sh b/docker/dev_env/entrypoint.sh new file mode 100755 index 00000000000..53305cfbe85 --- /dev/null +++ b/docker/dev_env/entrypoint.sh @@ -0,0 +1,53 @@ +#! /usr/bin/env bash + +set -e + +if [ ! -d "$OPENVERSE_PROJECT"/.git ]; then + printf "Repository not mounted to container!\n" + exit 1 +fi + +cd "$OPENVERSE_PROJECT" || exit 1 + +corepack install 1>/dev/null + +if [ -z "$(n ls 2>/dev/null)" ]; then + printf "Installing the specific Node JS version required by Openverse frontend; this is only necessary the first time the toolkit runs\n" + n install auto +fi + +if [ -n "$PNPM_HOME" ]; then + export PATH="$PNPM_HOME:$PATH" +fi + +pdm config python.install_root "/opt/pdm/python" +pdm config venv.location "/opt/pdm/venvs" + +_python3s=(/opt/pdm/python/cpython@3.11.*/bin/python3) + +if [ ! -x "${_python3s[0]}" ]; then + printf "Installing the specific Python version required for pipenv environments; this is only necessary the first time the toolkit runs\n" + pdm python install 3.11 + _python3s=(/opt/pdm/python/cpython@3.11.*/bin/python3) +fi + +PYTHON_311=${_python3s[0]} +export PYTHON_311 + +mkdir -p "$PDM_PYTHONS" + +if [ ! -x "$PDM_PYTHONS"/python3.11 ]; then + ln -s "$PYTHON_311" "$PDM_PYTHONS"/python3.11 +fi + +if [ -z "$(command -v pipenv)" ]; then + # Install pipenv with the specific python version required by environments + # still using it, otherwise it gets confused about which dependencies to use + pipx install pipenv --python "$PYTHON_311" +fi + +if [ -n "$PDM_CACHE_DIR" ]; then + pdm config install.cache on +fi + +bash -c "$*" diff --git a/docker/dev_env/hooks/pre-commit b/docker/dev_env/hooks/pre-commit new file mode 100644 index 00000000000..185a73052cf --- /dev/null +++ b/docker/dev_env/hooks/pre-commit @@ -0,0 +1,5 @@ +#! /usr/bin/env bash + +export TERM=xterm + +ov hook pre-commit "$@" diff --git a/docker/dev_env/hooks/pre-push b/docker/dev_env/hooks/pre-push new file mode 100644 index 00000000000..b26b2c1b8fd --- /dev/null +++ b/docker/dev_env/hooks/pre-push @@ -0,0 +1,5 @@ +#! /usr/bin/env bash + +export TERM=xterm + +ov hook pre-push "$@" diff --git a/docker/dev_env/run.sh b/docker/dev_env/run.sh new file mode 100755 index 00000000000..c4a087aca0a --- /dev/null +++ b/docker/dev_env/run.sh @@ -0,0 +1,74 @@ +#! /usr/bin/env bash + +set -e + +volume_name="openverse-dev-env" + +if ! docker volume inspect openverse-dev-env &>/dev/null; then + docker volume create "$volume_name" 1>/dev/null +fi + +run_args=( + -i + --rm + --env "OPENVERSE_PROJECT=$OPENVERSE_PROJECT" + --env "TERM=xterm-256color" + --network host + # Bind the repo to the same exact location inside the container so that pre-commit + # and others don't get confused about where files are supposed to be + -v "$OPENVERSE_PROJECT:$OPENVERSE_PROJECT:rw,z" + --workdir "$OPENVERSE_PROJECT" + # Save the /opt directory of the container so we can reuse it each time + --mount "type=volume,src=$volume_name,target=/opt" + # Expose the host's docker socket to the container so the container can run docker/compose etc + -v /var/run/docker.sock:/var/run/docker.sock +) + +# When running `ov` directly, `-t 0` will show that stdin is available, so +# we should provision a TTY in the docker container (making it possible to +# interact with the container directly) +# However, when running in pre-commit (for example), there is no TTY, and +# docker run will complain if `-t` requests a TTY when the execution +# environment doesn't have one to attach. +# In other words, only tell Docker to attach a TTY to the container when +# there's one to attach in the first place. +if [ -t 0 ]; then + run_args+=(-t) +fi + +case "$OSTYPE" in +linux*) + run_args+=(--user "$UID:$(getent group docker | cut -d: -f3)") + ;; +darwin*) + # noop, just catching them to avoid the fall-through error case + ;; +*) + printf "Openverse development is only supported on Linux and macOS hosts. Please use WSL to run the Openverse development environment under Linux on Windows computers." >/dev/stderr + exit 1 + ;; +esac + +host_pnpm_store="$(pnpm store path 2>/dev/null || echo)" + +# Share the pnpm cache with the container, if it's available locally +if [ "$host_pnpm_store" != "" ]; then + pnpm_home="$(dirname "$host_pnpm_store")" + run_args+=( + --env PNPM_HOME="$pnpm_home" + -v "$pnpm_home:$pnpm_home:rw,z" + ) +fi + +# Share the PDM cache with the container, if it's available locally +# --quiet so PDM doesn't repeatedly fill the console with update messages +# if they're enabled +if [ "$(pdm config --quiet install.cache)" == "True" ]; then + host_pdm_cache="$(pdm config --quiet cache_dir)" + run_args+=( + --env "PDM_CACHE_DIR=$host_pdm_cache" + -v "$host_pdm_cache:$host_pdm_cache:rw,z" + ) +fi + +docker run "${run_args[@]}" openverse-dev-env:latest "$@" diff --git a/justfile b/justfile index abb327c071e..45c6dd140bd 100644 --- a/justfile +++ b/justfile @@ -62,6 +62,10 @@ install: just node-install just py-install +# Install `ov`-based git hooks +@install-hooks: + bash -c "cp ./docker/dev_env/hooks/* ./.git/hooks" + # Setup pre-commit as a Git hook precommit: #!/usr/bin/env bash diff --git a/ov b/ov new file mode 100755 index 00000000000..f260fee473a --- /dev/null +++ b/ov @@ -0,0 +1,85 @@ +#! /usr/bin/env bash + +set -e + +# https://stackoverflow.com/a/1482133 +OPENVERSE_PROJECT="$(dirname "$(readlink -f -- "$0")")" +export OPENVERSE_PROJECT + +_self="$OPENVERSE_PROJECT/ov" + +_cmd="$1" + +dev_env="$OPENVERSE_PROJECT"/docker/dev_env + +if [[ $_cmd == "help" || -z $_cmd ]]; then + cat <<-'EOF' +Openverse development toolkit + +USAGE + ov [args] + +COMMANDS + ov init + Initialise the Openverse development toolkit for the first time + Alias for: + + ov build && ov just install-hooks + + ov build + Build the Openverse development toolkit Docker image + + ov clean + Remove the Openverse development toolkit Docker image and volume + Use in conjunction with `ov init` to recreate the environment from + scratch: + + ov clean && ov init + + ov hook HOOK + Run Git hooks through pre-commit inside the development toolkit container + + ov COMMAND + Run COMMAND inside the development toolkit container + Hints: The toolkit comes loaded with many tools for working with Openverse! Try + some of the following: + + - ov just + - ov pdm + - ov pnpm + - ov python + - ov bash +EOF + + exit 0 +fi + +case "$_cmd" in +init) + "$_self" build + "$_self" just install-hooks + ;; + +build) + docker build -t openverse-dev-env:latest "$dev_env" + ;; + +clean) + docker volume rm openverse-dev-env + ;; + +hook) + # Arguments match the implementation of hooks installed by pre-commit + "$_self" pre-commit hook-impl \ + --config=.pre-commit-config.yaml \ + --hook-type="$2" \ + --hook-dir "$OPENVERSE_PROJECT"/.git/hooks \ + --color=always \ + -- "${@:3}" + ;; + +*) + "$dev_env"/run.sh "$@" + ;; + +esac