From 51981b8044abede295d4309b82097aa7eff3b1b8 Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Fri, 20 Sep 2024 09:23:17 +1000 Subject: [PATCH 1/7] Fix just argument splitting on pass-through recipes with complex arguments --- api/justfile | 22 +++++--- automations/js/justfile | 10 ++-- automations/python/justfile | 6 ++- catalog/justfile | 9 ++-- docker/cache/justfile | 3 +- documentation/justfile | 3 +- frontend/justfile | 6 ++- indexer_worker/justfile | 10 ++-- justfile | 54 +++++++++++-------- packages/js/k6/justfile | 5 +- .../python/openverse-attribution/justfile | 3 +- 11 files changed, 81 insertions(+), 50 deletions(-) diff --git a/api/justfile b/api/justfile index 9bddb7b6e7b..d724387bb6f 100644 --- a/api/justfile +++ b/api/justfile @@ -45,12 +45,14 @@ install *args: ###### # Bring up services specific to the API profile +[positional-arguments] up *flags: - env COMPOSE_PROFILES="api" just ../up {{ flags }} + env COMPOSE_PROFILES="api" just ../up "$@" # Bring up services specific to the API profile, in addition to the API server +[positional-arguments] up-extra *flags: - env COMPOSE_PROFILES="api,api_extra" just ../up {{ flags }} + env COMPOSE_PROFILES="api,api_extra" just ../up "$@" # Wait for all profile services to be up wait-up: up @@ -103,12 +105,14 @@ pgcli db_user_pass="deploy" db_name="openledger": up ######################### # Run Django administrative commands locally -dj-local +args="": - pdm run python manage.py {{ args }} +[positional-arguments] +dj-local *args: + pdm run python manage.py "$@" # Run Django administrative commands inside the Docker container -dj +args="": wait-up - env DC_USER="{{ env_var_or_default("DC_USER", "opener") }}" just ../exec web python manage.py {{ args }} +[positional-arguments] +dj *args: wait-up + env DC_USER="{{ env_var_or_default("DC_USER", "opener") }}" just ../exec web python manage.py "$@" # Get IPython shell inside the Docker container ipython: @@ -152,12 +156,14 @@ generate-docs doc="media-props" fail_on_diff="true": ######### # Run API tests inside the Docker container +[positional-arguments] test *args: wait-up - env DC_USER="opener" just ../exec web pytest {{ args }} + env DC_USER="opener" just ../exec web pytest "$@" # Run API tests locally +[positional-arguments] test-local *args: - pdm run pytest {{ args }} + pdm run pytest "$@" # Run smoke test for the API docs doc-test: wait-up diff --git a/automations/js/justfile b/automations/js/justfile index e23171a23bd..aeab802c625 100644 --- a/automations/js/justfile +++ b/automations/js/justfile @@ -10,8 +10,9 @@ NO_COLOR := "\\033[0m" just --list --unsorted # Run a script +[positional-arguments] run script *args: - pnpm --filter automations exec node src/{{ script }} {{ args }} + pnpm --filter automations exec node src/{{ script }} "${@:2}" ################# # Weekly report # @@ -26,14 +27,15 @@ report: ########## render-release-drafter: - #! /usr/bin/env sh + #! /usr/bin/env bash render_release_drafter() { slug=$1 name=$2 + confjson=$(printf '{"app": "%s"}' "$name") just run render-jinja.js \ templates/release-drafter.yml.jinja \ - .github/release-drafter-$slug.yml \ - "'{ \"app\": \"$name\" }'" + .github/release-drafter-"$slug".yml \ + "$confjson" } render_release_drafter api API render_release_drafter ingestion_server "Ingestion Server" diff --git a/automations/python/justfile b/automations/python/justfile index 29dbc3ffa65..c6c4020db5f 100644 --- a/automations/python/justfile +++ b/automations/python/justfile @@ -11,9 +11,11 @@ NO_COLOR := "\\033[0m" # Install dependencies +[positional-arguments] install *args: - pdm install {{ args }} + pdm install "$@" # Run a script +[positional-arguments] run script *args: - pdm run {{ script }} {{ args }} + pdm run "$@" diff --git a/catalog/justfile b/catalog/justfile index cb6bffcabb2..fcc0e3d31e3 100644 --- a/catalog/justfile +++ b/catalog/justfile @@ -56,12 +56,14 @@ install *args: check-py-version ###### # Bring up services specific to the catalog profile +[positional-arguments] up *flags: - env COMPOSE_PROFILES="catalog" just ../up {{ flags }} + env COMPOSE_PROFILES="catalog" just ../up "$@" # Bring up services specific to the catalog profile, except Airflow +[positional-arguments] up-deps *flags: - env COMPOSE_PROFILES="catalog_dependencies" just ../up {{ flags }} + env COMPOSE_PROFILES="catalog_dependencies" just ../up "$@" # Load sample data into upstream DB init: up @@ -111,8 +113,9 @@ test-session: just _mount-test bash # Run pytest in a test container under `SERVICE` +[positional-arguments] test *args: - just _mount-test "bash -c \'pytest {{ args }}\'" + just _mount-test "bash -c \'pytest ""$@""\'" ############# # Utilities # diff --git a/docker/cache/justfile b/docker/cache/justfile index eb0bcb84625..51f8a933f7c 100644 --- a/docker/cache/justfile +++ b/docker/cache/justfile @@ -14,8 +14,9 @@ NO_COLOR := "\\033[0m" ####### # Open the Redis CLI as an interactive REPL +[positional-arguments] cli *args: - just ../../exec cache redis-cli {{ args }} + just ../../exec cache redis-cli "$@" # Delete all data cached in Redis flushall: diff --git a/documentation/justfile b/documentation/justfile index 7203d690f3d..4a997abcbe0 100644 --- a/documentation/justfile +++ b/documentation/justfile @@ -72,8 +72,9 @@ live port="50230": clean pdm run sphinx-autobuild -b html . _serve/ --port {{ port }} # Compile the documentation into a static site +[positional-arguments] build *args: clean - pdm run sphinx-build {{ args }} -b html . _build/ + pdm run sphinx-build "$@" -b html . _build/ # Serve the compiled docs using a simple HTTP server serve port="50231": diff --git a/frontend/justfile b/frontend/justfile index eeb402bb24f..77479adf644 100644 --- a/frontend/justfile +++ b/frontend/justfile @@ -55,8 +55,9 @@ run-img tag="openverse-frontend:local": ###### # Bring up services specific to the frontend profile +[positional-arguments] up *flags: - env COMPOSE_PROFILES="frontend" just ../up {{ flags }} + env COMPOSE_PROFILES="frontend" just ../up "$@" # Wait for all profile services to be up wait-up: up @@ -67,8 +68,9 @@ init: wait-up cd .. && ./setup_plausible.sh # Run a package.json script via pnpm +[positional-arguments] run *args: - pnpm run {{ args }} + pnpm run "$@" # Generate the specified kind of documentation generate-docs doc="media-props" fail_on_diff="true": diff --git a/indexer_worker/justfile b/indexer_worker/justfile index 4669b9e0451..47cdd69d76b 100644 --- a/indexer_worker/justfile +++ b/indexer_worker/justfile @@ -33,8 +33,9 @@ install *args="--dev": ###### # Bring up services specific to the indexer worker profile +[positional-arguments] up *flags: - env COMPOSE_PROFILES="catalog_indexer_worker" just ../up {{ flags }} + env COMPOSE_PROFILES="catalog_indexer_worker" just ../up "$@" wait-up: up just wait @@ -77,9 +78,12 @@ curl-post data host="localhost:50281": # Tests # ######### +# Run indexer worker tests in the container +[positional-arguments] test *args: wait-up - just ../exec catalog_indexer_worker pytest {{ args }} + just ../exec catalog_indexer_worker pytest "$@" # Run indexer-worker tests locally +[positional-arguments] test-local *args: - pdm run pytest {{ args }} + pdm run pytest "$@" diff --git a/justfile b/justfile index 851b31f39ed..a489bae0bd0 100644 --- a/justfile +++ b/justfile @@ -4,9 +4,6 @@ set dotenv-load := false # @ - Quiet recipes (https://github.com/casey/just#quiet-recipes) # _ - Private recipes (https://github.com/casey/just#private-recipes) -IS_PROD := env_var_or_default("PROD", env_var_or_default("IS_PROD", "")) -# `PROD_ENV` can be "ingestion_server" or "catalog" -PROD_ENV := env_var_or_default("PROD_ENV", "") IS_CI := env_var_or_default("CI", "") DC_USER := env_var_or_default("DC_USER", "opener") @@ -137,8 +134,9 @@ precommit: fi # Run pre-commit to lint and reformat files +[positional-arguments] lint hook="" *files="": precommit - python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files" } }} {{ files }} + python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files {{ files }}" } }} # Run codeowners validator locally. Only enable experimental hooks if there are no uncommitted changes. lint-codeowners checks="stable": @@ -171,14 +169,7 @@ lint-codeowners checks="stable": # Docker # ########## -DOCKER_FILE := "-f " + ( - if IS_PROD == "true" { - if PROD_ENV == "ingestion_server" { "ingestion_server/docker-compose.yml" } - else if PROD_ENV == "catalog" { "catalog/docker-compose.yml" } - else { "docker-compose.yml" } - } - else { "docker-compose.yml" } -) +DOCKER_FILE := "-f docker-compose.yml" EXEC_DEFAULTS := if IS_CI == "" { "" } else { "-T" } export CATALOG_PY_VERSION := `just catalog/py-version` @@ -207,13 +198,15 @@ versions: EOF # Run `docker compose` configured with the correct files and environment +[positional-arguments] dc *args: @{{ if IS_CI != "" { "just env" } else { "true" } }} - env COMPOSE_PROFILES="{{ env_var_or_default("COMPOSE_PROFILES", "api,ingestion_server,frontend,catalog") }}" docker compose {{ DOCKER_FILE }} {{ args }} + env COMPOSE_PROFILES="{{ env_var_or_default("COMPOSE_PROFILES", "api,ingestion_server,frontend,catalog") }}" docker compose {{ DOCKER_FILE }} "$@" # Build all (or specified) services +[positional-arguments] build *args: - just dc build {{ args }} + just dc build "$@" # List all services and their URLs and ports @ps: @@ -223,11 +216,12 @@ build *args: # Also see `up` recipe in sub-justfiles # Bring all Docker services up, in all profiles +[positional-arguments] up *flags: env && ps #!/usr/bin/env bash set -eo pipefail while true; do - if just dc up {{ if IS_CI != "" { "--quiet-pull" } else { "" } }} -d {{ flags }} ; then + if just dc up {{ if IS_CI != "" { "--quiet-pull" } else { "" } }} -d "$@" ; then break fi ((c++)) && ((c==3)) && break @@ -249,8 +243,9 @@ init: just frontend/init # Take all Docker services down, in all profiles +[positional-arguments] down *flags: - just dc down {{ flags }} + just dc down "$@" # Take all services down then call the specified app's up recipe. ex.: `just dup catalog` is useful for restarting the catalog with new environment variables dup app: @@ -277,12 +272,14 @@ attach service: docker attach $(just dc ps | awk '{print $1}' | grep {{ service }}) # Execute statement in service containers using Docker Compose +[positional-arguments] exec +args: - just dc exec -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} {{ args }} + just dc exec -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} "$@" # Execute statement in a new service container using Docker Compose +[positional-arguments] run +args: - just dc run --rm -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} "{{ args }}" + just dc run --rm -u {{ env_var_or_default("DC_USER", "root") }} {{ EXEC_DEFAULTS }} "$@" # Execute pgcli against one of the database instances _pgcli container db_user_pass db_name db_host db_port="5432": @@ -352,20 +349,31 @@ f: just frontend/run dev # alias for `pnpm --filter {package} run {script}` -p package script +args="": - pnpm --filter {{ package }} run {{ script }} {{ args }} +[positional-arguments] +p package script *args: + pnpm --filter {{ package }} run {{ script }} "${@:3}" # Run eslint with --fix and default file selection enabled; used to enable easy file overriding whilst retaining the defaults when running --all-files -eslint *files="frontend automations/js packages/js .pnpmfile.cjs .eslintrc.js prettier.config.js tsconfig.base.json": +[positional-arguments] +eslint *args: + #! /usr/bin/env bash just p '@openverse/eslint-plugin' build + if [[ "$@" ]]; then + files=("$@") + else + # default files + files=(frontend automations/js packages/js .pnpmfile.cjs .eslintrc.js prettier.config.js tsconfig.base.json) + fi + pnpm exec eslint \ --ext .js,.ts,.vue,.json,.json5 \ --ignore-path .gitignore \ --ignore-path .eslintignore \ --max-warnings=0 \ --fix \ - {{ files }} + "${files[@]}" # Alias for `just packages/js/k6/run` or `just p k6 run` +[positional-arguments] @k6 *args: - just packages/js/k6/run {{ args }} + just packages/js/k6/run "$@" diff --git a/packages/js/k6/justfile b/packages/js/k6/justfile index 64c59654a9f..dc6c2e6090a 100644 --- a/packages/js/k6/justfile +++ b/packages/js/k6/justfile @@ -1,4 +1,5 @@ # Build and run K6 script by namespace and scenario -run namespace scenario +extra_args="": +[positional-arguments] +run namespace scenario *extra_args: pnpm run build --input src/{{ namespace }}/{{ scenario }}.test.ts - k6 run {{ extra_args }} ./dist/{{ namespace }}/{{ scenario }}.test.js + k6 run "${@:3}" ./dist/{{ namespace }}/{{ scenario }}.test.js diff --git a/packages/python/openverse-attribution/justfile b/packages/python/openverse-attribution/justfile index 56cb7e75a92..250ef73c0fa 100644 --- a/packages/python/openverse-attribution/justfile +++ b/packages/python/openverse-attribution/justfile @@ -14,5 +14,6 @@ NO_COLOR := "\\033[0m" ########### # Install dependencies +[positional-arguments] install *args: - pdm install {{ args }} + pdm install "$@" From 97aeed8c3a48c16b29bddb9bf14202e55524d7a8 Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Fri, 20 Sep 2024 10:19:22 +1000 Subject: [PATCH 2/7] Use bash instead of sh for consistent behaviour in just recipes and other scripts --- .vale/justfile | 2 +- frontend/bin/playwright.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.vale/justfile b/.vale/justfile index 5fbd468cd88..d8d9331c540 100644 --- a/.vale/justfile +++ b/.vale/justfile @@ -14,7 +14,7 @@ IGNORE_FILES_PATTERN := ".pnpm|changelogs|projects/proposals|node_modules|.?venv # of the current working directory. If a custom separator is specified, newlines are # replaced with this separator before outputting the file list. _files separator="\n": - #! /usr/bin/env sh + #! /usr/bin/env bash files=$( find "$PWD/.." -type f \( -name "*.md" -o -name "*.mdx" \) \ | grep -vE "{{ IGNORE_FILES_PATTERN }}" diff --git a/frontend/bin/playwright.sh b/frontend/bin/playwright.sh index cdfa8a6b659..00554c3f8ce 100755 --- a/frontend/bin/playwright.sh +++ b/frontend/bin/playwright.sh @@ -1,4 +1,4 @@ -#! /usr/bin/env sh +#! /usr/bin/env bash set -e version() { From accf415fcc914047cca20a677b1237d2a0ffc3e5 Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Fri, 20 Sep 2024 10:40:29 +1000 Subject: [PATCH 3/7] Simplify render-release-drafter recipes --- automations/js/justfile | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/automations/js/justfile b/automations/js/justfile index aeab802c625..98537b05f19 100644 --- a/automations/js/justfile +++ b/automations/js/justfile @@ -26,22 +26,14 @@ report: # Render # ########## +_render slug name: + just run render-jinja.js \ + templates/release-drafter.yml.jinja \ + .github/release-drafter-{{ slug }}.yml \ + '{"app": "{{ name }}"}' + render-release-drafter: - #! /usr/bin/env bash - render_release_drafter() { - slug=$1 - name=$2 - confjson=$(printf '{"app": "%s"}' "$name") - just run render-jinja.js \ - templates/release-drafter.yml.jinja \ - .github/release-drafter-"$slug".yml \ - "$confjson" - } - render_release_drafter api API - render_release_drafter ingestion_server "Ingestion Server" - render_release_drafter frontend Frontend - render_release_drafter catalog Catalog - -# Render all templates (shortcut for easy iteration) -render-templates: - just render-release-drafter + just _render api API + just _render ingestion_server "Ingestion Server" + just _render frontend Frontend + just _render catalog Catalog From e83422953ff64717273b8b8bcaf057b7cee6a2d8 Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Fri, 20 Sep 2024 10:57:32 +1000 Subject: [PATCH 4/7] Fix recreate --- justfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/justfile b/justfile index a489bae0bd0..b71e8b3d46e 100644 --- a/justfile +++ b/justfile @@ -136,7 +136,12 @@ precommit: # Run pre-commit to lint and reformat files [positional-arguments] lint hook="" *files="": precommit - python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files {{ files }}" } }} + #! /usr/bin/env bash + if [[ "$files" ]]; then + python3 pre-commit.pyz run {{ hook }} --files "${@:2}" + else + python3 pre-commit.pyz run {{ hook }} --all-files + fi # Run codeowners validator locally. Only enable experimental hooks if there are no uncommitted changes. lint-codeowners checks="stable": @@ -254,7 +259,7 @@ dup app: # Recreate all volumes and containers from scratch recreate: just down -v - just up "--force-recreate --build" + just up --force-recreate --build just init # Bust pnpm cache and reinstall Node.js dependencies From 3a9fd328342e33360541d798d59d3c8baa5492f1 Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Fri, 20 Sep 2024 10:58:04 +1000 Subject: [PATCH 5/7] Try another fix for render-release-drafter --- automations/js/justfile | 1 + 1 file changed, 1 insertion(+) diff --git a/automations/js/justfile b/automations/js/justfile index 98537b05f19..4a8235f6069 100644 --- a/automations/js/justfile +++ b/automations/js/justfile @@ -12,6 +12,7 @@ NO_COLOR := "\\033[0m" # Run a script [positional-arguments] run script *args: + #! /usr/bin/env bash pnpm --filter automations exec node src/{{ script }} "${@:2}" ################# From f97a5e612ba6046e718a617b919ca817ee3d5da0 Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Fri, 20 Sep 2024 11:11:32 +1000 Subject: [PATCH 6/7] Fix catalog/test --- catalog/justfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/catalog/justfile b/catalog/justfile index fcc0e3d31e3..f81655bc060 100644 --- a/catalog/justfile +++ b/catalog/justfile @@ -99,13 +99,14 @@ pgcli db_user_pass="deploy" db_name="openledger": up ######### # Run a command in a test container under `SERVICE` -_mount-test command: up-deps +[positional-arguments] +_mount-test *command: up-deps env DC_USER="airflow" just ../run \ -e AIRFLOW_VAR_INGESTION_LIMIT=1000000 \ -w /opt/airflow/catalog \ - --volume \'{{ justfile_directory() }}/../docker\':/opt/airflow/docker/ \ + --volume {{ justfile_directory() }}/../docker:/opt/airflow/docker/ \ {{ SERVICE }} \ - {{ command }} + "$@" # Launch a Bash shell in a test container under `SERVICE` # Run pytest with `--pdb` to workaround xdist breaking pdb.set_trace() @@ -115,7 +116,7 @@ test-session: # Run pytest in a test container under `SERVICE` [positional-arguments] test *args: - just _mount-test "bash -c \'pytest ""$@""\'" + just _mount-test bash -c "pytest $@" ############# # Utilities # From 9904123368d245050350cd9111a12b221b44beb3 Mon Sep 17 00:00:00 2001 From: sarayourfriend Date: Fri, 20 Sep 2024 12:20:54 +1000 Subject: [PATCH 7/7] Fix generate-docs and lint recipe --- catalog/justfile | 13 ++++++++----- justfile | 8 +------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/catalog/justfile b/catalog/justfile index f81655bc060..135a8f5e601 100644 --- a/catalog/justfile +++ b/catalog/justfile @@ -84,11 +84,12 @@ shell: env DC_USER="airflow" just ../exec {{ SERVICE }} /bin/bash # Launch an IPython shell in a new container under `SERVICE` +[positional-arguments] ipython *args: up-deps env DC_USER="airflow" just ../run \ --workdir /opt/airflow/catalog/dags \ {{ SERVICE }} \ - bash -c \'ipython {{ args }}\' + bash -c "ipython ${@:2}" # Launch a `pgcli` shell in the PostgreSQL container pgcli db_user_pass="deploy" db_name="openledger": up @@ -124,7 +125,7 @@ test *args: # Generate the documentation (either "dag" or "media-props") generate-docs doc="dag" fail_on_diff="false": - #!/bin/bash + #! /usr/bin/env bash set -e if [ "{{ doc }}" == "dag" ]; then SCRIPT_PATH="catalog/utilities/dag_doc_gen/dag_doc_generation.py" @@ -140,18 +141,20 @@ generate-docs doc="dag" fail_on_diff="false": echo "Invalid documentation type specified, use \`dag\` or \`media-props\`. Exiting." exit 1 fi + GENERATED_ABS_PATH="/opt/airflow/catalog/${GENERATED_REL_PATH}" just ../run \ --volume {{ justfile_directory() }}/../docker:/opt/airflow/docker/ \ -e PYTHONPATH=/opt/airflow/catalog:/opt/airflow/catalog/dags \ {{ SERVICE }} \ - "bash -c 'python $SCRIPT_PATH && chmod 666 $GENERATED_ABS_PATH'" + bash -c "python $SCRIPT_PATH && chmod 666 $GENERATED_ABS_PATH" + TEMP="documentation/meta/temp" - mv ../catalog/$GENERATED_REL_PATH ../$TEMP.md + mv ../catalog/"$GENERATED_REL_PATH" ../"$TEMP".md echo "Moved the generated file to ../$TEMP.md" echo -n "Running linting..." # Linting step afterwards is necessary since the generated output differs greatly from what prettier expects - just ../lint prettier $TEMP.md &>/dev/null || true + just ../lint prettier "$TEMP".md &>/dev/null || true echo "Linting done!" echo -n "Replacing linted md
'---' with '----' required by sphinx..." diff --git a/justfile b/justfile index b71e8b3d46e..037ef385ab9 100644 --- a/justfile +++ b/justfile @@ -134,14 +134,8 @@ precommit: fi # Run pre-commit to lint and reformat files -[positional-arguments] lint hook="" *files="": precommit - #! /usr/bin/env bash - if [[ "$files" ]]; then - python3 pre-commit.pyz run {{ hook }} --files "${@:2}" - else - python3 pre-commit.pyz run {{ hook }} --all-files - fi + python3 pre-commit.pyz run {{ hook }} {{ if files == "" { "--all-files" } else { "--files" } }} {{ files }} # Run codeowners validator locally. Only enable experimental hooks if there are no uncommitted changes. lint-codeowners checks="stable":