From 9b1aa7762f37646b2c6b4a1dc5c8feb493f5bb4d Mon Sep 17 00:00:00 2001 From: Dhruv Bhanushali Date: Sat, 11 May 2024 10:12:59 +0400 Subject: [PATCH] Make API image smaller by not including dev dependencies (#4285) --- .github/workflows/ci_cd.yml | 35 ++++++- .../python/workflows/set_matrix_images.py | 98 ++++++++++++------- documentation/meta/ci_cd/jobs/api.md | 4 + 3 files changed, 100 insertions(+), 37 deletions(-) diff --git a/.github/workflows/ci_cd.yml b/.github/workflows/ci_cd.yml index 65dd42a3b87..e148ae73361 100644 --- a/.github/workflows/ci_cd.yml +++ b/.github/workflows/ci_cd.yml @@ -233,14 +233,14 @@ jobs: uses: docker/build-push-action@v5 with: context: ${{ matrix.context }} - target: ${{ matrix.target || '' }} + target: ${{ matrix.target }} push: false tags: openverse-${{ matrix.image }} file: ${{ matrix.file }} cache-from: type=gha,scope=${{ matrix.image }} cache-to: type=gha,scope=${{ matrix.image }} outputs: type=docker,dest=/tmp/${{ matrix.image }}.tar - build-contexts: ${{ matrix.build-contexts || '' }} + build-contexts: ${{ matrix.build-contexts }} build-args: | SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }} CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }} @@ -250,6 +250,7 @@ jobs: FRONTEND_NODE_VERSION=${{ steps.prepare-build-args.outputs.frontend_node_version }} FRONTEND_PNPM_VERSION=${{ steps.prepare-build-args.outputs.frontend_pnpm_version }} PGCLI_VERSION=${{ steps.prepare-build-args.outputs.pgcli_version }} + ${{ matrix.build-args || '' }} - name: Upload image `${{ matrix.image }}` id: upload-img @@ -398,6 +399,7 @@ jobs: needs: - get-changes - build-images + - get-image-tag steps: - name: Checkout repository @@ -416,7 +418,34 @@ jobs: uses: ./.github/actions/load-img with: run_id: ${{ github.run_id }} - setup_images: upstream_db ingestion_server api + setup_images: upstream_db ingestion_server + + # Sets build args specifying versions needed to build Docker image. + - name: Prepare build args + id: prepare-build-args + run: | + just versions | tee "$GITHUB_OUTPUT" + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + install: true + + - name: Build API dev image + uses: docker/build-push-action@v5 + with: + context: api + target: api + push: false + load: true + tags: openverse-api + cache-from: type=gha,scope=api_dev + cache-to: type=gha,scope=api_dev + build-contexts: packages=./packages/python + build-args: | + SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }} + API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }} + PDM_INSTALL_ARGS=--dev - name: Start API, ingest and index test data run: just api/init diff --git a/automations/python/workflows/set_matrix_images.py b/automations/python/workflows/set_matrix_images.py index 59415ad6a22..ba7eb6dae41 100644 --- a/automations/python/workflows/set_matrix_images.py +++ b/automations/python/workflows/set_matrix_images.py @@ -6,10 +6,42 @@ import json import os +from dataclasses import asdict, dataclass from shared.actions import write_to_github_output +@dataclass +class Include: + image: str + target: str + + context: str + file: str + build_contexts: str + build_args: str + + def __init__( + self, + image: str, + target: str, + context: str | None = None, + file: str | None = None, + build_contexts: str | None = None, + build_args: str | None = None, + ): + self.image = image + self.target = target + self.context = context or image + self.file = file or f"{self.context}/Dockerfile" + self.build_contexts = build_contexts or "" + self.build_args = build_args or "" + + @property + def asdict(self) -> dict[str, str]: + return {k.replace("_", "-"): v for k, v in asdict(self).items()} + + changes = json.loads(os.environ.get("CHANGES")) @@ -21,32 +53,37 @@ def ser_set(x): build_matrix = {"image": set()} publish_matrix = {"image": set()} -includes = { - "upstream_db": { - "image": "upstream_db", - "context": "docker/upstream_db", - "target": "db", - }, - "catalog": {"image": "catalog", "target": "cat"}, - "ingestion_server": {"image": "ingestion_server", "target": "ing"}, - "api": { - "image": "api", - "target": "api", - "build-contexts": "packages=./packages/python", - }, - "api_nginx": { - "image": "api_nginx", - "context": "api", - "target": "nginx", - "build-contexts": "packages=./packages/python", - }, - "frontend": {"image": "frontend", "target": "app", "build-contexts": "repo_root=."}, - "frontend_nginx": { - "image": "frontend_nginx", - "context": "frontend", - "file": "frontend/Dockerfile.nginx", - "target": "nginx", - }, +includes: dict[str, Include] = { + "upstream_db": Include( + image="upstream_db", + target="db", + context="docker/upstream_db", + ), + "catalog": Include(image="catalog", target="cat"), + "ingestion_server": Include(image="ingestion_server", target="ing"), + "api": Include( + image="api", + target="api", + build_contexts="packages=./packages/python", + build_args="PDM_INSTALL_ARGS=--prod", + ), + "api_nginx": Include( + image="api_nginx", + target="nginx", + context="api", + build_contexts="packages=./packages/python", + ), + "frontend": Include( + image="frontend", + target="app", + build_contexts="repo_root=.", + ), + "frontend_nginx": Include( + image="frontend_nginx", + target="nginx", + context="frontend", + file="frontend/Dockerfile.nginx", + ), } if "ci_cd" in changes: @@ -65,14 +102,7 @@ def ser_set(x): publish_matrix["image"] |= {"frontend", "frontend_nginx"} -build_matrix["include"] = [includes[item] for item in build_matrix["image"]] - -for item in build_matrix["include"]: - if "context" not in item: - item["context"] = item["image"] - - if "file" not in item: - item["file"] = f"{item['context']}/Dockerfile" +build_matrix["include"] = [includes[item].asdict for item in build_matrix["image"]] do_build = "true" if len(build_matrix["image"]) else "false" do_publish = "true" if len(publish_matrix["image"]) else "false" diff --git a/documentation/meta/ci_cd/jobs/api.md b/documentation/meta/ci_cd/jobs/api.md index 7e615491dbb..14a31decad4 100644 --- a/documentation/meta/ci_cd/jobs/api.md +++ b/documentation/meta/ci_cd/jobs/api.md @@ -15,6 +15,10 @@ Initialises the API using the `api/init` recipe and runs tests for the API using the `api/test` recipe. Tests are run inside a Docker container so neither Python nor Node.js needs to be installed. +This job creates a special separate API image that includes the dev dependencies +required for running tests. It does not use the `api` image created in the +[`build-images`](/meta/ci_cd/jobs/docker.md#build-images) job. + This job is skipped if the API codebase has not changed. Its counterparts are - [`test-cat`](/meta/ci_cd/jobs/catalog.md#test-cat) for the catalog