Make API image smaller by not including dev dependencies (#4285)

.github/workflows/ci_cd.yml

@@ -233,14 +233,14 @@ jobs:
         uses: docker/build-push-action@v5
         with:
           context: ${{ matrix.context }}
-          target: ${{ matrix.target || '' }}
+          target: ${{ matrix.target }}
           push: false
           tags: openverse-${{ matrix.image }}
           file: ${{ matrix.file }}
           cache-from: type=gha,scope=${{ matrix.image }}
           cache-to: type=gha,scope=${{ matrix.image }}
           outputs: type=docker,dest=/tmp/${{ matrix.image }}.tar
-          build-contexts: ${{ matrix.build-contexts || '' }}
+          build-contexts: ${{ matrix.build-contexts }}
           build-args: |
             SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }}
             CATALOG_PY_VERSION=${{ steps.prepare-build-args.outputs.catalog_py_version }}
@@ -250,6 +250,7 @@ jobs:
             FRONTEND_NODE_VERSION=${{ steps.prepare-build-args.outputs.frontend_node_version }}
             FRONTEND_PNPM_VERSION=${{ steps.prepare-build-args.outputs.frontend_pnpm_version }}
             PGCLI_VERSION=${{ steps.prepare-build-args.outputs.pgcli_version }}
+            ${{ matrix.build-args || '' }}
 
       - name: Upload image `${{ matrix.image }}`
         id: upload-img
@@ -398,6 +399,7 @@ jobs:
     needs:
       - get-changes
       - build-images
+      - get-image-tag
 
     steps:
       - name: Checkout repository
@@ -416,7 +418,34 @@ jobs:
         uses: ./.github/actions/load-img
         with:
           run_id: ${{ github.run_id }}
-          setup_images: upstream_db ingestion_server api
+          setup_images: upstream_db ingestion_server
+
+      # Sets build args specifying versions needed to build Docker image.
+      - name: Prepare build args
+        id: prepare-build-args
+        run: |
+          just versions | tee "$GITHUB_OUTPUT"
+
+      - name: Setup Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          install: true
+
+      - name: Build API dev image
+        uses: docker/build-push-action@v5
+        with:
+          context: api
+          target: api
+          push: false
+          load: true
+          tags: openverse-api
+          cache-from: type=gha,scope=api_dev
+          cache-to: type=gha,scope=api_dev
+          build-contexts: packages=./packages/python
+          build-args: |
+            SEMANTIC_VERSION=${{ needs.get-image-tag.outputs.image_tag }}
+            API_PY_VERSION=${{ steps.prepare-build-args.outputs.api_py_version }}
+            PDM_INSTALL_ARGS=--dev
 
       - name: Start API, ingest and index test data
         run: just api/init

automations/python/workflows/set_matrix_images.py

@@ -6,10 +6,42 @@
 
 import json
 import os
+from dataclasses import asdict, dataclass
 
 from shared.actions import write_to_github_output
 
 
+@dataclass
+class Include:
+    image: str
+    target: str
+
+    context: str
+    file: str
+    build_contexts: str
+    build_args: str
+
+    def __init__(
+        self,
+        image: str,
+        target: str,
+        context: str | None = None,
+        file: str | None = None,
+        build_contexts: str | None = None,
+        build_args: str | None = None,
+    ):
+        self.image = image
+        self.target = target
+        self.context = context or image
+        self.file = file or f"{self.context}/Dockerfile"
+        self.build_contexts = build_contexts or ""
+        self.build_args = build_args or ""
+
+    @property
+    def asdict(self) -> dict[str, str]:
+        return {k.replace("_", "-"): v for k, v in asdict(self).items()}
+
+
 changes = json.loads(os.environ.get("CHANGES"))
 
 
@@ -21,32 +53,37 @@ def ser_set(x):
 build_matrix = {"image": set()}
 publish_matrix = {"image": set()}
 
-includes = {
-    "upstream_db": {
-        "image": "upstream_db",
-        "context": "docker/upstream_db",
-        "target": "db",
-    },
-    "catalog": {"image": "catalog", "target": "cat"},
-    "ingestion_server": {"image": "ingestion_server", "target": "ing"},
-    "api": {
-        "image": "api",
-        "target": "api",
-        "build-contexts": "packages=./packages/python",
-    },
-    "api_nginx": {
-        "image": "api_nginx",
-        "context": "api",
-        "target": "nginx",
-        "build-contexts": "packages=./packages/python",
-    },
-    "frontend": {"image": "frontend", "target": "app", "build-contexts": "repo_root=."},
-    "frontend_nginx": {
-        "image": "frontend_nginx",
-        "context": "frontend",
-        "file": "frontend/Dockerfile.nginx",
-        "target": "nginx",
-    },
+includes: dict[str, Include] = {
+    "upstream_db": Include(
+        image="upstream_db",
+        target="db",
+        context="docker/upstream_db",
+    ),
+    "catalog": Include(image="catalog", target="cat"),
+    "ingestion_server": Include(image="ingestion_server", target="ing"),
+    "api": Include(
+        image="api",
+        target="api",
+        build_contexts="packages=./packages/python",
+        build_args="PDM_INSTALL_ARGS=--prod",
+    ),
+    "api_nginx": Include(
+        image="api_nginx",
+        target="nginx",
+        context="api",
+        build_contexts="packages=./packages/python",
+    ),
+    "frontend": Include(
+        image="frontend",
+        target="app",
+        build_contexts="repo_root=.",
+    ),
+    "frontend_nginx": Include(
+        image="frontend_nginx",
+        target="nginx",
+        context="frontend",
+        file="frontend/Dockerfile.nginx",
+    ),
 }
 
 if "ci_cd" in changes:
@@ -65,14 +102,7 @@ def ser_set(x):
     publish_matrix["image"] |= {"frontend", "frontend_nginx"}
 
 
-build_matrix["include"] = [includes[item] for item in build_matrix["image"]]
-
-for item in build_matrix["include"]:
-    if "context" not in item:
-        item["context"] = item["image"]
-
-    if "file" not in item:
-        item["file"] = f"{item['context']}/Dockerfile"
+build_matrix["include"] = [includes[item].asdict for item in build_matrix["image"]]
 
 do_build = "true" if len(build_matrix["image"]) else "false"
 do_publish = "true" if len(publish_matrix["image"]) else "false"

documentation/meta/ci_cd/jobs/api.md

@@ -15,6 +15,10 @@ Initialises the API using the `api/init` recipe and runs tests for the API using
 the `api/test` recipe. Tests are run inside a Docker container so neither Python
 nor Node.js needs to be installed.
 
+This job creates a special separate API image that includes the dev dependencies
+required for running tests. It does not use the `api` image created in the
+[`build-images`](/meta/ci_cd/jobs/docker.md#build-images) job.
+
 This job is skipped if the API codebase has not changed. Its counterparts are
 
 - [`test-cat`](/meta/ci_cd/jobs/catalog.md#test-cat) for the catalog