dockerfile: support multiarch image build (#1697)

Signed-off-by: saintube <[email protected]>
koordinator-sh · Oct 9, 2023 · a317f1e · a317f1e
1 parent 9a80fe4
commit a317f1e
Show file tree

Hide file tree

Showing 11 changed files with 82 additions and 22 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -78,7 +78,6 @@ jobs:
 
   build-and-push:
     needs: [ golangci-lint, unit-tests ]
-    if: github.ref == 'refs/heads/main'
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -96,22 +95,42 @@ jobs:
           registry: ${{ env.GITHUB_REG }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Login to AliyunCS_BJ
+      - name: Build the image
+        uses: docker/build-push-action@v5
+        with:
+          platforms: linux/amd64
+          push: false
+          pull: true
+          file: docker/${{ matrix.target }}.dockerfile
+          labels: |
+            org.opencontainers.image.title=${{ matrix.target }}
+            org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}
+            org.opencontainers.image.revision=${{ github.sha }}
+            org.opencontainers.image.created=${{ github.event.repository.updated_at}}
+            org.opencontainers.image.licenses=Apache-2.0
+          tags: |
+            ${{ env.GITHUB_REG }}/${{ github.repository_owner }}/${{ matrix.target }}:${{ github.sha }}
+          cache-from: type=gha,scope=build-${{ matrix.target }}
+          cache-to: type=gha,mode=max,scope=build-${{ matrix.target }}
+      - if: github.ref == 'refs/heads/main'
+        name: Login to AliyunCS_BJ
         uses: docker/login-action@v3
         with:
           registry: ${{ env.ALIYUN_BJ_REG }}
           username: ${{ secrets.ALIYUN_USERNAME }}
           password: ${{ secrets.ALIYUN_PWD }}
-      - name: Login to AliyunCS_HZ
+      - if: github.ref == 'refs/heads/main'
+        name: Login to AliyunCS_HZ
         uses: docker/login-action@v3
         with:
           registry: ${{ env.ALIYUN_HZ_REG }}
           username: ${{ secrets.ALIYUN_USERNAME }}
           password: ${{ secrets.ALIYUN_PWD }}
-      - name: Build and push
+      - if: github.ref == 'refs/heads/main'
+        name: Build and push the multi-arch images
         uses: docker/build-push-action@v5
         with:
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: true
           pull: true
           file: docker/${{ matrix.target }}.dockerfile

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -43,7 +43,7 @@ jobs:
       - name: Build and push
         uses: docker/build-push-action@v5
         with:
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64
           push: true
           pull: true
           file: docker/${{ matrix.target }}.dockerfile

diff --git a/Makefile b/Makefile
@@ -232,7 +232,7 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
-	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest
 
 .PHONY: golangci-lint
 golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.

diff --git a/docker/koord-descheduler.dockerfile b/docker/koord-descheduler.dockerfile
@@ -1,6 +1,12 @@
-FROM golang:1.18 as builder
+FROM --platform=$TARGETPLATFORM golang:1.18 as builder
 WORKDIR /go/src/github.com/koordinator-sh/koordinator
 
+ARG VERSION
+ARG TARGETARCH
+ENV VERSION $VERSION
+ENV GOOS linux
+ENV GOARCH $TARGETARCH
+
 COPY go.mod go.mod
 COPY go.sum go.sum
 
@@ -10,9 +16,9 @@ COPY apis/ apis/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o koord-descheduler cmd/koord-descheduler/main.go
+RUN CGO_ENABLED=0 go build -a -o koord-descheduler cmd/koord-descheduler/main.go
 
-FROM gcr.io/distroless/static:latest
+FROM --platform=$TARGETPLATFORM gcr.io/distroless/static:latest
 WORKDIR /
 COPY --from=builder /go/src/github.com/koordinator-sh/koordinator/koord-descheduler .
 ENTRYPOINT ["/koord-descheduler"]
diff --git a/docker/koord-manager.dockerfile b/docker/koord-manager.dockerfile
@@ -1,6 +1,12 @@
-FROM golang:1.18 as builder
+FROM --platform=$TARGETPLATFORM golang:1.18 as builder
 WORKDIR /go/src/github.com/koordinator-sh/koordinator
 
+ARG VERSION
+ARG TARGETARCH
+ENV VERSION $VERSION
+ENV GOOS linux
+ENV GOARCH $TARGETARCH
+
 COPY go.mod go.mod
 COPY go.sum go.sum
 
@@ -10,9 +16,9 @@ COPY apis/ apis/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o koord-manager cmd/koord-manager/main.go
+RUN CGO_ENABLED=0 go build -a -o koord-manager cmd/koord-manager/main.go
 
-FROM gcr.io/distroless/static:latest
+FROM --platform=$TARGETPLATFORM gcr.io/distroless/static:latest
 WORKDIR /
 COPY --from=builder /go/src/github.com/koordinator-sh/koordinator/koord-manager .
 ENTRYPOINT ["/koord-manager"]
diff --git a/docker/koord-runtimeproxy.dockerfile b/docker/koord-runtimeproxy.dockerfile
@@ -1,6 +1,12 @@
-FROM golang:1.18 as builder
+FROM --platform=$TARGETPLATFORM golang:1.18 as builder
 WORKDIR /go/src/github.com/koordinator-sh/koordinator
 
+ARG VERSION
+ARG TARGETARCH
+ENV VERSION $VERSION
+ENV GOOS linux
+ENV GOARCH $TARGETARCH
+
 COPY go.mod go.mod
 COPY go.sum go.sum
 
@@ -10,9 +16,9 @@ COPY apis/ apis/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o koord-runtime-proxy cmd/koord-runtime-proxy/main.go
+RUN CGO_ENABLED=0 go build -a -o koord-runtime-proxy cmd/koord-runtime-proxy/main.go
 
-FROM gcr.io/distroless/static:latest
+FROM --platform=$TARGETPLATFORM gcr.io/distroless/static:latest
 WORKDIR /
 COPY --from=builder /go/src/github.com/koordinator-sh/koordinator/koord-runtime-proxy .
 ENTRYPOINT ["/koord-runtime-proxy"]
diff --git a/docker/koord-scheduler.dockerfile b/docker/koord-scheduler.dockerfile
@@ -1,6 +1,12 @@
-FROM golang:1.18 as builder
+FROM --platform=$TARGETPLATFORM golang:1.18 as builder
 WORKDIR /go/src/github.com/koordinator-sh/koordinator
 
+ARG VERSION
+ARG TARGETARCH
+ENV VERSION $VERSION
+ENV GOOS linux
+ENV GOARCH $TARGETARCH
+
 COPY go.mod go.mod
 COPY go.sum go.sum
 
@@ -10,9 +16,9 @@ COPY apis/ apis/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
 
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o koord-scheduler cmd/koord-scheduler/main.go
+RUN CGO_ENABLED=0 go build -a -o koord-scheduler cmd/koord-scheduler/main.go
 
-FROM gcr.io/distroless/static:latest
+FROM --platform=$TARGETPLATFORM gcr.io/distroless/static:latest
 WORKDIR /
 COPY --from=builder /go/src/github.com/koordinator-sh/koordinator/koord-scheduler .
 ENTRYPOINT ["/koord-scheduler"]
diff --git a/docker/koordlet.dockerfile b/docker/koordlet.dockerfile
@@ -1,6 +1,12 @@
-FROM golang:1.18 as builder
+FROM --platform=$TARGETPLATFORM golang:1.18 as builder
 WORKDIR /go/src/github.com/koordinator-sh/koordinator
 
+ARG VERSION
+ARG TARGETARCH
+ENV VERSION $VERSION
+ENV GOOS linux
+ENV GOARCH $TARGETARCH
+
 COPY go.mod go.mod
 COPY go.sum go.sum
 
@@ -18,15 +24,15 @@ RUN wget https://sourceforge.net/projects/perfmon2/files/libpfm4/libpfm-4.13.0.t
 RUN export DBG="-g -Wall" && \
   make -e -C libpfm-4.13.0 && \
   make install -C libpfm-4.13.0
-RUN GOOS=linux GOARCH=amd64 go build -a -o koordlet cmd/koordlet/main.go
+RUN go build -a -o koordlet cmd/koordlet/main.go
 
 # The CUDA container images provide an easy-to-use distribution for CUDA supported platforms and architectures.
 # NVIDIA provides rich images in https://hub.docker.com/r/nvidia/cuda/tags, literally cover all kinds of CUDA version
 # and system architecture. Please replace the following base image according to your Kubernetes/System environment.
 # For more details about how those images got built, you might wanna check the original Dockerfile in
 # https://gitlab.com/nvidia/container-images/cuda/-/tree/master/dist.
 
-FROM nvidia/cuda:11.2.2-base-ubuntu20.04
+FROM --platform=$TARGETPLATFORM nvidia/cuda:11.6.2-base-ubuntu20.04
 WORKDIR /
 RUN apt-get update && apt-get install -y lvm2 && rm -rf /var/lib/apt/lists/*
 COPY --from=builder /go/src/github.com/koordinator-sh/koordinator/koordlet .

diff --git a/go.mod b/go.mod
@@ -57,6 +57,7 @@ require (
 	k8s.io/kubernetes v1.24.15
 	k8s.io/utils v0.0.0-20221128185143-99ec85e7a448
 	sigs.k8s.io/controller-runtime v0.12.3
+	sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20231005234617-5771399a8ce5
 	sigs.k8s.io/descheduler v0.26.0
 	sigs.k8s.io/scheduler-plugins v0.22.6
 	sigs.k8s.io/yaml v1.3.0
@@ -112,6 +113,7 @@ require (
 	github.com/go-kit/log v0.2.1 // indirect
 	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
+	github.com/go-logr/zapr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
@@ -177,6 +179,7 @@ require (
 	github.com/rubiojr/go-vhd v0.0.0-20200706105327-02e210299021 // indirect
 	github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
 	github.com/storageos/go-api v2.2.0+incompatible // indirect
 	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect

diff --git a/go.sum b/go.sum
@@ -486,6 +486,7 @@ github.com/go-logr/logr v1.2.0 h1:QK40JKJyMdUDz+h+xvCsru/bJhvG0UxvePV0ufL/AcE=
 github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
 github.com/go-logr/zapr v1.2.0/go.mod h1:Qa4Bsj2Vb+FAVeAKsLD8RLQ+YRJB8YDmOAKxaBQf7Ro=
 github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
+github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
 github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
 github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY=
 github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg=
@@ -1106,6 +1107,7 @@ github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
@@ -2023,6 +2025,8 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37 h1:fAPTNEpzQMOLM
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.37/go.mod h1:vfnxT4FXNT8eGvO+xi/DsyC/qHmdujqwrUa1WSspCsk=
 sigs.k8s.io/controller-runtime v0.12.3 h1:FCM8xeY/FI8hoAfh/V4XbbYMY20gElh9yh+A98usMio=
 sigs.k8s.io/controller-runtime v0.12.3/go.mod h1:qKsk4WE6zW2Hfj0G4v10EnNB2jMG1C+NTb8h+DwCoU0=
+sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20231005234617-5771399a8ce5 h1:q6JvS/AwlDfe9sHMTo1KOMdI376+mlB21pV7Xhfy5uA=
+sigs.k8s.io/controller-runtime/tools/setup-envtest v0.0.0-20231005234617-5771399a8ce5/go.mod h1:B6HLcvOy2S1qq2eWOFm9xepiKPMIc8Z9OXSPsnUDaR4=
 sigs.k8s.io/descheduler v0.26.1-0.20230402001301-90905d2c2194 h1:xPrRjhoJr6pst13/3UHNwATYAsJROWcv+vvGRa+Fk1Q=
 sigs.k8s.io/descheduler v0.26.1-0.20230402001301-90905d2c2194/go.mod h1:/z7jjqyhgYDSd+LclGulcqX/JgfGIOTNB7ohFlGNQAQ=
 sigs.k8s.io/json v0.0.0-20211208200746-9f7c6b3444d2/go.mod h1:B+TnT182UBxE84DiCz4CVE26eOSDAeYCpfDnC2kdKMY=

diff --git a/hack/tools.go b/hack/tools.go
@@ -22,4 +22,8 @@ package hack
 import (
 	// To generate clientset/informers/listers for apis
 	_ "k8s.io/code-generator"
+
+	// submodule test dependencies
+	// https://github.com/kubernetes-sigs/controller-runtime/issues/1670
+	_ "sigs.k8s.io/controller-runtime/tools/setup-envtest"
 )