Build gardener/gardener dev images (#181)

* Build gardener/gardener dev images * Nits, add gardener-build-dev-images-release * Drop overengineered antiaffinity for build pods * Remove concourse-ci/publish from branch protection Co-authored-by: Tim Ebert <[email protected]>
gardener · Apr 12, 2022 · a235426 · a235426
1 parent 306bc17
commit a235426
Show file tree

Hide file tree

Showing 6 changed files with 202 additions and 25 deletions.
diff --git a/.../jobs/gardener/build-gardener-images.yaml → ...s/gardener/gardener-build-dev-images.yaml b/.../jobs/gardener/build-gardener-images.yaml → ...s/gardener/gardener-build-dev-images.yaml
@@ -1,16 +1,14 @@
 postsubmits:
   gardener/gardener:
-  - name: post-gardener-build-images
+  - name: post-gardener-build-dev-images
     cluster: gardener-prow-trusted
+    skip_if_only_changed: '^VERSION$'
     branches:
     - ^master$
     annotations:
-      description: Testing gardener image build on master branch
+      description: Gardener development image build on master branch
     decorate: true
     max_concurrency: 1
-    reporter_config:
-      slack:
-        channel: "gardener-prow-alerts"
     spec:
       serviceAccountName: image-builder
       containers:
@@ -20,9 +18,9 @@ postsubmits:
         - /image-builder
         args:
         - --log-level=info
-        - --docker-config-secret=k8s-playground-docker-config
-        - --registry=eu.gcr.io/sap-cloud-platform-dev1/gardener
-        - --cache-registry=eu.gcr.io/sap-cloud-platform-dev1/kaniko-cache
+        - --docker-config-secret=gardener-prow-gcr-docker-config
+        - --registry=eu.gcr.io/gardener-project/gardener
+        - --cache-registry=eu.gcr.io/gardener-project/ci-infra/kaniko-cache
         - --target=apiserver
         - --target=controller-manager
         - --target=scheduler
@@ -33,7 +31,6 @@ postsubmits:
         - --target=gardener-extension-provider-local
         - --add-version-tag=true
         - --add-version-sha-tag=true
-        - --add-fixed-tag=latest
         # image-builder is the pod which is "scheduled" to a node. The pods created by image-builder have an affinity rule
         # which schedules them to the same node as their parent image-builder. This needs to be done, that PVCs could be mounted
         # to multiple build pods in parallel.
@@ -52,17 +49,3 @@ postsubmits:
         operator: "Equal"
         value: "high-cpu"
         effect: "NoSchedule"
-      # Affinity is not copied to build pods
-      # Unpacking cache layers makes build quite IO intense. Thus, try to schedule builds to different nodes if available
-      affinity:
-        podAntiAffinity:
-          preferredDuringSchedulingIgnoredDuringExecution:
-          - weight: 100
-            podAffinityTerm:
-              labelSelector:
-                matchExpressions:
-                - key: prow.k8s.io/job
-                  operator: In
-                  values:
-                  - post-build-gardener-images
-              topologyKey: kubernetes.io/hostname
diff --git a/config/jobs/gardener/gardener-test-builds.yaml b/config/jobs/gardener/gardener-test-builds.yaml
@@ -0,0 +1,71 @@
+presubmits:
+  gardener/gardener:
+  - name: pull-gardener-verify-image-build
+    cluster: gardener-prow-build
+    always_run: true
+    skip_branches:
+    - release-v\d+.\d+ # don't run on release branches for now (add a job per branch later)
+    annotations:
+      description: Verify gardener image build on pull requests to master branch
+    decorate: true
+    spec:
+      containers:
+      - name: kaniko
+        image: gcr.io/kaniko-project/executor:v1.8.1
+        command:
+        - /kaniko/executor
+        args:
+        - --context=/home/prow/go/src/github.com/gardener/gardener
+        - --dockerfile=Dockerfile
+        - --no-push
+        resources:
+          requests:
+            cpu: 6
+            memory: 9Gi
+  - name: pull-gardener-publish-test-images
+    cluster: gardener-prow-trusted
+    skip_branches:
+    - release-v\d+.\d+ # don't run on release branches for now (add a job per branch later)
+    annotations:
+      description: Publish gardener development images on pull requests
+    decorate: true
+    optional: true
+    spec:
+      serviceAccountName: image-builder
+      containers:
+      - name: image-builder
+        image: eu.gcr.io/gardener-project/ci-infra/image-builder:v20220407-1704ddf
+        command:
+        - /image-builder
+        args:
+        - --log-level=info
+        - --docker-config-secret=gardener-prow-gcr-docker-config
+        - --registry=eu.gcr.io/gardener-project/gardener
+        - --cache-registry=eu.gcr.io/gardener-project/ci-infra/kaniko-cache
+        - --target=apiserver
+        - --target=controller-manager
+        - --target=scheduler
+        - --target=gardenlet
+        - --target=admission-controller
+        - --target=seed-admission-controller
+        - --target=resource-manager
+        - --target=gardener-extension-provider-local
+        - --add-version-sha-tag=true
+        # image-builder is the pod which is "scheduled" to a node. The pods created by image-builder have an affinity rule
+        # which schedules them to the same node as their parent image-builder. This needs to be done, that PVCs could be mounted
+        # to multiple build pods in parallel.
+        # For a proper scheduling the combined resource requests of all build pods are assigned to this pod, even though it does not
+        # use them. The resource requests of build pods themselves are "0"
+        resources:
+          requests:
+            cpu: 6
+            memory: 9Gi
+      # Node selector is copied to build pods
+      nodeSelector:
+        dedicated: high-cpu
+      # Tolerations are copied to build pods
+      tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "high-cpu"
+        effect: "NoSchedule"
diff --git a/config/jobs/gardener/release/gardener-build-dev-images-release.yaml b/config/jobs/gardener/release/gardener-build-dev-images-release.yaml
@@ -0,0 +1,51 @@
+postsubmits:
+  gardener/gardener:
+  - name: post-gardener-build-dev-images-release
+    cluster: gardener-prow-trusted
+    skip_if_only_changed: '^VERSION$'
+    branches:
+    - release-v\d+.\d+
+    annotations:
+      description: Gardener development image build on release branch
+    decorate: true
+    max_concurrency: 1
+    spec:
+      serviceAccountName: image-builder
+      containers:
+      - name: image-builder
+        image: eu.gcr.io/gardener-project/ci-infra/image-builder:v20220407-1704ddf
+        command:
+        - /image-builder
+        args:
+        - --log-level=info
+        - --docker-config-secret=gardener-prow-gcr-docker-config
+        - --registry=eu.gcr.io/gardener-project/gardener
+        - --cache-registry=eu.gcr.io/gardener-project/ci-infra/kaniko-cache
+        - --target=apiserver
+        - --target=controller-manager
+        - --target=scheduler
+        - --target=gardenlet
+        - --target=admission-controller
+        - --target=seed-admission-controller
+        - --target=resource-manager
+        - --target=gardener-extension-provider-local
+        - --add-version-tag=true
+        - --add-version-sha-tag=true
+        # image-builder is the pod which is "scheduled" to a node. The pods created by image-builder have an affinity rule
+        # which schedules them to the same node as their parent image-builder. This needs to be done, that PVCs could be mounted
+        # to multiple build pods in parallel.
+        # For a proper scheduling the combined resource requests of all build pods are assigned to this pod, even though it does not
+        # use them. The resource requests of build pods themselves are "0"
+        resources:
+          requests:
+            cpu: 6
+            memory: 9Gi
+      # Node selector is copied to build pods
+      nodeSelector:
+        dedicated: high-cpu
+      # Tolerations are copied to build pods
+      tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "high-cpu"
+        effect: "NoSchedule"
diff --git a/config/jobs/gardener/release/gardener-test-builds-release.yaml b/config/jobs/gardener/release/gardener-test-builds-release.yaml
@@ -0,0 +1,73 @@
+presubmits:
+  gardener/gardener:
+  - name: pull-gardener-verify-image-build-release
+    cluster: gardener-prow-build
+    always_run: true
+    # Run on release branches / adapt this setting and create a new job in case of incompatible changes in tests or go version between the releases
+    branches:
+    - release-v\d+.\d+
+    annotations:
+      description: Verify gardener image build on pull requests to release branch
+    decorate: true
+    spec:
+      containers:
+      - name: kaniko
+        image: gcr.io/kaniko-project/executor:v1.8.1
+        command:
+        - /kaniko/executor
+        args:
+        - --context=/home/prow/go/src/github.com/gardener/gardener
+        - --dockerfile=Dockerfile
+        - --no-push
+        resources:
+          requests:
+            cpu: 6
+            memory: 9Gi
+  - name: pull-gardener-publish-test-images-release
+    cluster: gardener-prow-trusted
+    # Run on release branches / adapt this setting and create a new job in case of incompatible changes in tests or go version between the releases
+    branches:
+    - release-v\d+.\d+
+    annotations:
+      description: Publish gardener development images on pull requests
+    decorate: true
+    optional: true
+    spec:
+      serviceAccountName: image-builder
+      containers:
+      - name: image-builder
+        image: eu.gcr.io/gardener-project/ci-infra/image-builder:v20220407-1704ddf
+        command:
+        - /image-builder
+        args:
+        - --log-level=info
+        - --docker-config-secret=gardener-prow-gcr-docker-config
+        - --registry=eu.gcr.io/gardener-project/gardener
+        - --cache-registry=eu.gcr.io/gardener-project/ci-infra/kaniko-cache
+        - --target=apiserver
+        - --target=controller-manager
+        - --target=scheduler
+        - --target=gardenlet
+        - --target=admission-controller
+        - --target=seed-admission-controller
+        - --target=resource-manager
+        - --target=gardener-extension-provider-local
+        - --add-version-sha-tag=true
+        # image-builder is the pod which is "scheduled" to a node. The pods created by image-builder have an affinity rule
+        # which schedules them to the same node as their parent image-builder. This needs to be done, that PVCs could be mounted
+        # to multiple build pods in parallel.
+        # For a proper scheduling the combined resource requests of all build pods are assigned to this pod, even though it does not
+        # use them. The resource requests of build pods themselves are "0"
+        resources:
+          requests:
+            cpu: 6
+            memory: 9Gi
+      # Node selector is copied to build pods
+      nodeSelector:
+        dedicated: high-cpu
+      # Tolerations are copied to build pods
+      tolerations:
+      - key: "dedicated"
+        operator: "Equal"
+        value: "high-cpu"
+        effect: "NoSchedule"
diff --git a/config/prow/config.yaml b/config/prow/config.yaml
@@ -142,7 +142,6 @@ branch-protection:
           required_status_checks:
             contexts:
             - license/cla
-            - concourse-ci/publish
             - "Check Release Milestone"
           restrictions: # prevent everyone from pushing/merging (except admins)
             # NB: tide is running as GitHub App, which currently cannot be configured here to be excluded from branch

diff --git a/prow/cmd/image-builder/main.go b/prow/cmd/image-builder/main.go
@@ -93,7 +93,7 @@ func gatherOptions() options {
 	fs.Var(&o.kanikoArgs, "kaniko-arg", "kaniko-arg for the build")
 	fs.StringVar(&o.registry, "registry", "", "container registry where build artifacts are being pushed. Cache is disabled for empty value")
 	fs.StringVar(&o.cacheRegistry, "cache-registry", "", "container registry where cache artifacts are being pushed")
-	fs.StringVar(&o.kanikoImage, "kaniko-image", "gcr.io/kaniko-project/executor:v1.8.0", "kaniko image for kaniko build")
+	fs.StringVar(&o.kanikoImage, "kaniko-image", "gcr.io/kaniko-project/executor:v1.8.1", "kaniko image for kaniko build")
 	fs.BoolVar(&o.addVersionTag, "add-version-tag", false, "Add label from VERSION file of git root directory to image tags")
 	fs.BoolVar(&o.addVersionSHATag, "add-version-sha-tag", false, "Add label from VERSION file of git root directory plus SHA from git HEAD to image tags")
 	fs.BoolVar(&o.addDateSHATag, "add-date-sha-tag", false, "Using vYYYYMMDD-<rev short> scheme which is compatible to autobumper")