Resync 20240926

Makefile

@@ -12,19 +12,28 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARCHS = amd64 arm64
-COMMONENVVAR=GOOS=$(shell uname -s | tr A-Z a-z)
-BUILDENVVAR=CGO_ENABLED=0
+GO_VERSION := $(shell awk '/^go /{print $$2}' go.mod|head -n1)
 INTEGTESTENVVAR=SCHED_PLUGINS_TEST_VERBOSE=1
 
-# RELEASE_REGISTRY is the container registry to push
+# Manage platform and builders
+PLATFORMS ?= linux/amd64,linux/arm64,linux/s390x,linux/ppc64le
+BUILDER ?= docker
+ifeq ($(BUILDER),podman)
+	ALL_FLAG=--all
+else
+	ALL_FLAG=
+endif
+
+# REGISTRY is the container registry to push
 # into. The default is to push to the staging
 # registry, not production(registry.k8s.io).
-RELEASE_REGISTRY?=gcr.io/k8s-staging-scheduler-plugins
+REGISTRY?=gcr.io/k8s-staging-scheduler-plugins
 RELEASE_VERSION?=v$(shell date +%Y%m%d)-$(shell git describe --tags --match "v*")
 RELEASE_IMAGE:=kube-scheduler:$(RELEASE_VERSION)
 RELEASE_CONTROLLER_IMAGE:=controller:$(RELEASE_VERSION)
-GO_BASE_IMAGE?=golang
+GO_BASE_IMAGE?=golang:$(GO_VERSION)
+DISTROLESS_BASE_IMAGE?=gcr.io/distroless/static:nonroot
+EXTRA_ARGS=""
 
 # VERSION is the scheduler's version
 #
@@ -40,77 +49,37 @@ all: build
 .PHONY: build
 build: build-controller build-scheduler
 
-.PHONY: build.amd64
-build.amd64: build-controller.amd64 build-scheduler.amd64
-
-.PHONY: build.arm64v8
-build.arm64v8: build-controller.arm64v8 build-scheduler.arm64v8
-
 .PHONY: build-controller
 build-controller:
-	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-w' -o bin/controller cmd/controller/controller.go
-
-.PHONY: build-controller.amd64
-build-controller.amd64:
-	$(COMMONENVVAR) $(BUILDENVVAR) GOARCH=amd64 go build -ldflags '-w' -o bin/controller cmd/controller/controller.go
-
-.PHONY: build-controller.arm64v8
-build-controller.arm64v8:
-	$(COMMONENVVAR) $(BUILDENVVAR) GOARCH=arm64 go build -ldflags '-w' -o bin/controller cmd/controller/controller.go
+	$(GO_BUILD_ENV) go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/controller cmd/controller/controller.go
 
 .PHONY: build-scheduler
 build-scheduler:
-	$(COMMONENVVAR) $(BUILDENVVAR) go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/kube-scheduler cmd/scheduler/main.go
-
-.PHONY: build-scheduler.amd64
-build-scheduler.amd64:
-	$(COMMONENVVAR) $(BUILDENVVAR) GOARCH=amd64 go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/kube-scheduler cmd/scheduler/main.go
+	$(GO_BUILD_ENV) go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/kube-scheduler cmd/scheduler/main.go
 
-.PHONY: build-scheduler.arm64v8
-build-scheduler.arm64v8:
-	$(COMMONENVVAR) $(BUILDENVVAR) GOARCH=arm64 go build -ldflags '-X k8s.io/component-base/version.gitVersion=$(VERSION) -w' -o bin/kube-scheduler cmd/scheduler/main.go
-
-.PHONY: local-image
-local-image: clean
-	RELEASE_VERSION=$(RELEASE_VERSION) hack/build-images.sh
-
-.PHONY: release-image.amd64
-release-image.amd64: clean
-	ARCH="amd64" \
+.PHONY: build-images
+build-images:
+	BUILDER=$(BUILDER) \
+	PLATFORMS=$(PLATFORMS) \
 	RELEASE_VERSION=$(RELEASE_VERSION) \
-	REGISTRY=$(RELEASE_REGISTRY) \
-	IMAGE=$(RELEASE_IMAGE)-amd64 \
-	CONTROLLER_IMAGE=$(RELEASE_CONTROLLER_IMAGE)-amd64 \
+	REGISTRY=$(REGISTRY) \
+	IMAGE=$(RELEASE_IMAGE) \
+	CONTROLLER_IMAGE=$(RELEASE_CONTROLLER_IMAGE) \
 	GO_BASE_IMAGE=$(GO_BASE_IMAGE) \
-	ALPINE_BASE_IMAGE=$(ALPINE_BASE_IMAGE) \
-	hack/build-images.sh
+	DISTROLESS_BASE_IMAGE=$(DISTROLESS_BASE_IMAGE) \
+	DOCKER_BUILDX_CMD=$(DOCKER_BUILDX_CMD) \
+	EXTRA_ARGS=$(EXTRA_ARGS) hack/build-images.sh
 
-.PHONY: release-image.arm64v8
-release-image.arm64v8: clean
-	ARCH="arm64" \
-	RELEASE_VERSION=$(RELEASE_VERSION) \
-	REGISTRY=$(RELEASE_REGISTRY) \
-	IMAGE=$(RELEASE_IMAGE)-arm64 \
-	CONTROLLER_IMAGE=$(RELEASE_CONTROLLER_IMAGE)-arm64 \
-	GO_BASE_IMAGE=$(GO_BASE_IMAGE) \
-	ALPINE_BASE_IMAGE=$(ALPINE_BASE_IMAGE) \
-	hack/build-images.sh
-
-.PHONY: push-release-images
-push-release-images: release-image.amd64 release-image.arm64v8
-	gcloud auth configure-docker
-	for arch in $(ARCHS); do \
-		docker push $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-$${arch} ;\
-		docker push $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE)-$${arch} ;\
-	done
-	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) $(addprefix --amend $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-, $(ARCHS))
-	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest create $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE) $(addprefix --amend $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE)-, $(ARCHS))
-	for arch in $(ARCHS); do \
-		DOCKER_CLI_EXPERIMENTAL=enabled docker manifest annotate --arch $${arch} $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) $(RELEASE_REGISTRY)/$(RELEASE_IMAGE)-$${arch} ;\
-		DOCKER_CLI_EXPERIMENTAL=enabled docker manifest annotate --arch $${arch} $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE) $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE)-$${arch} ;\
-	done
-	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push $(RELEASE_REGISTRY)/$(RELEASE_IMAGE) ;\
-	DOCKER_CLI_EXPERIMENTAL=enabled docker manifest push $(RELEASE_REGISTRY)/$(RELEASE_CONTROLLER_IMAGE) ;\
+.PHONY: local-image
+local-image: PLATFORMS="linux/$$(uname -m)"
+local-image: RELEASE_VERSION="v0.0.0"
+local-image: REGISTRY="localhost:5000/scheduler-plugins"
+local-image: EXTRA_ARGS="--load"
+local-image: clean build-images
+
+.PHONY: release-images
+push-images: EXTRA_ARGS="--push"
+push-images: build-images
 
 .PHONY: update-vendor
 update-vendor:

README.md

@@ -31,7 +31,7 @@ The kube-scheduler binary includes the below list of plugins. They can be config
 * [Node Resources](pkg/noderesources/README.md)
 * [Node Resource Topology](pkg/noderesourcetopology/README.md)
 * [Preemption Toleration](pkg/preemptiontoleration/README.md)
-* [Trimaran](pkg/trimaran/README.md)
+* [Trimaran (Load-Aware Scheduling)](pkg/trimaran/README.md)
 * [Network-Aware Scheduling](pkg/networkaware/README.md)
 
 Additionally, the kube-scheduler binary includes the below list of sample plugins. These plugins are not intended for use in production
@@ -57,34 +57,47 @@ on demand, (e.g., `v0.18.800`) are used to indicated that the k8s client package
 changed since the previous release, and that only scheduler plugins code (features or bug fixes) was
 changed.
 
-| Scheduler Plugins | Compiled With k8s Version | Container Image                                           | Arch           |
-|-------------------|---------------------------|-----------------------------------------------------------|----------------|
-| v0.28.9           | v1.28.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.28.9  | AMD64<br>ARM64 |
-| v0.27.8           | v1.27.8                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.27.8  | AMD64<br>ARM64 |
-| v0.26.7           | v1.26.7                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.26.7  | AMD64<br>ARM64 |
-| v0.25.12          | v1.25.12                  | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.25.12 | AMD64<br>ARM64 |
-| v0.24.9           | v1.24.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.24.9  | AMD64<br>ARM64 |
-| v0.23.10          | v1.23.10                  | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.23.10 | AMD64<br>ARM64 |
-| v0.22.6           | v1.22.6                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.22.6  | AMD64<br>ARM64 |
-| v0.21.6           | v1.21.6                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.21.6  | AMD64<br>ARM64 |
-| v0.20.10          | v1.20.10                  | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.20.10 | AMD64<br>ARM64 |
-| v0.19.9           | v1.19.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.19.9  | AMD64<br>ARM64 |
-| v0.19.8           | v1.19.8                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.19.8  | AMD64<br>ARM64 |
-| v0.18.9           | v1.18.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.18.9  | AMD64          |
-
-| Controller | Compiled With k8s Version | Container Image                                       | Arch           |
-|------------|---------------------------|-------------------------------------------------------|----------------|
-| v0.28.9    | v1.28.9                   | registry.k8s.io/scheduler-plugins/controller:v0.28.9  | AMD64<br>ARM64 |
-| v0.27.8    | v1.27.8                   | registry.k8s.io/scheduler-plugins/controller:v0.27.8  | AMD64<br>ARM64 |
-| v0.26.7    | v1.26.7                   | registry.k8s.io/scheduler-plugins/controller:v0.26.7  | AMD64<br>ARM64 |
-| v0.25.12   | v1.25.12                  | registry.k8s.io/scheduler-plugins/controller:v0.25.12 | AMD64<br>ARM64 |
-| v0.24.9    | v1.24.9                   | registry.k8s.io/scheduler-plugins/controller:v0.24.9  | AMD64<br>ARM64 |
-| v0.23.10   | v1.23.10                  | registry.k8s.io/scheduler-plugins/controller:v0.23.10 | AMD64<br>ARM64 |
-| v0.22.6    | v1.22.6                   | registry.k8s.io/scheduler-plugins/controller:v0.22.6  | AMD64<br>ARM64 |
-| v0.21.6    | v1.21.6                   | registry.k8s.io/scheduler-plugins/controller:v0.21.6  | AMD64<br>ARM64 |
-| v0.20.10   | v1.20.10                  | registry.k8s.io/scheduler-plugins/controller:v0.20.10 | AMD64<br>ARM64 |
-| v0.19.9    | v1.19.9                   | registry.k8s.io/scheduler-plugins/controller:v0.19.9  | AMD64<br>ARM64 |
-| v0.19.8    | v1.19.8                   | registry.k8s.io/scheduler-plugins/controller:v0.19.8  | AMD64<br>ARM64 |
+| Scheduler Plugins | Compiled With k8s Version | Container Image                                           | Arch                                                       |
+|-------------------|---------------------------|-----------------------------------------------------------|------------------------------------------------------------|
+| v0.29.7           | v1.29.7                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.29.7  | linux/amd64<br>linux/arm64<br>linux/s390x<br>linux/ppc64le |
+| v0.28.9           | v1.28.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.28.9  | linux/amd64<br>linux/arm64                                 |
+| v0.27.8           | v1.27.8                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.27.8  | linux/amd64<br>linux/arm64                                 |
+
+| Controller | Compiled With k8s Version | Container Image                                       | Arch                                                       |
+|------------|---------------------------|-------------------------------------------------------|------------------------------------------------------------|
+| v0.29.7    | v1.29.7                   | registry.k8s.io/scheduler-plugins/controller:v0.29.7  | linux/amd64<br>linux/arm64<br>linux/s390x<br>linux/ppc64le |
+| v0.28.9    | v1.28.9                   | registry.k8s.io/scheduler-plugins/controller:v0.28.9  | linux/amd64<br>linux/arm64                                 |
+| v0.27.8    | v1.27.8                   | registry.k8s.io/scheduler-plugins/controller:v0.27.8  | linux/amd64<br>linux/arm64                                 |
+
+<details>
+<summary>Older releases</summary>
+
+| Scheduler Plugins | Compiled With k8s Version | Container Image                                           | Arch                       |
+|-------------------|---------------------------|-----------------------------------------------------------|----------------------------|
+| v0.26.7           | v1.26.7                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.26.7  | linux/amd64<br>linux/arm64 |
+| v0.25.12          | v1.25.12                  | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.25.12 | linux/amd64<br>linux/arm64 |
+| v0.24.9           | v1.24.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.24.9  | linux/amd64<br>linux/arm64 |
+| v0.23.10          | v1.23.10                  | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.23.10 | linux/amd64<br>linux/arm64 |
+| v0.22.6           | v1.22.6                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.22.6  | linux/amd64<br>linux/arm64 |
+| v0.21.6           | v1.21.6                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.21.6  | linux/amd64<br>linux/arm64 |
+| v0.20.10          | v1.20.10                  | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.20.10 | linux/amd64<br>linux/arm64 |
+| v0.19.9           | v1.19.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.19.9  | linux/amd64<br>linux/arm64 |
+| v0.19.8           | v1.19.8                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.19.8  | linux/amd64<br>linux/arm64 |
+| v0.18.9           | v1.18.9                   | registry.k8s.io/scheduler-plugins/kube-scheduler:v0.18.9  | linux/amd64                |
+
+| Controller | Compiled With k8s Version | Container Image                                       | Arch                       |
+|------------|---------------------------|-------------------------------------------------------|----------------------------|
+| v0.26.7    | v1.26.7                   | registry.k8s.io/scheduler-plugins/controller:v0.26.7  | linux/amd64<br>linux/arm64 |
+| v0.25.12   | v1.25.12                  | registry.k8s.io/scheduler-plugins/controller:v0.25.12 | linux/amd64<br>linux/arm64 |
+| v0.24.9    | v1.24.9                   | registry.k8s.io/scheduler-plugins/controller:v0.24.9  | linux/amd64<br>linux/arm64 |
+| v0.23.10   | v1.23.10                  | registry.k8s.io/scheduler-plugins/controller:v0.23.10 | linux/amd64<br>linux/arm64 |
+| v0.22.6    | v1.22.6                   | registry.k8s.io/scheduler-plugins/controller:v0.22.6  | linux/amd64<br>linux/arm64 |
+| v0.21.6    | v1.21.6                   | registry.k8s.io/scheduler-plugins/controller:v0.21.6  | linux/amd64<br>linux/arm64 |
+| v0.20.10   | v1.20.10                  | registry.k8s.io/scheduler-plugins/controller:v0.20.10 | linux/amd64<br>linux/arm64 |
+| v0.19.9    | v1.19.9                   | registry.k8s.io/scheduler-plugins/controller:v0.19.9  | linux/amd64<br>linux/arm64 |
+| v0.19.8    | v1.19.8                   | registry.k8s.io/scheduler-plugins/controller:v0.19.8  | linux/amd64<br>linux/arm64 |
+
+</details>
 
 ## Community, discussion, contribution, and support
 

RESYNC.log.md

@@ -1,5 +1,6 @@
 | Resync Date | Merge With Upstream Tag/Commit                                                                       | Author      |
 |-------------|------------------------------------------------------------------------------------------------------|-------------|
+| 2024.09.26  | https://github.com/kubernetes-sigs/scheduler-plugins/commit/bb56af11184a0f6ed33e2fc8b189a5b1ccfc60e4 | ffromani    |
 | 2024.06.24  | https://github.com/kubernetes-sigs/scheduler-plugins/commit/2c1c0cfe6134c5d55a23dae1726264664a943f4b | ffromani    |
 | 2024.05.29  | https://github.com/kubernetes-sigs/scheduler-plugins/commit/0834feb92676712cebe8290615ce1c47537fe078 | ffromani    |
 | 2024.05.07  | https://github.com/kubernetes-sigs/scheduler-plugins/commit/70981813a19f16c4202f6f74a2525bf917b63685 | ffromani    |

apis/scheduling/v1alpha1/types.go

@@ -146,6 +146,8 @@ type PodGroupSpec struct {
 	// MinMember defines the minimal number of members/tasks to run the pod group;
 	// if there's not enough resources to start all tasks, the scheduler
 	// will not start any.
+	// The minimum is 1
+	// +kubebuilder:validation:Minimum=1
 	MinMember int32 `json:"minMember,omitempty"`
 
 	// MinResources defines the minimal resource of members/tasks to run the pod group;

build/controller/Dockerfile

@@ -11,19 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG ARCH
-ARG GO_BASE_IMAGE=golang
-ARG ALPINE_BASE_IMAGE=$ARCH/alpine
-FROM $GO_BASE_IMAGE:1.21
+ARG GO_BASE_IMAGE
+ARG DISTROLESS_BASE_IMAGE=gcr.io/distroless/static:nonroot
+FROM --platform=${BUILDPLATFORM} $GO_BASE_IMAGE AS builder
 
-WORKDIR /go/src/sigs.k8s.io/scheduler-plugins
+WORKDIR /workspace
 COPY . .
-ARG ARCH
-RUN make build-controller.$ARCH
+ARG TARGETARCH
+RUN make build-controller GO_BUILD_ENV='CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH}'
 
-FROM $ALPINE_BASE_IMAGE:3.16
-
-COPY --from=0 /go/src/sigs.k8s.io/scheduler-plugins/bin/controller /bin/controller
+FROM --platform=${BUILDPLATFORM} $DISTROLESS_BASE_IMAGE
 
 WORKDIR /bin
-CMD ["controller"]
+COPY --from=builder /workspace/bin/controller .
+USER 65532:65532
+
+ENTRYPOINT ["/bin/controller"]

build/scheduler/Dockerfile

@@ -11,20 +11,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-ARG ARCH
-ARG GO_BASE_IMAGE=golang
-ARG ALPINE_BASE_IMAGE=$ARCH/alpine
-FROM $GO_BASE_IMAGE:1.21
+ARG GO_BASE_IMAGE
+ARG DISTROLESS_BASE_IMAGE=gcr.io/distroless/static:nonroot
+FROM --platform=${BUILDPLATFORM} $GO_BASE_IMAGE AS builder
 
-WORKDIR /go/src/sigs.k8s.io/scheduler-plugins
+WORKDIR /workspace
 COPY . .
-ARG ARCH
-ARG RELEASE_VERSION
-RUN RELEASE_VERSION=${RELEASE_VERSION} make build-scheduler.$ARCH
+ARG TARGETARCH
+RUN make build-scheduler GO_BUILD_ENV='CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH}'
 
-FROM $ALPINE_BASE_IMAGE:3.16
-
-COPY --from=0 /go/src/sigs.k8s.io/scheduler-plugins/bin/kube-scheduler /bin/kube-scheduler
+FROM --platform=${BUILDPLATFORM} $DISTROLESS_BASE_IMAGE
 
 WORKDIR /bin
-CMD ["kube-scheduler"]
+COPY --from=builder /workspace/bin/kube-scheduler .
+USER 65532:65532
+
+ENTRYPOINT ["/bin/kube-scheduler"]

cloudbuild.yaml

@@ -1,20 +1,22 @@
 # See https://cloud.google.com/cloud-build/docs/build-config
 
 # this must be specified in seconds. If omitted, defaults to 600s (10 mins)
-timeout: 2400s
+timeout: 3000s
 # this prevents errors if you don't use both _GIT_TAG and _PULL_BASE_REF,
 # or any new substitutions added in the future.
 options:
   substitution_option: ALLOW_LOOSE
+  machineType: 'N1_HIGHCPU_8'
 steps:
-  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20211118-2f2d816b90'
+  # see https://github.com/kubernetes/test-infra/tree/master/config/jobs/image-pushing
+  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20240718-5ef92b5c36'
     entrypoint: make
     env:
-    - DOCKER_CLI_EXPERIMENTAL=enabled
     - RELEASE_VERSION=$_GIT_TAG
     - BASE_REF=$_PULL_BASE_REF
+    - DOCKER_BUILDX_CMD=/buildx-entrypoint
     args:
-    - push-release-images
+    - push-images
 substitutions:
   # _GIT_TAG will be filled with a git-based tag for the image, of the form vYYYYMMDD-hash, and
   # can be used as a substitution