Create node-specific-sizing from an existing webhook repo. It's subst…

…antially different from upstream's.

.gitignore

@@ -15,4 +15,5 @@ vendor/*
 build/_output
 
 # GOPATH
-.go
+.go
+.idea

.mise.toml

@@ -0,0 +1,7 @@
+[tools]
+go = "1.23"
+cmctl = "cmd/ctl/v1.14.7"
+
+[env]
+LOG_DEVEL = "true"
+LOG_LEVEL = "debug"

Dockerfile

@@ -1,34 +1,37 @@
-# Build the sidecar-injector binary
-FROM golang:1.17 as builder
+# Build the manager binary
+FROM --platform=$BUILDPLATFORM golang:1.23 AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
+COPY go.mod go.sum ./
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN go mod download
+ENV GOCACHE=/root/.cache/go-build
+RUN \
+  --mount=type=cache,target=/root/.cache/go-build \
+  go mod download
 
 # Copy the go source
 COPY cmd/ cmd/
 
-# Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o sidecar-injector ./cmd
-
-
-FROM alpine:latest
-
-# install curl for prestop script
-RUN apk --no-cache add curl
+ARG TARGETOS
+ARG TARGETARCH
+ARG TARGETVARIANT
 
+# Build
+# the GOARCH has not a default value to allow the binary be built according to the host where the command
+# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
+# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
+# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} GOARM=${TARGETVARIANT} go build -a -o manager cmd/*
+
+# Use distroless as minimal base image to package the manager binary
+# Refer to https://github.com/GoogleContainerTools/distroless for more details
+FROM gcr.io/distroless/static:nonroot
 WORKDIR /
-
-# install binary
-COPY --from=builder /workspace/sidecar-injector .
-
-# install the prestop script
-COPY ./prestop.sh .
-
+COPY --from=builder /workspace/manager .
 USER 65532:65532
 
-ENTRYPOINT ["/sidecar-injector"]
+ENTRYPOINT ["/manager"]

Makefile

@@ -19,7 +19,7 @@ KUSTOMIZE_VERSION?=v3.5.4
 KUSTOMIZE_ARCHIVE_NAME?=kustomize_$(KUSTOMIZE_VERSION)_$(GOHOSTOS)_$(GOHOSTARCH).tar.gz
 kustomize_dir:=$(dir $(KUSTOMIZE))
 
-IMAGE = quay.io/morvencao/sidecar-injector:latest
+IMAGE = node-specific-sizing:latest
 
 all: build
 .PHONY: all
@@ -59,7 +59,7 @@ test: fmt vet ## Run tests.
 
 .PHONY: build
 build: fmt vet ## Build binary.
-	go build -o bin/sidecar-injector ./cmd/
+	go build -o bin/node-specific-sizing ./cmd/
 
 .PHONY: docker-build
 docker-build: test ## Build docker image.
@@ -72,10 +72,19 @@ docker-push: ## Push docker image.
 ##@ Deployment
 
 deploy: kustomize
-	cp deploy/kustomization.yaml deploy/kustomization.yaml.tmp
-	cd deploy && $(KUSTOMIZE) edit set image sidecar-injector=$(IMAGE)
-	$(KUSTOMIZE) build deploy | $(KUBECTL) apply -f -
-	mv deploy/kustomization.yaml.tmp deploy/kustomization.yaml
+#	cp deploy/kustomization.yaml deploy/kustomization.yaml.tmp
+#	cd deploy && $(KUSTOMIZE) edit set image node-specific-sizing=$(IMAGE)
+	$(KUSTOMIZE) build deploy > deploy.yaml
+	$(KUBECTL) apply -f deploy.yaml
+	#mv deploy/kustomization.yaml.tmp deploy/kustomization.yaml
+
+.PHONY: docker-kind-load
+docker-kind-load: ## Share image with kind cluster
+	kind load docker-image -n ${KIND_CLUSTER} ${IMAGE}
+
+.PHONY: docker-k3d-load
+docker-k3d-load: ## Share image with kind cluster
+	k3d image import -m direct --trace -c ${K3D_CLUSTER} ${IMAGE}
 
 undeploy: kustomize
 	$(KUSTOMIZE) build deploy | $(KUBECTL) delete --ignore-not-found -f -

README.md

@@ -1,88 +1,77 @@
-# kube-sidecar-injector
+# kube-node-specific-sizing
 
-This repo is used for [a tutorial at Medium](https://medium.com/ibm-cloud/diving-into-kubernetes-mutatingadmissionwebhook-6ef3c5695f74) to create a Kubernetes [MutatingAdmissionWebhook](https://kubernetes.io/docs/admin/admission-controllers/#mutatingadmissionwebhook-beta-in-19) that injects a nginx sidecar container into pod prior to persistence of the object.
+Helps you resize pods created by a DaemonSet depending on the amount of allocatable resources present on the node.
 
-## Prerequisites
+## How to use
 
-- [git](https://git-scm.com/downloads)
-- [go](https://golang.org/dl/) version v1.17+
-- [docker](https://docs.docker.com/install/) version 19.03+
-- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.19+
-- Access to a Kubernetes v1.19+ cluster with the `admissionregistration.k8s.io/v1` API enabled. Verify that by the following command:
+1. Add the `node-specific-sizing.manomano.tech/enabled: "true"` label any pod you'd like to size depending on the node.
+   Only the `"true"` string works.
+   For DaemonSets - the intended use-case - this should therefore go in `spec: metadata: labels:`
 
-```
-kubectl api-versions | grep admissionregistration.k8s.io
-```
-The result should be:
-```
-admissionregistration.k8s.io/v1
-admissionregistration.k8s.io/v1beta1
-```
+2. Override pod CPU/Memory Request/Limit based on node resources using the following annotations.
+    - `node-specific-sizing.manomano.tech/request-mcpu-per-node-cpu: 0.1`
+    - `node-specific-sizing.manomano.tech/limit-mcpu-per-node-cpu: 0.1`
+    - `node-specific-sizing.manomano.tech/request-memory-per-node-memory: 0.1`
+    - `node-specific-sizing.manomano.tech/limit-memory-per-node-memory: 0.1`
 
-> Note: In addition, the `MutatingAdmissionWebhook` and `ValidatingAdmissionWebhook` admission controllers should be added and listed in the correct order in the admission-control flag of kube-apiserver.
+3. Optionally set absolute minimums, maximums and exclusions
+   NB: Only pods with the `node-specific-sizing.manomano.tech/enabled: "true"` label will see their resource modified.
+   - `node-specific-sizing.manomano.tech/minimum-cpu-request: 0.5`
+   - `node-specific-sizing.manomano.tech/minimum-cpu-limit: 0.5`
+   - `node-specific-sizing.manomano.tech/maximum-cpu-request: 0.5`
+   - `node-specific-sizing.manomano.tech/maximum-cpu-limit: 0.5`
+
 
-## Build and Deploy
+4. Optionally exclude some containers from dynamic-sizing.
+    - `node-specific-sizing.manomano.tech/exclude-containers: istio-init,istio-proxy`
 
-1. Build and push docker image:
+5. Take care of the following:
+    - In some instances, if limit ends up being below request it will be adjusted to be equal to the request.
+    - WARNING: We have not tested all cases of partial configuration or weird mish-mashes. 
+    - You're safer defining both requests and limits, or just requests if the underlying DaemonSet does not have limits.
+    - Having some containers define a request or limit while others do not is unsupported.
 
-```bash
-make docker-build docker-push IMAGE=quay.io/<your_quayio_username>/sidecar-injector:latest
-```
+## Resource Sizing Algorithm
 
-2. Deploy the kube-sidecar-injector to kubernetes cluster:
+Assuming a pod is eligible for dynamic sizing, the mutating webhook computes new resources by following these steps:
 
-```bash
-make deploy IMAGE=quay.io/<your_quayio_username>/sidecar-injector:latest
-```
+- For each container in the pod, and for each tunable [1], compute the tunable's relative value per container.
+  For any given container, `relative_tunable = container_tunable / (sum(container_tunables) - sum(excluded_container_tunables))` 
+- Derive a `pod_tunable_budget = allocatable_tunable_on_node * configured_pod_proportion - sum(excluded_container_tunables)`. This represents the resources that will be given to the pod.
+- Clamp `pod_tunable_budget` if minimums and/or maximums are set for that tunable.
+- Finally, `new_absolute_tunable = pod_tunable_budget * relative_tunable` spreads the budget around.
 
-3. Verify the kube-sidecar-injector is up and running:
+If no containers are excluded from sizing, the requests/limits proportions between the different containers stays the same.
 
-```bash
-# kubectl -n sidecar-injector get pod
-# kubectl -n sidecar-injector get pod
-NAME                                READY   STATUS    RESTARTS   AGE
-sidecar-injector-7c8bc5f4c9-28c84   1/1     Running   0          30s
-```
+## Development
 
-## How to use
+### Prerequisites
 
-1. Create a new namespace `test-ns` and label it with `sidecar-injector=enabled`:
+- [git](https://git-scm.com/downloads)
+- [go](https://golang.org/dl/) version v1.17+
+- [docker](https://docs.docker.com/install/) version 19.03+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.19+
+- [k3d](...) recommended.
 
-```
-# kubectl create ns test-ns
-# kubectl label namespace test-ns sidecar-injection=enabled
-# kubectl get namespace -L sidecar-injection
-NAME                 STATUS   AGE   SIDECAR-INJECTION
-default              Active   26m
-test-ns              Active   13s   enabled
-kube-public          Active   26m
-kube-system          Active   26m
-sidecar-injector     Active   17m
-```
+## Build and Deploy
 
-2. Deploy an app in Kubernetes cluster, take `alpine` app as an example
+1. Build and push docker image:
 
 ```bash
-kubectl -n test-ns run alpine \
-    --image=alpine \
-    --restart=Never \
-    --command -- sleep infinity
+make docker-build docker-push IMAGE=quay.io/<your_quayio_username>/node-specific-sizing:latest
 ```
 
-3. Verify sidecar container is injected:
+2. Deploy the kube-node-specific-sizing to kubernetes cluster:
 
-```
-# kubectl -n test-ns get pod
-NAME                     READY     STATUS        RESTARTS   AGE
-alpine                   2/2       Running       0          10s
-# kubectl -n test-ns get pod alpine -o jsonpath="{.spec.containers[*].name}"
-alpine sidecar-nginx
+```bash
+make deploy IMAGE=quay.io/<your_quayio_username>/node-specific-sizing:latest
 ```
 
-## Troubleshooting
+3. Verify the kube-node-specific-sizing deployment is up and running:
 
-Sometimes you may find that pod is injected with sidecar container as expected, check the following items:
-
-1. The sidecar-injector pod is in running state and no error logs.
-2. The namespace in which application pod is deployed has the correct labels(`sidecar-injector=enabled`) as configured in `mutatingwebhookconfiguration`.
-3. Check if the application pod has annotation `sidecar-injector-webhook.morven.me/inject:"yes"`.
+```bash
+# kubectl -n node-specific-sizing get pod
+# kubectl -n node-specific-sizing get pod
+NAME                                READY   STATUS    RESTARTS   AGE
+node-specific-sizing-dc75b5d95-spqs7   1/1     Running   0          30s
+```

bin/dev_toggle.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+SCRIPT_DIR="$(dirname -- "${BASH_SOURCE[0]}")"
+DEFAULT_KUSTOMIZE="$(realpath "${SCRIPT_DIR}/kustomize")"
+KUSTOMIZE="${KUSTOMIZE:-$DEFAULT_KUSTOMIZE}"
+
+if [[ ! -x "$KUSTOMIZE" ]]; then
+  echo "ERROR: Make sure kustomize is available at $KUSTOMIZE by running 'make kustomize' or set the KUSTOMIZE environment variable appropriately" >&2
+fi
+
+# Figure out if we're in dev mode or in normal mode
+_is_dev_mode=$(grep service-dev.yaml "${SCRIPT_DIR}/../deploy/kustomization.yaml" >/dev/null 2>&1 || echo nope)
+if [[ "$_is_dev_mode" == "nope" ]]; then
+  echo "Switching to 'development' mode (webhooks runs on workstation)"
+
+  "${SCRIPT_DIR}/extract_k8s_secret.sh" -n kube-system -s node-specific-sizing-cert
+
+  # Figure out the bridge address (that's going to be fun to port to MacOS ...)
+  _gateway_ip=$(docker inspect k3d-knss-server-0 | jq '.[0].NetworkSettings.Networks | to_entries | .[0].value.Gateway')
+
+  cd "${SCRIPT_DIR}/../deploy"
+  sed -ri "s/- ip: \"[^\"]+\"/- ip: ${_gateway_ip}/" "service-dev.yaml"
+  "$KUSTOMIZE" edit add resource service-dev.yaml
+  "$KUSTOMIZE" edit remove resource service.yaml
+
+  cd .. && make deploy
+else
+  echo "Switching to 'regular service' mode (webhooks runs inside kubernetes)"
+
+  cd "${SCRIPT_DIR}/../deploy"
+  "$KUSTOMIZE" edit remove resource service-dev.yaml
+  "$KUSTOMIZE" edit add resource service.yaml
+
+  kubectl -n kube-system
+  cd .. && make deploy
+fi
+
+
+
+

bin/extract_k8s_secret.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+# Initialize variables
+SECRET_NAME=""
+NAMESPACE=${NAMESPACE:-kube-system}
+OUTPUT_DIR=${OUTPUT_DIR:-/tmp/k8s-webhook-server/serving-certs}
+
+# Helper function
+show_help() {
+  echo "Usage: $0 --secret-name SECRET_NAME --namespace NAMESPACE --output-dir OUTPUT_DIR"
+  echo ""
+  echo "Options:"
+  echo "  -s, --secret-name NAME     Name of the Kubernetes secret"
+  echo "  -n, --namespace NAMESPACE  Kubernetes namespace"
+  echo "  -o, --output-dir DIR       Directory to save the output"
+  echo "  -h, --help                 Show this help message"
+}
+
+# Extract options
+while true; do
+  case "$1" in
+    -s | --secret-name ) SECRET_NAME="$2"; shift; shift ;;
+    -n | --namespace ) NAMESPACE="$2"; shift; shift ;;
+    -o | --output-dir ) OUTPUT_DIR="$2"; shift; shift ;;
+    -h | --help ) show_help; exit 0 ;;
+    -- ) shift; break ;;
+    * ) break ;;
+  esac
+done
+
+# Check if the required options are set
+if [[ -z "$SECRET_NAME" || -z "$NAMESPACE" || -z "$OUTPUT_DIR" ]]; then
+    echo "All arguments are required."
+    show_help
+    exit 1
+fi
+
+if ! kubectl -n "${NAMESPACE}" get secret "${SECRET_NAME}" >/dev/null 2>&1; then
+  echo "can't find certs, make sure you ./kind-testx.sh once to have them generated"
+  exit 1
+fi
+
+_secret_files_json=$(kubectl -n "${NAMESPACE}" get secret -ojson "${SECRET_NAME}" | jq -r '.data | to_entries')
+_max_i=$(echo "$_secret_files_json" | jq '. | length - 1')
+
+_dir=${OUTPUT_DIR}
+mkdir -pv "$_dir"
+
+for i in $(seq 0 "${_max_i}"); do
+  _filename=$(echo "$_secret_files_json" | jq -r "\"${_dir}/\\(.[$i].key)\"")
+  _file_contents=$(echo "$_secret_files_json" | jq -r ".[$i].value | @base64d")
+  echo "$_file_contents" > "$_filename"
+  echo "Wrote $_filename"
+done