From 0c02b79b3be4f07ee2d8ce0301b05485f6263171 Mon Sep 17 00:00:00 2001 From: Carlos Salas Date: Thu, 19 Dec 2024 10:55:18 +0100 Subject: [PATCH] add clusterclass controller image building and tilt deployment Signed-off-by: Carlos Salas --- Makefile | 79 +++++++++++++++++++++++++++ Tiltfile | 17 +++++- exp/clusterclass/Dockerfile | 47 +++++++++------- scripts/turtles-dev.sh | 103 ++++++++++++++++++++++-------------- 4 files changed, 184 insertions(+), 62 deletions(-) diff --git a/Makefile b/Makefile index e43a082e..5ef0a1d6 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,7 @@ TEST_DIR := test TOOLS_DIR := hack/tools TOOLS_BIN_DIR := $(abspath $(TOOLS_DIR)/$(BIN_DIR)) EXP_ETCDRESTORE_DIR := exp/etcdrestore +EXP_CLUSTERCLASS_DIR := exp/clusterclass $(TOOLS_BIN_DIR): mkdir -p $@ @@ -184,6 +185,14 @@ CONTROLLER_IMG ?= $(REGISTRY)/$(ORG)/$(CONTROLLER_IMAGE_NAME) CONTROLLER_IMAGE_VERSION ?= $(shell git describe --abbrev=0 2>/dev/null) IID_FILE ?= $(shell mktemp) +# etcdrestore +ETCDRESTORE_IMAGE_NAME ?= turtles-etcd-snapshot-restore +ETCDRESTORE_IMG ?= $(REGISTRY)/$(ORG)/$(ETCDRESTORE_IMAGE_NAME) + +# clusterclass +CLUSTERCLASS_IMAGE_NAME ?= turtles-clusterclass-operations +CLUSTERCLASS_IMG ?= $(REGISTRY)/$(ORG)/$(CLUSTERCLASS_IMAGE_NAME) + # Release # Exclude tags with the prefix 'test/' RELEASE_TAG ?= $(shell git describe --abbrev=0 --exclude 'test/*' 2>/dev/null) @@ -261,11 +270,19 @@ generate-exp-etcdrestore-manifests-api: controller-gen ## Generate ClusterRole a output:webhook:dir=./exp/etcdrestore/config/webhook \ webhook +.PHONY: generate-exp-clusterclass-manifests-api +generate-exp-clusterclass-manifests-api: controller-gen ## Generate ClusterRole and CustomResourceDefinition objects for experimental API. + $(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./exp/etcdrestore/api/v1alpha1/..." \ + paths=./exp/clusterclass/internal/controller/... \ + output:crd:artifacts:config=./exp/clusterclass/config/crd/bases \ + output:rbac:dir=./exp/clusterclass/config/rbac + .PHONY: generate-modules generate-modules: ## Run go mod tidy to ensure modules are up to date go mod tidy cd $(TEST_DIR); go mod tidy cd $(EXP_ETCDRESTORE_DIR); go mod tidy + cd $(EXP_CLUSTERCLASS_DIR); go mod tidy .PHONY: generate-go-deepcopy generate-go-deepcopy: ## Run deepcopy generation @@ -332,6 +349,10 @@ test: $(SETUP_ENVTEST) manifests test-exp-etcdrestore ## Run all generators and test-exp-etcdrestore: $(SETUP_ENVTEST) ## Run tests for experimental etcdrestore API. cd $(EXP_ETCDRESTORE_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS) +.PHONY: test-exp-clusterclass +test-exp-clusterclass: $(SETUP_ENVTEST) ## Run tests for experimental clusterclass API. + cd $(EXP_CLUSTERCLASS_DIR); KUBEBUILDER_ASSETS="$(KUBEBUILDER_ASSETS)" go test ./... $(TEST_ARGS) + ##@ Build .PHONY: build @@ -384,6 +405,64 @@ docker-build-and-push: buildx-machine docker-pull-prerequisites ## Run docker-bu --build-arg package=. \ --build-arg ldflags="$(LDFLAGS)" . -t $(CONTROLLER_IMG):$(TAG) +## -------------------------------------- +## Docker - etcdrestore +## -------------------------------------- + +.PHONY: docker-build-etcdrestore ## Build the docker image for etcdrestore +docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture + ## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368) + # buildx does not support using local registry for multi-architecture images + cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ + --platform $(ARCH) \ + --load \ + --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ + --build-arg goproxy=$(GOPROXY) \ + --build-arg package=./exp/etcdrestore \ + --build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain + +.PHONY: docker-build-and-push-etcdrestore +docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-etcdrestore targets for all architectures + cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ + --platform $(TARGET_PLATFORMS) \ + --push \ + --sbom=true \ + --attest type=provenance,mode=max \ + --iidfile=$(IID_FILE) \ + --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ + --build-arg goproxy=$(GOPROXY) \ + --build-arg package=./exp/etcdrestore \ + --build-arg ldflags="$(LDFLAGS)" . -t $(ETCDRESTORE_IMG):$(TAG) --file - --progress=plain + +## -------------------------------------- +## Docker - clusterclass +## -------------------------------------- + +.PHONY: docker-build-clusterclass ## Build the docker image for clusterclass +docker-build-clusterclass: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture + ## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368) + # buildx does not support using local registry for multi-architecture images + cat $(EXP_CLUSTERCLASS_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ + --platform $(ARCH) \ + --load \ + --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ + --build-arg goproxy=$(GOPROXY) \ + --build-arg package=./exp/clusterclass \ + --build-arg ldflags="$(LDFLAGS)" . -t $(CLUSTERCLASS_IMG):$(TAG) --file - --progress=plain + +.PHONY: docker-build-and-push-clusterclass +docker-build-and-push-clusterclass: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-clusterclass targets for all architectures + cat $(EXP_CLUSTERCLASS_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ + --platform $(TARGET_PLATFORMS) \ + --push \ + --sbom=true \ + --attest type=provenance,mode=max \ + --iidfile=$(IID_FILE) \ + --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ + --build-arg goproxy=$(GOPROXY) \ + --build-arg package=./exp/clusterclass \ + --build-arg ldflags="$(LDFLAGS)" . -t $(CLUSTERCLASS_IMG):$(TAG) --file - --progress=plain + docker-list-all: @echo $(CONTROLLER_IMG):${TAG} diff --git a/Tiltfile b/Tiltfile index 5a3cc029..c3652c7a 100644 --- a/Tiltfile +++ b/Tiltfile @@ -35,7 +35,7 @@ if settings.get("trigger_mode") == "manual": if settings.get("default_registry") != "": default_registry(settings.get("default_registry")) -always_enable_projects = ["turtles", "turtles-capiproviders", "turtles-etcdsnapshotrestore"] +always_enable_projects = ["turtles", "turtles-capiproviders", "turtles-etcdsnapshotrestore", "turtles-clusterclass-operations"] projects = { "turtles": { @@ -74,6 +74,19 @@ projects = { "kustomize_dir": "config/capiproviders", "label": "turtles-capiproviders", "op": "apply" + "label": "turtles-etcdsnapshotrestore" + }, + "turtles-clusterclass-operations": { + "context": "exp/clusterclass", + "image": "ghcr.io/rancher/turtles-clusterclass-operations:dev", + "live_reload_deps": [ + "main.go", + "go.mod", + "go.sum", + "internal/", + ], + "kustomize_dir": "config/default", + "label": "turtles-clusterclass-operations" } } @@ -101,4 +114,4 @@ def get_projects(): include_user_tilt_files() -enable_projects() \ No newline at end of file +enable_projects() diff --git a/exp/clusterclass/Dockerfile b/exp/clusterclass/Dockerfile index 79875e29..46fc17db 100644 --- a/exp/clusterclass/Dockerfile +++ b/exp/clusterclass/Dockerfile @@ -1,37 +1,46 @@ +# syntax=docker/dockerfile:1.4 + +# Copyright 2024 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Build the manager binary ARG builder_image -# Build architecture -ARG ARCH - FROM ${builder_image} as builder WORKDIR /workspace +# Run this with docker build --build-arg goproxy=$(go env GOPROXY) to override the goproxy ARG goproxy=https://proxy.golang.org +# Run this with docker build --build-arg package=./exp/clusterclass ENV GOPROXY=$goproxy -COPY go.mod go.mod -COPY go.sum go.sum - -RUN --mount=type=cache,target=/go/pkg/mod \ - go mod download - -# Copy the go source COPY ./ ./ -ARG package=. -ARG ARCH +# Build ARG ldflags -RUN --mount=type=cache,target=/go/pkg/mod \ - CGO_ENABLED=0 GOOS=linux GOARCH=${ARCH} \ - go build -trimpath -ldflags "${ldflags} -extldflags '-static'" \ - -o manager ${package} +# Do not force rebuild of up-to-date packages (do not use -a) and use the compiler cache folder +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + CGO_ENABLED=0 GOOS=linux \ + sh -c "cd exp/clusterclass && ls && go build -trimpath -ldflags \"${ldflags} -extldflags '-static'\" -o manager ${package}" -FROM gcr.io/distroless/static:nonroot-${ARCH} -LABEL org.opencontainers.image.source=https://github.com/rancher/turtles/exp/clusterclass +# Use distroless as minimal base image to package the manager binary +# Refer to https://github.com/GoogleContainerTools/distroless for more details +FROM gcr.io/distroless/static:nonroot WORKDIR / -COPY --from=builder /workspace/manager . +COPY --from=builder /workspace/exp/clusterclass/manager . # Use uid of nonroot user (65532) because kubernetes expects numeric user when applying pod security policies USER 65532 ENTRYPOINT ["/manager"] diff --git a/scripts/turtles-dev.sh b/scripts/turtles-dev.sh index 333bde74..ea2711c0 100755 --- a/scripts/turtles-dev.sh +++ b/scripts/turtles-dev.sh @@ -16,14 +16,18 @@ RANCHER_HOSTNAME=$1 if [ -z "$RANCHER_HOSTNAME" ]; then - echo "You must pass a rancher host name" - exit 1 + echo "You must pass a rancher host name" + exit 1 fi RANCHER_VERSION=${RANCHER_VERSION:-v2.10.0} +CAPI_VERSION=${CAPI_VERSION:-v1.8.5} +CAPRKE2_VERSION=${CAPRKE2_VERSION:-v0.9.0} CLUSTER_NAME=${CLUSTER_NAME:-capi-test} ETCD_CONTROLLER_IMAGE=${ETCD_CONTROLLER_IMAGE:-ghcr.io/rancher/turtles} ETCD_CONTROLLER_IMAGE_TAG=${ETCD_CONTROLLER_IMAGE_TAG:-dev} +CC_CONTROLLER_IMAGE=${CC_CONTROLLER_IMAGE:-ghcr.io/rancher/turtles-clusterclass-operations} +CC_CONTROLLER_IMAGE_TAG=${CC_CONTROLLER_IMAGE_TAG:-dev} USE_TILT_DEV=${USE_TILT_DEV:-true} BASEDIR=$(dirname "$0") @@ -39,61 +43,78 @@ helm repo add ngrok https://charts.ngrok.com helm repo update helm install cert-manager jetstack/cert-manager \ - --namespace cert-manager \ - --create-namespace \ - --set crds.enabled=true + --namespace cert-manager \ + --create-namespace \ + --set crds.enabled=true + +export EXP_CLUSTER_RESOURCE_SET=true +export CLUSTER_TOPOLOGY=true + +helm install capi-operator capi-operator/cluster-api-operator \ + --create-namespace -n capi-operator-system \ + --set infrastructure=docker:$CAPI_VERSION \ + --set core=cluster-api:$CAPI_VERSION \ + --set controlPlane=rke2:$CAPRKE2_VERSION \ + --set bootstrap=rke2:$CAPRKE2_VERSION \ + --timeout 90s --wait + +kubectl rollout status deployment capi-operator-cluster-api-operator -n capi-operator-system --timeout=180s helm upgrade ngrok ngrok/kubernetes-ingress-controller \ - --install \ - --wait \ - --timeout 5m \ - --set credentials.apiKey=$NGROK_API_KEY \ - --set credentials.authtoken=$NGROK_AUTHTOKEN + --install \ + --wait \ + --timeout 5m \ + --set credentials.apiKey=$NGROK_API_KEY \ + --set credentials.authtoken=$NGROK_AUTHTOKEN kubectl apply -f test/e2e/data/rancher/ingress-class-patch.yaml helm install rancher rancher-latest/rancher \ - --namespace cattle-system \ - --create-namespace \ - --set bootstrapPassword=rancheradmin \ - --set replicas=1 \ - --set hostname="$RANCHER_HOSTNAME" \ - --version="$RANCHER_VERSION" \ - --wait + --namespace cattle-system \ + --create-namespace \ + --set bootstrapPassword=rancheradmin \ + --set replicas=1 \ + --set hostname="$RANCHER_HOSTNAME" \ + --version="$RANCHER_VERSION" \ + --wait kubectl rollout status deployment rancher -n cattle-system --timeout=180s kubectl apply -f test/e2e/data/rancher/rancher-service-patch.yaml -envsubst < test/e2e/data/rancher/rancher-setting-patch.yaml | kubectl apply -f - +envsubst