e2e test

Signed-off-by: Artem Bortnikov <[email protected]>

.github/workflows/test_e2e.yml

@@ -0,0 +1,25 @@
+name: Test E2E
+
+on: pull_request
+
+jobs:
+  test-e2e:
+    name: Test E2E
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version-file: go.mod
+    - uses: docker/[email protected]
+    - uses: tale/[email protected]
+      with:
+        kubectl-version: v1.30.0
+        # Empty kubeconfig file
+        base64-kube-config: "YXBpVmVyc2lvbjogdjEKa2luZDogQ29uZmlnCnByZWZlcmVuY2VzOiB7fQo="
+    - name: Run tests
+      run: make test-e2e

.gitignore

@@ -25,3 +25,5 @@ go.work
 *.swp
 *.swo
 *~
+
+bin

Makefile

@@ -42,7 +42,11 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: ## Run tests.
-	@go run github.com/onsi/ginkgo/v2/ginkgo -r --race --randomize-suites --keep-going --randomize-all --repeat=11
+	@go run github.com/onsi/ginkgo/v2/ginkgo -r --skip-package e2e --race --randomize-suites --keep-going --randomize-all --repeat=11
+
+.PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
+test-e2e:
+	go run github.com/onsi/ginkgo/v2/ginkgo -r --skip-package controller -v
 
 .PHONY: lint
 lint: ## Run golangci-lint linter & yamllint.
@@ -71,3 +75,79 @@ install: manifests ## Install CRDs into the K8s cluster specified in ~/.kube/con
 .PHONY: uninstall
 uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
 	@go run sigs.k8s.io/kustomize/kustomize/v5 build config/crd | kubectl delete --ignore-not-found=true -f -
+
+##@ E2E Tests
+
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+IMG ?= ghcr.io/ironcore-dev/metal:latest
+KIND_CLUSTER_NAME ?= ironcore-metal-e2e
+NAMESPACE ?= metal-system
+KUBECTL ?= kubectl
+KIND ?= $(LOCALBIN)/kind
+KUSTOMIZE ?= $(LOCALBIN)/kustomize
+CONTAINER_TOOL ?= docker
+YQ = $(LOCALBIN)/yq
+
+K8S_VERSION ?= v1.30.0
+CERT_MANAGER_VERSION ?= v1.15.0
+YQ_VERSION ?= v4.44.1
+KUSTOMIZE_VERSION ?= v5.3.0
+KIND_VERSION ?= v0.23.0
+
+KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
+
+.PHONY: docker-build
+docker-build: ## Build docker image with the manager.
+	@$(CONTAINER_TOOL) build -t ${IMG} .
+
+.PHONY: deploy
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | $(KUBECTL) -n $(NAMESPACE) apply -f -
+	$(KUBECTL) wait deployment.apps/metal-controller-manager --for condition=Available --namespace $(NAMESPACE) --timeout 5m
+
+.PHONY: kind-create
+kind-create: kind yq ## Create kubernetes cluster using Kind.
+	@if ! $(KIND) get clusters | grep -q $(KIND_CLUSTER_NAME); then \
+		$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION) --config test/e2e/config.yaml; \
+	elif ! $(CONTAINER_TOOL) container inspect $$($(KIND) get nodes --name $(KIND_CLUSTER_NAME)) | $(YQ) e '.[0].Config.Image' | grep -q $(K8S_VERSION); then \
+  		$(KIND) delete cluster --name $(KIND_CLUSTER_NAME); \
+		$(KIND) create cluster --name $(KIND_CLUSTER_NAME) --image kindest/node:$(K8S_VERSION) --config test/e2e/config.yaml; \
+	fi
+
+.PHONY: kind-delete
+kind-delete: kind ## Create kubernetes cluster using Kind.
+	@if $(KIND) get clusters | grep -q $(KIND_CLUSTER_NAME); then \
+		$(KIND) delete cluster --name $(KIND_CLUSTER_NAME); \
+	fi
+
+.PHONY: kind-prepare
+kind-prepare: kind-create
+	# Install cert-manager operator
+	$(KUBECTL) apply --server-side -f "https://github.com/jetstack/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml"
+	$(KUBECTL) wait deployment.apps/cert-manager-webhook --for condition=Available --namespace cert-manager --timeout 5m
+
+.PHONY: kind-load
+kind-load: kind ## Build and upload docker image to the local Kind cluster.
+	$(KIND) load docker-image ${IMG} --name $(KIND_CLUSTER_NAME)
+
+.PHONY: kustomize
+kustomize: $(LOCALBIN)
+	@if test -x $(KUSTOMIZE) && ! $(KUSTOMIZE) version | grep -q $(KUSTOMIZE_VERSION); then \
+		rm -f $(KUSTOMIZE); \
+	fi
+	@test -x $(KUSTOMIZE) || { curl -Ss $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN); }
+
+.PHONY: kind
+kind: $(LOCALBIN)
+	@test -x $(KIND) && $(KIND) version | grep -q $(KIND_VERSION) || \
+	GOBIN=$(LOCALBIN) go install sigs.k8s.io/kind@$(KIND_VERSION)
+
+.PHONY: yq
+yq: $(LOCALBIN)
+	@test -x $(YQ) && $(YQ) version | grep -q $(YQ_VERSION) || \
+	GOBIN=$(LOCALBIN) go install github.com/mikefarah/yq/v4@$(YQ_VERSION)

cmd/main.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/go-logr/logr"
 	ipamv1alpha1 "github.com/ironcore-dev/ipam/api/ipam/v1alpha1"
+	"github.com/ironcore-dev/metal/internal/bmc"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -59,6 +60,7 @@ type params struct {
 	oobShutdownTimeout           time.Duration
 	oobUsernamePrefix            string
 	oobTemporaryPasswordSecret   string
+	testingMode                  bool
 }
 
 func parseCmdLine() params {
@@ -87,6 +89,7 @@ func parseCmdLine() params {
 	pflag.Duration("oob-shutdown-timeout", 7*time.Minute, "Wait this long before issuing an immediate shutdown, if graceful shutdown has not succeeded.")
 	pflag.String("oob-username-prefix", "metal-", "OOB: Use a prefix when creating BMC users. Cannot be empty.")
 	pflag.String("oob-temporary-password-secret", "bmc-temporary-password", "OOB: Secret to store a temporary password in. Will be generated if it does not exist.")
+	pflag.Bool("testing-mode", false, "Enable testing mode.")
 
 	var help bool
 	pflag.BoolVarP(&help, "help", "h", false, "Show this help message.")
@@ -126,6 +129,7 @@ func parseCmdLine() params {
 		oobShutdownTimeout:           viper.GetDuration("oob-shutdown-timeout"),
 		oobUsernamePrefix:            viper.GetString("oob-username-prefix"),
 		oobTemporaryPasswordSecret:   viper.GetString("oob-temporary-password-secret"),
+		testingMode:                  viper.GetBool("testing-mode"),
 	}
 }
 
@@ -156,6 +160,13 @@ func main() {
 	defer stop()
 	log.Info(ctx, "Starting Metal")
 
+	if p.testingMode {
+		log.Info(ctx, "Running testing mode: fake bmc registered")
+		bmc.RegisterFake()
+	} else {
+		log.Info(ctx, "Running normal mode")
+	}
+
 	defer func() {
 		log.Info(ctx, "Exiting", "exitCode", exitCode)
 	}()

config/manager/kustomization.yaml

@@ -1,2 +1,8 @@
 resources:
 - manager.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+images:
+- name: controller
+  newName: ghcr.io/ironcore-dev/metal
+  newTag: latest

internal/controller/oob_controller_test.go

@@ -4,12 +4,11 @@
 package controller
 
 import (
-	"crypto/rand"
 	"fmt"
-	"net/netip"
 	"time"
 
 	ipamv1alpha1 "github.com/ironcore-dev/ipam/api/ipam/v1alpha1"
+	"github.com/ironcore-dev/metal/test/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -32,9 +31,9 @@ var _ = PDescribe("OOB Controller", Serial, func() {
 
 	BeforeEach(func(ctx SpecContext) {
 		var err error
-		mac, err = generateMacAddress()
+		mac, err = utils.GenerateMacAddress(macPrefix)
 		Expect(err).NotTo(HaveOccurred())
-		ipAddr, err = generateIpAddress()
+		ipAddr, err = utils.GenerateIpAddress()
 		Expect(err).NotTo(HaveOccurred())
 
 		Eventually(ObjectList(&ipamv1alpha1.IPList{}, &client.ListOptions{
@@ -911,32 +910,6 @@ var _ = PDescribe("OOB Controller", Serial, func() {
 	})
 })
 
-func generateMacAddress() (string, error) {
-	buf := make([]byte, 3)
-	_, err := rand.Read(buf)
-	if err != nil {
-		return "", fmt.Errorf("cannot generate MAC address: %w", err)
-	}
-	mac := fmt.Sprintf("%s%02x%02x%02x", macPrefix, buf[0], buf[1], buf[2])
-	return mac, nil
-}
-
-func generateIpAddress() (ipamv1alpha1.IPAddr, error) {
-	buf := make([]byte, 3)
-	_, err := rand.Read(buf)
-	if err != nil {
-		return ipamv1alpha1.IPAddr{}, fmt.Errorf("cannot generate IP address: %w", err)
-	}
-	var ip netip.Addr
-	ip, err = netip.ParseAddr(fmt.Sprintf("10.%d.%d.%d", buf[0], buf[1], buf[2]))
-	if err != nil {
-		return ipamv1alpha1.IPAddr{}, fmt.Errorf("cannot generate IP address: %w", err)
-	}
-	return ipamv1alpha1.IPAddr{
-		Net: ip,
-	}, nil
-}
-
 func readyReason(o client.Object) (string, error) {
 	oob, ok := o.(*metalv1alpha1.OOB)
 	if !ok {

test/e2e/config.yaml

@@ -0,0 +1,4 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  kubeProxyMode: "ipvs"