[CRDB-47257] Testing Infrastructure Enhancements

.github/workflows/ci.yaml

@@ -2,8 +2,7 @@ name: Helm Chart Package CI
 on:
   pull_request:
     branches:
-      - 'master'
-      - 'cert-manager-feature-branch'
+      - '*'
 
 jobs:
 
@@ -49,14 +48,11 @@ jobs:
           repository: ${{github.event.pull_request.head.repo.full_name}}
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v6
         with:
-          version: v1.50.1
+          version: v1.64
           working-directory: .
           args: --timeout=5m
-          skip-build-cache: true
-          skip-pkg-cache: true
-          skip-go-installation: true
 
   # pre job to run helm lint
   helm:

Makefile

@@ -1,4 +1,5 @@
 UNAME_S := $(shell uname -s)
+NC := $(shell tput sgr0) # No Color
 ifeq ($(UNAME_S),Linux)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
@@ -21,7 +22,11 @@ ifeq ($(UNAME_S),Darwin)
 endif
 
 K3D_CLUSTER ?= chart-testing
-REPOSITORY ?= gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert
+REGISTRY ?= gcr.io
+REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
+DOCKER_NETWORK_NAME ?= "k3d-${K3D_CLUSTER}"
+LOCAL_REGISTRY ?= "localhost:5000"
+CLUSTER_SIZE ?= 1
 
 export BUNDLE_IMAGE ?= cockroach-operator-bundle
 export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
@@ -56,7 +61,7 @@ build/chart: bin/helm ## build the helm chart to build/artifacts
 build/self-signer: bin/yq ## build the self-signer image
 	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Release
 
@@ -66,39 +71,54 @@ release: ## publish the build artifacts to S3
 build-and-push/self-signer: bin/yq ## push the self-signer image
 	@docker buildx build --platform=linux/amd64,linux/arm64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Dev
 dev/clean: ## remove built artifacts
 	@rm -r build/artifacts/
 
+## Setup/teardown registries for easier local dev
+dev/registries/up: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" = "" ]; then \
+		echo "$(CYAN)Starting local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		./tests/k3d/registries.sh up $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/registries/down: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" != "" ]; then \
+		echo "$(CYAN)Stopping local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/registries/bounce: bin/k3d dev/registries/down dev/registries/up
+
+dev/push/local: dev/registries/up
+	@echo "$(CYAN)Pushing image to local registry...$(NC)"
+	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
+          	--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
+          	-t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+	@docker push "${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml)"
+
 ##@ Test
-test/cluster: bin/k3d test/cluster_up ## start a local k3d cluster for testing
+test/cluster: bin/k3d test/cluster/up ## start a local k3d cluster for testing
+
+test/cluster/bounce: bin/k3d test/cluster/down test/cluster/up ## restart a local k3d cluster for testing
 
-test/cluster_up: bin/k3d
-	@bin/k3d cluster list | grep $(K3D_CLUSTER) || bin/k3d cluster create $(K3D_CLUSTER)
+test/cluster/up: bin/k3d
+	@bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(CLUSTER_SIZE)
 
-test/cluster_down: bin/k3d
-	bin/k3d cluster delete $(K3D_CLUSTER)
+test/cluster/down: bin/k3d
+	./tests/k3d/dev-cluster.sh down --name "$(K3D_CLUSTER)"
 
 test/e2e/%: PKG=$*
-test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate)
-	@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/$(PKG)/...
+test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/cluster ## run e2e tests for package (e.g. install or rotate)
+	@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/... || EXIT_CODE=$$?; \
+	$(MAKE) test/cluster/down; \
+	exit $${EXIT_CODE:-0}
 
 test/lint: bin/helm ## lint the helm chart
 	@build/lint.sh && bin/helm lint cockroachdb
 
-IMAGE_LIST = cockroachdb/cockroach:v23.2.0 quay.io/jetstack/cert-manager-cainjector:v1.11.0 quay.io/jetstack/cert-manager-webhook:v1.11.0 quay.io/jetstack/cert-manager-controller:v1.11.0 quay.io/jetstack/cert-manager-ctl:v1.11.0
-test/publish-images-to-k3d: bin/yq test/cluster ## publish signer and cockroach image to local k3d registry
-	for i in $(IMAGE_LIST); do \
-		docker pull $$i; \
-		bin/k3d image import $$i -c $(K3D_CLUSTER); \
-	done
-	docker pull ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
-	bin/k3d image import \
-		${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
-		-c $(K3D_CLUSTER)
-
 test/template: bin/cockroach bin/helm ## Run template tests
 	@PATH="$(PWD)/bin:${PATH}" go test -v ./tests/template/...
 

build/templates/values.yaml

@@ -674,7 +674,7 @@ tls:
     # Image Placeholder for the selfSigner utility. This will be changed once the CI workflows for the image is in place.
     image:
       repository: cockroachlabs-helm-charts/cockroach-self-signer-cert
-      tag: "1.5"
+      tag: "1.6"
       pullPolicy: IfNotPresent
       credentials: {}
       registry: gcr.io

cockroachdb/values.yaml

@@ -675,7 +675,7 @@ tls:
     # Image Placeholder for the selfSigner utility. This will be changed once the CI workflows for the image is in place.
     image:
       repository: cockroachlabs-helm-charts/cockroach-self-signer-cert
-      tag: "1.5"
+      tag: "1.6"
       pullPolicy: IfNotPresent
       credentials: {}
       registry: gcr.io

pkg/resource/cleanup.go

@@ -18,6 +18,7 @@ package resource
 
 import (
 	"context"
+
 	"github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"

tests/e2e/install/cockroachdb_helm_e2e_test.go

@@ -13,6 +13,9 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach-operator/pkg/kube"
+	"github.com/cockroachdb/helm-charts/pkg/security"
+	util "github.com/cockroachdb/helm-charts/pkg/utils"
+	"github.com/cockroachdb/helm-charts/tests/testutil"
 	"github.com/gruntwork-io/terratest/modules/helm"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
@@ -21,10 +24,6 @@ import (
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-
-	"github.com/cockroachdb/helm-charts/pkg/security"
-	util "github.com/cockroachdb/helm-charts/pkg/utils"
-	"github.com/cockroachdb/helm-charts/tests/testutil"
 )
 
 var (
@@ -52,7 +51,7 @@ func TestCockroachDbHelmInstall(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	const testDBName = "testdb"
 
@@ -124,7 +123,7 @@ func TestCockroachDbHelmInstallWithCAProvided(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	certOutput, err := shell.RunCommandAndGetOutputE(t, cmd)
 	t.Log(certOutput)
@@ -250,7 +249,7 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// Make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	cmds := []shell.Command{cmdCa, cmdNode, cmdClient}
 	for i := range cmds {
@@ -309,7 +308,7 @@ func TestCockroachDbHelmMigration(t *testing.T) {
 			"statefulset.updateStrategy.type": "OnDelete",
 		}),
 		ExtraArgs: map[string][]string{
-			"upgrade": []string{
+			"upgrade": {
 				"--timeout=20m",
 			},
 		},
@@ -357,7 +356,7 @@ func TestCockroachDbWithInsecureMode(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	// Setup the args. For this test, we will set the following input values:
 	options := &helm.Options{
@@ -393,7 +392,7 @@ func TestCockroachDbWithCertManager(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	certManagerHelmOptions := &helm.Options{
 		KubectlOptions: k8s.NewKubectlOptions("", "", "cert-manager"),
@@ -519,7 +518,7 @@ func testWALFailoverExistingCluster(t *testing.T, additionalValues map[string]st
 	}
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	// Print the debug logs in case of test failure.
 	defer func() {

tests/e2e/rotate/cockroachdb_rotate_certs_test.go

@@ -1,8 +1,7 @@
-package rotate
+package integration
 
 import (
 	"fmt"
-	"github.com/gruntwork-io/terratest/modules/shell"
 	"path"
 	"path/filepath"
 	"strings"
@@ -13,6 +12,7 @@ import (
 	"github.com/gruntwork-io/terratest/modules/helm"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/gruntwork-io/terratest/modules/random"
+	"github.com/gruntwork-io/terratest/modules/shell"
 	"github.com/stretchr/testify/require"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -55,7 +55,7 @@ func TestCockroachDbRotateCertificates(t *testing.T) {
 
 	k8s.CreateNamespace(t, kubectlOptions, namespaceName)
 	// ... and make sure to delete the namespace at the end of the test
-	defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
+	defer testutil.DeleteNamespace(t, k8sClient, namespaceName)
 
 	// Setup the args. For this test, we will set the following input values:
 	helmValues := map[string]string{