Skip to content

[CRDB-47257] Testing Infrastructure Enhancements #455

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 5, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 3 additions & 7 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,7 @@ name: Helm Chart Package CI
on:
pull_request:
branches:
- 'master'
- 'cert-manager-feature-branch'
- '*'

jobs:

Expand Down Expand Up @@ -49,14 +48,11 @@ jobs:
repository: ${{github.event.pull_request.head.repo.full_name}}

- name: Run golangci-lint
uses: golangci/golangci-lint-action@v3
uses: golangci/golangci-lint-action@v6
with:
version: v1.50.1
version: v1.64
working-directory: .
args: --timeout=5m
skip-build-cache: true
skip-pkg-cache: true
skip-go-installation: true

# pre job to run helm lint
helm:
Expand Down
62 changes: 41 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
UNAME_S := $(shell uname -s)
NC := $(shell tput sgr0) # No Color
ifeq ($(UNAME_S),Linux)
COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
Expand All @@ -21,7 +22,11 @@ ifeq ($(UNAME_S),Darwin)
endif

K3D_CLUSTER ?= chart-testing
REPOSITORY ?= gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert
REGISTRY ?= gcr.io
REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
DOCKER_NETWORK_NAME ?= "k3d-${K3D_CLUSTER}"
LOCAL_REGISTRY ?= "localhost:5000"
CLUSTER_SIZE ?= 1

export BUNDLE_IMAGE ?= cockroach-operator-bundle
export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
Expand Down Expand Up @@ -56,7 +61,7 @@ build/chart: bin/helm ## build the helm chart to build/artifacts
build/self-signer: bin/yq ## build the self-signer image
@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .

##@ Release

Expand All @@ -66,39 +71,54 @@ release: ## publish the build artifacts to S3
build-and-push/self-signer: bin/yq ## push the self-signer image
@docker buildx build --platform=linux/amd64,linux/arm64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .

##@ Dev
dev/clean: ## remove built artifacts
@rm -r build/artifacts/

## Setup/teardown registries for easier local dev
dev/registries/up: bin/k3d
@if [ "`docker ps -f name=registry.localhost -q`" = "" ]; then \
echo "$(CYAN)Starting local Docker registry (for fast offline image push/pull)...$(NC)"; \
./tests/k3d/registries.sh up $(DOCKER_NETWORK_NAME); \
fi

dev/registries/down: bin/k3d
@if [ "`docker ps -f name=registry.localhost -q`" != "" ]; then \
echo "$(CYAN)Stopping local Docker registry (for fast offline image push/pull)...$(NC)"; \
./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \
fi

dev/registries/bounce: bin/k3d dev/registries/down dev/registries/up

dev/push/local: dev/registries/up
@echo "$(CYAN)Pushing image to local registry...$(NC)"
@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
@docker push "${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml)"

##@ Test
test/cluster: bin/k3d test/cluster_up ## start a local k3d cluster for testing
test/cluster: bin/k3d test/cluster/up ## start a local k3d cluster for testing

test/cluster/bounce: bin/k3d test/cluster/down test/cluster/up ## restart a local k3d cluster for testing

test/cluster_up: bin/k3d
@bin/k3d cluster list | grep $(K3D_CLUSTER) || bin/k3d cluster create $(K3D_CLUSTER)
test/cluster/up: bin/k3d
@bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(CLUSTER_SIZE)

test/cluster_down: bin/k3d
bin/k3d cluster delete $(K3D_CLUSTER)
test/cluster/down: bin/k3d
./tests/k3d/dev-cluster.sh down --name "$(K3D_CLUSTER)"

test/e2e/%: PKG=$*
test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate)
@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/$(PKG)/...
test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/cluster ## run e2e tests for package (e.g. install or rotate)
@PATH="$(PWD)/bin:${PATH}" go test -timeout 30m -v ./tests/e2e/... || EXIT_CODE=$$?; \
$(MAKE) test/cluster/down; \
exit $${EXIT_CODE:-0}

test/lint: bin/helm ## lint the helm chart
@build/lint.sh && bin/helm lint cockroachdb

IMAGE_LIST = cockroachdb/cockroach:v23.2.0 quay.io/jetstack/cert-manager-cainjector:v1.11.0 quay.io/jetstack/cert-manager-webhook:v1.11.0 quay.io/jetstack/cert-manager-controller:v1.11.0 quay.io/jetstack/cert-manager-ctl:v1.11.0
test/publish-images-to-k3d: bin/yq test/cluster ## publish signer and cockroach image to local k3d registry
for i in $(IMAGE_LIST); do \
docker pull $$i; \
bin/k3d image import $$i -c $(K3D_CLUSTER); \
done
docker pull ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
bin/k3d image import \
${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
-c $(K3D_CLUSTER)

test/template: bin/cockroach bin/helm ## Run template tests
@PATH="$(PWD)/bin:${PATH}" go test -v ./tests/template/...

Expand Down
2 changes: 1 addition & 1 deletion build/templates/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -674,7 +674,7 @@ tls:
# Image Placeholder for the selfSigner utility. This will be changed once the CI workflows for the image is in place.
image:
repository: cockroachlabs-helm-charts/cockroach-self-signer-cert
tag: "1.5"
tag: "1.6"
pullPolicy: IfNotPresent
credentials: {}
registry: gcr.io
Expand Down
2 changes: 1 addition & 1 deletion cockroachdb/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -675,7 +675,7 @@ tls:
# Image Placeholder for the selfSigner utility. This will be changed once the CI workflows for the image is in place.
image:
repository: cockroachlabs-helm-charts/cockroach-self-signer-cert
tag: "1.5"
tag: "1.6"
pullPolicy: IfNotPresent
credentials: {}
registry: gcr.io
Expand Down
1 change: 1 addition & 0 deletions pkg/resource/cleanup.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package resource

import (
"context"

"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
Expand Down
21 changes: 10 additions & 11 deletions tests/e2e/install/cockroachdb_helm_e2e_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,9 @@ import (
"time"

"github.com/cockroachdb/cockroach-operator/pkg/kube"
"github.com/cockroachdb/helm-charts/pkg/security"
util "github.com/cockroachdb/helm-charts/pkg/utils"
"github.com/cockroachdb/helm-charts/tests/testutil"
"github.com/gruntwork-io/terratest/modules/helm"
"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/gruntwork-io/terratest/modules/random"
Expand All @@ -21,10 +24,6 @@ import (
_ "k8s.io/client-go/plugin/pkg/client/auth"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"

"github.com/cockroachdb/helm-charts/pkg/security"
util "github.com/cockroachdb/helm-charts/pkg/utils"
"github.com/cockroachdb/helm-charts/tests/testutil"
)

var (
Expand Down Expand Up @@ -52,7 +51,7 @@ func TestCockroachDbHelmInstall(t *testing.T) {

k8s.CreateNamespace(t, kubectlOptions, namespaceName)
// ... and make sure to delete the namespace at the end of the test
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
defer testutil.DeleteNamespace(t, k8sClient, namespaceName)

const testDBName = "testdb"

Expand Down Expand Up @@ -124,7 +123,7 @@ func TestCockroachDbHelmInstallWithCAProvided(t *testing.T) {

k8s.CreateNamespace(t, kubectlOptions, namespaceName)
// ... and make sure to delete the namespace at the end of the test
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
defer testutil.DeleteNamespace(t, k8sClient, namespaceName)

certOutput, err := shell.RunCommandAndGetOutputE(t, cmd)
t.Log(certOutput)
Expand Down Expand Up @@ -250,7 +249,7 @@ func TestCockroachDbHelmMigration(t *testing.T) {

k8s.CreateNamespace(t, kubectlOptions, namespaceName)
// Make sure to delete the namespace at the end of the test
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
defer testutil.DeleteNamespace(t, k8sClient, namespaceName)

cmds := []shell.Command{cmdCa, cmdNode, cmdClient}
for i := range cmds {
Expand Down Expand Up @@ -309,7 +308,7 @@ func TestCockroachDbHelmMigration(t *testing.T) {
"statefulset.updateStrategy.type": "OnDelete",
}),
ExtraArgs: map[string][]string{
"upgrade": []string{
"upgrade": {
"--timeout=20m",
},
},
Expand Down Expand Up @@ -357,7 +356,7 @@ func TestCockroachDbWithInsecureMode(t *testing.T) {

k8s.CreateNamespace(t, kubectlOptions, namespaceName)
// ... and make sure to delete the namespace at the end of the test
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
defer testutil.DeleteNamespace(t, k8sClient, namespaceName)

// Setup the args. For this test, we will set the following input values:
options := &helm.Options{
Expand Down Expand Up @@ -393,7 +392,7 @@ func TestCockroachDbWithCertManager(t *testing.T) {

k8s.CreateNamespace(t, kubectlOptions, namespaceName)
// ... and make sure to delete the namespace at the end of the test
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
defer testutil.DeleteNamespace(t, k8sClient, namespaceName)

certManagerHelmOptions := &helm.Options{
KubectlOptions: k8s.NewKubectlOptions("", "", "cert-manager"),
Expand Down Expand Up @@ -519,7 +518,7 @@ func testWALFailoverExistingCluster(t *testing.T, additionalValues map[string]st
}

k8s.CreateNamespace(t, kubectlOptions, namespaceName)
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
defer testutil.DeleteNamespace(t, k8sClient, namespaceName)

// Print the debug logs in case of test failure.
defer func() {
Expand Down
6 changes: 3 additions & 3 deletions tests/e2e/rotate/cockroachdb_rotate_certs_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
package rotate
package integration

import (
"fmt"
"github.com/gruntwork-io/terratest/modules/shell"
"path"
"path/filepath"
"strings"
Expand All @@ -13,6 +12,7 @@ import (
"github.com/gruntwork-io/terratest/modules/helm"
"github.com/gruntwork-io/terratest/modules/k8s"
"github.com/gruntwork-io/terratest/modules/random"
"github.com/gruntwork-io/terratest/modules/shell"
"github.com/stretchr/testify/require"
_ "k8s.io/client-go/plugin/pkg/client/auth"
ctrl "sigs.k8s.io/controller-runtime"
Expand Down Expand Up @@ -55,7 +55,7 @@ func TestCockroachDbRotateCertificates(t *testing.T) {

k8s.CreateNamespace(t, kubectlOptions, namespaceName)
// ... and make sure to delete the namespace at the end of the test
defer k8s.DeleteNamespace(t, kubectlOptions, namespaceName)
defer testutil.DeleteNamespace(t, k8sClient, namespaceName)

// Setup the args. For this test, we will set the following input values:
helmValues := map[string]string{
Expand Down
Loading