diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 734733e5..8607d144 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -2,8 +2,7 @@ name: Helm Chart Package CI on: pull_request: branches: - - 'master' - - 'cert-manager-feature-branch' + - '*' jobs: diff --git a/Makefile b/Makefile index 63bef87e..d9ea3642 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ UNAME_S := $(shell uname -s) +NC := $(shell tput sgr0) # No Color ifeq ($(UNAME_S),Linux) COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz @@ -21,7 +22,11 @@ ifeq ($(UNAME_S),Darwin) endif K3D_CLUSTER ?= chart-testing -REPOSITORY ?= gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert +REGISTRY ?= gcr.io +REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert +DOCKER_NETWORK_NAME ?= "k3d-${K3D_CLUSTER}" +LOCAL_REGISTRY ?= "localhost:5000" +CLUSTER_SIZE ?= 1 export BUNDLE_IMAGE ?= cockroach-operator-bundle export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator @@ -56,7 +61,7 @@ build/chart: bin/helm ## build the helm chart to build/artifacts build/self-signer: bin/yq ## build the self-signer image @docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \ --build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \ - -t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) . + -t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) . ##@ Release @@ -66,20 +71,44 @@ release: ## publish the build artifacts to S3 build-and-push/self-signer: bin/yq ## push the self-signer image @docker buildx build --platform=linux/amd64,linux/arm64 -f build/docker-image/self-signer-cert-utility/Dockerfile \ --build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \ - -t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) . + -t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) . ##@ Dev dev/clean: ## remove built artifacts @rm -r build/artifacts/ +## Setup/teardown registries for easier local dev +dev/registries/up: bin/k3d + @if [ "`docker ps -f name=registry.localhost -q`" = "" ]; then \ + echo "$(CYAN)Starting local Docker registry (for fast offline image push/pull)...$(NC)"; \ + cd ../../bin/k3d; ./tests/k3d/registries.sh up $(DOCKER_NETWORK_NAME); \ + fi + +dev/registries/down: bin/k3d + @if [ "`docker ps -f name=registry.localhost -q`" != "" ]; then \ + echo "$(CYAN)Stopping local Docker registry (for fast offline image push/pull)...$(NC)"; \ + cd ../../bin/k3d; ./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \ + fi + +dev/registries/bounce: bin/k3d dev/registries/down dev/registries/up + +dev/push/local: dev/registries/up + @echo "$(CYAN)Pushing image to local registry...$(NC)" + @docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \ + --build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \ + -t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) . + @docker push "${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml)" + ##@ Test -test/cluster: bin/k3d test/cluster_up ## start a local k3d cluster for testing +test/cluster: bin/k3d test/cluster/up ## start a local k3d cluster for testing + +test/cluster/bounce: bin/k3d test/cluster/down test/cluster/up ## restart a local k3d cluster for testing -test/cluster_up: bin/k3d - @bin/k3d cluster list | grep $(K3D_CLUSTER) || bin/k3d cluster create $(K3D_CLUSTER) +test/cluster/up: bin/k3d + @bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(CLUSTER_SIZE) -test/cluster_down: bin/k3d - bin/k3d cluster delete $(K3D_CLUSTER) +test/cluster/down: bin/k3d + ./tests/k3d/dev-cluster.sh down --name "$(K3D_CLUSTER)" test/e2e/%: PKG=$* test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate) @@ -94,9 +123,9 @@ test/publish-images-to-k3d: bin/yq test/cluster ## publish signer and cockroach docker pull $$i; \ bin/k3d image import $$i -c $(K3D_CLUSTER); \ done - docker pull ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \ + docker pull ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \ bin/k3d image import \ - ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \ + ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \ -c $(K3D_CLUSTER) test/template: bin/cockroach bin/helm ## Run template tests diff --git a/tests/k3d/dev-cluster.sh b/tests/k3d/dev-cluster.sh new file mode 100755 index 00000000..90f39668 --- /dev/null +++ b/tests/k3d/dev-cluster.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +region="us-east-1" +zones=3 + +K3D_PATH="./bin/k3d" + +if [ $# -eq 0 ]; then + echo "No arguments supplied: " + echo " up: Start cluster." + echo " --nodes x: The cluster should have x nodes (default 1)" + echo " --version x: The version of Kubernetes (default 1.24.14)" + echo " --name x: The name of the cluster (default local)" + echo " --network_name x: The name of the cluster's network (default k3d-\${name})" + echo " --region x: The name of the cluster's region for node labels topology.kubernetes.io/region (default us-east-1)" + echo " --zones x: The number of zones in the region for node labels topology.kubernetes.io/zone (default 3)" + + echo " down: Delete cluster." + + exit 1 +fi + +COMMAND="${1-}" +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +nodes=${environment:-1} +version=${version:-1.31.2} + +while [ $# -gt 0 ]; do + + if [[ $1 == *"--"* ]]; then + param="${1/--/}" + declare $param="$2" + # echo $1 $2 // Optional to see the parameter:value result + fi + + shift +done + +name=${name:-local} +network_name=${network_name:-"k3d-${name}"} + +# Function to set topology.kubernetes.io/zone labels in a round-robin fashion +set_node_labels() { + local nodes=$1 + local region=$2 + local zones=$3 + local labels="" + local az=(a b c d e f g h i j k l m n o p q r s t u v w x y z) + + for ((i = 0; i < nodes; i++)); do + zone="${region}${az[$((i % zones))]}" + labels+="--k3s-node-label topology.kubernetes.io/zone=${zone}@agent:${i} " + labels+="--k3s-node-label topology.kubernetes.io/region=${region}@agent:${i} " + done + + echo "${labels}" +} + +case $COMMAND in +up) + node_labels=$(set_node_labels ${nodes} ${region} ${zones}) + ${K3D_PATH} cluster create ${name} \ + --network ${network_name} \ + --registry-config "$SCRIPT_DIR/registries.yaml" \ + --image rancher/k3s:v${version}-k3s1 \ + --agents ${nodes} \ + --k3s-node-label "topology.kubernetes.io/region=${region}@server:0" \ + ${node_labels} + ;; +down) + ${K3D_PATH} cluster delete ${name} + ;; +*) + echo "Unknown command: $COMMAND" + exit 1 + ;; +esac \ No newline at end of file diff --git a/tests/k3d/docker-compose.yaml b/tests/k3d/docker-compose.yaml new file mode 100644 index 00000000..e83283ae --- /dev/null +++ b/tests/k3d/docker-compose.yaml @@ -0,0 +1,65 @@ +version: "3.3" + +# +# Each proxy needs a 'dns' section added. When it wasn't there, the registries were unable to resolve +# remote DNS names, only DNS names set up within the network of the docker-compose. Here the dns is pointed +# at the Google DNS servers. +# + +services: + registry-localhost: + image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2" + restart: "always" + volumes: + - "registry:/var/lib/registry" + ports: + - "5000:5000" + + registry-quayio: + image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2" + restart: "always" + dns: + - 8.8.8.8 + volumes: + - "registry:/var/lib/registry" + environment: + REGISTRY_PROXY_REMOTEURL: "https://quay.io/repository" + REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED: "true" + + registry-dockerio: + image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2" + restart: "always" + dns: + - 8.8.8.8 + volumes: + - "registry:/var/lib/registry" + environment: + REGISTRY_PROXY_REMOTEURL: "http://registry-1.docker.io" + + registry-us-gcr-io: + image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2" + restart: "always" + dns: + - 8.8.8.8 + volumes: + - "registry:/var/lib/registry" + environment: + REGISTRY_PROXY_REMOTEURL: "https://us.gcr.io" + + us-docker-pkg-dev: + image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2" + restart: "always" + dns: + - 8.8.8.8 + volumes: + - "registry:/var/lib/registry" + environment: + REGISTRY_PROXY_REMOTEURL: "https://us-docker.pkg.dev" + +volumes: + registry: { } + +networks: + default: + external: true + name: ${DOCKER_NETWORK_NAME} diff --git a/tests/k3d/registries.sh b/tests/k3d/registries.sh new file mode 100755 index 00000000..e36404c6 --- /dev/null +++ b/tests/k3d/registries.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -euxo pipefail + +# Figure out, regardless of any symlinks, aliases, etc, where this script +# is located. +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done +DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + + +COMMAND="${1-}" + +DOCKER_REGISTRY_PROJECT_NAME=${2:-k3d-local} + +# The name of the docker network. This must change if it changes in the docker-compose.yaml file. +DOCKER_REGISTRY_NETWORK_NAME=${2:-k3d-local} + + +case $COMMAND in + up) + docker network create --driver bridge ${DOCKER_REGISTRY_NETWORK_NAME} || true + DOCKER_NETWORK_NAME=${DOCKER_REGISTRY_NETWORK_NAME} docker-compose -p ${DOCKER_REGISTRY_PROJECT_NAME} -f ${DIR}/docker-compose.yaml up -d + ;; + down) + DOCKER_NETWORK_NAME=${DOCKER_REGISTRY_NETWORK_NAME} docker-compose -p ${DOCKER_REGISTRY_PROJECT_NAME} -f ${DIR}/docker-compose.yaml down + ;; + *) + echo "Unknown command: $COMMAND" + exit 1; + ;; +esac diff --git a/tests/k3d/registries.yaml b/tests/k3d/registries.yaml new file mode 100644 index 00000000..4faaa7b5 --- /dev/null +++ b/tests/k3d/registries.yaml @@ -0,0 +1,16 @@ +mirrors: + "localhost:5000": + endpoint: + - "http://registry-localhost:5000" + quay.io: + endpoint: + - "http://registry-quayio:5000" + docker.io: + endpoint: + - "http://registry-dockerio:5000" + us.gcr.io: + endpoint: + - "http://registry-us-gcr-io:5000" + us-docker.pkg.dev: + endpoint: + - "http://us-docker-pkg-dev:5000" \ No newline at end of file