Nishanth/k3d registry

.github/workflows/ci.yaml

@@ -2,8 +2,7 @@ name: Helm Chart Package CI
 on:
   pull_request:
     branches:
-      - 'master'
-      - 'cert-manager-feature-branch'
+      - '*'
 
 jobs:
 

Makefile

@@ -1,4 +1,5 @@
 UNAME_S := $(shell uname -s)
+NC := $(shell tput sgr0) # No Color
 ifeq ($(UNAME_S),Linux)
   COCKROACH_BIN ?= https://binaries.cockroachdb.com/cockroach-v23.2.0.linux-amd64.tgz
   HELM_BIN ?= https://get.helm.sh/helm-v3.14.0-linux-amd64.tar.gz
@@ -21,7 +22,11 @@ ifeq ($(UNAME_S),Darwin)
 endif
 
 K3D_CLUSTER ?= chart-testing
-REPOSITORY ?= gcr.io/cockroachlabs-helm-charts/cockroach-self-signer-cert
+REGISTRY ?= gcr.io
+REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
+DOCKER_NETWORK_NAME ?= "k3d-${K3D_CLUSTER}"
+LOCAL_REGISTRY ?= "localhost:5000"
+CLUSTER_SIZE ?= 1
 
 export BUNDLE_IMAGE ?= cockroach-operator-bundle
 export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
@@ -56,7 +61,7 @@ build/chart: bin/helm ## build the helm chart to build/artifacts
 build/self-signer: bin/yq ## build the self-signer image
 	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Release
 
@@ -66,20 +71,44 @@ release: ## publish the build artifacts to S3
 build-and-push/self-signer: bin/yq ## push the self-signer image
 	@docker buildx build --platform=linux/amd64,linux/arm64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
 		--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) --push \
-		-t ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+		-t ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
 
 ##@ Dev
 dev/clean: ## remove built artifacts
 	@rm -r build/artifacts/
 
+## Setup/teardown registries for easier local dev
+dev/registries/up: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" = "" ]; then \
+		echo "$(CYAN)Starting local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		cd ../../bin/k3d; ./tests/k3d/registries.sh up $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/registries/down: bin/k3d
+	@if [ "`docker ps -f name=registry.localhost -q`" != "" ]; then \
+		echo "$(CYAN)Stopping local Docker registry (for fast offline image push/pull)...$(NC)"; \
+		cd ../../bin/k3d; ./tests/k3d/registries.sh down $(DOCKER_NETWORK_NAME); \
+	fi
+
+dev/registries/bounce: bin/k3d dev/registries/down dev/registries/up
+
+dev/push/local: dev/registries/up
+	@echo "$(CYAN)Pushing image to local registry...$(NC)"
+	@docker build --platform=linux/amd64 -f build/docker-image/self-signer-cert-utility/Dockerfile \
+          	--build-arg COCKROACH_VERSION=$(shell bin/yq '.appVersion' ./cockroachdb/Chart.yaml) \
+          	-t ${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) .
+	@docker push "${LOCAL_REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml)"
+
 ##@ Test
-test/cluster: bin/k3d test/cluster_up ## start a local k3d cluster for testing
+test/cluster: bin/k3d test/cluster/up ## start a local k3d cluster for testing
+
+test/cluster/bounce: bin/k3d test/cluster/down test/cluster/up ## restart a local k3d cluster for testing
 
-test/cluster_up: bin/k3d
-	@bin/k3d cluster list | grep $(K3D_CLUSTER) || bin/k3d cluster create $(K3D_CLUSTER)
+test/cluster/up: bin/k3d
+	@bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(CLUSTER_SIZE)
 
-test/cluster_down: bin/k3d
-	bin/k3d cluster delete $(K3D_CLUSTER)
+test/cluster/down: bin/k3d
+	./tests/k3d/dev-cluster.sh down --name "$(K3D_CLUSTER)"
 
 test/e2e/%: PKG=$*
 test/e2e/%: bin/cockroach bin/kubectl bin/helm build/self-signer test/publish-images-to-k3d ## run e2e tests for package (e.g. install or rotate)
@@ -94,9 +123,9 @@ test/publish-images-to-k3d: bin/yq test/cluster ## publish signer and cockroach
 		docker pull $$i; \
 		bin/k3d image import $$i -c $(K3D_CLUSTER); \
 	done
-	docker pull ${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
+	docker pull ${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml); \
 	bin/k3d image import \
-		${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
+		${REGISTRY}/${REPOSITORY}:$(shell bin/yq '.tls.selfSigner.image.tag' ./cockroachdb/values.yaml) \
 		-c $(K3D_CLUSTER)
 
 test/template: bin/cockroach bin/helm ## Run template tests

tests/k3d/dev-cluster.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+region="us-east-1"
+zones=3
+
+K3D_PATH="./bin/k3d"
+
+if [ $# -eq 0 ]; then
+  echo "No arguments supplied: "
+  echo "  up: Start cluster."
+  echo "     --nodes x: The cluster should have x nodes (default 1)"
+  echo "     --version x: The version of Kubernetes (default 1.24.14)"
+  echo "     --name x: The name of the cluster (default local)"
+  echo "     --network_name x: The name of the cluster's network (default k3d-\${name})"
+  echo "     --region x: The name of the cluster's region for node labels topology.kubernetes.io/region (default us-east-1)"
+  echo "     --zones x: The number of zones in the region for node labels topology.kubernetes.io/zone (default 3)"
+
+  echo "  down: Delete cluster."
+
+  exit 1
+fi
+
+COMMAND="${1-}"
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+nodes=${environment:-1}
+version=${version:-1.31.2}
+
+while [ $# -gt 0 ]; do
+
+  if [[ $1 == *"--"* ]]; then
+    param="${1/--/}"
+    declare $param="$2"
+    # echo $1 $2 // Optional to see the parameter:value result
+  fi
+
+  shift
+done
+
+name=${name:-local}
+network_name=${network_name:-"k3d-${name}"}
+
+# Function to set topology.kubernetes.io/zone labels in a round-robin fashion
+set_node_labels() {
+  local nodes=$1
+  local region=$2
+  local zones=$3
+  local labels=""
+  local az=(a b c d e f g h i j k l m n o p q r s t u v w x y z)
+
+  for ((i = 0; i < nodes; i++)); do
+    zone="${region}${az[$((i % zones))]}"
+    labels+="--k3s-node-label topology.kubernetes.io/zone=${zone}@agent:${i} "
+    labels+="--k3s-node-label topology.kubernetes.io/region=${region}@agent:${i} "
+  done
+
+  echo "${labels}"
+}
+
+case $COMMAND in
+up)
+  node_labels=$(set_node_labels ${nodes} ${region} ${zones})
+  ${K3D_PATH} cluster create ${name} \
+    --network ${network_name} \
+    --registry-config "$SCRIPT_DIR/registries.yaml" \
+    --image rancher/k3s:v${version}-k3s1 \
+    --agents ${nodes} \
+    --k3s-node-label "topology.kubernetes.io/region=${region}@server:0" \
+    ${node_labels}
+  ;;
+down)
+  ${K3D_PATH} cluster delete ${name}
+  ;;
+*)
+  echo "Unknown command: $COMMAND"
+  exit 1
+  ;;
+esac

tests/k3d/docker-compose.yaml

@@ -0,0 +1,65 @@
+version: "3.3"
+
+#
+# Each proxy needs a 'dns' section added. When it wasn't there, the registries were unable to resolve
+# remote DNS names, only DNS names set up within the network of the docker-compose. Here the dns is pointed
+# at the Google DNS servers.
+#
+
+services:
+  registry-localhost:
+    image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
+    restart: "always"
+    volumes:
+      - "registry:/var/lib/registry"
+    ports:
+      - "5000:5000"
+
+  registry-quayio:
+    image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
+    restart: "always"
+    dns:
+      - 8.8.8.8
+    volumes:
+      - "registry:/var/lib/registry"
+    environment:
+      REGISTRY_PROXY_REMOTEURL: "https://quay.io/repository"
+      REGISTRY_COMPATIBILITY_SCHEMA1_ENABLED: "true"
+
+  registry-dockerio:
+    image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
+    restart: "always"
+    dns:
+      - 8.8.8.8
+    volumes:
+      - "registry:/var/lib/registry"
+    environment:
+      REGISTRY_PROXY_REMOTEURL: "http://registry-1.docker.io"
+
+  registry-us-gcr-io:
+    image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
+    restart: "always"
+    dns:
+      - 8.8.8.8
+    volumes:
+      - "registry:/var/lib/registry"
+    environment:
+      REGISTRY_PROXY_REMOTEURL: "https://us.gcr.io"
+
+  us-docker-pkg-dev:
+    image: "us-east1-docker.pkg.dev/crl-docker-sync/docker-io/library/registry:2"
+    restart: "always"
+    dns:
+      - 8.8.8.8
+    volumes:
+      - "registry:/var/lib/registry"
+    environment:
+      REGISTRY_PROXY_REMOTEURL: "https://us-docker.pkg.dev"
+
+volumes:
+  registry: { }
+
+networks:
+  default:
+    external: true
+    name: ${DOCKER_NETWORK_NAME}

tests/k3d/registries.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -euxo pipefail
+
+# Figure out, regardless of any symlinks, aliases, etc, where this script
+# is located.
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ] ; do SOURCE="$(readlink "$SOURCE")"; done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+
+COMMAND="${1-}"
+
+DOCKER_REGISTRY_PROJECT_NAME=${2:-k3d-local}
+
+# The name of the docker network. This must change if it changes in the docker-compose.yaml file.
+DOCKER_REGISTRY_NETWORK_NAME=${2:-k3d-local}
+
+
+case $COMMAND in
+  up)
+    docker network create --driver bridge ${DOCKER_REGISTRY_NETWORK_NAME} || true
+    DOCKER_NETWORK_NAME=${DOCKER_REGISTRY_NETWORK_NAME} docker-compose -p ${DOCKER_REGISTRY_PROJECT_NAME} -f ${DIR}/docker-compose.yaml up -d
+  ;;
+  down)
+    DOCKER_NETWORK_NAME=${DOCKER_REGISTRY_NETWORK_NAME} docker-compose -p ${DOCKER_REGISTRY_PROJECT_NAME} -f ${DIR}/docker-compose.yaml down
+  ;;
+  *)
+    echo "Unknown command: $COMMAND"
+    exit 1;
+  ;;
+esac

tests/k3d/registries.yaml

@@ -0,0 +1,16 @@
+mirrors:
+  "localhost:5000":
+    endpoint:
+      - "http://registry-localhost:5000"
+  quay.io:
+    endpoint:
+      - "http://registry-quayio:5000"
+  docker.io:
+    endpoint:
+      - "http://registry-dockerio:5000"
+  us.gcr.io:
+    endpoint:
+      - "http://registry-us-gcr-io:5000"
+  us-docker.pkg.dev:
+    endpoint:
+      - "http://us-docker-pkg-dev:5000"