Deploy using the operator PoC

Makefile

@@ -24,8 +24,9 @@ endif
 K3D_CLUSTER ?= chart-testing
 REGISTRY ?= gcr.io
 REPOSITORY ?= cockroachlabs-helm-charts/cockroach-self-signer-cert
-DOCKER_NETWORK_NAME ?= ${K3D_CLUSTER}
+DOCKER_NETWORK_NAME ?= "k3d-${K3D_CLUSTER}"
 LOCAL_REGISTRY ?= "localhost:5000"
+CLUSTER_SIZE ?= 1
 
 export BUNDLE_IMAGE ?= cockroach-operator-bundle
 export HELM_OPERATOR_IMAGE ?= cockroach-helm-operator
@@ -103,7 +104,7 @@ test/cluster: bin/k3d test/cluster/up ## start a local k3d cluster for testing
 test/cluster/bounce: bin/k3d test/cluster/down test/cluster/up ## restart a local k3d cluster for testing
 
 test/cluster/up: bin/k3d
-	@bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)"
+	@bin/k3d cluster list | grep $(K3D_CLUSTER) || ./tests/k3d/dev-cluster.sh up --name "$(K3D_CLUSTER)" --nodes $(CLUSTER_SIZE)
 
 test/cluster/down: bin/k3d
 	./tests/k3d/dev-cluster.sh down --name "$(K3D_CLUSTER)"

cockroachdb/Chart.yaml

@@ -2,7 +2,7 @@
 apiVersion: v1
 name: cockroachdb
 home: https://www.cockroachlabs.com
-version: 14.0.5
+version: 14.0.6
 appVersion: 24.2.4
 description: CockroachDB is a scalable, survivable, strongly-consistent SQL database.
 icon: https://raw.githubusercontent.com/cockroachdb/cockroach/master/docs/media/cockroach_db.png

cockroachdb/templates/_helpers.tpl

@@ -289,3 +289,23 @@ Validate that if user enabled tls, then either self-signed certificates or certi
 {{- end }}
 {{- end }}
 {{- end }}
+
+
+{{/* Common labels that are applied to all managed objects. */}}
+{{- define "cluster.labels" -}}
+helm.sh/chart: {{ include "cockroachdb.chart" . }}
+{{ include "cluster.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+  Selector labels defines the set of labels that can be used as selectors for 
+  crdb nodes.
+*/}}
+{{- define "cluster.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "cockroachdb.clusterfullname" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

cockroachdb/templates/crdb.yaml

@@ -0,0 +1,48 @@
+{{- if .Values.operator.enabled }}
+---
+apiVersion: crdb.cockroachlabs.com/v1alpha1
+kind: CrdbCluster
+metadata:
+  name: {{ .Release.Name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "cluster.labels" . | nindent 4 }}
+  {{- with .Values.statefulset.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+spec:
+  {{- with .Values.operator.clusterSettings }}
+  clusterSettings: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.operator.regions }}
+  regions: {{- toYaml . | nindent 4 }}
+  {{- end }}
+  features:
+    - reconcile
+    - reconcile-beta
+  template:
+    spec:
+      image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}"
+      certificates:
+        externalCertificates:
+          clientCaConfigMapName: {{ .Values.operator.certificates.externalCertificates.clientCaConfigMapName | default (printf "%s-client-ca" $.Release.Name) }}
+          nodeCaConfigMapName: {{ .Values.operator.certificates.externalCertificates.nodeCaConfigMapName | default (printf "%s-node-ca" $.Release.Name) }}
+          httpSecretName: {{ .Values.operator.certificates.externalCertificates.httpSecretName | default (printf "%s-ui-certs" $.Release.Name) }} 
+          nodeClientSecretName: {{ .Values.operator.certificates.externalCertificates.nodeClientSecretName | default (printf "%s-node-client-certs" $.Release.Name) }} 
+          nodeSecretName: {{ .Values.operator.certificates.externalCertificates.nodeSecretName | default (printf "%s-node-certs" $.Release.Name) }} 
+          rootSqlClientSecretName: {{ .Values.operator.certificates.externalCertificates.rootSqlClientSecretName | default (printf "%s-client-certs" $.Release.Name) }}
+    {{- with .Values.operator.resources }}
+      resourceRequirements: {{- toYaml . | nindent 8 }}
+    {{- end }}
+      serviceAccountName: {{ default .Release.Name .Values.operator.rbac.serviceAccountName }}
+    {{- if .Values.operator.loggingConf }}
+      loggingConfigMapName: {{ .Release.Name }}-logging
+    {{- end }}
+  # All properties below are solely to pass validation. They aren't used by the
+  # betaclusterctrl controller so the values don't matter so long as they're
+  # valid.
+  dataStore: {}
+{{- end }}

cockroachdb/templates/job.init.yaml

@@ -1,6 +1,6 @@
 {{ $isClusterInitEnabled := and (eq (len .Values.conf.join) 0) (not (index .Values.conf `single-node`)) }}
 {{ $isDatabaseProvisioningEnabled := .Values.init.provisioning.enabled }}
-{{- if or $isClusterInitEnabled $isDatabaseProvisioningEnabled }}
+{{- if and (or $isClusterInitEnabled $isDatabaseProvisioningEnabled) (not .Values.operator.enabled) }}
   {{ template "cockroachdb.tlsValidation" . }}
 kind: Job
 apiVersion: batch/v1

cockroachdb/templates/poddisruptionbudget.yaml

@@ -1,3 +1,4 @@
+{{- if (not .Values.operator.enabled) }}
 kind: PodDisruptionBudget
 {{- if or (.Capabilities.APIVersions.Has "policy/v1") (semverCompare ">=1.21-0" .Capabilities.KubeVersion.Version) }}
 apiVersion: policy/v1
@@ -24,3 +25,4 @@ spec:
       {{- toYaml . | nindent 6 }}
     {{- end }}
   maxUnavailable: {{ .Values.statefulset.budget.maxUnavailable | int64 }}
+{{- end }}

cockroachdb/templates/role-certSelfSigner.yaml

@@ -30,4 +30,7 @@ rules:
   - apiGroups: [""]
     resources: ["pods"]
     verbs: ["delete", "get"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["create", "get", "update", "delete"]
 {{- end }}

cockroachdb/templates/role.yaml

@@ -20,4 +20,25 @@ rules:
     {{- else }}
     verbs: ["create", "get"]
     {{- end }}
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "cockroachdb.fullname" . }}-node-reader
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    helm.sh/chart: {{ template "cockroachdb.chart" . }}
+    app.kubernetes.io/name: {{ template "cockroachdb.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+  {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
 {{- end }}

cockroachdb/templates/rolebinding.yaml

@@ -20,4 +20,26 @@ subjects:
   - kind: ServiceAccount
     name: {{ template "cockroachdb.serviceAccount.name" . }}
     namespace: {{ .Release.Namespace | quote }}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ template "cockroachdb.fullname" . }}-node-reader
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    helm.sh/chart: {{ template "cockroachdb.chart" . }}
+    app.kubernetes.io/name: {{ template "cockroachdb.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name | quote }}
+    app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+  {{- with .Values.labels }}
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "cockroachdb.fullname" . }}-node-reader
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "cockroachdb.serviceAccount.name" . }}
+    namespace: {{ .Release.Namespace | quote }}
 {{- end }}

cockroachdb/templates/statefulset.yaml

@@ -1,3 +1,4 @@
+{{- if not .Values.operator.enabled }}
 kind: StatefulSet
 apiVersion: {{ template "cockroachdb.statefulset.apiVersion" . }}
 metadata:
@@ -433,3 +434,4 @@ spec:
           requests:
             storage: {{ .Values.storage.persistentVolume.size | quote }}
 {{- end }}
+{{- end }}

cockroachdb/values.yaml

@@ -613,10 +613,10 @@ tls:
     # Image Placeholder for the selfSigner utility. This will be changed once the CI workflows for the image is in place.
     image:
       repository: cockroachlabs-helm-charts/cockroach-self-signer-cert
-      tag: "1.5"
+      tag: "1.6"
       pullPolicy: IfNotPresent
       credentials: {}
-      registry: gcr.io
+      registry: "localhost:5000"
       # username: john_doe
       # password: changeme
 
@@ -652,4 +652,98 @@ iap:
 
 # Use the CRDB Operator to manage the CRDB clusters
 operator:
- enabled: false
+  enabled: true
+  # Default values for the cluster chart.
+  image:
+    repository: cockroachdb/cockroach
+    pullPolicy: IfNotPresent
+    # Overrides the image tag whose default is the cluster chart's appVersion.
+    tag: ""
+
+  nameOverride: ""
+  fullnameOverride: ""
+
+  # A map of CRDB cluster settings.
+  # See https://www.cockroachlabs.com/docs/stable/cluster-settings.html
+  clusterSettings: ~
+
+  # Regions controls the number of CRDB nodes that are deployed per region.
+  #regions: ~
+    # - code: us-central1
+      # nodes: 3
+
+  # loggingConf is the logging configuration used by cockroach.
+  # More details: https://www.cockroachlabs.com/docs/stable/logging-overview.html
+  loggingConf: ~
+    # sinks:
+      # stderr:
+        # channels: [health, dev]
+        # filter: INFO
+
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as K3D. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the tilde after 'resources:'.
+  resources: ~
+    # limits:
+      # cpu: 100m
+      # memory: 128Mi
+    # requests:
+      # cpu: 100m
+      # memory: 128Mi
+
+  certificates:
+    # Any extra alt names that should be added to the node certs.
+    extraNodeAltNames: []
+      # - somevalue
+      # - somevalue.default
+      # - somevalue.default.svc.local
+    # the number of days generated certs are valid for
+    # validForDays: 3650
+
+    # External certificates for the CRDB cluster.
+    externalCertificates: 
+      clientCaConfigMapName: my-release-cockroachdb-ca-secret-crt
+      nodeCaConfigMapName: my-release-cockroachdb-ca-secret-crt
+      httpSecretName: my-release-cockroachdb-client-secret
+      nodeClientSecretName: my-release-cockroachdb-client-secret
+      nodeSecretName: my-release-cockroachdb-node-secret
+      rootSqlClientSecretName: my-release-cockroachdb-client-secret
+
+  # RBAC settings for CRDB nodes
+  rbac:
+    # By default the service account will be the resource name. It will
+    # be created during the installation along with a namespaced role and
+    # a cluster role with the policy rules below.
+    #
+    # Uncomment the line below to use a custom SA. If a custom SA is used,
+    # no roles or bindings will be created.
+    # serviceAccountName: my-custom-sa
+
+    # Rules for the namespaced role bound to the service account.
+    #
+    # E.g.
+    # permissions:
+    #   - apiGroup: [""]
+    #     resources: ["secrets"]
+    #     verbs: ["create", "get"]
+    rules: []
+
+    # Rules for the cluster role bound to the service account.
+    clusterRules:
+      # Get nodes allows the locality container to work as expected. It pulls the
+      # failure-domain.beta.kubernetes.io/zone label to determine node locality.
+      - apiGroups: [""]
+        resources: ["nodes"]
+        verbs: ["get"]
+    serviceAccountName: ~
+
+  regions:
+  - code: us-east-1
+    nodes: 3
+    cloudProvider: k3d
+    namespace: default
+
+  extras:
+    # Add a container with dnsutils (nslookup, dig, ping, etc.) installed.
+    dnsutils: false