Support user configs, user secrets and separate environments for cassandra and sidecar (#218)

* Initial work for clouds secrets and user-defined configmap

Signed-off-by: Alex Lourie <[email protected]>

* PR comments fixes

Fixes #213
Fixes #208

* Fixed some comments from PR review
* Added Env to CRD to allow specifying environment for containers
(exists in java version)
* Added userConfigMap handling (#213)
* Restored PrivelegedSupported handling (#208)
* Backup secret volume allows providing GOOGLE_APPLICATION_CREDENTIALS
in a secret
* Cloud providers creds can be set using Env field in CRD

Signed-off-by: Alex Lourie <[email protected]>

* Support TLS certificates for internal communication

Signed-off-by: Alex Lourie <[email protected]>

* PR comments, docs updates

Signed-off-by: Alex Lourie <[email protected]>

* Cleanups

Signed-off-by: Alex Lourie <[email protected]>

* Path naming update

Signed-off-by: Alex Lourie <[email protected]>

* Cleanups

* go fmt

Signed-off-by: Alex Lourie <[email protected]>

Author: alourie

deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml

@@ -29,6 +29,12 @@ spec:
           type: object
         spec:
           properties:
+            backupSecretVolumeSource:
+              type: object
+            cassandraEnv:
+              items:
+                type: object
+              type: array
             cassandraImage:
               type: string
             cluster:
@@ -46,15 +52,25 @@ spec:
             nodes:
               format: int32
               type: integer
+            privilegedSupported:
+              type: boolean
             prometheusSupport:
               type: boolean
             resources:
               type: object
+            sidecarEnv:
+              items:
+                type: object
+              type: array
             serviceAccountName:
               description: ServiceAccount to assign to pods created by the operator
               type: string
             sidecarImage:
               type: string
+            userConfigMapVolumeSource:
+              type: object
+            userSecretVolumeSource:
+              type: object
           required:
           - nodes
           - cassandraImage

doc/backup_restore.md

@@ -22,7 +22,7 @@ You can inspect the secret created via `kubectl describe secrets/awsbackuptest`
 Create a `CassandraDataCenter` CRD that injects the secret as environment variables that matches the AWS client libraries expected env variables:
 
 ```yaml
-  env:
+  sidecarEnv:
     - name: AWS_ACCESS_KEY_ID
       valueFrom:
         secretKeyRef:
@@ -65,7 +65,7 @@ spec:
     resources:
       requests:
         storage: 100Mi
-  env:
+  sidecarEnv:
     - name: AWS_ACCESS_KEY_ID
       valueFrom:
         secretKeyRef:
@@ -85,6 +85,9 @@ spec:
 To create a cluster using this yaml file use `kubectl apply -f myBackupCluster.yaml`
 
 ## Configuring GCP Object Storage via environment variables
+The backup credentials will be added to the sidecar container at the `/tmp/backup-creds` location. 
+Use this location to set GOOGLE_APPLICATION_CREDENTIALS environment variable to the key json file stored in the secret.
+
 First create a secret in kubernetes to hold a Google service account token/file (assuming they are stored in files named access and secret respectively).
 
 `kubectl create secret generic gcp-auth-reference --from-file=my_service_key.json`
@@ -118,14 +121,14 @@ spec:
     resources:
       requests:
         storage: 100Mi
-  userSecretSource:
+  backupSecretVolumeSource:
     name: gcp-auth-reference
     items:
       - key: my_service_key.json
         path:  my_service_key.json
-  env:
+  sidecarEnv:
     - name: GOOGLE_APPLICATION_CREDENTIALS
-      value: "/tmp/user-secret/my_service_key.json"
+      value: "/tmp/backup-creds/my_service_key.json"
     - name: GOOGLE_CLOUD_PROJECT
       value: "cassandra-operator"
     - name: BUCKET_NAME

examples/go/example-datacenter.yaml

@@ -10,6 +10,27 @@ spec:
   cassandraImage: "gcr.io/cassandra-operator/cassandra:3.11.3"
   sidecarImage: "gcr.io/cassandra-operator/cassandra-sidecar:latest"
   imagePullPolicy: IfNotPresent
+  imagePullSecrets:
+    - name: regcred
+  backupSecretVolumeSource:
+    # example from doc/backup_restore.md
+    secretName: gcp-auth-reference
+    #     type is a workaround for https://github.com/kubernetes/kubernetes/issues/68466
+    type: array
+    items:
+      - key: my-service-key.json
+        path: my-service-key.json
+  sidecarEnv:
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/tmp/backup-creds/my-service-key.json"
+  userConfigMapVolumeSource:
+    # example from doc/providers/pks.md
+    # the name of the ConfigMap
+    name: concurrent-data
+    # ConfigMap keys -> file paths (relative to /etc/cassandra)
+    items:
+      - key: 100-concurrent-yaml
+        path: cassandra.yaml.d/100-concurrent.yaml
   resources:
     limits:
       memory: 1Gi

pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter_types.go

@@ -10,19 +10,21 @@ import (
 type CassandraDataCenterSpec struct {
 	// Cluster is either a string or v1.LocalObjectReference
 	//Cluster interface{} `json:"cluster,omitempty"`
-	Cluster          string                    `json:"cluster,omitempty"`
-	Nodes            int32                     `json:"nodes"`
-	CassandraImage   string                    `json:"cassandraImage"`
-	SidecarImage     string                    `json:"sidecarImage"`
-	ImagePullPolicy  v1.PullPolicy             `json:"imagePullPolicy"`
-	ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
-
-	Resources v1.ResourceRequirements `json:"resources"`
-
-	DataVolumeClaimSpec v1.PersistentVolumeClaimSpec `json:"dataVolumeClaimSpec"`
-
-	PrometheusSupport bool `json:"prometheusSupport"`
-
+	Cluster                   string                       `json:"cluster,omitempty"`
+	Nodes                     int32                        `json:"nodes"`
+	CassandraImage            string                       `json:"cassandraImage"`
+	SidecarImage              string                       `json:"sidecarImage"`
+	ImagePullPolicy           v1.PullPolicy                `json:"imagePullPolicy"`
+	ImagePullSecrets          []v1.LocalObjectReference    `json:"imagePullSecrets,omitempty"`
+	BackupSecretVolumeSource  *v1.SecretVolumeSource       `json:"backupSecretVolumeSource,omitempty"`
+	UserSecretVolumeSource    *v1.SecretVolumeSource       `json:"userSecretVolumeSource,omitempty"`
+	UserConfigMapVolumeSource *v1.ConfigMapVolumeSource    `json:"userConfigMapVolumeSource,omitempty"`
+	Resources                 v1.ResourceRequirements      `json:"resources"`
+	DataVolumeClaimSpec       v1.PersistentVolumeClaimSpec `json:"dataVolumeClaimSpec"`
+	PrivilegedSupported       bool                         `json:"privilegedSupported,omitempty"`
+	PrometheusSupport         bool                         `json:"prometheusSupport"`
+	SidecarEnv                []v1.EnvVar                  `json:"sidecarEnv,omitempty"`
+	CassandraEnv              []v1.EnvVar                  `json:"cassandraEnv,omitempty"`
 	// ServiceAccount to assign to pods created by the operator
 	ServiceAccountName string `json:"serviceAccountName,omitempty"`
 }

pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go

@@ -309,6 +309,21 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm
 							},
 						},
 					},
+					"backupSecretVolumeSource": {
+						SchemaProps: spec.SchemaProps{
+							Ref: ref("k8s.io/api/core/v1.SecretVolumeSource"),
+						},
+					},
+					"userSecretVolumeSource": {
+						SchemaProps: spec.SchemaProps{
+							Ref: ref("k8s.io/api/core/v1.SecretVolumeSource"),
+						},
+					},
+					"userConfigMapVolumeSource": {
+						SchemaProps: spec.SchemaProps{
+							Ref: ref("k8s.io/api/core/v1.ConfigMapVolumeSource"),
+						},
+					},
 					"resources": {
 						SchemaProps: spec.SchemaProps{
 							Ref: ref("k8s.io/api/core/v1.ResourceRequirements"),
@@ -319,12 +334,42 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm
 							Ref: ref("k8s.io/api/core/v1.PersistentVolumeClaimSpec"),
 						},
 					},
+					"privilegedSupported": {
+						SchemaProps: spec.SchemaProps{
+							Type:   []string{"boolean"},
+							Format: "",
+						},
+					},
 					"prometheusSupport": {
 						SchemaProps: spec.SchemaProps{
 							Type:   []string{"boolean"},
 							Format: "",
 						},
 					},
+					"sidecarEnv": {
+						SchemaProps: spec.SchemaProps{
+							Type: []string{"array"},
+							Items: &spec.SchemaOrArray{
+								Schema: &spec.Schema{
+									SchemaProps: spec.SchemaProps{
+										Ref: ref("k8s.io/api/core/v1.EnvVar"),
+									},
+								},
+							},
+						},
+					},
+					"cassandraEnv": {
+						SchemaProps: spec.SchemaProps{
+							Type: []string{"array"},
+							Items: &spec.SchemaOrArray{
+								Schema: &spec.Schema{
+									SchemaProps: spec.SchemaProps{
+										Ref: ref("k8s.io/api/core/v1.EnvVar"),
+									},
+								},
+							},
+						},
+					},
 					"serviceAccountName": {
 						SchemaProps: spec.SchemaProps{
 							Description: "ServiceAccount to assign to pods created by the operator",
@@ -337,7 +382,7 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm
 			},
 		},
 		Dependencies: []string{
-			"k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaimSpec", "k8s.io/api/core/v1.ResourceRequirements"},
+			"k8s.io/api/core/v1.ConfigMapVolumeSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaimSpec", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecretVolumeSource"},
 	}
 }
 

pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go

@@ -38,6 +38,7 @@ func createOrUpdateOperatorConfigMap(rctx *reconciliationRequestContext, seedNod
 
 		addPrometheusSupport(rctx.cdc, addFileFn)
 
+
 		if err := controllerutil.SetControllerReference(rctx.cdc, configMap, rctx.scheme); err != nil {
 			return err
 		}