diff --git a/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml b/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml index 1e0bf76a..83251998 100644 --- a/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml +++ b/deploy/crds/cassandraoperator_v1alpha1_cassandradatacenter_crd.yaml @@ -27,6 +27,8 @@ spec: type: object spec: properties: + backupSecretVolume: + type: object cassandraImage: type: string cluster: @@ -35,6 +37,10 @@ spec: type: string dataVolumeClaimSpec: type: object + env: + items: + type: object + type: array imagePullPolicy: type: string imagePullSecrets: @@ -44,12 +50,16 @@ spec: nodes: format: int32 type: integer + privilegedSupported: + type: boolean prometheusSupport: type: boolean resources: type: object sidecarImage: type: string + userConfigMapVolumeSource: + type: object required: - nodes - cassandraImage diff --git a/examples/go/example-datacenter.yaml b/examples/go/example-datacenter.yaml index 8b3a5921..f91dc92e 100644 --- a/examples/go/example-datacenter.yaml +++ b/examples/go/example-datacenter.yaml @@ -7,9 +7,29 @@ metadata: spec: cluster: "test-cluster" nodes: 3 + racks: 3 cassandraImage: "gcr.io/cassandra-operator/cassandra:3.11.3" sidecarImage: "gcr.io/cassandra-operator/cassandra-sidecar:latest" imagePullPolicy: IfNotPresent + imagePullSecrets: + - name: regcred + backupSecretVolume: + secretName: backup-secret + # type is a workaround for https://github.com/kubernetes/kubernetes/issues/68466 + type: array + items: + - key: creds.json + path: creds.json + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: "/etc/google/creds.json" + userConfigMapVolumeSource: + name: concurrent-data + # type is a workaround for https://github.com/kubernetes/kubernetes/issues/68466 + type: array + items: + - key: "concurrent.yaml" + path: "cassandra.yaml.d/100-user.yaml" resources: limits: memory: 1Gi diff --git a/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter.go b/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter.go index 9f358ef1..18f667cb 100644 --- a/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter.go +++ b/pkg/apis/cassandraoperator/v1alpha1/cassandradatacenter.go @@ -10,18 +10,19 @@ import ( type CassandraDataCenterSpec struct { // Cluster is either a string or v1.LocalObjectReference //Cluster interface{} `json:"cluster,omitempty"` - Cluster string `json:"cluster,omitempty"` - Nodes int32 `json:"nodes"` - CassandraImage string `json:"cassandraImage"` - SidecarImage string `json:"sidecarImage"` - ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy"` - ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - - Resources v1.ResourceRequirements `json:"resources"` - - DataVolumeClaimSpec v1.PersistentVolumeClaimSpec `json:"dataVolumeClaimSpec"` - - PrometheusSupport bool `json:"prometheusSupport"` + Cluster string `json:"cluster,omitempty"` + Nodes int32 `json:"nodes"` + CassandraImage string `json:"cassandraImage"` + SidecarImage string `json:"sidecarImage"` + ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy"` + ImagePullSecrets []v1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + BackupSecretVolume v1.SecretVolumeSource `json:"backupSecretVolume,omitempty"` + UserConfigMapVolumeSource v1.ConfigMapVolumeSource `json:"userConfigMapVolumeSource,omitempty"` + Resources v1.ResourceRequirements `json:"resources"` + DataVolumeClaimSpec v1.PersistentVolumeClaimSpec `json:"dataVolumeClaimSpec"` + PrivilegedSupported bool `json:"privilegedSupported,omitempty"` + PrometheusSupport bool `json:"prometheusSupport"` + Env []v1.EnvVar `json:"env,omitempty"` } // CassandraDataCenterStatus defines the observed state of CassandraDataCenter diff --git a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go index 8e254cef..9d759b42 100644 --- a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.deepcopy.go @@ -14,8 +14,22 @@ func (in *CassandraBackup) DeepCopyInto(out *CassandraBackup) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec - out.Status = in.Status + in.Spec.DeepCopyInto(&out.Spec) + if in.Status != nil { + in, out := &in.Status, &out.Status + *out = make(map[string]*CassandraBackupStatus, len(*in)) + for key, val := range *in { + var outVal *CassandraBackupStatus + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = new(CassandraBackupStatus) + **out = **in + } + (*out)[key] = outVal + } + } return } @@ -73,6 +87,11 @@ func (in *CassandraBackupList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CassandraBackupSpec) DeepCopyInto(out *CassandraBackupSpec) { *out = *in + if in.Keyspaces != nil { + in, out := &in.Keyspaces, &out.Keyspaces + *out = make([]string, len(*in)) + copy(*out, *in) + } return } @@ -264,8 +283,17 @@ func (in *CassandraDataCenterSpec) DeepCopyInto(out *CassandraDataCenterSpec) { *out = make([]v1.LocalObjectReference, len(*in)) copy(*out, *in) } + in.BackupSecretVolume.DeepCopyInto(&out.BackupSecretVolume) + in.UserConfigMapVolumeSource.DeepCopyInto(&out.UserConfigMapVolumeSource) in.Resources.DeepCopyInto(&out.Resources) in.DataVolumeClaimSpec.DeepCopyInto(&out.DataVolumeClaimSpec) + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make([]v1.EnvVar, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } return } diff --git a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go index 69a1a88e..280325e9 100644 --- a/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go +++ b/pkg/apis/cassandraoperator/v1alpha1/zz_generated.openapi.go @@ -311,6 +311,16 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm }, }, }, + "backupSecretVolume": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.SecretVolumeSource"), + }, + }, + "userConfigMapVolumeSource": { + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.ConfigMapVolumeSource"), + }, + }, "resources": { SchemaProps: spec.SchemaProps{ Ref: ref("k8s.io/api/core/v1.ResourceRequirements"), @@ -321,18 +331,36 @@ func schema_pkg_apis_cassandraoperator_v1alpha1_CassandraDataCenterSpec(ref comm Ref: ref("k8s.io/api/core/v1.PersistentVolumeClaimSpec"), }, }, + "privilegedSupported": { + SchemaProps: spec.SchemaProps{ + Type: []string{"boolean"}, + Format: "", + }, + }, "prometheusSupport": { SchemaProps: spec.SchemaProps{ Type: []string{"boolean"}, Format: "", }, }, + "env": { + SchemaProps: spec.SchemaProps{ + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/api/core/v1.EnvVar"), + }, + }, + }, + }, + }, }, Required: []string{"nodes", "cassandraImage", "sidecarImage", "imagePullPolicy", "resources", "dataVolumeClaimSpec", "prometheusSupport"}, }, }, Dependencies: []string{ - "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaimSpec", "k8s.io/api/core/v1.ResourceRequirements"}, + "k8s.io/api/core/v1.ConfigMapVolumeSource", "k8s.io/api/core/v1.EnvVar", "k8s.io/api/core/v1.LocalObjectReference", "k8s.io/api/core/v1.PersistentVolumeClaimSpec", "k8s.io/api/core/v1.ResourceRequirements", "k8s.io/api/core/v1.SecretVolumeSource"}, } } diff --git a/pkg/controller/cassandradatacenter/configmap.go b/pkg/controller/cassandradatacenter/configmap.go index 40e704ad..558eab11 100644 --- a/pkg/controller/cassandradatacenter/configmap.go +++ b/pkg/controller/cassandradatacenter/configmap.go @@ -37,6 +37,7 @@ func createOrUpdateOperatorConfigMap(rctx *reconciliationRequestContext, seedNod addPrometheusSupport(rctx.cdc, addFileFn) + if err := controllerutil.SetControllerReference(rctx.cdc, configMap, rctx.scheme); err != nil { return err } diff --git a/pkg/controller/cassandradatacenter/statefulset.go b/pkg/controller/cassandradatacenter/statefulset.go index 26617690..1f521fe5 100644 --- a/pkg/controller/cassandradatacenter/statefulset.go +++ b/pkg/controller/cassandradatacenter/statefulset.go @@ -12,21 +12,32 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "path" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "strings" "sync" ) -const DataVolumeMountPath = "/var/lib/cassandra" +const ( + DataVolumeMountPath = "/var/lib/cassandra" + OperatorConfigVolumeMountPath = "/tmp/operator-config" + UserConfigVolumeMountPath = "/tmp/user-config" +) const SidecarApiPort = 4567 +const GOOGLE_APPLICATION_CREDENTIALS = "GOOGLE_APPLICATION_CREDENTIALS" + var sidecarClientOptions = sidecar.ClientOptions{ Port: SidecarApiPort, Secure: false, } +func boolRef(b bool) *bool { + return &b +} + func createOrUpdateStatefulSet(rctx *reconciliationRequestContext, configVolume *corev1.Volume) (*v1beta2.StatefulSet, error) { statefulSet := &v1beta2.StatefulSet{ObjectMeta: DataCenterResourceMetadata(rctx.cdc)} @@ -39,11 +50,12 @@ func createOrUpdateStatefulSet(rctx *reconciliationRequestContext, configVolume } dataVolumeClaim := newDataVolumeClaim(&rctx.cdc.Spec.DataVolumeClaimSpec) - podInfoVolume := newPodInfoVolume() + backupSecretVolume := newBackupSecretVolume(rctx) + userConfigVolume := newUserConfigVolume(rctx) - cassandraContainer := newCassandraContainer(rctx.cdc, dataVolumeClaim, configVolume) - sidecarContainer := newSidecarContainer(rctx.cdc, dataVolumeClaim, podInfoVolume) + cassandraContainer := newCassandraContainer(rctx.cdc, dataVolumeClaim, configVolume, userConfigVolume) + sidecarContainer := newSidecarContainer(rctx.cdc, dataVolumeClaim, podInfoVolume, backupSecretVolume) sysctlLimitsContainer := newSysctlLimitsContainer(rctx.cdc) @@ -52,6 +64,14 @@ func createOrUpdateStatefulSet(rctx *reconciliationRequestContext, configVolume []corev1.Container{*cassandraContainer, *sidecarContainer}, []corev1.Container{*sysctlLimitsContainer}) + if backupSecretVolume != nil { + podSpec.Volumes = append(podSpec.Volumes, *backupSecretVolume) + } + + if userConfigVolume != nil { + podSpec.Volumes = append(podSpec.Volumes, *userConfigVolume) + } + statefulSetSpec := newStatefulSetSpec(rctx.cdc, podSpec, dataVolumeClaim) if statefulSet.CreationTimestamp.IsZero() { @@ -100,9 +120,7 @@ func newPodSpec(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, volumes []co return podSpec } -func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, configVolume *corev1.Volume) *corev1.Container { - const OperatorConfigVolumeMountPath = "/tmp/operator-config" - +func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, configVolume *corev1.Volume, userConfigVolume *corev1.Volume) *corev1.Container { container := &corev1.Container{ Name: "cassandra", Image: cdc.Spec.CassandraImage, @@ -137,6 +155,11 @@ func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, d }, } + if userConfigVolume != nil { + container.Args = append(container.Args, UserConfigVolumeMountPath) + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: userConfigVolume.Name, MountPath: UserConfigVolumeMountPath}) + } + if cdc.Spec.PrometheusSupport == true { container.Ports = append(container.Ports, corev1.ContainerPort{Name: "promql", ContainerPort: 9500}) } @@ -144,8 +167,8 @@ func newCassandraContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, d return container } -func newSidecarContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, podInfoVolume *corev1.Volume) *corev1.Container { - return &corev1.Container{ +func newSidecarContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dataVolumeClaim *corev1.PersistentVolumeClaim, podInfoVolume *corev1.Volume, backupSecretVolume *corev1.Volume) *corev1.Container { + container := &corev1.Container{ Name: "sidecar", Image: cdc.Spec.SidecarImage, ImagePullPolicy: cdc.Spec.ImagePullPolicy, @@ -157,6 +180,30 @@ func newSidecarContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter, dat {Name: podInfoVolume.Name, MountPath: "/etc/pod-info"}, }, } + + if len(cdc.Spec.Env) > 0 { + container.Env = cdc.Spec.Env + } + + if backupSecretVolume != nil { + // find GOOGLE_APPLICATION_CREDENTIALS in env: + google_creds_path := "/etc/gcp" + for _, env := range cdc.Spec.Env { + if env.Name == GOOGLE_APPLICATION_CREDENTIALS { + google_creds_path = path.Dir(env.Value) + } + } + + if google_creds_path == "/etc/gcp" { + // environment not set, log it, but it may be not an issue + log.Info("Warning: backupSecretVolume is set, but GOOGLE_APPLICATION_CREDENTIALS env is missing") + } + + // Use it for the mount + container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{Name: backupSecretVolume.Name, MountPath: google_creds_path}) + } + + return container } func newSysctlLimitsContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter) *corev1.Container { @@ -165,7 +212,7 @@ func newSysctlLimitsContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter Image: cdc.Spec.CassandraImage, ImagePullPolicy: cdc.Spec.ImagePullPolicy, SecurityContext: &corev1.SecurityContext{ - Privileged: func() *bool { b := true; return &b }(), + Privileged: boolRef(cdc.Spec.PrivilegedSupported), }, Command: []string{"bash", "-xuec"}, Args: []string{ @@ -174,6 +221,32 @@ func newSysctlLimitsContainer(cdc *cassandraoperatorv1alpha1.CassandraDataCenter } } +func newUserConfigVolume(rctx *reconciliationRequestContext) *corev1.Volume { + + // check if set + if len(rctx.cdc.Spec.UserConfigMapVolumeSource.Name) == 0 { + return nil + } + + return &corev1.Volume{ + Name: rctx.cdc.Spec.UserConfigMapVolumeSource.Name, + VolumeSource: corev1.VolumeSource{ConfigMap: &rctx.cdc.Spec.UserConfigMapVolumeSource}, + } +} + +func newBackupSecretVolume(rctx *reconciliationRequestContext) *corev1.Volume { + + // check if set + if len(rctx.cdc.Spec.BackupSecretVolume.SecretName) == 0 { + return nil + } + + return &corev1.Volume{ + Name: rctx.cdc.Spec.BackupSecretVolume.SecretName, + VolumeSource: corev1.VolumeSource{Secret: &rctx.cdc.Spec.BackupSecretVolume}, + } +} + func newPodInfoVolume() *corev1.Volume { return &corev1.Volume{ Name: "pod-info",