From 405986892e058f80853c1e538197a3f0de3dff91 Mon Sep 17 00:00:00 2001 From: Inel Pandzic Date: Mon, 25 Dec 2023 16:05:30 +0100 Subject: [PATCH] K8SPXC-1079: Standardize service exposure (#1538) * Update specs. * Deprecate * Depracate ServiceExpose.TrafficPolicy * Add new fields for 1.14.0 version and above. * Rename ExposeReplica to ExposeReplicas. * Fix ReplicasServiceEnabled. * Update default cr. * Fix tests. * Fix linter. * Add linter config. * Refactor * Fix PXC labels and annotations and update integration test. * Fix panic. * Fix panic * Fix wrong proxysql check. * Fix tests and add integration test for 1.13.0. * Don't focus the test. * Update test name. * Fix cr name in tests. --- .golangci.yml | 4 + ...pxc.percona.com_perconaxtradbclusters.yaml | 91 +++++++ deploy/bundle.yaml | 91 +++++++ deploy/cr.yaml | 66 +++-- deploy/crd.yaml | 91 +++++++ deploy/cw-bundle.yaml | 91 +++++++ e2e-tests/haproxy/run | 7 +- .../proxy-protocol/conf/proxy-protocol.yml | 5 +- pkg/apis/pxc/v1/pxc_types.go | 149 +++++++---- pkg/apis/pxc/v1/zz_generated.deepcopy.go | 32 ++- pkg/controller/pxc/controller.go | 64 +++-- pkg/controller/pxc/controller_test.go | 56 ++--- pkg/controller/pxc/replication.go | 19 +- pkg/controller/pxc/service_test.go | 235 +++++++++++++++++- pkg/controller/pxc/status.go | 19 +- pkg/controller/pxc/status_test.go | 39 ++- pkg/pxc/service.go | 177 ++++++++++--- 17 files changed, 1037 insertions(+), 199 deletions(-) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000..801b5aabda --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,4 @@ +linters-settings: + staticcheck: + checks: + - "-SA1019" \ No newline at end of file diff --git a/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml b/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml index 2ba75a45bc..987df2fd68 100644 --- a/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml +++ b/config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml @@ -1394,6 +1394,60 @@ spec: type: boolean envVarsSecret: type: string + exposePrimary: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object + exposeReplicas: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -4126,6 +4180,33 @@ spec: type: boolean envVarsSecret: type: string + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -6590,6 +6671,16 @@ spec: type: object enabled: type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string loadBalancerSourceRanges: items: type: string diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 761630acbe..2a85d14c86 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -2287,6 +2287,60 @@ spec: type: boolean envVarsSecret: type: string + exposePrimary: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object + exposeReplicas: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -5019,6 +5073,33 @@ spec: type: boolean envVarsSecret: type: string + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -7483,6 +7564,16 @@ spec: type: object enabled: type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string loadBalancerSourceRanges: items: type: string diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 88e94e6288..eaeae7851f 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -53,11 +53,15 @@ spec: # expose: # enabled: true # type: LoadBalancer -# trafficPolicy: Local +# externalTrafficPolicy: Local +# internalTrafficPolicy: Local # loadBalancerSourceRanges: # - 10.0.0.0/8 +# loadBalancerIP: 127.0.01 # annotations: # networking.gke.io/load-balancer-type: "Internal" +# labels: +# rack: rack-22 # replicationChannels: # - name: pxc1_to_pxc2 # isSource: true @@ -257,25 +261,30 @@ spec: # periodSeconds: 30 # successThreshold: 1 # failureThreshold: 4 -# serviceType: ClusterIP -# externalTrafficPolicy: Cluster -# loadBalancerSourceRanges: -# - 10.0.0.0/8 -# loadBalancerIP: 127.0.0.1 -# serviceAnnotations: -# service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http -# serviceLabels: -# rack: rack-23 -# replicasServiceEnabled: false -# replicasLoadBalancerSourceRanges: -# - 10.0.0.0/8 -# replicasLoadBalancerIP: 127.0.0.1 -# replicasServiceType: ClusterIP -# replicasExternalTrafficPolicy: Cluster -# replicasServiceAnnotations: -# service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http -# replicasServiceLabels: -# rack: rack-23 +# exposePrimary: +# enabled: false +# type: ClusterIP +# annotations: +# service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp +# externalTrafficPolicy: Cluster +# internalTrafficPolicy: Cluster +# labels: +# rack: rack-22 +# loadBalancerSourceRanges: +# - 10.0.0.0/8 +# loadBalancerIP: 127.0.0.1 +# exposeReplicas: +# enabled: false +# type: ClusterIP +# annotations: +# service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp +# externalTrafficPolicy: Cluster +# internalTrafficPolicy: Cluster +# labels: +# rack: rack-22 +# loadBalancerSourceRanges: +# - 10.0.0.0/8 +# loadBalancerIP: 127.0.0.1 # runtimeClassName: image-rc # sidecars: # - image: busybox @@ -407,11 +416,18 @@ spec: # iam.amazonaws.com/role: role-arn # labels: # rack: rack-22 -# serviceType: ClusterIP -# loadBalancerSourceRanges: -# - 10.0.0.0/8 -# loadBalancerIP: 127.0.0.1 -# externalTrafficPolicy: Cluster +# expose: +# enabled: false +# type: ClusterIP +# annotations: +# service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp +# externalTrafficPolicy: Cluster +# internalTrafficPolicy: Cluster +# labels: +# rack: rack-22 +# loadBalancerSourceRanges: +# - 10.0.0.0/8 +# loadBalancerIP: 127.0.0.1 # runtimeClassName: image-rc # sidecars: # - image: busybox diff --git a/deploy/crd.yaml b/deploy/crd.yaml index 0915addaf2..94ec5df967 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -2287,6 +2287,60 @@ spec: type: boolean envVarsSecret: type: string + exposePrimary: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object + exposeReplicas: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -5019,6 +5073,33 @@ spec: type: boolean envVarsSecret: type: string + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -7483,6 +7564,16 @@ spec: type: object enabled: type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string loadBalancerSourceRanges: items: type: string diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index 341c16ac6c..644ab76dd4 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -2287,6 +2287,60 @@ spec: type: boolean envVarsSecret: type: string + exposePrimary: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object + exposeReplicas: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -5019,6 +5073,33 @@ spec: type: boolean envVarsSecret: type: string + expose: + properties: + annotations: + additionalProperties: + type: string + type: object + enabled: + type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + trafficPolicy: + type: string + type: + type: string + type: object externalTrafficPolicy: type: string forceUnsafeBootstrap: @@ -7483,6 +7564,16 @@ spec: type: object enabled: type: boolean + externalTrafficPolicy: + type: string + internalTrafficPolicy: + type: string + labels: + additionalProperties: + type: string + type: object + loadBalancerIP: + type: string loadBalancerSourceRanges: items: type: string diff --git a/e2e-tests/haproxy/run b/e2e-tests/haproxy/run index bd0a49d5fd..77a4798636 100755 --- a/e2e-tests/haproxy/run +++ b/e2e-tests/haproxy/run @@ -3,18 +3,19 @@ set -o errexit test_default_replicas_service() { - kpatch_delete_field pxc $1 "/spec/haproxy/replicasServiceEnabled" && sleep 1 + kpatch_delete_field pxc $1 "/spec/haproxy/exposeReplicas" && sleep 1 compare_kubectl "service/$1-haproxy-replicas" } test_disable_replicas_service() { - kpatch_set_field pxc $1 "/spec/haproxy/replicasServiceEnabled" false && sleep 1 + kpatch_set_field pxc $1 "/spec/haproxy/exposeReplicas" {} && sleep 1 + kpatch_set_field pxc $1 "/spec/haproxy/exposeReplicas/enabled" false && sleep 1 wait_for_delete "svc/$1-haproxy-replicas" (kubectl_bin get svc "$1-haproxy-replicas" || :) 2>&1 | grep -e "not found$" >/dev/null } test_enable_replicas_service() { - kpatch_set_field pxc $1 "/spec/haproxy/replicasServiceEnabled" true && sleep 1 + kpatch_set_field pxc $1 "/spec/haproxy/exposeReplicas/enabled" true && sleep 1 compare_kubectl "service/$1-haproxy-replicas" } diff --git a/e2e-tests/proxy-protocol/conf/proxy-protocol.yml b/e2e-tests/proxy-protocol/conf/proxy-protocol.yml index e17d60b2fa..2886276bac 100644 --- a/e2e-tests/proxy-protocol/conf/proxy-protocol.yml +++ b/e2e-tests/proxy-protocol/conf/proxy-protocol.yml @@ -33,8 +33,9 @@ spec: size: 3 image: -haproxy serviceType: LoadBalancer - serviceAnnotations: - service.beta.kubernetes.io/aws-load-balancer-type: nlb + exposePrimary: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: nlb externalTrafficPolicy: Local resources: requests: diff --git a/pkg/apis/pxc/v1/pxc_types.go b/pkg/apis/pxc/v1/pxc_types.go index 087e5949d7..63068ec37f 100644 --- a/pkg/apis/pxc/v1/pxc_types.go +++ b/pkg/apis/pxc/v1/pxc_types.go @@ -36,7 +36,7 @@ type PerconaXtraDBClusterSpec struct { LogCollectorSecretName string `json:"logCollectorSecretName,omitempty"` TLS *TLSSpec `json:"tls,omitempty"` PXC *PXCSpec `json:"pxc,omitempty"` - ProxySQL *PodSpec `json:"proxysql,omitempty"` + ProxySQL *ProxySQLSpec `json:"proxysql,omitempty"` HAProxy *HAProxySpec `json:"haproxy,omitempty"` PMM *PMMSpec `json:"pmm,omitempty"` LogCollector *LogCollectorSpec `json:"logcollector,omitempty"` @@ -70,8 +70,14 @@ type ServiceExpose struct { Enabled bool `json:"enabled,omitempty"` Type corev1.ServiceType `json:"type,omitempty"` LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"` + LoadBalancerIP string `json:"loadBalancerIP,omitempty"` Annotations map[string]string `json:"annotations,omitempty"` - TrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"trafficPolicy,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ExternalTrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"externalTrafficPolicy,omitempty"` + InternalTrafficPolicy corev1.ServiceInternalTrafficPolicy `json:"internalTrafficPolicy,omitempty"` + + // Deprecated: Use ExternalTrafficPolicy instead + TrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"trafficPolicy,omitempty"` } type ReplicationChannel struct { @@ -403,59 +409,82 @@ func (list *PerconaXtraDBClusterList) HasUnfinishedFinalizers() bool { } type PodSpec struct { - Enabled bool `json:"enabled,omitempty"` - Size int32 `json:"size,omitempty"` - Image string `json:"image,omitempty"` - Resources corev1.ResourceRequirements `json:"resources,omitempty"` - SidecarResources corev1.ResourceRequirements `json:"sidecarResources,omitempty"` - VolumeSpec *VolumeSpec `json:"volumeSpec,omitempty"` - Affinity *PodAffinity `json:"affinity,omitempty"` - NodeSelector map[string]string `json:"nodeSelector,omitempty"` - Tolerations []corev1.Toleration `json:"tolerations,omitempty"` - PriorityClassName string `json:"priorityClassName,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` - Configuration string `json:"configuration,omitempty"` - PodDisruptionBudget *PodDisruptionBudgetSpec `json:"podDisruptionBudget,omitempty"` - VaultSecretName string `json:"vaultSecretName,omitempty"` - SSLSecretName string `json:"sslSecretName,omitempty"` - SSLInternalSecretName string `json:"sslInternalSecretName,omitempty"` - EnvVarsSecretName string `json:"envVarsSecret,omitempty"` - TerminationGracePeriodSeconds *int64 `json:"gracePeriod,omitempty"` - ForceUnsafeBootstrap bool `json:"forceUnsafeBootstrap,omitempty"` - ServiceType corev1.ServiceType `json:"serviceType,omitempty"` - ReplicasServiceType corev1.ServiceType `json:"replicasServiceType,omitempty"` - ExternalTrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"externalTrafficPolicy,omitempty"` + Enabled bool `json:"enabled,omitempty"` + Size int32 `json:"size,omitempty"` + Image string `json:"image,omitempty"` + Resources corev1.ResourceRequirements `json:"resources,omitempty"` + SidecarResources corev1.ResourceRequirements `json:"sidecarResources,omitempty"` + VolumeSpec *VolumeSpec `json:"volumeSpec,omitempty"` + Affinity *PodAffinity `json:"affinity,omitempty"` + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + PriorityClassName string `json:"priorityClassName,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"` + Configuration string `json:"configuration,omitempty"` + PodDisruptionBudget *PodDisruptionBudgetSpec `json:"podDisruptionBudget,omitempty"` + VaultSecretName string `json:"vaultSecretName,omitempty"` + SSLSecretName string `json:"sslSecretName,omitempty"` + SSLInternalSecretName string `json:"sslInternalSecretName,omitempty"` + EnvVarsSecretName string `json:"envVarsSecret,omitempty"` + TerminationGracePeriodSeconds *int64 `json:"gracePeriod,omitempty"` + ForceUnsafeBootstrap bool `json:"forceUnsafeBootstrap,omitempty"` + + // Deprecated: Use ServiceExpose.Type instead + ServiceType corev1.ServiceType `json:"serviceType,omitempty"` + // Deprecated: Use ServiceExpose.Type instead + ReplicasServiceType corev1.ServiceType `json:"replicasServiceType,omitempty"` + // Deprecated: Use ServiceExpose.ExternalTrafficPolicy instead + ExternalTrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"externalTrafficPolicy,omitempty"` + // Deprecated: Use ServiceExpose.ExternalTrafficPolicy instead ReplicasExternalTrafficPolicy corev1.ServiceExternalTrafficPolicyType `json:"replicasExternalTrafficPolicy,omitempty"` - LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"` - LoadBalancerIP string `json:"loadBalancerIP,omitempty"` - ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` - ServiceLabels map[string]string `json:"serviceLabels,omitempty"` - ReplicasServiceAnnotations map[string]string `json:"replicasServiceAnnotations,omitempty"` - ReplicasServiceLabels map[string]string `json:"replicasServiceLabels,omitempty"` - SchedulerName string `json:"schedulerName,omitempty"` - ReadinessInitialDelaySeconds *int32 `json:"readinessDelaySec,omitempty"` - ReadinessProbes corev1.Probe `json:"readinessProbes,omitempty"` - LivenessInitialDelaySeconds *int32 `json:"livenessDelaySec,omitempty"` - LivenessProbes corev1.Probe `json:"livenessProbes,omitempty"` - PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` - ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` - ServiceAccountName string `json:"serviceAccountName,omitempty"` - ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` - Sidecars []corev1.Container `json:"sidecars,omitempty"` - SidecarVolumes []corev1.Volume `json:"sidecarVolumes,omitempty"` - SidecarPVCs []corev1.PersistentVolumeClaim `json:"sidecarPVCs,omitempty"` - RuntimeClassName *string `json:"runtimeClassName,omitempty"` - HookScript string `json:"hookScript,omitempty"` - TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` + // Deprecated: Use ServiceExpose.LoadBalancerSourceRanges instead + LoadBalancerSourceRanges []string `json:"loadBalancerSourceRanges,omitempty"` + // Deprecated: Use ServiceExpose.LoadBalancerIP instead + LoadBalancerIP string `json:"loadBalancerIP,omitempty"` + // Deprecated: Use ServiceExpose.Annotations instead + ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` + // Deprecated: Use ServiceExpose.Labels instead + ServiceLabels map[string]string `json:"serviceLabels,omitempty"` + // Deprecated: Use ServiceExpose.Annotations instead + ReplicasServiceAnnotations map[string]string `json:"replicasServiceAnnotations,omitempty"` + // Deprecated: Use ServiceExpose.Labels instead + ReplicasServiceLabels map[string]string `json:"replicasServiceLabels,omitempty"` + + SchedulerName string `json:"schedulerName,omitempty"` + ReadinessInitialDelaySeconds *int32 `json:"readinessDelaySec,omitempty"` + ReadinessProbes corev1.Probe `json:"readinessProbes,omitempty"` + LivenessInitialDelaySeconds *int32 `json:"livenessDelaySec,omitempty"` + LivenessProbes corev1.Probe `json:"livenessProbes,omitempty"` + PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` + ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` + ServiceAccountName string `json:"serviceAccountName,omitempty"` + ImagePullPolicy corev1.PullPolicy `json:"imagePullPolicy,omitempty"` + Sidecars []corev1.Container `json:"sidecars,omitempty"` + SidecarVolumes []corev1.Volume `json:"sidecarVolumes,omitempty"` + SidecarPVCs []corev1.PersistentVolumeClaim `json:"sidecarPVCs,omitempty"` + RuntimeClassName *string `json:"runtimeClassName,omitempty"` + HookScript string `json:"hookScript,omitempty"` + TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` +} + +type ProxySQLSpec struct { + PodSpec `json:",inline"` + Expose ServiceExpose `json:"expose,omitempty"` } type HAProxySpec struct { - PodSpec `json:",inline"` - ReplicasServiceEnabled *bool `json:"replicasServiceEnabled,omitempty"` + PodSpec `json:",inline"` + ExposePrimary ServiceExpose `json:"exposePrimary,omitempty"` + ExposeReplicas *ServiceExpose `json:"exposeReplicas,omitempty"` + + // Deprecated: Use ExposeReplica.Enabled instead + ReplicasServiceEnabled *bool `json:"replicasServiceEnabled,omitempty"` + // Deprecated: Use ExposeReplica.LoadBalancerSourceRanges instead ReplicasLoadBalancerSourceRanges []string `json:"replicasLoadBalancerSourceRanges,omitempty"` - ReplicasLoadBalancerIP string `json:"replicasLoadBalancerIP,omitempty"` + // Deprecated: Use ExposeReplica.LoadBalancerIP instead + ReplicasLoadBalancerIP string `json:"replicasLoadBalancerIP,omitempty"` } type PodDisruptionBudgetSpec struct { @@ -840,9 +869,17 @@ func (cr *PerconaXtraDBCluster) CheckNSetDefaults(serverVersion *version.ServerV } if c.HAProxyEnabled() { - if c.HAProxy.ReplicasServiceEnabled == nil { - t := true - c.HAProxy.ReplicasServiceEnabled = &t + if cr.CompareVersionWith("1.14.0") >= 0 { + if c.HAProxy.ExposeReplicas == nil { + c.HAProxy.ExposeReplicas = &ServiceExpose{ + Enabled: true, + } + } + } else { + if c.HAProxy.ReplicasServiceEnabled == nil { + t := true + c.HAProxy.ReplicasServiceEnabled = &t + } } if len(c.HAProxy.ImagePullPolicy) == 0 { @@ -1318,7 +1355,11 @@ func (cr *PerconaXtraDBCluster) HAProxyEnabled() bool { } func (cr *PerconaXtraDBCluster) HAProxyReplicasServiceEnabled() bool { - return *cr.Spec.HAProxy.ReplicasServiceEnabled + if cr.CompareVersionWith("1.14.0") < 0 { + return *cr.Spec.HAProxy.ReplicasServiceEnabled + } + + return cr.Spec.HAProxy.ExposeReplicas.Enabled } func (cr *PerconaXtraDBCluster) ProxySQLEnabled() bool { diff --git a/pkg/apis/pxc/v1/zz_generated.deepcopy.go b/pkg/apis/pxc/v1/zz_generated.deepcopy.go index 0bf8967869..9abc98e88d 100644 --- a/pkg/apis/pxc/v1/zz_generated.deepcopy.go +++ b/pkg/apis/pxc/v1/zz_generated.deepcopy.go @@ -243,6 +243,12 @@ func (in *ComponentStatus) DeepCopy() *ComponentStatus { func (in *HAProxySpec) DeepCopyInto(out *HAProxySpec) { *out = *in in.PodSpec.DeepCopyInto(&out.PodSpec) + in.ExposePrimary.DeepCopyInto(&out.ExposePrimary) + if in.ExposeReplicas != nil { + in, out := &in.ExposeReplicas, &out.ExposeReplicas + *out = new(ServiceExpose) + (*in).DeepCopyInto(*out) + } if in.ReplicasServiceEnabled != nil { in, out := &in.ReplicasServiceEnabled, &out.ReplicasServiceEnabled *out = new(bool) @@ -790,7 +796,7 @@ func (in *PerconaXtraDBClusterSpec) DeepCopyInto(out *PerconaXtraDBClusterSpec) } if in.ProxySQL != nil { in, out := &in.ProxySQL, &out.ProxySQL - *out = new(PodSpec) + *out = new(ProxySQLSpec) (*in).DeepCopyInto(*out) } if in.HAProxy != nil { @@ -1088,6 +1094,23 @@ func (in *PodSpec) DeepCopy() *PodSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProxySQLSpec) DeepCopyInto(out *ProxySQLSpec) { + *out = *in + in.PodSpec.DeepCopyInto(&out.PodSpec) + in.Expose.DeepCopyInto(&out.Expose) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxySQLSpec. +func (in *ProxySQLSpec) DeepCopy() *ProxySQLSpec { + if in == nil { + return nil + } + out := new(ProxySQLSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ReplicationChannel) DeepCopyInto(out *ReplicationChannel) { *out = *in @@ -1194,6 +1217,13 @@ func (in *ServiceExpose) DeepCopyInto(out *ServiceExpose) { (*out)[key] = val } } + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceExpose. diff --git a/pkg/controller/pxc/controller.go b/pkg/controller/pxc/controller.go index 2f596e098e..bb459894bc 100644 --- a/pkg/controller/pxc/controller.go +++ b/pkg/controller/pxc/controller.go @@ -326,7 +326,7 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(ctx context.Context, request r saveOldSvcMeta := true if o.CompareVersionWith("1.14.0") >= 0 { - saveOldSvcMeta = len(o.Spec.PXC.ServiceLabels) == 0 && len(o.Spec.PXC.ServiceAnnotations) == 0 + saveOldSvcMeta = len(o.Spec.PXC.Expose.Labels) == 0 && len(o.Spec.PXC.Expose.Annotations) == 0 } err = r.createOrUpdateService(o, pxc.NewServicePXC(o), saveOldSvcMeta) if err != nil { @@ -368,15 +368,24 @@ func (r *ReconcilePerconaXtraDBCluster) Reconcile(ctx context.Context, request r pxc.MergeTemplateAnnotations(proxysqlSet.StatefulSet(), userReconcileResult.proxyAnnotations) if o.Spec.ProxySQLEnabled() { - err = r.updatePod(ctx, proxysqlSet, o.Spec.ProxySQL, o, proxyInits) + err = r.updatePod(ctx, proxysqlSet, &o.Spec.ProxySQL.PodSpec, o, proxyInits) if err != nil { return reconcile.Result{}, errors.Wrap(err, "ProxySQL upgrade error") } svc := pxc.NewServiceProxySQL(o) - err = r.createOrUpdateService(o, svc, len(o.Spec.ProxySQL.ServiceLabels) == 0 && len(o.Spec.ProxySQL.ServiceAnnotations) == 0) - if err != nil { - return reconcile.Result{}, errors.Wrapf(err, "%s upgrade error", svc.Name) + + if o.CompareVersionWith("1.14.0") >= 0 { + err = r.createOrUpdateService(o, svc, len(o.Spec.ProxySQL.Expose.Labels) == 0 && len(o.Spec.ProxySQL.Expose.Annotations) == 0) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "%s upgrade error", svc.Name) + } + } else { + err = r.createOrUpdateService(o, svc, len(o.Spec.ProxySQL.ServiceLabels) == 0 && len(o.Spec.ProxySQL.ServiceAnnotations) == 0) + if err != nil { + return reconcile.Result{}, errors.Wrapf(err, "%s upgrade error", svc.Name) + } } + svc = pxc.NewServiceProxySQLUnready(o) err = r.createOrUpdateService(o, svc, true) if err != nil { @@ -461,16 +470,40 @@ func (r *ReconcilePerconaXtraDBCluster) reconcileHAProxy(ctx context.Context, cr } svc := pxc.NewServiceHAProxy(cr) podSpec := cr.Spec.HAProxy.PodSpec - err := r.createOrUpdateService(cr, svc, len(podSpec.ServiceLabels) == 0 && len(podSpec.ServiceAnnotations) == 0) - if err != nil { - return errors.Wrapf(err, "%s upgrade error", svc.Name) + expose := cr.Spec.HAProxy.ExposePrimary + + if cr.CompareVersionWith("1.14.0") >= 0 { + err := r.createOrUpdateService(cr, svc, len(expose.Labels) == 0 && len(expose.Annotations) == 0) + if err != nil { + return errors.Wrapf(err, "%s upgrade error", svc.Name) + } + } else { + err := r.createOrUpdateService(cr, svc, len(podSpec.ServiceLabels) == 0 && len(podSpec.ServiceAnnotations) == 0) + if err != nil { + return errors.Wrapf(err, "%s upgrade error", svc.Name) + } } + if cr.HAProxyReplicasServiceEnabled() { svc := pxc.NewServiceHAProxyReplicas(cr) - err = r.createOrUpdateService(cr, svc, len(podSpec.ReplicasServiceLabels) == 0 && len(podSpec.ReplicasServiceAnnotations) == 0) + err := setControllerReference(cr, svc, r.scheme) if err != nil { - return errors.Wrapf(err, "%s upgrade error", svc.Name) + return errors.Wrapf(err, "%s setControllerReference", svc.Name) + } + + if cr.CompareVersionWith("1.14.0") >= 0 { + e := cr.Spec.HAProxy.ExposeReplicas + err = r.createOrUpdateService(cr, svc, len(e.Labels) == 0 && len(e.Annotations) == 0) + if err != nil { + return errors.Wrapf(err, "%s upgrade error", svc.Name) + } + } else { + err = r.createOrUpdateService(cr, svc, len(podSpec.ReplicasServiceLabels) == 0 && len(podSpec.ReplicasServiceAnnotations) == 0) + if err != nil { + return errors.Wrapf(err, "%s upgrade error", svc.Name) + } } + } else { if err := r.deleteServices(pxc.NewServiceHAProxyReplicas(cr)); err != nil { return errors.Wrap(err, "delete HAProxy replica service") @@ -673,7 +706,7 @@ func (r *ReconcilePerconaXtraDBCluster) deploy(ctx context.Context, cr *api.Perc if cr.Spec.ProxySQLEnabled() { sfsProxy := statefulset.NewProxy(cr) - proxySet, err := pxc.StatefulSet(sfsProxy, cr.Spec.ProxySQL, cr, secrets, proxyInits, log, r.getConfigVolume) + proxySet, err := pxc.StatefulSet(sfsProxy, &cr.Spec.ProxySQL.PodSpec, cr, secrets, proxyInits, log, r.getConfigVolume) if err != nil { return errors.Wrap(err, "create ProxySQL Service") } @@ -1294,7 +1327,7 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdate(cr *api.PerconaXtraDBClus return nil } -func setIgnoredAnnotationsAndLabels(cr *api.PerconaXtraDBCluster, obj, oldObject client.Object) error { +func setIgnoredAnnotationsAndLabels(cr *api.PerconaXtraDBCluster, obj, oldObject client.Object) { oldAnnotations := oldObject.GetAnnotations() if oldAnnotations == nil { oldAnnotations = make(map[string]string) @@ -1309,6 +1342,7 @@ func setIgnoredAnnotationsAndLabels(cr *api.PerconaXtraDBCluster, obj, oldObject } } obj.SetAnnotations(annotations) + oldLabels := oldObject.GetLabels() if oldLabels == nil { oldLabels = make(map[string]string) @@ -1323,7 +1357,6 @@ func setIgnoredAnnotationsAndLabels(cr *api.PerconaXtraDBCluster, obj, oldObject } } obj.SetLabels(labels) - return nil } func mergeMaps(x, y map[string]string) map[string]string { @@ -1362,9 +1395,8 @@ func (r *ReconcilePerconaXtraDBCluster) createOrUpdateService(cr *api.PerconaXtr svc.SetAnnotations(mergeMaps(svc.GetAnnotations(), oldSvc.GetAnnotations())) svc.SetLabels(mergeMaps(svc.GetLabels(), oldSvc.GetLabels())) } - if err = setIgnoredAnnotationsAndLabels(cr, svc, oldSvc); err != nil { - return errors.Wrap(err, "set ignored annotations and labels") - } + setIgnoredAnnotationsAndLabels(cr, svc, oldSvc) + return r.createOrUpdate(cr, svc) } diff --git a/pkg/controller/pxc/controller_test.go b/pkg/controller/pxc/controller_test.go index 378bbe7bf5..96490b8950 100644 --- a/pkg/controller/pxc/controller_test.go +++ b/pkg/controller/pxc/controller_test.go @@ -805,11 +805,11 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - cr.Spec.HAProxy.ServiceAnnotations = make(map[string]string) - cr.Spec.HAProxy.ServiceLabels = make(map[string]string) + cr.Spec.HAProxy.ExposePrimary.Annotations = make(map[string]string) + cr.Spec.HAProxy.ExposePrimary.Labels = make(map[string]string) - cr.Spec.HAProxy.ServiceAnnotations["crAnnotation"] = "true" - cr.Spec.HAProxy.ServiceLabels["crLabel"] = "true" + cr.Spec.HAProxy.ExposePrimary.Annotations["crAnnotation"] = "true" + cr.Spec.HAProxy.ExposePrimary.Labels["crLabel"] = "true" err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) @@ -845,8 +845,8 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - delete(cr.Spec.HAProxy.ServiceAnnotations, "crAnnotation") - delete(cr.Spec.HAProxy.ServiceLabels, "crLabel") + delete(cr.Spec.HAProxy.ExposePrimary.Annotations, "crAnnotation") + delete(cr.Spec.HAProxy.ExposePrimary.Labels, "crLabel") err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) @@ -880,14 +880,14 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - cr.Spec.HAProxy.ServiceAnnotations = make(map[string]string) - cr.Spec.HAProxy.ServiceLabels = make(map[string]string) + cr.Spec.HAProxy.ExposePrimary.Annotations = make(map[string]string) + cr.Spec.HAProxy.ExposePrimary.Labels = make(map[string]string) - cr.Spec.HAProxy.ServiceAnnotations["secondCrAnnotation"] = "true" - cr.Spec.HAProxy.ServiceAnnotations["thirdCrAnnotation"] = "true" + cr.Spec.HAProxy.ExposePrimary.Annotations["secondCrAnnotation"] = "true" + cr.Spec.HAProxy.ExposePrimary.Annotations["thirdCrAnnotation"] = "true" - cr.Spec.HAProxy.ServiceLabels["secondCrLabel"] = "true" - cr.Spec.HAProxy.ServiceLabels["thirdCrLabel"] = "true" + cr.Spec.HAProxy.ExposePrimary.Labels["secondCrLabel"] = "true" + cr.Spec.HAProxy.ExposePrimary.Labels["thirdCrLabel"] = "true" err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) @@ -925,8 +925,8 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - delete(cr.Spec.HAProxy.ServiceAnnotations, "secondCrAnnotation") - delete(cr.Spec.HAProxy.ServiceLabels, "secondCrLabel") + delete(cr.Spec.HAProxy.ExposePrimary.Annotations, "secondCrAnnotation") + delete(cr.Spec.HAProxy.ExposePrimary.Labels, "secondCrLabel") err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) @@ -1098,11 +1098,11 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - cr.Spec.ProxySQL.ServiceAnnotations = make(map[string]string) - cr.Spec.ProxySQL.ServiceLabels = make(map[string]string) + cr.Spec.ProxySQL.Expose.Annotations = make(map[string]string) + cr.Spec.ProxySQL.Expose.Labels = make(map[string]string) - cr.Spec.ProxySQL.ServiceAnnotations["crAnnotation"] = "true" - cr.Spec.ProxySQL.ServiceLabels["crLabel"] = "true" + cr.Spec.ProxySQL.Expose.Annotations["crAnnotation"] = "true" + cr.Spec.ProxySQL.Expose.Labels["crLabel"] = "true" err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) @@ -1138,8 +1138,8 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - delete(cr.Spec.ProxySQL.ServiceAnnotations, "crAnnotation") - delete(cr.Spec.ProxySQL.ServiceLabels, "crLabel") + delete(cr.Spec.ProxySQL.Expose.Annotations, "crAnnotation") + delete(cr.Spec.ProxySQL.Expose.Labels, "crLabel") err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) @@ -1173,14 +1173,14 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - cr.Spec.ProxySQL.ServiceAnnotations = make(map[string]string) - cr.Spec.ProxySQL.ServiceLabels = make(map[string]string) + cr.Spec.ProxySQL.Expose.Annotations = make(map[string]string) + cr.Spec.ProxySQL.Expose.Labels = make(map[string]string) - cr.Spec.ProxySQL.ServiceAnnotations["secondCrAnnotation"] = "true" - cr.Spec.ProxySQL.ServiceAnnotations["thirdCrAnnotation"] = "true" + cr.Spec.ProxySQL.Expose.Annotations["secondCrAnnotation"] = "true" + cr.Spec.ProxySQL.Expose.Annotations["thirdCrAnnotation"] = "true" - cr.Spec.ProxySQL.ServiceLabels["secondCrLabel"] = "true" - cr.Spec.ProxySQL.ServiceLabels["thirdCrLabel"] = "true" + cr.Spec.ProxySQL.Expose.Labels["secondCrLabel"] = "true" + cr.Spec.ProxySQL.Expose.Labels["thirdCrLabel"] = "true" err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) @@ -1218,8 +1218,8 @@ var _ = Describe("Ignore labels and annotations", Ordered, func() { orig := cr.DeepCopy() - delete(cr.Spec.ProxySQL.ServiceAnnotations, "secondCrAnnotation") - delete(cr.Spec.ProxySQL.ServiceLabels, "secondCrLabel") + delete(cr.Spec.ProxySQL.Expose.Annotations, "secondCrAnnotation") + delete(cr.Spec.ProxySQL.Expose.Labels, "secondCrLabel") err = k8sClient.Patch(ctx, cr, client.MergeFrom(orig)) Expect(err).NotTo(HaveOccurred()) diff --git a/pkg/controller/pxc/replication.go b/pkg/controller/pxc/replication.go index d8d8f835f0..f7cf4cacd0 100644 --- a/pkg/controller/pxc/replication.go +++ b/pkg/controller/pxc/replication.go @@ -531,11 +531,20 @@ func NewExposedPXCService(svcName string, cr *api.PerconaXtraDBCluster) *corev1. if cr.Spec.PXC.Expose.Type == corev1.ServiceTypeNodePort || cr.Spec.PXC.Expose.Type == corev1.ServiceTypeLoadBalancer { - switch cr.Spec.PXC.Expose.TrafficPolicy { - case corev1.ServiceExternalTrafficPolicyTypeLocal, corev1.ServiceExternalTrafficPolicyTypeCluster: - svc.Spec.ExternalTrafficPolicy = cr.Spec.PXC.Expose.TrafficPolicy - default: - svc.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeCluster + if cr.CompareVersionWith("1.14.0") >= 0 { + switch cr.Spec.PXC.Expose.ExternalTrafficPolicy { + case corev1.ServiceExternalTrafficPolicyTypeLocal, corev1.ServiceExternalTrafficPolicyTypeCluster: + svc.Spec.ExternalTrafficPolicy = cr.Spec.PXC.Expose.ExternalTrafficPolicy + default: + svc.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeCluster + } + } else { + switch cr.Spec.PXC.Expose.TrafficPolicy { + case corev1.ServiceExternalTrafficPolicyTypeLocal, corev1.ServiceExternalTrafficPolicyTypeCluster: + svc.Spec.ExternalTrafficPolicy = cr.Spec.PXC.Expose.TrafficPolicy + default: + svc.Spec.ExternalTrafficPolicy = corev1.ServiceExternalTrafficPolicyTypeCluster + } } } diff --git a/pkg/controller/pxc/service_test.go b/pkg/controller/pxc/service_test.go index 80460424fa..1afc4d9728 100644 --- a/pkg/controller/pxc/service_test.go +++ b/pkg/controller/pxc/service_test.go @@ -2,6 +2,7 @@ package pxc import ( "context" + "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -24,7 +25,7 @@ var _ = Describe("Service labels and annotations", Ordered, func() { Namespace: ns, }, } - crName := ns + "-reconciler" + crName := ns + "-cr" crNamespacedName := types.NamespacedName{Name: crName, Namespace: ns} cr, err := readDefaultCR(crName, ns) It("should read default cr.yaml", func() { @@ -98,14 +99,14 @@ var _ = Describe("Service labels and annotations", Ordered, func() { Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed()) cr.Spec.IgnoreAnnotations = []string{"ignored-annotation"} cr.Spec.IgnoreLabels = []string{"ignored-label"} - cr.Spec.PXC.ServiceLabels = map[string]string{"cr-label": "test"} - cr.Spec.PXC.ServiceAnnotations = map[string]string{"cr-annotation": "test"} - cr.Spec.HAProxy.ServiceLabels = map[string]string{"cr-label": "test"} - cr.Spec.HAProxy.ServiceAnnotations = map[string]string{"cr-annotation": "test"} - cr.Spec.HAProxy.ReplicasServiceLabels = map[string]string{"cr-label": "test"} - cr.Spec.HAProxy.ReplicasServiceAnnotations = map[string]string{"cr-annotation": "test"} - cr.Spec.ProxySQL.ServiceLabels = map[string]string{"cr-label": "test"} - cr.Spec.ProxySQL.ServiceAnnotations = map[string]string{"cr-annotation": "test"} + cr.Spec.PXC.Expose.Labels = map[string]string{"cr-label": "test"} + cr.Spec.PXC.Expose.Annotations = map[string]string{"cr-annotation": "test"} + cr.Spec.HAProxy.ExposePrimary.Labels = map[string]string{"cr-label": "test"} + cr.Spec.HAProxy.ExposePrimary.Annotations = map[string]string{"cr-annotation": "test"} + cr.Spec.HAProxy.ExposeReplicas.Labels = map[string]string{"cr-label": "test"} + cr.Spec.HAProxy.ExposeReplicas.Annotations = map[string]string{"cr-annotation": "test"} + cr.Spec.ProxySQL.Expose.Labels = map[string]string{"cr-label": "test"} + cr.Spec.ProxySQL.Expose.Annotations = map[string]string{"cr-annotation": "test"} Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) }) It("should reconcile PerconaXtraDBCluster", func() { @@ -187,8 +188,220 @@ var _ = Describe("Service labels and annotations", Ordered, func() { cr.Spec.HAProxy.Enabled = false cr.Spec.ProxySQL.Enabled = true - cr.Spec.PXC.ServiceLabels = nil - cr.Spec.PXC.ServiceAnnotations = nil + cr.Spec.PXC.Expose.Labels = nil + cr.Spec.PXC.Expose.Annotations = nil + cr.Spec.HAProxy.ExposePrimary.Labels = nil + cr.Spec.HAProxy.ExposePrimary.Annotations = nil + cr.Spec.HAProxy.ExposeReplicas.Labels = nil + cr.Spec.HAProxy.ExposeReplicas.Annotations = nil + cr.Spec.ProxySQL.Expose.Labels = nil + cr.Spec.ProxySQL.Expose.Annotations = nil + Expect(k8sClient.Update(ctx, cr)).To(Succeed()) + }) + It("should reconcile PerconaXtraDBCluster", func() { + _, err := reconciler().Reconcile(ctx, reconcile.Request{ + NamespacedName: crNamespacedName, + }) + Expect(err).To(Succeed()) + }) + + Context("check proxysql cluster", func() { + checkLabelsAndAnnotations([]*corev1.Service{ + pxc.NewServicePXC(cr), + pxc.NewServiceProxySQL(cr), + }) + }) +}) + +var _ = Describe("Service labels and annotations for 1.13.0", Ordered, func() { + ctx := context.Background() + const ns = "svc-ls-an-1-13-0" + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + Namespace: ns, + }, + } + crName := ns + "-cr" + crNamespacedName := types.NamespacedName{Name: crName, Namespace: ns} + cr, err := readDefaultCR(crName, ns) + It("should read default cr.yaml", func() { + Expect(err).NotTo(HaveOccurred()) + }) + cr.Spec.CRVersion = "1.13.0" + + BeforeAll(func() { + By("Creating the Namespace to perform the tests") + err := k8sClient.Create(ctx, namespace) + Expect(err).To(Not(HaveOccurred())) + }) + + AfterAll(func() { + // TODO(user): Attention if you improve this code by adding other context test you MUST + // be aware of the current delete namespace limitations. More info: https://book.kubebuilder.io/reference/envtest.html#testing-considerations + By("Deleting the Namespace to perform the tests") + _ = k8sClient.Delete(ctx, namespace) + }) + + Context("Create Percona XtraDB cluster", func() { + It("Should create PerconaXtraDBCluster", func() { + Expect(k8sClient.Create(ctx, cr)).Should(Succeed()) + }) + }) + + It("should reconcile PerconaXtraDBCluster", func() { + _, err := reconciler().Reconcile(ctx, reconcile.Request{ + NamespacedName: crNamespacedName, + }) + Expect(err).To(Succeed()) + }) + + checkLabelsAndAnnotations := func(services []*corev1.Service) { + Context("update service labels manually", func() { + It("should update service labels manually", func() { + for i := range services { + svc := new(corev1.Service) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(services[i]), svc)).To(Succeed()) + + svc.Labels["manual-label"] = "test" + svc.Labels["ignored-label"] = "test" + svc.Annotations["manual-annotation"] = "test" + svc.Annotations["ignored-annotation"] = "test" + Expect(k8sClient.Update(ctx, svc)).To(Succeed()) + } + }) + + It("should reconcile PerconaXtraDBCluster", func() { + _, err := reconciler().Reconcile(ctx, reconcile.Request{ + NamespacedName: crNamespacedName, + }) + Expect(err).To(Succeed()) + }) + It("should check if manual labels and annotations are still there", func() { + for i := range services { + svc := new(corev1.Service) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(services[i]), svc)).To(Succeed()) + + Expect(svc.Labels["manual-label"]).To(Equal("test")) + Expect(svc.Annotations["manual-annotation"]).To(Equal("test")) + Expect(svc.Labels["ignored-label"]).To(Equal("test")) + Expect(svc.Annotations["ignored-annotation"]).To(Equal("test")) + } + }) + }) + + Context("set service labels and annotations", func() { + It("should update cr", func() { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed()) + cr.Spec.IgnoreAnnotations = []string{"ignored-annotation"} + cr.Spec.IgnoreLabels = []string{"ignored-label"} + + cr.Spec.HAProxy.ServiceLabels = map[string]string{"cr-label": "test"} + cr.Spec.HAProxy.ServiceAnnotations = map[string]string{"cr-annotation": "test"} + cr.Spec.HAProxy.ReplicasServiceLabels = map[string]string{"cr-label": "test"} + cr.Spec.HAProxy.ReplicasServiceAnnotations = map[string]string{"cr-annotation": "test"} + cr.Spec.ProxySQL.ServiceLabels = map[string]string{"cr-label": "test"} + cr.Spec.ProxySQL.ServiceAnnotations = map[string]string{"cr-annotation": "test"} + Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) + }) + It("should reconcile PerconaXtraDBCluster", func() { + _, err := reconciler().Reconcile(ctx, reconcile.Request{ + NamespacedName: crNamespacedName, + }) + Expect(err).To(Succeed()) + }) + It("check labels and annotations", func() { + for i := range services { + svc := new(corev1.Service) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(services[i]), svc)).To(Succeed()) + + if strings.Contains(svc.Name, "pxc") { + Expect(svc.Labels["manual-label"]).To(Equal("test")) + Expect(svc.Annotations["manual-annotation"]).To(Equal("test")) + Expect(svc.Labels["ignored-label"]).To(Equal("test")) + Expect(svc.Annotations["ignored-annotation"]).To(Equal("test")) + + Expect(svc.Labels["cr-label"]).To(BeEmpty()) + Expect(svc.Annotations["cr-annotation"]).To(BeEmpty()) + } else { + Expect(svc.Labels["manual-label"]).To(Equal("")) + Expect(svc.Annotations["manual-annotation"]).To(Equal("")) + Expect(svc.Labels["ignored-label"]).To(Equal("test")) + Expect(svc.Annotations["ignored-annotation"]).To(Equal("test")) + Expect(svc.Labels["cr-label"]).To(Equal("test")) + Expect(svc.Annotations["cr-annotation"]).To(Equal("test")) + } + } + }) + }) + Context("remove ignored labels and annotations", func() { + It("should update cr", func() { + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed()) + cr.Spec.IgnoreAnnotations = []string{} + cr.Spec.IgnoreLabels = []string{} + Expect(k8sClient.Update(ctx, cr)).Should(Succeed()) + }) + It("should reconcile PerconaXtraDBCluster", func() { + _, err := reconciler().Reconcile(ctx, reconcile.Request{ + NamespacedName: crNamespacedName, + }) + Expect(err).To(Succeed()) + }) + It("should check if there are no ignored labels and annotations", func() { + for i := range services { + svc := new(corev1.Service) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(services[i]), svc)).To(Succeed()) + + if strings.Contains(svc.Name, "pxc") { + Expect(svc.Labels["ignored-label"]).To(Equal("test")) + Expect(svc.Annotations["ignored-annotation"]).To(Equal("test")) + Expect(svc.Labels["cr-label"]).To(BeEmpty()) + Expect(svc.Annotations["cr-annotation"]).To(BeEmpty()) + } else { + Expect(svc.Labels["ignored-label"]).To(Equal("")) + Expect(svc.Annotations["ignored-annotation"]).To(Equal("")) + Expect(svc.Labels["cr-label"]).To(Equal("test")) + Expect(svc.Annotations["cr-annotation"]).To(Equal("test")) + } + } + }) + }) + } + + services := []*corev1.Service{ + pxc.NewServicePXC(cr), + pxc.NewServiceHAProxy(cr), + pxc.NewServiceHAProxyReplicas(cr), + } + + Context("check haproxy cluster", func() { + checkLabelsAndAnnotations(services) + }) + + It("should delete services", func() { + for _, svc := range services { + Expect(k8sClient.Delete(ctx, svc)).To(Succeed()) + } + }) + + It("should switch to ProxySQL and remove serviceLabels, serviceAnnotations", func() { + haproxySts := &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: cr.Name + "-haproxy", + Namespace: cr.Namespace, + }, + } + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(haproxySts), haproxySts)).To(Succeed()) + Expect(k8sClient.Delete(ctx, haproxySts)).To(Succeed()) + + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(cr), cr)).To(Succeed()) + cr.Spec.HAProxy.Enabled = false + cr.Spec.ProxySQL.Enabled = true + cr.Spec.HAProxy.ServiceLabels = nil cr.Spec.HAProxy.ServiceAnnotations = nil cr.Spec.HAProxy.ReplicasServiceLabels = nil diff --git a/pkg/controller/pxc/status.go b/pkg/controller/pxc/status.go index f194d11e27..e45e320a98 100644 --- a/pkg/controller/pxc/status.go +++ b/pkg/controller/pxc/status.go @@ -48,6 +48,7 @@ func (r *ReconcilePerconaXtraDBCluster) updateStatus(cr *api.PerconaXtraDBCluste app api.StatefulApp status *api.AppStatus spec *api.PodSpec + expose *api.ServiceExpose } // Maintaining the order of this slice is important! @@ -58,6 +59,7 @@ func (r *ReconcilePerconaXtraDBCluster) updateStatus(cr *api.PerconaXtraDBCluste app: statefulset.NewNode(cr), status: &cr.Status.PXC, spec: cr.Spec.PXC.PodSpec, + expose: &cr.Spec.PXC.Expose, }, } @@ -71,6 +73,7 @@ func (r *ReconcilePerconaXtraDBCluster) updateStatus(cr *api.PerconaXtraDBCluste app: statefulset.NewHAProxy(cr), status: &cr.Status.HAProxy, spec: &cr.Spec.HAProxy.PodSpec, + expose: &cr.Spec.HAProxy.ExposePrimary, }) } @@ -83,7 +86,8 @@ func (r *ReconcilePerconaXtraDBCluster) updateStatus(cr *api.PerconaXtraDBCluste apps = append(apps, sfsstatus{ app: statefulset.NewProxy(cr), status: &cr.Status.ProxySQL, - spec: cr.Spec.ProxySQL, + spec: &cr.Spec.ProxySQL.PodSpec, + expose: &cr.Spec.ProxySQL.Expose, }) } @@ -102,7 +106,7 @@ func (r *ReconcilePerconaXtraDBCluster) updateStatus(cr *api.PerconaXtraDBCluste } *a.status = status - host, err := r.appHost(cr, a.app, a.spec) + host, err := r.appHost(cr, a.app, a.spec, a.expose) if err != nil { return errors.Wrapf(err, "get %s host", a.app.Name()) } @@ -235,12 +239,19 @@ func (r *ReconcilePerconaXtraDBCluster) appStatus(app api.StatefulApp, namespace return status, nil } -func (r *ReconcilePerconaXtraDBCluster) appHost(cr *api.PerconaXtraDBCluster, app api.StatefulApp, podSpec *api.PodSpec) (string, error) { +func (r *ReconcilePerconaXtraDBCluster) appHost(cr *api.PerconaXtraDBCluster, app api.StatefulApp, + podSpec *api.PodSpec, expose *api.ServiceExpose) (string, error) { svcName := app.Service() if app.Name() == "proxysql" { svcName = cr.Name + "-proxysql" } - if podSpec.ServiceType != corev1.ServiceTypeLoadBalancer { + + svcType := expose.Type + if cr.CompareVersionWith("1.14.0") < 0 { + svcType = podSpec.ServiceType + } + + if svcType != corev1.ServiceTypeLoadBalancer { return svcName + "." + cr.Namespace, nil } diff --git a/pkg/controller/pxc/status_test.go b/pkg/controller/pxc/status_test.go index f79094507d..8c50af6f98 100644 --- a/pkg/controller/pxc/status_test.go +++ b/pkg/controller/pxc/status_test.go @@ -42,15 +42,33 @@ func newCR(name, namespace string) *api.PerconaXtraDBCluster { Enabled: true, Size: 3, }, + Expose: api.ServiceExpose{ + Enabled: false, + Type: corev1.ServiceTypeClusterIP, + }, }, HAProxy: &api.HAProxySpec{ PodSpec: api.PodSpec{ Enabled: true, Size: 3, }, + ExposePrimary: api.ServiceExpose{ + Enabled: false, + Type: corev1.ServiceTypeClusterIP, + }, + ExposeReplicas: &api.ServiceExpose{ + Enabled: false, + Type: corev1.ServiceTypeClusterIP, + }, }, - ProxySQL: &api.PodSpec{ - Enabled: false, + ProxySQL: &api.ProxySQLSpec{ + PodSpec: api.PodSpec{ + Enabled: false, + }, + Expose: api.ServiceExpose{ + Enabled: false, + Type: corev1.ServiceTypeClusterIP, + }, }, }, Status: api.PerconaXtraDBClusterStatus{}, @@ -218,7 +236,7 @@ func TestAppHostNoLoadBalancer(t *testing.T) { r := buildFakeClient([]runtime.Object{cr, pxcSfs, haproxySfs}) - host, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec) + host, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec, &cr.Spec.HAProxy.ExposePrimary) if err != nil { t.Error(err) } @@ -231,17 +249,18 @@ func TestAppHostNoLoadBalancer(t *testing.T) { func TestAppHostLoadBalancerNoSvc(t *testing.T) { cr := newCR("cr-mock", "pxc") + cr.Spec.CRVersion = "1.14.0" pxc := statefulset.NewNode(cr) pxcSfs := pxc.StatefulSet() haproxy := statefulset.NewHAProxy(cr) haproxySfs := haproxy.StatefulSet() - cr.Spec.HAProxy.ServiceType = corev1.ServiceTypeLoadBalancer + cr.Spec.HAProxy.ExposePrimary.Type = corev1.ServiceTypeLoadBalancer r := buildFakeClient([]runtime.Object{cr, pxcSfs, haproxySfs}) - _, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec) + _, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec, &cr.Spec.HAProxy.ExposePrimary) if err == nil { t.Error("want err, got nil") } @@ -249,13 +268,14 @@ func TestAppHostLoadBalancerNoSvc(t *testing.T) { func TestAppHostLoadBalancerOnlyIP(t *testing.T) { cr := newCR("cr-mock", "pxc") + cr.Spec.CRVersion = "1.14.0" pxc := statefulset.NewNode(cr) pxcSfs := pxc.StatefulSet() haproxy := statefulset.NewHAProxy(cr) haproxySfs := haproxy.StatefulSet() - cr.Spec.HAProxy.ServiceType = corev1.ServiceTypeLoadBalancer + cr.Spec.HAProxy.ExposePrimary.Type = corev1.ServiceTypeLoadBalancer ip := "99.99.99.99" haproxySvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -271,7 +291,7 @@ func TestAppHostLoadBalancerOnlyIP(t *testing.T) { r := buildFakeClient([]runtime.Object{cr, pxcSfs, haproxySfs, haproxySvc}) - host, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec) + host, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec, &cr.Spec.HAProxy.ExposePrimary) if err != nil { t.Error(err) } @@ -283,13 +303,14 @@ func TestAppHostLoadBalancerOnlyIP(t *testing.T) { func TestAppHostLoadBalancerWithHostname(t *testing.T) { cr := newCR("cr-mock", "pxc") + cr.Spec.CRVersion = "1.14.0" pxc := statefulset.NewNode(cr) pxcSfs := pxc.StatefulSet() haproxy := statefulset.NewHAProxy(cr) haproxySfs := haproxy.StatefulSet() - cr.Spec.HAProxy.ServiceType = corev1.ServiceTypeLoadBalancer + cr.Spec.HAProxy.ExposePrimary.Type = corev1.ServiceTypeLoadBalancer wantHost := "cr-mock.haproxy.test" haproxySvc := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -305,7 +326,7 @@ func TestAppHostLoadBalancerWithHostname(t *testing.T) { r := buildFakeClient([]runtime.Object{cr, pxcSfs, haproxySfs, haproxySvc}) - gotHost, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec) + gotHost, err := r.appHost(cr, haproxy, &cr.Spec.HAProxy.PodSpec, &cr.Spec.HAProxy.ExposePrimary) if err != nil { t.Error(err) } diff --git a/pkg/pxc/service.go b/pkg/pxc/service.go index abfdfef39c..eca7a9515d 100644 --- a/pkg/pxc/service.go +++ b/pkg/pxc/service.go @@ -69,8 +69,8 @@ func NewServicePXC(cr *api.PerconaXtraDBCluster) *corev1.Service { if cr.CompareVersionWith("1.14.0") >= 0 { if cr.Spec.PXC != nil { - obj.Annotations = cr.Spec.PXC.ServiceAnnotations - obj.Labels = fillServiceLabels(obj.Labels, cr.Spec.PXC.ServiceLabels) + obj.Annotations = cr.Spec.PXC.Expose.Annotations + obj.Labels = fillServiceLabels(obj.Labels, cr.Spec.PXC.Expose.Labels) } } @@ -141,8 +141,8 @@ func NewServicePXCUnready(cr *api.PerconaXtraDBCluster) *corev1.Service { if cr.CompareVersionWith("1.14.0") >= 0 { if cr.Spec.PXC != nil { - obj.Annotations = cr.Spec.PXC.ServiceAnnotations - obj.Labels = fillServiceLabels(obj.Labels, cr.Spec.PXC.ServiceLabels) + obj.Annotations = cr.Spec.PXC.Expose.Annotations + obj.Labels = fillServiceLabels(obj.Labels, cr.Spec.PXC.Expose.Labels) } } @@ -212,9 +212,15 @@ func NewServiceProxySQLUnready(cr *api.PerconaXtraDBCluster) *corev1.Service { func NewServiceProxySQL(cr *api.PerconaXtraDBCluster) *corev1.Service { svcType := corev1.ServiceTypeClusterIP - if cr.Spec.ProxySQL != nil && len(cr.Spec.ProxySQL.ServiceType) > 0 { - svcType = cr.Spec.ProxySQL.ServiceType + + if cr.Spec.ProxySQL != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && len(cr.Spec.ProxySQL.Expose.Type) > 0 { + svcType = cr.Spec.ProxySQL.Expose.Type + } else if len(cr.Spec.ProxySQL.ServiceType) > 0 { + svcType = cr.Spec.ProxySQL.ServiceType + } } + serviceAnnotations := make(map[string]string) serviceLabels := map[string]string{ "app.kubernetes.io/name": "percona-xtradb-cluster", @@ -222,12 +228,21 @@ func NewServiceProxySQL(cr *api.PerconaXtraDBCluster) *corev1.Service { } loadBalancerSourceRanges := []string{} loadBalancerIP := "" + if cr.Spec.ProxySQL != nil { - serviceAnnotations = cr.Spec.ProxySQL.ServiceAnnotations - serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.ProxySQL.ServiceLabels) - loadBalancerSourceRanges = cr.Spec.ProxySQL.LoadBalancerSourceRanges - loadBalancerIP = cr.Spec.ProxySQL.LoadBalancerIP + if cr.CompareVersionWith("1.14.0") >= 0 { + serviceAnnotations = cr.Spec.ProxySQL.Expose.Annotations + serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.ProxySQL.Expose.Labels) + loadBalancerSourceRanges = cr.Spec.ProxySQL.Expose.LoadBalancerSourceRanges + loadBalancerIP = cr.Spec.ProxySQL.Expose.LoadBalancerIP + } else { + serviceAnnotations = cr.Spec.ProxySQL.ServiceAnnotations + serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.ProxySQL.ServiceLabels) + loadBalancerSourceRanges = cr.Spec.ProxySQL.LoadBalancerSourceRanges + loadBalancerIP = cr.Spec.ProxySQL.LoadBalancerIP + } } + obj := &corev1.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -259,17 +274,29 @@ func NewServiceProxySQL(cr *api.PerconaXtraDBCluster) *corev1.Service { if svcType == corev1.ServiceTypeLoadBalancer || svcType == corev1.ServiceTypeNodePort { svcTrafficPolicyType := corev1.ServiceExternalTrafficPolicyTypeCluster - if cr.Spec.ProxySQL != nil && len(cr.Spec.ProxySQL.ExternalTrafficPolicy) > 0 { - svcTrafficPolicyType = cr.Spec.ProxySQL.ExternalTrafficPolicy + + if cr.Spec.ProxySQL != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && len(cr.Spec.ProxySQL.Expose.ExternalTrafficPolicy) > 0 { + svcTrafficPolicyType = cr.Spec.ProxySQL.Expose.ExternalTrafficPolicy + } else if len(cr.Spec.ProxySQL.ExternalTrafficPolicy) > 0 { + svcTrafficPolicyType = cr.Spec.ProxySQL.ExternalTrafficPolicy + } } obj.Spec.ExternalTrafficPolicy = svcTrafficPolicyType } - if cr.Spec.ProxySQL != nil && cr.Spec.ProxySQL.ServiceAnnotations != nil { - if cr.Spec.ProxySQL.ServiceAnnotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { - obj.Annotations[HeadlessServiceAnnotation] = "true" - obj.Spec.ClusterIP = corev1.ClusterIPNone + if cr.Spec.ProxySQL != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && cr.Spec.ProxySQL.Expose.Annotations != nil { + if cr.Spec.ProxySQL.Expose.Annotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { + obj.Annotations[HeadlessServiceAnnotation] = "true" + obj.Spec.ClusterIP = corev1.ClusterIPNone + } + } else if cr.Spec.ProxySQL.ServiceAnnotations != nil { + if cr.Spec.ProxySQL.ServiceAnnotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { + obj.Annotations[HeadlessServiceAnnotation] = "true" + obj.Spec.ClusterIP = corev1.ClusterIPNone + } } } @@ -294,8 +321,13 @@ func NewServiceProxySQL(cr *api.PerconaXtraDBCluster) *corev1.Service { func NewServiceHAProxy(cr *api.PerconaXtraDBCluster) *corev1.Service { svcType := corev1.ServiceTypeClusterIP - if cr.Spec.HAProxy != nil && len(cr.Spec.HAProxy.ServiceType) > 0 { - svcType = cr.Spec.HAProxy.ServiceType + + if cr.Spec.HAProxy != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && len(cr.Spec.HAProxy.ExposePrimary.Type) > 0 { + svcType = cr.Spec.HAProxy.ExposePrimary.Type + } else if len(cr.Spec.HAProxy.ServiceType) > 0 { + svcType = cr.Spec.HAProxy.ServiceType + } } serviceAnnotations := make(map[string]string) @@ -308,11 +340,19 @@ func NewServiceHAProxy(cr *api.PerconaXtraDBCluster) *corev1.Service { } loadBalancerSourceRanges := []string{} loadBalancerIP := "" + if cr.Spec.HAProxy != nil { - serviceAnnotations = cr.Spec.HAProxy.ServiceAnnotations - serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.HAProxy.PodSpec.ServiceLabels) - loadBalancerSourceRanges = cr.Spec.HAProxy.LoadBalancerSourceRanges - loadBalancerIP = cr.Spec.HAProxy.LoadBalancerIP + if cr.CompareVersionWith("1.14.0") >= 0 { + serviceAnnotations = cr.Spec.HAProxy.ExposePrimary.Annotations + serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.HAProxy.ExposePrimary.Labels) + loadBalancerSourceRanges = cr.Spec.HAProxy.ExposePrimary.LoadBalancerSourceRanges + loadBalancerIP = cr.Spec.HAProxy.ExposePrimary.LoadBalancerIP + } else { + serviceAnnotations = cr.Spec.HAProxy.ServiceAnnotations + serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.HAProxy.PodSpec.ServiceLabels) + loadBalancerSourceRanges = cr.Spec.HAProxy.LoadBalancerSourceRanges + loadBalancerIP = cr.Spec.HAProxy.LoadBalancerIP + } } obj := &corev1.Service{ @@ -352,8 +392,13 @@ func NewServiceHAProxy(cr *api.PerconaXtraDBCluster) *corev1.Service { if svcType == corev1.ServiceTypeLoadBalancer || svcType == corev1.ServiceTypeNodePort { svcTrafficPolicyType := corev1.ServiceExternalTrafficPolicyTypeCluster - if cr.Spec.HAProxy != nil && len(cr.Spec.HAProxy.ExternalTrafficPolicy) > 0 { - svcTrafficPolicyType = cr.Spec.HAProxy.ExternalTrafficPolicy + + if cr.Spec.HAProxy != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && len(cr.Spec.HAProxy.ExposePrimary.ExternalTrafficPolicy) > 0 { + svcTrafficPolicyType = cr.Spec.HAProxy.ExposePrimary.ExternalTrafficPolicy + } else if len(cr.Spec.HAProxy.ExternalTrafficPolicy) > 0 { + svcTrafficPolicyType = cr.Spec.HAProxy.ExternalTrafficPolicy + } } obj.Spec.ExternalTrafficPolicy = svcTrafficPolicyType @@ -381,10 +426,21 @@ func NewServiceHAProxy(cr *api.PerconaXtraDBCluster) *corev1.Service { ) } - if cr.Spec.HAProxy != nil && cr.Spec.HAProxy.ServiceAnnotations != nil { - if cr.Spec.HAProxy.ServiceAnnotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { - obj.Annotations[HeadlessServiceAnnotation] = "true" - obj.Spec.ClusterIP = corev1.ClusterIPNone + if cr.Spec.HAProxy != nil { + if cr.CompareVersionWith("1.14.0") >= 0 { + if cr.Spec.HAProxy.ExposePrimary.Annotations != nil { + if cr.Spec.HAProxy.ExposePrimary.Annotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { + obj.Annotations[HeadlessServiceAnnotation] = "true" + obj.Spec.ClusterIP = corev1.ClusterIPNone + } + } + } else { + if cr.Spec.HAProxy.ServiceAnnotations != nil { + if cr.Spec.HAProxy.ServiceAnnotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { + obj.Annotations[HeadlessServiceAnnotation] = "true" + obj.Spec.ClusterIP = corev1.ClusterIPNone + } + } } } @@ -392,10 +448,23 @@ func NewServiceHAProxy(cr *api.PerconaXtraDBCluster) *corev1.Service { } func NewServiceHAProxyReplicas(cr *api.PerconaXtraDBCluster) *corev1.Service { + if cr.CompareVersionWith("1.14.0") >= 0 && cr.Spec.HAProxy != nil { + if cr.Spec.HAProxy.ExposeReplicas == nil { + cr.Spec.HAProxy.ExposeReplicas = &api.ServiceExpose{ + Enabled: true, + } + } + } + svcType := corev1.ServiceTypeClusterIP - if cr.Spec.HAProxy != nil && len(cr.Spec.HAProxy.ReplicasServiceType) > 0 { - svcType = cr.Spec.HAProxy.ReplicasServiceType + if cr.Spec.HAProxy != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && len(cr.Spec.HAProxy.ExposeReplicas.Type) > 0 { + svcType = cr.Spec.HAProxy.ExposeReplicas.Type + } else if len(cr.Spec.HAProxy.ReplicasServiceType) > 0 { + svcType = cr.Spec.HAProxy.ReplicasServiceType + } } + serviceAnnotations := make(map[string]string) serviceLabels := map[string]string{ "app.kubernetes.io/name": "percona-xtradb-cluster", @@ -407,20 +476,34 @@ func NewServiceHAProxyReplicas(cr *api.PerconaXtraDBCluster) *corev1.Service { loadBalancerSourceRanges := []string{} loadBalancerIP := "" if cr.Spec.HAProxy != nil { - if cr.CompareVersionWith("1.12.0") >= 0 { + if cr.CompareVersionWith("1.14.0") >= 0 { + serviceAnnotations = cr.Spec.HAProxy.ExposeReplicas.Annotations + serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.HAProxy.ExposeReplicas.Labels) + } else if cr.CompareVersionWith("1.12.0") >= 0 { serviceAnnotations = cr.Spec.HAProxy.ReplicasServiceAnnotations serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.HAProxy.PodSpec.ReplicasServiceLabels) } else { serviceAnnotations = cr.Spec.HAProxy.ServiceAnnotations serviceLabels = fillServiceLabels(serviceLabels, cr.Spec.HAProxy.PodSpec.ServiceLabels) } - if cr.Spec.HAProxy.ReplicasLoadBalancerSourceRanges != nil { - loadBalancerSourceRanges = cr.Spec.HAProxy.ReplicasLoadBalancerSourceRanges + + if cr.CompareVersionWith("1.14.0") >= 0 { + if cr.Spec.HAProxy.ExposeReplicas.LoadBalancerSourceRanges != nil { + loadBalancerSourceRanges = cr.Spec.HAProxy.ExposeReplicas.LoadBalancerSourceRanges + } else { + loadBalancerSourceRanges = cr.Spec.HAProxy.ExposePrimary.LoadBalancerSourceRanges + } + loadBalancerIP = cr.Spec.HAProxy.ExposeReplicas.LoadBalancerIP } else { - loadBalancerSourceRanges = cr.Spec.HAProxy.LoadBalancerSourceRanges + if cr.Spec.HAProxy.ReplicasLoadBalancerSourceRanges != nil { + loadBalancerSourceRanges = cr.Spec.HAProxy.ReplicasLoadBalancerSourceRanges + } else { + loadBalancerSourceRanges = cr.Spec.HAProxy.LoadBalancerSourceRanges + } + loadBalancerIP = cr.Spec.HAProxy.ReplicasLoadBalancerIP } - loadBalancerIP = cr.Spec.HAProxy.ReplicasLoadBalancerIP } + obj := &corev1.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", @@ -453,17 +536,29 @@ func NewServiceHAProxyReplicas(cr *api.PerconaXtraDBCluster) *corev1.Service { if svcType == corev1.ServiceTypeLoadBalancer || svcType == corev1.ServiceTypeNodePort { svcTrafficPolicyType := corev1.ServiceExternalTrafficPolicyTypeCluster - if cr.Spec.HAProxy != nil && len(cr.Spec.HAProxy.ReplicasExternalTrafficPolicy) > 0 { - svcTrafficPolicyType = cr.Spec.HAProxy.ReplicasExternalTrafficPolicy + + if cr.Spec.HAProxy != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && len(cr.Spec.HAProxy.ExposeReplicas.ExternalTrafficPolicy) > 0 { + svcTrafficPolicyType = cr.Spec.HAProxy.ExposeReplicas.ExternalTrafficPolicy + } else if len(cr.Spec.HAProxy.ReplicasExternalTrafficPolicy) > 0 { + svcTrafficPolicyType = cr.Spec.HAProxy.ReplicasExternalTrafficPolicy + } } obj.Spec.ExternalTrafficPolicy = svcTrafficPolicyType } - if cr.Spec.HAProxy != nil && cr.Spec.HAProxy.ReplicasServiceAnnotations != nil { - if cr.Spec.HAProxy.ReplicasServiceAnnotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { - obj.Annotations[HeadlessServiceAnnotation] = "true" - obj.Spec.ClusterIP = corev1.ClusterIPNone + if cr.Spec.HAProxy != nil { + if cr.CompareVersionWith("1.14.0") >= 0 && cr.Spec.HAProxy.ExposeReplicas.Annotations != nil { + if cr.Spec.HAProxy.ExposeReplicas.Annotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { + obj.Annotations[HeadlessServiceAnnotation] = "true" + obj.Spec.ClusterIP = corev1.ClusterIPNone + } + } else if cr.Spec.HAProxy.ReplicasServiceAnnotations != nil { + if cr.Spec.HAProxy.ReplicasServiceAnnotations[HeadlessServiceAnnotation] == "true" && svcType == corev1.ServiceTypeClusterIP { + obj.Annotations[HeadlessServiceAnnotation] = "true" + obj.Spec.ClusterIP = corev1.ClusterIPNone + } } }