(feat) update status ConnectionStatus field

Sveltos periodically attempts to connect to managed SveltosClusters. Upon successful initial connection, the `Ready` status is set to true and remains unchanged. To indicate connection health, this PR introduces a `ConnectionStatus` field within the Status. This field can be either `Healthy` or `Down`. The status transitions to `Down` after a configurable number of consecutive connection failures, which defaults to three.
projectsveltos · Aug 5, 2024 · d59860b · d59860b
1 parent 1e359f7
commit d59860b
Show file tree

Hide file tree

Showing 9 changed files with 110 additions and 9 deletions.
diff --git a/Makefile b/Makefile
@@ -25,7 +25,7 @@ ARCH ?= amd64
 OS ?= $(shell uname -s | tr A-Z a-z)
 K8S_LATEST_VER ?= $(shell curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)
 export CONTROLLER_IMG ?= $(REGISTRY)/$(IMAGE_NAME)
-TAG ?= main
+TAG ?= dev
 
 .PHONY: all
 all: build

diff --git a/config/default/manager_image_patch.yaml b/config/default/manager_image_patch.yaml
@@ -8,5 +8,5 @@ spec:
     spec:
       containers:
       # Change the value of image field below to your controller image URL
-      - image: docker.io/projectsveltos/sveltoscluster-manager:main
+      - image: docker.io/projectsveltos/sveltoscluster-manager:dev
         name: manager
diff --git a/controllers/sveltoscluster_controller.go b/controllers/sveltoscluster_controller.go
@@ -141,11 +141,14 @@ func (r *SveltosClusterReconciler) reconcileNormal(
 	logger := sveltosClusterScope.Logger
 	logger.V(logs.LogInfo).Info("Reconciling SveltosCluster")
 
+	defer handleAutomaticPauseUnPause(sveltosClusterScope.SveltosCluster, time.Now(), logger)
+
 	s := runtime.NewScheme()
 	if err := clientgoscheme.AddToScheme(s); err != nil {
 		errorMessage := err.Error()
 		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get scheme: %v", err))
 		sveltosClusterScope.SveltosCluster.Status.FailureMessage = &errorMessage
+		updateConnectionStatus(sveltosClusterScope, logger)
 		return
 	}
 
@@ -156,6 +159,7 @@ func (r *SveltosClusterReconciler) reconcileNormal(
 		errorMessage := err.Error()
 		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get client: %v", err))
 		sveltosClusterScope.SveltosCluster.Status.FailureMessage = &errorMessage
+		updateConnectionStatus(sveltosClusterScope, logger)
 		return
 	}
 
@@ -166,6 +170,7 @@ func (r *SveltosClusterReconciler) reconcileNormal(
 		errorMessage := err.Error()
 		logger.V(logs.LogInfo).Info(fmt.Sprintf("failed to get client: %v", err))
 		sveltosClusterScope.SveltosCluster.Status.FailureMessage = &errorMessage
+		updateConnectionStatus(sveltosClusterScope, logger)
 		return
 	}
 
@@ -205,7 +210,21 @@ func (r *SveltosClusterReconciler) reconcileNormal(
 		}
 	}
 
-	handleAutomaticPauseUnPause(sveltosClusterScope.SveltosCluster, time.Now(), logger)
+	updateConnectionStatus(sveltosClusterScope, logger)
+}
+
+func updateConnectionStatus(sveltosClusterScope *scope.SveltosClusterScope, logger logr.Logger) {
+	if sveltosClusterScope.SveltosCluster.Status.FailureMessage != nil {
+		logger.V(logs.LogDebug).Info("increasing connectionFailures")
+		sveltosClusterScope.SveltosCluster.Status.ConnectionFailures++
+		if sveltosClusterScope.SveltosCluster.Status.ConnectionFailures >= sveltosClusterScope.SveltosCluster.Spec.ConsecutiveFailureThreshold {
+			logger.V(logs.LogDebug).Info("connectionFailures is higher than consecutiveFailureThreshold. Set connectionStatus to down")
+			sveltosClusterScope.SveltosCluster.Status.ConnectionStatus = libsveltosv1beta1.ConnectionDown
+		}
+	} else {
+		sveltosClusterScope.SveltosCluster.Status.ConnectionStatus = libsveltosv1beta1.ConnectionHealthy
+		sveltosClusterScope.SveltosCluster.Status.ConnectionFailures = 0
+	}
 }
 
 // SetupWithManager sets up the controller with the Manager.

diff --git a/controllers/sveltoscluster_controller_test.go b/controllers/sveltoscluster_controller_test.go
@@ -89,6 +89,87 @@ var _ = Describe("SveltosCluster: Reconciler", func() {
 			return err == nil &&
 				currentSveltosCluster.Status.Ready
 		}, timeout, pollingInterval).Should(BeTrue())
+
+		currentSveltosCluster := &libsveltosv1beta1.SveltosCluster{}
+		err = testEnv.Get(context.TODO(), sveltosClusterName, currentSveltosCluster)
+		Expect(err).To(BeNil())
+		Expect(currentSveltosCluster.Status.ConnectionFailures).To(Equal(0))
+		Expect(currentSveltosCluster.Status.ConnectionStatus).To(Equal(libsveltosv1beta1.ConnectionHealthy))
+	})
+
+	It("reconcile set connection down after enough consecutive failed connection", func() {
+		ns := &corev1.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: sveltosCluster.Namespace,
+			},
+		}
+
+		Expect(testEnv.Create(context.TODO(), ns)).To(Succeed())
+		Expect(waitForObject(context.TODO(), testEnv.Client, ns)).To(Succeed())
+
+		// Create Secret containing Kubeconfig to access SveltosCluster
+
+		sveltosSecret := corev1.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Namespace: sveltosCluster.Namespace,
+				Name:      sveltosCluster.Name + "-sveltos-kubeconfig",
+			},
+			Data: map[string][]byte{
+				"data": []byte("not a valid kubeconfig"),
+			},
+		}
+
+		Expect(testEnv.Create(context.TODO(), &sveltosSecret)).To(Succeed())
+		Expect(waitForObject(context.TODO(), testEnv.Client, &sveltosSecret)).To(Succeed())
+
+		Expect(testEnv.Create(context.TODO(), sveltosCluster)).To(Succeed())
+		Expect(waitForObject(context.TODO(), testEnv.Client, sveltosCluster)).To(Succeed())
+
+		reconciler := getClusterProfileReconciler(testEnv.Client)
+
+		sveltosClusterName := client.ObjectKey{
+			Name:      sveltosCluster.Name,
+			Namespace: sveltosCluster.Namespace,
+		}
+		_, err := reconciler.Reconcile(context.TODO(), ctrl.Request{
+			NamespacedName: sveltosClusterName,
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() bool {
+			currentSveltosCluster := &libsveltosv1beta1.SveltosCluster{}
+			err := testEnv.Get(context.TODO(), sveltosClusterName, currentSveltosCluster)
+			return err == nil &&
+				!currentSveltosCluster.Status.Ready &&
+				currentSveltosCluster.Status.ConnectionFailures == 1
+		}, timeout, pollingInterval).Should(BeTrue())
+
+		_, err = reconciler.Reconcile(context.TODO(), ctrl.Request{
+			NamespacedName: sveltosClusterName,
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() bool {
+			currentSveltosCluster := &libsveltosv1beta1.SveltosCluster{}
+			err := testEnv.Get(context.TODO(), sveltosClusterName, currentSveltosCluster)
+			return err == nil &&
+				!currentSveltosCluster.Status.Ready &&
+				currentSveltosCluster.Status.ConnectionFailures == 2
+		}, timeout, pollingInterval).Should(BeTrue())
+
+		_, err = reconciler.Reconcile(context.TODO(), ctrl.Request{
+			NamespacedName: sveltosClusterName,
+		})
+		Expect(err).ToNot(HaveOccurred())
+
+		Eventually(func() bool {
+			currentSveltosCluster := &libsveltosv1beta1.SveltosCluster{}
+			err := testEnv.Get(context.TODO(), sveltosClusterName, currentSveltosCluster)
+			return err == nil &&
+				!currentSveltosCluster.Status.Ready &&
+				currentSveltosCluster.Status.ConnectionFailures == 3 &&
+				currentSveltosCluster.Status.ConnectionStatus == libsveltosv1beta1.ConnectionDown
+		}, timeout, pollingInterval).Should(BeTrue())
 	})
 
 	It("shouldRenewTokenRequest returns true when enough time has passed since last TokenRequest renewal", func() {

diff --git a/go.mod b/go.mod
@@ -9,7 +9,7 @@ require (
 	github.com/onsi/ginkgo/v2 v2.19.0
 	github.com/onsi/gomega v1.33.1
 	github.com/pkg/errors v0.9.1
-	github.com/projectsveltos/libsveltos v0.35.1-0.20240728165049-3f913227fee9
+	github.com/projectsveltos/libsveltos v0.35.1-0.20240805114830-bfe0e2b5e213
 	github.com/robfig/cron/v3 v3.0.1
 	github.com/spf13/pflag v1.0.5
 	k8s.io/api v0.30.3

diff --git a/go.sum b/go.sum
@@ -131,8 +131,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/projectsveltos/libsveltos v0.35.1-0.20240728165049-3f913227fee9 h1:u07JFnTTLl/xQG5Xykj+UeGfPJGjMR+8sWXsZuRxTPE=
-github.com/projectsveltos/libsveltos v0.35.1-0.20240728165049-3f913227fee9/go.mod h1:8cr9lSt8i0fRQ47AItTElqxsiD/ni80GJALVQgxfdG4=
+github.com/projectsveltos/libsveltos v0.35.1-0.20240805114830-bfe0e2b5e213 h1:TU8nNO5kQhjkRN29a74sLGypriQCE7PL4Lw2DiFrkPw=
+github.com/projectsveltos/libsveltos v0.35.1-0.20240805114830-bfe0e2b5e213/go.mod h1:8cr9lSt8i0fRQ47AItTElqxsiD/ni80GJALVQgxfdG4=
 github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk=
 github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA=
 github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=

diff --git a/manifest/deployment-shard.yaml b/manifest/deployment-shard.yaml
@@ -24,7 +24,7 @@ spec:
         - --v=5
         command:
         - /manager
-        image: docker.io/projectsveltos/sveltoscluster-manager:main
+        image: docker.io/projectsveltos/sveltoscluster-manager:dev
         livenessProbe:
           failureThreshold: 3
           httpGet:

diff --git a/manifest/manifest.yaml b/manifest/manifest.yaml
@@ -103,7 +103,7 @@ spec:
         - --v=5
         command:
         - /manager
-        image: docker.io/projectsveltos/sveltoscluster-manager:main
+        image: docker.io/projectsveltos/sveltoscluster-manager:dev
         livenessProbe:
           failureThreshold: 3
           httpGet:

diff --git a/test/fv/renew_toke_request_test.go b/test/fv/renew_toke_request_test.go
@@ -55,7 +55,8 @@ var _ = Describe("Renew TokenRequest", func() {
 			if err != nil {
 				return false
 			}
-			return currentSveltosCluster.Status.Ready
+			return currentSveltosCluster.Status.Ready &&
+				currentSveltosCluster.Status.ConnectionStatus == libsveltosv1beta1.ConnectionHealthy
 		}, timeout, pollingInterval).Should(BeTrue())
 
 		// Get Secret with SveltosCluster kubeconfig