refactor: reconcile on statefulset update

controllers/rediscluster_controller.go

@@ -18,15 +18,14 @@
 
 import (
 	"context"
-	"strconv"
 	"time"
 
 	"github.com/OT-CONTAINER-KIT/redis-operator/api/status"
 	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
 	"github.com/OT-CONTAINER-KIT/redis-operator/k8sutils"
 	intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil"
 	"github.com/go-logr/logr"
-	"k8s.io/apimachinery/pkg/api/errors"
+	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
@@ -123,15 +122,6 @@
 		return intctrlutil.RequeueWithError(err, reqLogger, "")
 	}
 
-	// todo: remove me after watch statefulset in controller
-	redisLeaderInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName()+"-leader")
-	if err != nil {
-		if errors.IsNotFound(err) {
-			return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "")
-		}
-		return intctrlutil.RequeueWithError(err, reqLogger, "")
-	}
-
 	if r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") {
 		// Mark the cluster status as initializing if there are no follower nodes
 		if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) ||
@@ -157,21 +147,9 @@
 			return intctrlutil.RequeueWithError(err, reqLogger, "")
 		}
 	}
-	// todo: remove me after watch statefulset in controller
-	redisFollowerInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName()+"-follower")
-	if err != nil {
-		if errors.IsNotFound(err) {
-			return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "")
-		}
-		return intctrlutil.RequeueWithError(err, reqLogger, "")
-	}
-
-	if leaderReplicas == 0 {
-		return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis leaders Cannot be 0", "Ready.Replicas", strconv.Itoa(int(redisLeaderInfo.Status.ReadyReplicas)), "Expected.Replicas", leaderReplicas)
-	}
 
 	if !(r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-follower")) {
-		return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis leader and follower nodes are not ready yet")
+		return intctrlutil.Reconciled()
 	}
 
 	// Mark the cluster status as bootstrapping if all the leader and follower nodes are ready
@@ -183,7 +161,7 @@
 	}
 
 	if nc := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, r.Log, instance, ""); nc != totalReplicas {
-		reqLogger.Info("Creating redis cluster by executing cluster creation commands", "Leaders.Ready", redisLeaderInfo.Status.ReadyReplicas, "Followers.Ready", redisFollowerInfo.Status.ReadyReplicas)
+		reqLogger.Info("Creating redis cluster by executing cluster creation commands")
 		leaderCount := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, r.Log, instance, "leader")
 		if leaderCount != leaderReplicas {
 			reqLogger.Info("Not all leader are part of the cluster...", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas)
@@ -199,7 +177,7 @@
 				}
 			}
 		} else {
-			if followerReplicas > 0 && redisFollowerInfo.Status.ReadyReplicas == followerReplicas {
+			if followerReplicas > 0 {
 				reqLogger.Info("All leader are part of the cluster, adding follower/replicas", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas, "Follower.Replicas", followerReplicas)
 				k8sutils.ExecuteRedisReplicationCommand(ctx, r.K8sClient, r.Log, instance)
 			} else {
@@ -239,5 +217,6 @@
 func (r *RedisClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&redisv1beta2.RedisCluster{}).
+		Owns(&appsv1.StatefulSet{}).
 		Complete(r)
 }

controllers/redisreplication_controller.go

@@ -2,13 +2,13 @@
 
 import (
 	"context"
-	"strconv"
 	"time"
 
 	redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2"
 	"github.com/OT-CONTAINER-KIT/redis-operator/k8sutils"
 	intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil"
 	"github.com/go-logr/logr"
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/dynamic"
@@ -21,6 +21,7 @@
 type RedisReplicationReconciler struct {
 	client.Client
 	k8sutils.Pod
+	k8sutils.StatefulSet
 	K8sClient  kubernetes.Interface
 	Dk8sClient dynamic.Interface
 	Log        logr.Logger
@@ -46,10 +47,6 @@
 		return intctrlutil.RequeueAfter(reqLogger, time.Second*10, "found skip reconcile annotation")
 	}
 
-	leaderReplicas := int32(1)
-	followerReplicas := instance.Spec.GetReplicationCounts("replication") - leaderReplicas
-	totalReplicas := leaderReplicas + followerReplicas
-
 	if err = k8sutils.AddFinalizer(instance, k8sutils.RedisReplicationFinalizer, r.Client); err != nil {
 		return intctrlutil.RequeueWithError(err, reqLogger, "")
 	}
@@ -63,22 +60,14 @@
 		return intctrlutil.RequeueWithError(err, reqLogger, "")
 	}
 
-	// Set Pod distruptiuon Budget Later
-
-	redisReplicationInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName())
-	if err != nil {
-		return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "")
-	}
-
-	// Check that the Leader and Follower are ready in redis replication
-	if redisReplicationInfo.Status.ReadyReplicas != totalReplicas {
-		return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis replication nodes are not ready yet", "Ready.Replicas", redisReplicationInfo.Status.ReadyReplicas, "Expected.Replicas", totalReplicas)
+	if !r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name) {
+		return intctrlutil.Reconciled()
 	}
 
 	var realMaster string
 	masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master")
-	if len(masterNodes) > int(leaderReplicas) {
-		reqLogger.Info("Creating redis replication by executing replication creation commands", "Replication.Ready", strconv.Itoa(int(redisReplicationInfo.Status.ReadyReplicas)))
+	if len(masterNodes) > 1 {
+		reqLogger.Info("Creating redis replication by executing replication creation commands")
 		slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "slave")
 		realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes)
 		if len(slaveNodes) == 0 {
@@ -138,5 +127,6 @@
 func (r *RedisReplicationReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&redisv1beta2.RedisReplication{}).
+		Owns(&appsv1.StatefulSet{}).
 		Complete(r)
 }

controllers/suite_test.go

@@ -101,21 +101,24 @@ var _ = BeforeSuite(func() {
 	}).SetupWithManager(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
 
-	rrLog := ctrl.Log.WithName("controllers").WithName("RedisReplication")
+	rcLog := ctrl.Log.WithName("controllers").WithName("RedisCluster")
 	err = (&RedisClusterReconciler{
 		Client:      k8sManager.GetClient(),
 		K8sClient:   k8sClient,
 		Dk8sClient:  dk8sClient,
 		Scheme:      k8sManager.GetScheme(),
-		StatefulSet: k8sutils.NewStatefulSetService(k8sClient, rrLog),
+		StatefulSet: k8sutils.NewStatefulSetService(k8sClient, rcLog),
 	}).SetupWithManager(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
 
+	rrLog := ctrl.Log.WithName("controllers").WithName("RedisReplication")
 	err = (&RedisReplicationReconciler{
-		Client:     k8sManager.GetClient(),
-		K8sClient:  k8sClient,
-		Dk8sClient: dk8sClient,
-		Scheme:     k8sManager.GetScheme(),
+		Client:      k8sManager.GetClient(),
+		K8sClient:   k8sClient,
+		Dk8sClient:  dk8sClient,
+		Scheme:      k8sManager.GetScheme(),
+		StatefulSet: k8sutils.NewStatefulSetService(k8sClient, rrLog),
+		Pod:         k8sutils.NewPodService(k8sClient, rrLog),
 	}).SetupWithManager(k8sManager)
 	Expect(err).ToNot(HaveOccurred())
 

main.go

@@ -138,12 +138,13 @@ func main() {
 	}
 	rrLog := ctrl.Log.WithName("controllers").WithName("RedisReplication")
 	if err = (&controllers.RedisReplicationReconciler{
-		Client:     mgr.GetClient(),
-		K8sClient:  k8sclient,
-		Dk8sClient: dk8sClient,
-		Log:        rrLog,
-		Scheme:     mgr.GetScheme(),
-		Pod:        k8sutils.NewPodService(k8sclient, rrLog),
+		Client:      mgr.GetClient(),
+		K8sClient:   k8sclient,
+		Dk8sClient:  dk8sClient,
+		Log:         rrLog,
+		Scheme:      mgr.GetScheme(),
+		Pod:         k8sutils.NewPodService(k8sclient, rrLog),
+		StatefulSet: k8sutils.NewStatefulSetService(k8sclient, rrLog),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "RedisReplication")
 		os.Exit(1)

tests/_config/chainsaw-configuration.yaml

@@ -9,5 +9,5 @@ spec:
   timeouts:
     apply: 5m
     delete: 5m
-    assert: 20m
-    error: 20m
+    assert: 15m
+    error: 15m

tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/chainsaw-test.yaml

@@ -32,10 +32,7 @@ spec:
             selector: control-plane=redis-operator
             container: manager
             tail: -1  # tail all logs
-    - name: Sleep for five minutes
-      try:
-        - sleep:
-            duration: 5m
+
     - name: Ping Cluster Nodes
       try:
         - script:

tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/ready-cluster.yaml

@@ -6,3 +6,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml

@@ -20,16 +20,21 @@ spec:
             file: ready-pvc.yaml
         - assert:
             file: ready-pod.yaml
+      catch:
+        - description: Redis Operator Logs
+          podLogs:
+            namespace: redis-operator-system
+            selector: control-plane=redis-operator
+            container: manager
+            tail: -1  # tail all logs
+
     - name: Install Redis Cli
       try:
         - script:
             timeout: 5m
             content: |
               sudo apt install redis-tools -y
-    - name: Sleep for five minutes
-      try:
-        - sleep:
-            duration: 5m
+
     - name: Ping Redis Cluster from every node
       try:
         - script:

tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml

@@ -6,3 +6,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/ignore-annots/redis-cluster/ready-cluster.yaml

@@ -8,3 +8,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/keep-pvc/redis-cluster/ready-cluster.yaml

@@ -6,3 +6,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml

@@ -18,10 +18,14 @@ spec:
             file: ready-svc.yaml
         - assert:
             file: ready-pvc.yaml
-    - name: Sleep for five minutes
-      try:
-        - sleep:
-            duration: 5m
+      catch:
+        - description: Redis Operator Logs
+          podLogs:
+            namespace: redis-operator-system
+            selector: control-plane=redis-operator
+            container: manager
+            tail: -1  # tail all logs
+
     - name: Ping Cluster
       try:
         - script:

tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml

@@ -6,3 +6,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/password/redis-cluster/chainsaw-test.yaml

@@ -21,10 +21,7 @@ spec:
             file: ready-pvc.yaml
         - assert:
             file: secret.yaml
-    - name: Sleep for five minutes
-      try:
-        - sleep:
-            duration: 5m
+
     - name: Ping Cluster With Password
       try:
         - script:

tests/e2e-chainsaw/v1beta2/password/redis-cluster/ready-cluster.yaml

@@ -6,3 +6,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/pvc-name/redis-cluster/ready-cluster.yaml

@@ -6,3 +6,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml

@@ -4,7 +4,7 @@ kind: RedisCluster
 metadata:
   name: redis-cluster-v1beta2
 status:
-  state: Ready
-  reason: RedisCluster is ready
   readyLeaderReplicas: 3
   readyFollowerReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready

tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml

@@ -24,11 +24,7 @@ spec:
             selector: control-plane=redis-operator
             container: manager
             tail: -1  # tail all logs
-     # no need to wait for 5 minutes, when we have ready-cluster.yaml, we can proceed
-#    - name: Sleep for five minutes
-#      try:
-#        - sleep:
-#            duration: 3m
+
     - name: Ping Cluster
       try:
         - script:

tests/e2e-chainsaw/v1beta2/teardown/redis-cluster/ready-cluster.yaml

@@ -6,3 +6,5 @@ metadata:
 status:
   readyFollowerReplicas: 3
   readyLeaderReplicas: 3
+  state: Ready
+  reason: RedisCluster is ready