diff --git a/controllers/rediscluster_controller.go b/controllers/rediscluster_controller.go index ae7d8af9c..813d41ef8 100644 --- a/controllers/rediscluster_controller.go +++ b/controllers/rediscluster_controller.go @@ -18,7 +18,6 @@ package controllers import ( "context" - "strconv" "time" "github.com/OT-CONTAINER-KIT/redis-operator/api/status" @@ -26,7 +25,7 @@ import ( "github.com/OT-CONTAINER-KIT/redis-operator/k8sutils" intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil" "github.com/go-logr/logr" - "k8s.io/apimachinery/pkg/api/errors" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" @@ -123,15 +122,6 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return intctrlutil.RequeueWithError(err, reqLogger, "") } - // todo: remove me after watch statefulset in controller - redisLeaderInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName()+"-leader") - if err != nil { - if errors.IsNotFound(err) { - return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "") - } - return intctrlutil.RequeueWithError(err, reqLogger, "") - } - if r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") { // Mark the cluster status as initializing if there are no follower nodes if (instance.Status.ReadyLeaderReplicas == 0 && instance.Status.ReadyFollowerReplicas == 0) || @@ -157,21 +147,9 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request return intctrlutil.RequeueWithError(err, reqLogger, "") } } - // todo: remove me after watch statefulset in controller - redisFollowerInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName()+"-follower") - if err != nil { - if errors.IsNotFound(err) { - return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "") - } - return intctrlutil.RequeueWithError(err, reqLogger, "") - } - - if leaderReplicas == 0 { - return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis leaders Cannot be 0", "Ready.Replicas", strconv.Itoa(int(redisLeaderInfo.Status.ReadyReplicas)), "Expected.Replicas", leaderReplicas) - } if !(r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-leader") && r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name+"-follower")) { - return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis leader and follower nodes are not ready yet") + return intctrlutil.Reconciled() } // Mark the cluster status as bootstrapping if all the leader and follower nodes are ready @@ -183,7 +161,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } if nc := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, r.Log, instance, ""); nc != totalReplicas { - reqLogger.Info("Creating redis cluster by executing cluster creation commands", "Leaders.Ready", redisLeaderInfo.Status.ReadyReplicas, "Followers.Ready", redisFollowerInfo.Status.ReadyReplicas) + reqLogger.Info("Creating redis cluster by executing cluster creation commands") leaderCount := k8sutils.CheckRedisNodeCount(ctx, r.K8sClient, r.Log, instance, "leader") if leaderCount != leaderReplicas { reqLogger.Info("Not all leader are part of the cluster...", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas) @@ -199,7 +177,7 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request } } } else { - if followerReplicas > 0 && redisFollowerInfo.Status.ReadyReplicas == followerReplicas { + if followerReplicas > 0 { reqLogger.Info("All leader are part of the cluster, adding follower/replicas", "Leaders.Count", leaderCount, "Instance.Size", leaderReplicas, "Follower.Replicas", followerReplicas) k8sutils.ExecuteRedisReplicationCommand(ctx, r.K8sClient, r.Log, instance) } else { @@ -239,5 +217,6 @@ func (r *RedisClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request func (r *RedisClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&redisv1beta2.RedisCluster{}). + Owns(&appsv1.StatefulSet{}). Complete(r) } diff --git a/controllers/redisreplication_controller.go b/controllers/redisreplication_controller.go index 0446b4dc3..d5a09e34c 100644 --- a/controllers/redisreplication_controller.go +++ b/controllers/redisreplication_controller.go @@ -2,13 +2,13 @@ package controllers import ( "context" - "strconv" "time" redisv1beta2 "github.com/OT-CONTAINER-KIT/redis-operator/api/v1beta2" "github.com/OT-CONTAINER-KIT/redis-operator/k8sutils" intctrlutil "github.com/OT-CONTAINER-KIT/redis-operator/pkg/controllerutil" "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/dynamic" @@ -21,6 +21,7 @@ import ( type RedisReplicationReconciler struct { client.Client k8sutils.Pod + k8sutils.StatefulSet K8sClient kubernetes.Interface Dk8sClient dynamic.Interface Log logr.Logger @@ -46,10 +47,6 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req return intctrlutil.RequeueAfter(reqLogger, time.Second*10, "found skip reconcile annotation") } - leaderReplicas := int32(1) - followerReplicas := instance.Spec.GetReplicationCounts("replication") - leaderReplicas - totalReplicas := leaderReplicas + followerReplicas - if err = k8sutils.AddFinalizer(instance, k8sutils.RedisReplicationFinalizer, r.Client); err != nil { return intctrlutil.RequeueWithError(err, reqLogger, "") } @@ -63,22 +60,14 @@ func (r *RedisReplicationReconciler) Reconcile(ctx context.Context, req ctrl.Req return intctrlutil.RequeueWithError(err, reqLogger, "") } - // Set Pod distruptiuon Budget Later - - redisReplicationInfo, err := k8sutils.GetStatefulSet(r.K8sClient, r.Log, instance.GetNamespace(), instance.GetName()) - if err != nil { - return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "") - } - - // Check that the Leader and Follower are ready in redis replication - if redisReplicationInfo.Status.ReadyReplicas != totalReplicas { - return intctrlutil.RequeueAfter(reqLogger, time.Second*60, "Redis replication nodes are not ready yet", "Ready.Replicas", redisReplicationInfo.Status.ReadyReplicas, "Expected.Replicas", totalReplicas) + if !r.IsStatefulSetReady(ctx, instance.Namespace, instance.Name) { + return intctrlutil.Reconciled() } var realMaster string masterNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "master") - if len(masterNodes) > int(leaderReplicas) { - reqLogger.Info("Creating redis replication by executing replication creation commands", "Replication.Ready", strconv.Itoa(int(redisReplicationInfo.Status.ReadyReplicas))) + if len(masterNodes) > 1 { + reqLogger.Info("Creating redis replication by executing replication creation commands") slaveNodes := k8sutils.GetRedisNodesByRole(ctx, r.K8sClient, r.Log, instance, "slave") realMaster = k8sutils.GetRedisReplicationRealMaster(ctx, r.K8sClient, r.Log, instance, masterNodes) if len(slaveNodes) == 0 { @@ -138,5 +127,6 @@ func (r *RedisReplicationReconciler) UpdateRedisPodRoleLabel(ctx context.Context func (r *RedisReplicationReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For(&redisv1beta2.RedisReplication{}). + Owns(&appsv1.StatefulSet{}). Complete(r) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index e6039fde5..63a73f01d 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -101,21 +101,24 @@ var _ = BeforeSuite(func() { }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) - rrLog := ctrl.Log.WithName("controllers").WithName("RedisReplication") + rcLog := ctrl.Log.WithName("controllers").WithName("RedisCluster") err = (&RedisClusterReconciler{ Client: k8sManager.GetClient(), K8sClient: k8sClient, Dk8sClient: dk8sClient, Scheme: k8sManager.GetScheme(), - StatefulSet: k8sutils.NewStatefulSetService(k8sClient, rrLog), + StatefulSet: k8sutils.NewStatefulSetService(k8sClient, rcLog), }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) + rrLog := ctrl.Log.WithName("controllers").WithName("RedisReplication") err = (&RedisReplicationReconciler{ - Client: k8sManager.GetClient(), - K8sClient: k8sClient, - Dk8sClient: dk8sClient, - Scheme: k8sManager.GetScheme(), + Client: k8sManager.GetClient(), + K8sClient: k8sClient, + Dk8sClient: dk8sClient, + Scheme: k8sManager.GetScheme(), + StatefulSet: k8sutils.NewStatefulSetService(k8sClient, rrLog), + Pod: k8sutils.NewPodService(k8sClient, rrLog), }).SetupWithManager(k8sManager) Expect(err).ToNot(HaveOccurred()) diff --git a/main.go b/main.go index 556b169ad..4da3274e6 100644 --- a/main.go +++ b/main.go @@ -138,12 +138,13 @@ func main() { } rrLog := ctrl.Log.WithName("controllers").WithName("RedisReplication") if err = (&controllers.RedisReplicationReconciler{ - Client: mgr.GetClient(), - K8sClient: k8sclient, - Dk8sClient: dk8sClient, - Log: rrLog, - Scheme: mgr.GetScheme(), - Pod: k8sutils.NewPodService(k8sclient, rrLog), + Client: mgr.GetClient(), + K8sClient: k8sclient, + Dk8sClient: dk8sClient, + Log: rrLog, + Scheme: mgr.GetScheme(), + Pod: k8sutils.NewPodService(k8sclient, rrLog), + StatefulSet: k8sutils.NewStatefulSetService(k8sclient, rrLog), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "RedisReplication") os.Exit(1) diff --git a/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/chainsaw-test.yaml index 92a6da4b8..2fa090a12 100644 --- a/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/chainsaw-test.yaml @@ -32,10 +32,7 @@ spec: selector: control-plane=redis-operator container: manager tail: -1 # tail all logs - - name: Sleep for five minutes - try: - - sleep: - duration: 5m + - name: Ping Cluster Nodes try: - script: diff --git a/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/ready-cluster.yaml index 004498aff..a0708b5a5 100644 --- a/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/acl-user/redis-cluster/ready-cluster.yaml @@ -6,3 +6,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml index c3a37ab9c..b67582d7d 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/chainsaw-test.yaml @@ -26,10 +26,7 @@ spec: timeout: 5m content: | sudo apt install redis-tools -y - - name: Sleep for five minutes - try: - - sleep: - duration: 5m + - name: Ping Redis Cluster from every node try: - script: diff --git a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml index 004498aff..a0708b5a5 100644 --- a/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/hostnetwork/redis-cluster/ready-cluster.yaml @@ -6,3 +6,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/ignore-annots/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/ignore-annots/redis-cluster/ready-cluster.yaml index fb6f17085..8228c1660 100644 --- a/tests/e2e-chainsaw/v1beta2/ignore-annots/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/ignore-annots/redis-cluster/ready-cluster.yaml @@ -8,3 +8,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/keep-pvc/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/keep-pvc/redis-cluster/ready-cluster.yaml index 004498aff..a0708b5a5 100644 --- a/tests/e2e-chainsaw/v1beta2/keep-pvc/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/keep-pvc/redis-cluster/ready-cluster.yaml @@ -6,3 +6,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml index 51161bc06..84755055c 100644 --- a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/chainsaw-test.yaml @@ -18,10 +18,7 @@ spec: file: ready-svc.yaml - assert: file: ready-pvc.yaml - - name: Sleep for five minutes - try: - - sleep: - duration: 5m + - name: Ping Cluster try: - script: diff --git a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml index 004498aff..a0708b5a5 100644 --- a/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/nodeport/redis-cluster/ready-cluster.yaml @@ -6,3 +6,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/password/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/password/redis-cluster/chainsaw-test.yaml index b7974b9db..1cc5674af 100644 --- a/tests/e2e-chainsaw/v1beta2/password/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/password/redis-cluster/chainsaw-test.yaml @@ -21,10 +21,7 @@ spec: file: ready-pvc.yaml - assert: file: secret.yaml - - name: Sleep for five minutes - try: - - sleep: - duration: 5m + - name: Ping Cluster With Password try: - script: diff --git a/tests/e2e-chainsaw/v1beta2/password/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/password/redis-cluster/ready-cluster.yaml index 004498aff..a0708b5a5 100644 --- a/tests/e2e-chainsaw/v1beta2/password/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/password/redis-cluster/ready-cluster.yaml @@ -6,3 +6,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/pvc-name/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/pvc-name/redis-cluster/ready-cluster.yaml index 004498aff..a0708b5a5 100644 --- a/tests/e2e-chainsaw/v1beta2/pvc-name/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/pvc-name/redis-cluster/ready-cluster.yaml @@ -6,3 +6,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml index 3d3d27ef6..1a5abdc74 100644 --- a/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml +++ b/tests/e2e-chainsaw/v1beta2/scaling/redis-cluster/cluster-status-00.yaml @@ -4,7 +4,7 @@ kind: RedisCluster metadata: name: redis-cluster-v1beta2 status: - state: Ready - reason: RedisCluster is ready readyLeaderReplicas: 3 readyFollowerReplicas: 3 + state: Ready + reason: RedisCluster is ready diff --git a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml index f3523e67d..a078dad75 100644 --- a/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml +++ b/tests/e2e-chainsaw/v1beta2/setup/redis-cluster/chainsaw-test.yaml @@ -24,11 +24,7 @@ spec: selector: control-plane=redis-operator container: manager tail: -1 # tail all logs - # no need to wait for 5 minutes, when we have ready-cluster.yaml, we can proceed -# - name: Sleep for five minutes -# try: -# - sleep: -# duration: 3m + - name: Ping Cluster try: - script: diff --git a/tests/e2e-chainsaw/v1beta2/teardown/redis-cluster/ready-cluster.yaml b/tests/e2e-chainsaw/v1beta2/teardown/redis-cluster/ready-cluster.yaml index 004498aff..a0708b5a5 100644 --- a/tests/e2e-chainsaw/v1beta2/teardown/redis-cluster/ready-cluster.yaml +++ b/tests/e2e-chainsaw/v1beta2/teardown/redis-cluster/ready-cluster.yaml @@ -6,3 +6,5 @@ metadata: status: readyFollowerReplicas: 3 readyLeaderReplicas: 3 + state: Ready + reason: RedisCluster is ready