CLOUDP-116155 Initial bootup with arbiters (#1024)

Author: slaskawi

Makefile

@@ -121,7 +121,10 @@ e2e-gh:
 	scripts/dev/run_e2e_gh.sh $(test)
 
 cleanup-e2e:
-	kubectl delete mdbc,all -l e2e-test=true -n ${TEST_NAMESPACE} | true
+	kubectl delete mdbc,all,secrets -l e2e-test=true -n ${TEST_NAMESPACE} || true
+	# Most of the tests use StatefulSets, which in turn use stable storage. In order to
+	# avoid interleaving tests with each other, we need to drop them all.
+	kubectl delete pvc --all || true
 
 # Generate code
 generate: controller-gen

controllers/replica_set_controller.go

@@ -405,6 +405,10 @@ func (r *ReplicaSetReconciler) shouldRunInOrder(mdb mdbv1.MongoDBCommunity) bool
 
 	// if we are scaling up, we need to make sure the StatefulSet is scaled up first.
 	if scale.IsScalingUp(mdb) || mdb.CurrentArbiters() < mdb.DesiredArbiters() {
+		if scale.HasZeroReplicas(mdb) {
+			r.log.Debug("Scaling up the ReplicaSet when there is no replicas, the Automation Config must be updated first")
+			return true
+		}
 		r.log.Debug("Scaling up the ReplicaSet, the StatefulSet must be updated first")
 		return false
 	}

pkg/util/scale/scale.go

@@ -45,6 +45,10 @@ func IsScalingUp(replicaSetScaler ReplicaSetScaler) bool {
 	return replicaSetScaler.DesiredReplicas() > replicaSetScaler.CurrentReplicas() && replicaSetScaler.CurrentReplicas() != 0
 }
 
+func HasZeroReplicas(replicaSetScaler ReplicaSetScaler) bool {
+	return replicaSetScaler.CurrentReplicas() == 0
+}
+
 // AnyAreStillScaling reports true if any of one the provided members is still scaling
 func AnyAreStillScaling(scalers ...ReplicaSetScaler) bool {
 	for _, s := range scalers {

test/e2e/mongodbtests/mongodbtests.go

@@ -97,7 +97,7 @@ func arbitersStatefulSetIsReady(mdb *mdbv1.MongoDBCommunity, opts ...wait.Config
 		if err != nil {
 			t.Fatal(err)
 		}
-		t.Logf("StatefulSet %s/%s is ready!", mdb.Namespace, mdb.Name)
+		t.Logf("Arbiters StatefulSet %s/%s is ready!", mdb.Namespace, mdb.Name)
 	}
 }
 
@@ -327,6 +327,16 @@ func CreateMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) fu
 	}
 }
 
+// DeleteMongoDBResource deletes the MongoDB resource
+func DeleteMongoDBResource(mdb *mdbv1.MongoDBCommunity, ctx *e2eutil.Context) func(*testing.T) {
+	return func(t *testing.T) {
+		if err := e2eutil.TestClient.Delete(context.TODO(), mdb); err != nil {
+			t.Fatal(err)
+		}
+		t.Logf("Deleted MongoDB resource %s/%s", mdb.Name, mdb.Namespace)
+	}
+}
+
 // GetConnectionStringSecret returnes the secret generated by the operator that is storing the connection string for a specific user
 func GetConnectionStringSecret(mdb mdbv1.MongoDBCommunity, user scram.User) corev1.Secret {
 	secret := corev1.Secret{}

test/e2e/replica_set_arbiter/replica_set_arbiter_test.go

@@ -2,17 +2,15 @@ package replica_set
 
 import (
 	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	. "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester"
-	"github.com/stretchr/testify/assert"
-
 	e2eutil "github.com/mongodb/mongodb-kubernetes-operator/test/e2e"
 	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/mongodbtests"
 	setup "github.com/mongodb/mongodb-kubernetes-operator/test/e2e/setup"
+	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/mongotester"
 	"github.com/mongodb/mongodb-kubernetes-operator/test/e2e/util/wait"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+	"time"
 )
 
 func TestMain(m *testing.M) {
@@ -23,71 +21,89 @@ func TestMain(m *testing.M) {
 	os.Exit(code)
 }
 
+func Test(t *testing.T) {
+
+}
+
 func TestReplicaSetArbiter(t *testing.T) {
 	ctx := setup.Setup(t)
 	defer ctx.Teardown()
 
-	// Invalid case 1
-	numberArbiters := 3
-	numberMembers := 3
-	desiredStatus := fmt.Sprintf("error validating new Spec: number of arbiters specified (%v) is greater or equal than the number of members in the replicaset (%v). At least one member must not be an arbiter", numberArbiters, numberMembers)
-	mdb, user := e2eutil.NewTestMongoDB(ctx, "mdb0", "")
-	mdb.Spec.Arbiters = numberArbiters
-	mdb.Spec.Members = numberMembers
-	_, err := setup.GeneratePasswordForUser(ctx, user, "")
-	if err != nil {
-		t.Fatal(err)
+	type args struct {
+		numberOfArbiters     int
+		scaleArbitersTo      int
+		numberOfMembers      int
+		expectedErrorMessage string
+		resourceName         string
 	}
-	t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
-	t.Run("Check status for case 1", mongodbtests.StatefulSetMessageIsReceived(&mdb, ctx, desiredStatus))
-
-	// Invalid case 2
-	numberArbiters = -1
-	numberMembers = 3
-	desiredStatus = "error validating new Spec: number of arbiters must be greater or equal than 0"
-	mdb, user = e2eutil.NewTestMongoDB(ctx, "mdb1", "")
-	mdb.Spec.Arbiters = numberArbiters
-	mdb.Spec.Members = numberMembers
-	_, err = setup.GeneratePasswordForUser(ctx, user, "")
-	if err != nil {
-		t.Fatal(err)
+	tests := map[string]args{
+		"Number of Arbiters must be less than number of nodes": {
+			numberOfArbiters:     3,
+			numberOfMembers:      3,
+			expectedErrorMessage: fmt.Sprintf("error validating new Spec: number of arbiters specified (%v) is greater or equal than the number of members in the replicaset (%v). At least one member must not be an arbiter", 3, 3),
+			resourceName:         "mdb0",
+		},
+		"Number of Arbiters must be greater than 0": {
+			numberOfArbiters:     -1,
+			numberOfMembers:      3,
+			expectedErrorMessage: "error validating new Spec: number of arbiters must be greater or equal than 0",
+			resourceName:         "mdb1",
+		},
+		"Scaling arbiters from 0 to 1": {
+			numberOfArbiters: 0,
+			scaleArbitersTo:  1,
+			numberOfMembers:  2,
+			resourceName:     "mdb2",
+		},
+		"Scaling Arbiters from 1 to 0": {
+			numberOfArbiters: 1,
+			scaleArbitersTo:  0,
+			numberOfMembers:  3,
+			resourceName:     "mdb3",
+		},
+		"Arbiters can be deployed in initial bootstrap": {
+			numberOfArbiters: 1,
+			scaleArbitersTo:  1,
+			numberOfMembers:  2,
+			resourceName:     "mdb4",
+		},
 	}
-	t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
-	t.Run("Check status for case 2", mongodbtests.StatefulSetMessageIsReceived(&mdb, ctx, desiredStatus))
+	for testName, _ := range tests {
+		t.Run(testName, func(t *testing.T) {
+			testConfig, _ := tests[testName]
+			mdb, user := e2eutil.NewTestMongoDB(ctx, testConfig.resourceName, "")
+			mdb.Spec.Arbiters = testConfig.numberOfArbiters
+			mdb.Spec.Members = testConfig.numberOfMembers
+			pwd, err := setup.GeneratePasswordForUser(ctx, user, "")
+			if err != nil {
+				t.Fatal(err)
+			}
+			t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
+			if len(testConfig.expectedErrorMessage) > 0 {
+				t.Run("Check status", mongodbtests.StatefulSetMessageIsReceived(&mdb, ctx, testConfig.expectedErrorMessage))
+			} else {
+				t.Run("Check that the stateful set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
+				t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, testConfig.numberOfArbiters))
 
-	numberArbiters = 0
-	numberMembers = 3
-	mdb, user = e2eutil.NewTestMongoDB(ctx, "mdb2", "")
-	mdb.Spec.Arbiters = numberArbiters
-	mdb.Spec.Members = numberMembers
-	pwd, err := setup.GeneratePasswordForUser(ctx, user, "")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	t.Run("Create MongoDB Resource", mongodbtests.CreateMongoDBResource(&mdb, ctx))
-	t.Run("Check that the stateful set becomes ready", mongodbtests.StatefulSetBecomesReady(&mdb, wait.Timeout(20*time.Minute)))
-	t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, numberArbiters))
-
-	// Arbiters need to be less than regular members
-	t.Run("Scale MongoDB Up to 2 Arbiters", mongodbtests.ScaleArbiters(&mdb, 2))
-	t.Run("Arbiters Stateful Set Scaled Up Correctly", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb))
-	t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
+				if testConfig.numberOfArbiters != testConfig.scaleArbitersTo {
+					t.Run(fmt.Sprintf("Scale Arbiters to %v", testConfig.scaleArbitersTo), mongodbtests.ScaleArbiters(&mdb, testConfig.scaleArbitersTo))
+					t.Run("Arbiters Stateful Set Scaled Correctly", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb))
+				}
 
-	t.Run("Test SRV Connectivity with generated connection string secret", func(t *testing.T) {
-		tester, err := FromResource(t, mdb)
-		if err != nil {
-			t.Fatal(err)
-		}
-		scramUser := mdb.GetScramUsers()[0]
-		expectedCnxStr := fmt.Sprintf("mongodb+srv://%s-user:%s@%s-svc.%s.svc.cluster.local/admin?replicaSet=mdb2&ssl=false", mdb.Name, pwd, mdb.Name, mdb.Namespace)
-		cnxStrSrv := mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)
-		assert.Equal(t, expectedCnxStr, cnxStrSrv)
-		tester.ConnectivitySucceeds(WithURI(cnxStrSrv))
-	})
-
-	t.Run("Scale MongoDB Up to 0 Arbiters", mongodbtests.ScaleArbiters(&mdb, 0))
-	t.Run("Arbiters Stateful Set Scaled Up Correctly", mongodbtests.ArbitersStatefulSetBecomesReady(&mdb))
-	t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
-	t.Run("Check the number of arbiters", mongodbtests.AutomationConfigReplicaSetsHaveExpectedArbiters(&mdb, 0))
+				t.Run("MongoDB Reaches Running Phase", mongodbtests.MongoDBReachesRunningPhase(&mdb))
+				t.Run("Test SRV Connectivity with generated connection string secret", func(t *testing.T) {
+					tester, err := mongotester.FromResource(t, mdb)
+					if err != nil {
+						t.Fatal(err)
+					}
+					scramUser := mdb.GetScramUsers()[0]
+					expectedCnxStr := fmt.Sprintf("mongodb+srv://%s-user:%s@%s-svc.%s.svc.cluster.local/admin?replicaSet=%s&ssl=false", mdb.Name, pwd, mdb.Name, mdb.Namespace, mdb.Name)
+					cnxStrSrv := mongodbtests.GetSrvConnectionStringForUser(mdb, scramUser)
+					assert.Equal(t, expectedCnxStr, cnxStrSrv)
+					tester.ConnectivitySucceeds(mongotester.WithURI(cnxStrSrv))
+				})
+			}
+			t.Run("Delete MongoDB Resource", mongodbtests.DeleteMongoDBResource(&mdb, ctx))
+		})
+	}
 }