diff --git a/pkg/cloudfabric-controller/controllerframework/cim_test.go b/pkg/cloudfabric-controller/controllerframework/cim_test.go index 1f73ff38634..2ceb98a0d42 100644 --- a/pkg/cloudfabric-controller/controllerframework/cim_test.go +++ b/pkg/cloudfabric-controller/controllerframework/cim_test.go @@ -23,7 +23,7 @@ import ( "testing" ) -func newControllerInstance(controllerType string, controllerKey int64, workloadNum int32, isLocked bool) *v1.ControllerInstance { +func newControllerInstance(cim *ControllerInstanceManager, controllerType string, controllerKey int64, workloadNum int32) *v1.ControllerInstance { controllerInstance := &v1.ControllerInstance{ ObjectMeta: metav1.ObjectMeta{ ResourceVersion: "100", @@ -33,7 +33,11 @@ func newControllerInstance(controllerType string, controllerKey int64, workloadN WorkloadNum: workloadNum, } - controllerInstance.Name = generateControllerName(nil) + GetInstanceHandler = func() *ControllerInstanceManager { + return cim + } + controllerInstance.Name = generateControllerName(controllerType, nil) + GetInstanceHandler = getControllerInstanceManager return controllerInstance } @@ -41,7 +45,7 @@ func newControllerInstance(controllerType string, controllerKey int64, workloadN func testAddEvent(t *testing.T, cim *ControllerInstanceManager, notifyTimes int) (*v1.ControllerInstance, string, map[string]v1.ControllerInstance) { // add event controllerType := "foo" - controllerInstance1 := newControllerInstance(controllerType, 10000, 999, false) + controllerInstance1 := newControllerInstance(cim, controllerType, 10000, 999) cim.addControllerInstance(controllerInstance1) controllerInstanceMap, err := cim.ListControllerInstances(controllerType) @@ -153,7 +157,7 @@ func TestDeleteControllerInstanceDoesNotExist(t *testing.T) { cim, _ := CreateTestControllerInstanceManager(stopCh) notifyTimes = 0 - controllerInstance1 := newControllerInstance("bar", 10000, 999, false) + controllerInstance1 := newControllerInstance(cim, "bar", 10000, 999) cim.deleteControllerInstance(controllerInstance1) controllerInstanceMap, err := cim.ListControllerInstances(controllerInstance1.ControllerType) @@ -166,16 +170,20 @@ func TestAddMultipleControllerInstancesForSameControllerType(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - cim, _ := CreateTestControllerInstanceManager(stopCh) + cim1, _ := CreateTestControllerInstanceManager(stopCh) + cim2, _ := CreateTestControllerInstanceManager(stopCh) notifyTimes = 0 // add event - controllerInstance1, controllerType1, _ := testAddEvent(t, cim, 1) - controllerInstance2, controllerType2, _ := testAddEvent(t, cim, 2) + controllerInstance1, controllerType1, _ := testAddEvent(t, cim1, 1) + controllerInstance2, controllerType2, _ := testAddEvent(t, cim2, 2) assert.Equal(t, controllerType1, controllerType2) assert.NotEqual(t, controllerInstance1.Name, controllerInstance2.Name) - controllerInstanceMap, err := cim.ListControllerInstances(controllerType1) + // cim 1 got controller 2 creation event + cim1.addControllerInstance(controllerInstance2) + + controllerInstanceMap, err := cim1.ListControllerInstances(controllerType1) assert.Nil(t, err) assert.NotNil(t, controllerInstanceMap) controllerInstanceRead1, isOK1 := controllerInstanceMap[controllerInstance1.Name] @@ -227,21 +235,27 @@ func TestErrorHandlingInListControllerInstances(t *testing.T) { cim, _ := CreateTestControllerInstanceManager(stopCh) notifyTimes = 0 - _, controllerType, _ := testAddEvent(t, cim, 1) + controllerInstance1, controllerType, _ := testAddEvent(t, cim, 1) testAddEvent(t, cim, 2) - controllerInstance3 := newControllerInstance("foo2", 10000, 999, false) - cim.addControllerInstance(controllerInstance3) + controllerInstance2 := newControllerInstance(cim, "foo2", 10000, 999) + cim.addControllerInstance(controllerInstance2) cim.isControllerListInitialized = false controllerInstanceMap1, err := cim.ListControllerInstances(controllerType) assert.Nil(t, err) assert.NotNil(t, controllerInstanceMap1) - assert.Equal(t, 2, len(controllerInstanceMap1)) + assert.Equal(t, 1, len(controllerInstanceMap1)) + instanceRead, isOK := controllerInstanceMap1[controllerInstance1.Name] + assert.True(t, isOK) + assert.Equal(t, controllerInstance1.ControllerKey, instanceRead.ControllerKey) controllerInstanceMap2, err := cim.ListControllerInstances("foo2") assert.Nil(t, err) assert.NotNil(t, controllerInstanceMap2) assert.Equal(t, 1, len(controllerInstanceMap2)) + instanceRead, isOK = controllerInstanceMap2[controllerInstance2.Name] + assert.True(t, isOK) + assert.Equal(t, controllerInstance2.ControllerKey, instanceRead.ControllerKey) } diff --git a/pkg/cloudfabric-controller/controllerframework/controller_framework.go b/pkg/cloudfabric-controller/controllerframework/controller_framework.go index 45396b3a00f..84a3a0882b5 100644 --- a/pkg/cloudfabric-controller/controllerframework/controller_framework.go +++ b/pkg/cloudfabric-controller/controllerframework/controller_framework.go @@ -18,12 +18,12 @@ package controllerframework import ( "fmt" + "strings" "sync" "github.com/grafov/bcast" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/uuid" clientset "k8s.io/client-go/kubernetes" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/cache" @@ -32,10 +32,6 @@ import ( "k8s.io/kubernetes/pkg/util/metrics" ) -const ( - controllerInstanceNamePrefix string = "ci" -) - var ResetFilterHandler = resetFilter type controllerInstanceLocal struct { @@ -92,7 +88,7 @@ func NewControllerBase(controllerType string, client clientset.Interface, cimUpd controller := &ControllerBase{ client: client, controllerType: controllerType, - controllerName: generateControllerName(controllerInstanceMap), + controllerName: generateControllerName(controllerType, controllerInstanceMap), controllerInstanceMap: controllerInstanceMap, curPos: -1, controllerInstanceUpdateCh: cimUpdateCh, @@ -441,18 +437,19 @@ func (c *ControllerBase) ReportHealth(client clientset.Interface) { } } -func generateControllerName(existedInstanceMap map[string]v1.ControllerInstance) string { - for { - uid := uuid.NewUUID() - name := fmt.Sprintf("%s-%v", controllerInstanceNamePrefix, uid) - _, isExist := existedInstanceMap[name] - if !isExist { - return name - } +func generateControllerName(controllerType string, existedInstanceMap map[string]v1.ControllerInstance) string { + cimInstanceId := GetInstanceId() + if cimInstanceId == "" { + klog.Fatalf("Controller Instance Manager not available.") + } - // Error - klog.Infof("Controller name %s conflict. Get a new one ", name) + name := fmt.Sprintf("%s-%v", strings.ToLower(controllerType), cimInstanceId) + _, isExist := existedInstanceMap[name] + if isExist { + klog.Fatalf("Controller instance name %s conflict. Need to restart process to get a new one ", name) } + + return name } // Get controller instances by controller type @@ -460,7 +457,7 @@ func generateControllerName(existedInstanceMap map[string]v1.ControllerInstance) func listControllerInstancesByType(controllerType string) (map[string]v1.ControllerInstance, error) { controllerInstanceMap := make(map[string]v1.ControllerInstance) - cim := GetControllerInstanceManager() + cim := GetInstanceHandler() if cim == nil { klog.Fatalf("Unexpected reference to uninitialized controller instance manager") } diff --git a/pkg/cloudfabric-controller/controllerframework/controller_framework_test.go b/pkg/cloudfabric-controller/controllerframework/controller_framework_test.go index 79eb7e51387..a35152ddf0e 100644 --- a/pkg/cloudfabric-controller/controllerframework/controller_framework_test.go +++ b/pkg/cloudfabric-controller/controllerframework/controller_framework_test.go @@ -69,7 +69,7 @@ func convertControllerBaseToControllerInstance(controllerBase *ControllerBase) * func TestGetControllerInstanceManager(t *testing.T) { instance = nil - cim := GetControllerInstanceManager() + cim := GetInstanceHandler() assert.Nil(t, cim) client := fake.NewSimpleClientset() @@ -103,6 +103,67 @@ func TestCreateControllerInstanceBase(t *testing.T) { assert.Equal(t, int64(math.MaxInt64), controllerInstanceBase2.sortedControllerInstancesLocal[0].controllerKey) } +func TestDeleteControllerInstance(t *testing.T) { + client := fake.NewSimpleClientset() + stopCh := make(chan struct{}) + defer close(stopCh) + + controllerType := "foo" + controllerInstanceBase, cim1 := createControllerInstanceBaseAndCIM(t, client, nil, controllerType, stopCh) + controllerInstance1 := convertControllerBaseToControllerInstance(controllerInstanceBase) + + // 1st controller instance for a type needs to cover all workload + assert.Equal(t, 0, controllerInstanceBase.curPos) + assert.Equal(t, 1, len(controllerInstanceBase.sortedControllerInstancesLocal)) + assert.Equal(t, int64(0), controllerInstanceBase.sortedControllerInstancesLocal[0].lowerboundKey) + assert.Equal(t, int64(math.MaxInt64), controllerInstanceBase.sortedControllerInstancesLocal[0].controllerKey) + + // 2nd controller instance will split workload space with 1st one + stopCh2 := make(chan struct{}) + controllerInstanceBase2, cim2 := createControllerInstanceBaseAndCIM(t, client, nil, controllerType, stopCh2) + controllerInstance2 := convertControllerBaseToControllerInstance(controllerInstanceBase2) + + // notify controller creation events + cim1.addControllerInstance(controllerInstance2) + cim2.addControllerInstance(controllerInstance1) + + controllerInstances, err := listControllerInstancesByType(controllerType) + assert.Nil(t, err) + assert.NotNil(t, controllerInstances) + controllerInstanceBase.updateCachedControllerInstances(controllerInstances) + + expectedPos := getPosFromControllerInstances(controllerInstance1, controllerInstance1, controllerInstance2) + assert.Equal(t, expectedPos, controllerInstanceBase.curPos) + + hashKey1 := int64(4611686018427387904) // mid point + assert.Equal(t, 2, len(controllerInstanceBase.sortedControllerInstancesLocal)) + assert.Equal(t, int64(0), controllerInstanceBase.sortedControllerInstancesLocal[0].lowerboundKey) + assert.Equal(t, hashKey1, controllerInstanceBase.sortedControllerInstancesLocal[0].controllerKey) + assert.Equal(t, hashKey1, controllerInstanceBase.sortedControllerInstancesLocal[1].lowerboundKey) + assert.Equal(t, int64(math.MaxInt64), controllerInstanceBase.sortedControllerInstancesLocal[1].controllerKey) + + // controller that takes the second half workload died, the left controller needs to take all workload + instanceNameToDel := controllerInstanceBase.sortedControllerInstancesLocal[1].instanceName + instanceBaseToCheck := controllerInstanceBase + if instanceNameToDel == controllerInstance1.Name { + cim1.deleteControllerInstance(controllerInstance1) + cim2.deleteControllerInstance(controllerInstance1) + instanceBaseToCheck = controllerInstanceBase2 + } else { + cim1.deleteControllerInstance(controllerInstance2) + cim2.deleteControllerInstance(controllerInstance2) + } + + controllerInstances, err = listControllerInstancesByType(controllerType) + assert.Nil(t, err) + assert.NotNil(t, controllerInstances) + instanceBaseToCheck.updateCachedControllerInstances(controllerInstances) + assert.Equal(t, 0, instanceBaseToCheck.curPos) + assert.Equal(t, 1, len(instanceBaseToCheck.sortedControllerInstancesLocal)) + assert.Equal(t, int64(0), instanceBaseToCheck.sortedControllerInstancesLocal[0].lowerboundKey) + assert.Equal(t, int64(math.MaxInt64), instanceBaseToCheck.sortedControllerInstancesLocal[0].controllerKey) +} + func TestCreateControllerInstanceBaseInRaceCondition_2(t *testing.T) { controllerType := "foo" @@ -170,14 +231,19 @@ func TestConsolidateControllerInstances_Sort(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - // 2nd controller instance will share same workload space with 1st one + // Test case : 2nd controller instance will split workload space with 1st one + // 1. create 1st controller controllerType := "foo" controllerInstanceBase, cim := createControllerInstanceBaseAndCIM(t, client, nil, controllerType, stopCh) controllerInstance1 := convertControllerBaseToControllerInstance(controllerInstanceBase) + // 2. create 2nd controller + cim2, _ := CreateTestControllerInstanceManager(stopCh) hashKey1 := int64(4611686018427387904) // mid point - controllerInstance1_2 := newControllerInstance(controllerType, int64(10000), int32(100), true) + controllerInstance1_2 := newControllerInstance(cim2, controllerType, int64(10000), int32(100)) cim.addControllerInstance(controllerInstance1_2) + cim2.addControllerInstance(controllerInstance1_2) + cim2.addControllerInstance(controllerInstance1) controllerInstances, err := listControllerInstancesByType(controllerType) assert.Nil(t, err) @@ -196,8 +262,13 @@ func TestConsolidateControllerInstances_Sort(t *testing.T) { // 3nd controller instance will share same workload space with the first 2 - each take 1/3 hashKey1 = int64(3074457345618258603) hashKey2 := int64(6148914691236517205) - controllerInstance1_3 := newControllerInstance("foo", int64(2000), 100, true) + cim3, _ := CreateTestControllerInstanceManager(stopCh) + controllerInstance1_3 := newControllerInstance(cim3, "foo", int64(2000), 100) cim.addControllerInstance(controllerInstance1_3) + cim2.addControllerInstance(controllerInstance1_3) + cim3.addControllerInstance(controllerInstance1_3) + cim3.addControllerInstance(controllerInstance1) + cim3.addControllerInstance(controllerInstance1_2) controllerInstances, err = listControllerInstancesByType(controllerType) assert.Nil(t, err) assert.NotNil(t, controllerInstances) @@ -232,6 +303,7 @@ func TestIsInRange(t *testing.T) { controllerType := "foo" controllerInstanceBase, cim := createControllerInstanceBaseAndCIM(t, client, nil, controllerType, stopCh) + controllerInstance1 := convertControllerBaseToControllerInstance(controllerInstanceBase) // check range assert.True(t, controllerInstanceBase.IsInRange(int64(0))) @@ -244,8 +316,11 @@ func TestIsInRange(t *testing.T) { controllerInstanceBase.sortedControllerInstancesLocal[0].workloadNum = workloadNum1 hashKey1 := int64(100000) - controllerInstance2 := newControllerInstance(controllerType, hashKey1, workloadNum1, true) + cim2, _ := CreateTestControllerInstanceManager(stopCh) + controllerInstance2 := newControllerInstance(cim2, controllerType, hashKey1, workloadNum1) cim.addControllerInstance(controllerInstance2) + cim2.addControllerInstance(controllerInstance2) + cim2.addControllerInstance(controllerInstance1) controllerInstanceBase.instanceUpdateProcess(controllerType) // check range diff --git a/pkg/cloudfabric-controller/controllerframework/controller_framework_utils.go b/pkg/cloudfabric-controller/controllerframework/controller_framework_utils.go index 1368ed14299..fb649f4c9bd 100644 --- a/pkg/cloudfabric-controller/controllerframework/controller_framework_utils.go +++ b/pkg/cloudfabric-controller/controllerframework/controller_framework_utils.go @@ -96,7 +96,7 @@ func CreateTestControllerInstanceManager(stopCh chan struct{}) (*ControllerInsta cim.controllerListerSynced = alwaysReady cim.notifyHandler = mockNotifyHander checkInstanceHandler = mockCheckInstanceHander - return GetControllerInstanceManager(), informers + return GetInstanceHandler(), informers } func MockCreateControllerInstanceAndResetChs(stopCh chan struct{}) (*bcast.Member, *bcast.Group) { @@ -104,7 +104,7 @@ func MockCreateControllerInstanceAndResetChs(stopCh chan struct{}) (*bcast.Membe cimUpdateCh := cimUpdateChGrp.Join() informersResetChGrp := bcast.NewGroup() - cim := GetControllerInstanceManager() + cim := GetInstanceHandler() if cim == nil { cim, _ = CreateTestControllerInstanceManager(stopCh) go cim.Run(stopCh) diff --git a/pkg/cloudfabric-controller/controllerframework/controllerinstancemanager.go b/pkg/cloudfabric-controller/controllerframework/controllerinstancemanager.go index f3c1c596d83..c7f57f2a4ed 100644 --- a/pkg/cloudfabric-controller/controllerframework/controllerinstancemanager.go +++ b/pkg/cloudfabric-controller/controllerframework/controllerinstancemanager.go @@ -59,8 +59,9 @@ type ControllerInstanceManager struct { var instance *ControllerInstanceManager var checkInstanceHandler = checkInstanceExistence +var GetInstanceHandler = getControllerInstanceManager -func GetControllerInstanceManager() *ControllerInstanceManager { +func getControllerInstanceManager() *ControllerInstanceManager { return instance } @@ -110,8 +111,13 @@ func NewControllerInstanceManager(coInformer coreinformers.ControllerInstanceInf return instance } -func (cim *ControllerInstanceManager) GetInstanceId() types.UID { - return cim.instanceId +func GetInstanceId() types.UID { + cim := GetInstanceHandler() + if cim != nil { + return cim.instanceId + } else { + return "" + } } func (cim *ControllerInstanceManager) addControllerInstance(obj interface{}) { diff --git a/pkg/cloudfabric-controller/replicaset/replica_set_test.go b/pkg/cloudfabric-controller/replicaset/replica_set_test.go index f9027c8e9f0..a432e6fae2b 100644 --- a/pkg/cloudfabric-controller/replicaset/replica_set_test.go +++ b/pkg/cloudfabric-controller/replicaset/replica_set_test.go @@ -66,7 +66,7 @@ func testNewReplicaSetControllerFromClient(client clientset.Interface, stopCh ch cimUpdateChGrp := bcast.NewGroup() cimUpdateCh := cimUpdateChGrp.Join() - cim := controllerframework.GetControllerInstanceManager() + cim := controllerframework.GetInstanceHandler() if cim == nil { cim, _ = controllerframework.CreateTestControllerInstanceManager(stopCh) go cim.Run(stopCh) @@ -481,7 +481,7 @@ func TestWatchControllers(t *testing.T) { stopCh := make(chan struct{}) defer close(stopCh) - cim := controllerframework.GetControllerInstanceManager() + cim := controllerframework.GetInstanceHandler() if cim == nil { cim, _ = controllerframework.CreateTestControllerInstanceManager(stopCh) go cim.Run(stopCh) diff --git a/test/integration/cloudfabriccontrollers/deployment_util.go b/test/integration/cloudfabriccontrollers/deployment_util.go index e8f5ae0c69f..618fe67bce9 100644 --- a/test/integration/cloudfabriccontrollers/deployment_util.go +++ b/test/integration/cloudfabriccontrollers/deployment_util.go @@ -164,7 +164,7 @@ func dcSetup(t *testing.T) (*httptest.Server, framework.CloseFunc, *replicaset.R informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(configs, "deployment-informers")), resyncPeriod) // controller instance manager set up - cim := controller.GetControllerInstanceManager() + cim := controller.GetInstanceHandler() if cim == nil { cimUpdateChGrp := bcast.NewGroup() go cimUpdateChGrp.Broadcast(0) diff --git a/test/integration/cloudfabriccontrollers/framework_test.go b/test/integration/cloudfabriccontrollers/framework_test.go index 0c152c760eb..0878fced972 100644 --- a/test/integration/cloudfabriccontrollers/framework_test.go +++ b/test/integration/cloudfabriccontrollers/framework_test.go @@ -23,12 +23,11 @@ import ( "math" "sort" "testing" - "time" ) func TestMultipleReplicaSetControllerLifeCycle(t *testing.T) { // case 1. start controller manager 1 - s, closeFn1, cim1, rsc1, informers1, client1 := RmSetup(t) + _, closeFn1, cim1, rsc1, informers1, client1 := RmSetup(t) defer closeFn1() stopCh1 := RunControllerAndInformers(t, cim1, rsc1, informers1, 0) defer close(stopCh1) @@ -44,93 +43,98 @@ func TestMultipleReplicaSetControllerLifeCycle(t *testing.T) { assert.NotNil(t, rsControllerInstance1.Name, "Nil controller instance name") assert.Equal(t, rsc1.GetControllerType(), rsControllerInstance1.ControllerType, "Unexpected controller type") - // case 2. start controller manager 2 - cim2, rsc2, informers2, client2 := RmSetupControllerMaster(t, s) - stopCh2 := RunControllerAndInformers(t, cim2, rsc2, informers2, 0) - defer close(stopCh2) - - // wait for controller instance to update controller key in registry - time.Sleep(5 * time.Second) - rsc1.ReportHealth(client1) - rsc2.ReportHealth(client2) - - // check replicaset controller status in controller manager 2 - t.Logf("rm 1 instance id: %v", rsc1.GetControllerName()) - t.Logf("rm 2 instance id: %v", rsc2.GetControllerName()) - - assert.NotEqual(t, rsc1.GetControllerName(), rsc2.GetControllerName()) - controllerInstanceList2, err := client2.CoreV1().ControllerInstances().List(metav1.ListOptions{}) - assert.Nil(t, err) - assert.NotNil(t, controllerInstanceList2) - assert.Equal(t, 2, len(controllerInstanceList2.Items), "number of controller instance") - t.Logf("new rms [%#v]", controllerInstanceList2) + /* Comment out as we can no longer test this case in integration test. + Also UT was able to simulate this scenario + */ - rsControllerInstanceRead1, err := client2.CoreV1().ControllerInstances().Get(rsc1.GetControllerName(), metav1.GetOptions{}) - assert.Nil(t, err) - rsControllerInstanceRead2, err := client2.CoreV1().ControllerInstances().Get(rsc2.GetControllerName(), metav1.GetOptions{}) - assert.Nil(t, err) + /* + // case 2. start controller manager 2 + cim2, rsc2, informers2, client2 := RmSetupControllerMaster(t, s) + stopCh2 := RunControllerAndInformers(t, cim2, rsc2, informers2, 0) + defer close(stopCh2) - instance1 := getInstanceFromControllerInstancesAndExpectedPos(0, rsControllerInstanceRead1, rsControllerInstanceRead2) - instance2 := getInstanceFromControllerInstancesAndExpectedPos(1, rsControllerInstanceRead1, rsControllerInstanceRead2) - t.Logf("sorted instance 1 %s controllerKey %v", instance1.Name, instance1.ControllerKey) - t.Logf("sorted instance 2 %s controllerKey %v", instance2.Name, instance2.ControllerKey) - // check controller instance updates - assert.Equal(t, rsControllerInstance1.ControllerType, instance1.ControllerType) - assert.Equal(t, rsControllerInstance1.ControllerType, instance2.ControllerType) - - assert.Equal(t, int64(4611686018427387904), instance1.ControllerKey) // consistent hash - assert.Equal(t, int64(math.MaxInt64), instance2.ControllerKey) - - // case 3. start controller manager 3 - cim3, rsc3, informers3, client3 := RmSetupControllerMaster(t, s) - stopCh3 := RunControllerAndInformers(t, cim3, rsc3, informers3, 0) - defer close(stopCh3) - time.Sleep(5 * time.Second) - t.Logf("rm 3 instance id: %v", rsc3.GetControllerName()) - rsc1.ReportHealth(client1) - rsc2.ReportHealth(client2) - rsc3.ReportHealth(client3) - - // check replicaset controller status in controller manager 2 - assert.NotEqual(t, rsc1.GetControllerName(), rsc3.GetControllerName()) - assert.NotEqual(t, rsc2.GetControllerName(), rsc3.GetControllerName()) - controllerInstanceList3, err := client2.CoreV1().ControllerInstances().List(metav1.ListOptions{}) - assert.Nil(t, err) - assert.NotNil(t, controllerInstanceList3) - assert.Equal(t, 3, len(controllerInstanceList3.Items), "number of controller instance") + // wait for controller instance to update controller key in registry + time.Sleep(5 * time.Second) + rsc1.ReportHealth(client1) + rsc2.ReportHealth(client2) - rsControllerInstanceRead1, err = client3.CoreV1().ControllerInstances().Get(rsc1.GetControllerName(), metav1.GetOptions{}) - assert.Nil(t, err) - rsControllerInstanceRead2, err = client3.CoreV1().ControllerInstances().Get(rsc2.GetControllerName(), metav1.GetOptions{}) - assert.Nil(t, err) - rsControllerInstanceRead3, err := client3.CoreV1().ControllerInstances().Get(rsc3.GetControllerName(), metav1.GetOptions{}) - assert.Nil(t, err) + // check replicaset controller status in controller manager 2 + t.Logf("rm 1 instance id: %v", rsc1.GetControllerName()) + t.Logf("rm 2 instance id: %v", rsc2.GetControllerName()) - instance1 = getInstanceFromControllerInstancesAndExpectedPos(0, rsControllerInstanceRead1, rsControllerInstanceRead2, rsControllerInstanceRead3) - instance2 = getInstanceFromControllerInstancesAndExpectedPos(1, rsControllerInstanceRead1, rsControllerInstanceRead2, rsControllerInstanceRead3) - instance3 := getInstanceFromControllerInstancesAndExpectedPos(2, rsControllerInstanceRead1, rsControllerInstanceRead2, rsControllerInstanceRead3) - t.Logf("sorted instance 1 %s controllerKey %v", instance1.Name, instance1.ControllerKey) - t.Logf("sorted instance 2 %s controllerKey %v", instance2.Name, instance2.ControllerKey) - t.Logf("sorted instance 3 %s controllerKey %v", instance3.Name, instance3.ControllerKey) + assert.NotEqual(t, rsc1.GetControllerName(), rsc2.GetControllerName()) + controllerInstanceList2, err := client2.CoreV1().ControllerInstances().List(metav1.ListOptions{}) + assert.Nil(t, err) + assert.NotNil(t, controllerInstanceList2) + assert.Equal(t, 2, len(controllerInstanceList2.Items), "number of controller instance") + t.Logf("new rms [%#v]", controllerInstanceList2) - // check controller instance updates - assert.True(t, instance1.ControllerKey < instance2.ControllerKey) - assert.True(t, instance2.ControllerKey < instance3.ControllerKey) - assert.Equal(t, int64(3074457345618258603), instance1.ControllerKey) - assert.Equal(t, int64(6148914691236517205), instance2.ControllerKey) - assert.Equal(t, int64(math.MaxInt64), instance3.ControllerKey) + rsControllerInstanceRead1, err := client2.CoreV1().ControllerInstances().Get(rsc1.GetControllerName(), metav1.GetOptions{}) + assert.Nil(t, err) + rsControllerInstanceRead2, err := client2.CoreV1().ControllerInstances().Get(rsc2.GetControllerName(), metav1.GetOptions{}) + assert.Nil(t, err) - assert.Equal(t, rsControllerInstance1.Name, rsControllerInstanceRead1.Name) - assert.Equal(t, rsControllerInstance1.ControllerType, rsControllerInstanceRead1.ControllerType) + instance1 := getInstanceFromControllerInstancesAndExpectedPos(0, rsControllerInstanceRead1, rsControllerInstanceRead2) + instance2 := getInstanceFromControllerInstancesAndExpectedPos(1, rsControllerInstanceRead1, rsControllerInstanceRead2) + t.Logf("sorted instance 1 %s controllerKey %v", instance1.Name, instance1.ControllerKey) + t.Logf("sorted instance 2 %s controllerKey %v", instance2.Name, instance2.ControllerKey) + // check controller instance updates + assert.Equal(t, rsControllerInstance1.ControllerType, instance1.ControllerType) + assert.Equal(t, rsControllerInstance1.ControllerType, instance2.ControllerType) + + assert.Equal(t, int64(4611686018427387904), instance1.ControllerKey) // consistent hash + assert.Equal(t, int64(math.MaxInt64), instance2.ControllerKey) + + // case 3. start controller manager 3 + cim3, rsc3, informers3, client3 := RmSetupControllerMaster(t, s) + stopCh3 := RunControllerAndInformers(t, cim3, rsc3, informers3, 0) + defer close(stopCh3) + time.Sleep(5 * time.Second) + t.Logf("rm 3 instance id: %v", rsc3.GetControllerName()) + rsc1.ReportHealth(client1) + rsc2.ReportHealth(client2) + rsc3.ReportHealth(client3) - assert.Equal(t, rsc1.GetControllerType(), rsControllerInstanceRead1.ControllerType, "Unexpected controller type") - assert.Equal(t, rsc2.GetControllerType(), rsControllerInstanceRead2.ControllerType, "Unexpected controller type") - assert.Equal(t, rsc3.GetControllerType(), rsControllerInstanceRead3.ControllerType, "Unexpected controller type") - t.Logf("new rms [%#v]", controllerInstanceList3) + // check replicaset controller status in controller manager 2 + assert.NotEqual(t, rsc1.GetControllerName(), rsc3.GetControllerName()) + assert.NotEqual(t, rsc2.GetControllerName(), rsc3.GetControllerName()) + controllerInstanceList3, err := client2.CoreV1().ControllerInstances().List(metav1.ListOptions{}) + assert.Nil(t, err) + assert.NotNil(t, controllerInstanceList3) + assert.Equal(t, 3, len(controllerInstanceList3.Items), "number of controller instance") - CleanupControllers(rsc1.ControllerBase, rsc2.ControllerBase, rsc3.ControllerBase) - //assert.False(t, rsControllerInstanceRead3.IsLocked, "Unexpected 3rd controller instance status") + rsControllerInstanceRead1, err = client3.CoreV1().ControllerInstances().Get(rsc1.GetControllerName(), metav1.GetOptions{}) + assert.Nil(t, err) + rsControllerInstanceRead2, err = client3.CoreV1().ControllerInstances().Get(rsc2.GetControllerName(), metav1.GetOptions{}) + assert.Nil(t, err) + rsControllerInstanceRead3, err := client3.CoreV1().ControllerInstances().Get(rsc3.GetControllerName(), metav1.GetOptions{}) + assert.Nil(t, err) + instance1 = getInstanceFromControllerInstancesAndExpectedPos(0, rsControllerInstanceRead1, rsControllerInstanceRead2, rsControllerInstanceRead3) + instance2 = getInstanceFromControllerInstancesAndExpectedPos(1, rsControllerInstanceRead1, rsControllerInstanceRead2, rsControllerInstanceRead3) + instance3 := getInstanceFromControllerInstancesAndExpectedPos(2, rsControllerInstanceRead1, rsControllerInstanceRead2, rsControllerInstanceRead3) + t.Logf("sorted instance 1 %s controllerKey %v", instance1.Name, instance1.ControllerKey) + t.Logf("sorted instance 2 %s controllerKey %v", instance2.Name, instance2.ControllerKey) + t.Logf("sorted instance 3 %s controllerKey %v", instance3.Name, instance3.ControllerKey) + + // check controller instance updates + assert.True(t, instance1.ControllerKey < instance2.ControllerKey) + assert.True(t, instance2.ControllerKey < instance3.ControllerKey) + assert.Equal(t, int64(3074457345618258603), instance1.ControllerKey) + assert.Equal(t, int64(6148914691236517205), instance2.ControllerKey) + assert.Equal(t, int64(math.MaxInt64), instance3.ControllerKey) + + assert.Equal(t, rsControllerInstance1.Name, rsControllerInstanceRead1.Name) + assert.Equal(t, rsControllerInstance1.ControllerType, rsControllerInstanceRead1.ControllerType) + + assert.Equal(t, rsc1.GetControllerType(), rsControllerInstanceRead1.ControllerType, "Unexpected controller type") + assert.Equal(t, rsc2.GetControllerType(), rsControllerInstanceRead2.ControllerType, "Unexpected controller type") + assert.Equal(t, rsc3.GetControllerType(), rsControllerInstanceRead3.ControllerType, "Unexpected controller type") + t.Logf("new rms [%#v]", controllerInstanceList3) + + CleanupControllers(rsc1.ControllerBase, rsc2.ControllerBase, rsc3.ControllerBase) + //assert.False(t, rsControllerInstanceRead3.IsLocked, "Unexpected 3rd controller instance status") + */ /* // case 4. 1st controller instance died - This needs to be done in unit test as integration test would be flaky close(stopCh1) diff --git a/test/integration/cloudfabriccontrollers/testutil.go b/test/integration/cloudfabriccontrollers/testutil.go index 127b783cd55..4f456788ff0 100644 --- a/test/integration/cloudfabriccontrollers/testutil.go +++ b/test/integration/cloudfabriccontrollers/testutil.go @@ -98,7 +98,7 @@ func RmSetupControllerMaster(t *testing.T, s *httptest.Server) (*controller.Cont informers := informers.NewSharedInformerFactory(clientset.NewForConfigOrDie(restclient.AddUserAgent(configs, "rs-informers")), resyncPeriod) // controller instance manager set up - cim := controller.GetControllerInstanceManager() + cim := controller.GetInstanceHandler() if cim == nil { cimUpdateChGrp := bcast.NewGroup() go cimUpdateChGrp.Broadcast(0)