diff --git a/pkg/instancegroups/manager.go b/pkg/instancegroups/manager.go index 7576dfca93..92652ddd6e 100644 --- a/pkg/instancegroups/manager.go +++ b/pkg/instancegroups/manager.go @@ -240,7 +240,7 @@ func (m *manager) List(logger klog.Logger) ([]string, error) { func (m *manager) splitNodesByZone(names []string, logger klog.Logger) map[string][]string { nodesByZone := map[string][]string{} for _, name := range names { - zone, err := m.ZoneGetter.ZoneForNode(name, logger) + zone, _, err := m.ZoneGetter.ZoneAndSubnetForNode(name, logger) if err != nil { logger.Error(err, "Failed to get zones for instance node, skipping", "name", name) continue diff --git a/pkg/l4lb/l4netlbcontroller_test.go b/pkg/l4lb/l4netlbcontroller_test.go index 795862a2db..c0e09337c3 100644 --- a/pkg/l4lb/l4netlbcontroller_test.go +++ b/pkg/l4lb/l4netlbcontroller_test.go @@ -74,6 +74,8 @@ const ( shortSessionAffinityIdleTimeout = int32(20) // 20 sec could be used for regular Session Affinity longSessionAffinityIdleTimeout = int32(2 * 60) // 2 min or 120 sec for Strong Session Affinity + + defaultTestSubnetURL = "https://www.googleapis.com/compute/v1/projects/proj/regions/us-central1/subnetworks/default" ) var ( @@ -323,8 +325,13 @@ func buildContext(vals gce.TestClusterValues) *ingctx.ControllerContext { } func newL4NetLBServiceController() *L4NetLBController { - stopCh := make(chan struct{}) vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + return createL4NetLBServiceController(vals) +} + +func createL4NetLBServiceController(vals gce.TestClusterValues) *L4NetLBController { + stopCh := make(chan struct{}) ctx := buildContext(vals) nodes, err := test.CreateAndInsertNodes(ctx.Cloud, []string{"instance-1", "instance-2"}, vals.ZoneName) if err != nil { @@ -1663,7 +1670,7 @@ func TestDualStackServiceNeedsUpdate(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { t.Parallel() - controller := newL4NetLBServiceController() + controller := createL4NetLBServiceController(gce.DefaultTestClusterValues()) controller.enableDualStack = true oldSvc := test.NewL4NetLBRBSService(8080) oldSvc.Spec.IPFamilies = tc.initialIPFamilies @@ -1862,7 +1869,7 @@ func TestCreateDeleteDualStackNetLBService(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - controller := newL4NetLBServiceController() + controller := createL4NetLBServiceController(gce.DefaultTestClusterValues()) controller.enableDualStack = true svc := test.NewL4NetLBRBSService(8080) svc.Spec.IPFamilies = tc.ipFamilies @@ -1907,7 +1914,7 @@ func TestCreateDeleteDualStackNetLBService(t *testing.T) { } func TestProcessDualStackNetLBServiceOnUserError(t *testing.T) { t.Parallel() - controller := newL4NetLBServiceController() + controller := createL4NetLBServiceController(gce.DefaultTestClusterValues()) controller.enableDualStack = true svc := test.NewL4NetLBRBSService(8080) svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol} diff --git a/pkg/neg/manager_test.go b/pkg/neg/manager_test.go index e8f28079ca..bd71a484cf 100644 --- a/pkg/neg/manager_test.go +++ b/pkg/neg/manager_test.go @@ -81,6 +81,7 @@ const ( labelValue2 = "v2" negName1 = "neg1" + defaultTestSubnet = "default" defaultTestSubnetURL = "https://www.googleapis.com/compute/v1/projects/proj/regions/us-central1/subnetworks/default" ) diff --git a/pkg/neg/metrics/metricscollector/metrics_collector.go b/pkg/neg/metrics/metricscollector/metrics_collector.go index 374ecc3806..246f0fd2a5 100644 --- a/pkg/neg/metrics/metricscollector/metrics_collector.go +++ b/pkg/neg/metrics/metricscollector/metrics_collector.go @@ -294,7 +294,7 @@ func (sm *SyncerMetrics) computeEndpointStateMetrics() (negtypes.StateCountMap, // CollectDualStackMigrationMetrics will be used by dualstack.Migrator to export // metrics. -func (sm *SyncerMetrics) CollectDualStackMigrationMetrics(key negtypes.NegSyncerKey, committedEndpoints map[string]negtypes.NetworkEndpointSet, migrationCount int) { +func (sm *SyncerMetrics) CollectDualStackMigrationMetrics(key negtypes.NegSyncerKey, committedEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, migrationCount int) { sm.updateMigrationStartAndEndTime(key, migrationCount) sm.updateEndpointsCountPerType(key, committedEndpoints, migrationCount) } @@ -337,7 +337,7 @@ func (sm *SyncerMetrics) updateMigrationStartAndEndTime(key negtypes.NegSyncerKe sm.dualStackMigrationStartTime[key] = sm.clock.Now() } -func (sm *SyncerMetrics) updateEndpointsCountPerType(key negtypes.NegSyncerKey, committedEndpoints map[string]negtypes.NetworkEndpointSet, migrationCount int) { +func (sm *SyncerMetrics) updateEndpointsCountPerType(key negtypes.NegSyncerKey, committedEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, migrationCount int) { sm.mu.Lock() defer sm.mu.Unlock() diff --git a/pkg/neg/metrics/metricscollector/metrics_collector_test.go b/pkg/neg/metrics/metricscollector/metrics_collector_test.go index a272e09f91..cc86dc8345 100644 --- a/pkg/neg/metrics/metricscollector/metrics_collector_test.go +++ b/pkg/neg/metrics/metricscollector/metrics_collector_test.go @@ -118,13 +118,13 @@ func TestUpdateMigrationStartAndEndTime(t *testing.T) { func TestUpdateEndpointsCountPerType(t *testing.T) { syncerKey := syncerKey(1) - inputCommittedEndpoints := map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputCommittedEndpoints := map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "ipv4only-1"}, {IP: "ipv4only-2"}, {IP: "ipv4only-3"}, {IPv6: "ipv6only-1"}, {IP: "dualStack-1a", IPv6: "dualStack-1b"}, {IP: "dualStack-2a", IPv6: "dualStack-2b"}, }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "ipv4only-4"}, {IP: "dualStack-3a", IPv6: "dualStack-3b"}, }...), diff --git a/pkg/neg/syncers/dualstack/migrator.go b/pkg/neg/syncers/dualstack/migrator.go index d2b8f90e4a..8084bb0729 100644 --- a/pkg/neg/syncers/dualstack/migrator.go +++ b/pkg/neg/syncers/dualstack/migrator.go @@ -22,6 +22,7 @@ import ( "time" "k8s.io/ingress-gce/pkg/neg/types" + negtypes "k8s.io/ingress-gce/pkg/neg/types" "k8s.io/klog/v2" "k8s.io/utils/clock" ) @@ -104,7 +105,7 @@ type errorStateChecker interface { } type MetricsCollector interface { - CollectDualStackMigrationMetrics(key types.NegSyncerKey, committedEndpoints map[string]types.NetworkEndpointSet, migrationCount int) + CollectDualStackMigrationMetrics(key types.NegSyncerKey, committedEndpoints map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, migrationCount int) } func NewMigrator(enableDualStackNEG bool, syncer syncable, syncerKey types.NegSyncerKey, metricsCollector MetricsCollector, errorStateChecker errorStateChecker, logger klog.Logger) *Migrator { @@ -130,13 +131,13 @@ func NewMigrator(enableDualStackNEG bool, syncer syncable, syncerKey types.NegSy // 2. If the migrator is not currently paused, it will also start the // detachment of a subset of migration-endpoints from a single zone. // -// The returned string represents the zone for which detachment was started. An -// empty return value signifies that detachment was not started (which is the -// case when there were no migration-endpoints to begin with, or the migrator -// was paused.) -func (d *Migrator) Filter(addEndpoints, removeEndpoints, committedEndpoints map[string]types.NetworkEndpointSet) string { +// The returned EndpointGroup represents the zone and subnet of NEG for which +// detachment was started on. An empty subnet and zone value signifies that +// detachment was not started (which is the case when there were no +// migration-endpoints to begin with, or the migrator was paused.) +func (d *Migrator) Filter(addEndpoints, removeEndpoints, committedEndpoints map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) negtypes.EndpointGroupInfo { if !d.enableDualStack { - return "" + return negtypes.EndpointGroupInfo{} } _, migrationEndpointsInRemoveSet := findAndFilterMigrationEndpoints(addEndpoints, removeEndpoints) @@ -147,7 +148,7 @@ func (d *Migrator) Filter(addEndpoints, removeEndpoints, committedEndpoints map[ paused := d.isPaused() if migrationCount == 0 || paused { d.logger.V(2).Info("Not starting migration detachments", "migrationCount", migrationCount, "paused", paused) - return "" + return negtypes.EndpointGroupInfo{} } return d.calculateMigrationEndpointsToDetach(addEndpoints, removeEndpoints, committedEndpoints, migrationEndpointsInRemoveSet) @@ -243,7 +244,7 @@ func (d *Migrator) isPaused() bool { // function) AND (2) we are in degraded mode OR the previous successful // detach was quite recent (as determined by the // tooLongSincePreviousDetach() function) -func (d *Migrator) calculateMigrationEndpointsToDetach(addEndpoints, removeEndpoints, committedEndpoints, migrationEndpoints map[string]types.NetworkEndpointSet) string { +func (d *Migrator) calculateMigrationEndpointsToDetach(addEndpoints, removeEndpoints, committedEndpoints, migrationEndpoints map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) negtypes.EndpointGroupInfo { addCount := endpointsCount(addEndpoints) committedCount := endpointsCount(committedEndpoints) migrationCount := endpointsCount(migrationEndpoints) @@ -258,19 +259,19 @@ func (d *Migrator) calculateMigrationEndpointsToDetach(addEndpoints, removeEndpo "previousDetach", d.previousDetach, "previousDetachThreshold", d.previousDetachThreshold, ) - return "" + return negtypes.EndpointGroupInfo{} } - // Find the zone which has the maximum number of migration-endpoints. - zone, maxZoneEndpointCount := "", 0 - for curZone, endpointSet := range migrationEndpoints { + // Find the zone and subnet which has the maximum number of migration-endpoints. + endpointGroupInfo, maxZoneEndpointCount := negtypes.EndpointGroupInfo{}, 0 + for curEndpointInfo, endpointSet := range migrationEndpoints { if endpointSet.Len() > maxZoneEndpointCount { maxZoneEndpointCount = endpointSet.Len() - zone = curZone + endpointGroupInfo = curEndpointInfo } } - if zone == "" { - return "" + if endpointGroupInfo.Zone == "" && endpointGroupInfo.Subnet == "" { + return negtypes.EndpointGroupInfo{} } currentlyMigratingCount := int(math.Ceil(float64(committedCount+migrationCount) * d.fractionOfMigratingEndpoints)) @@ -279,18 +280,18 @@ func (d *Migrator) calculateMigrationEndpointsToDetach(addEndpoints, removeEndpo } d.logger.V(2).Info("Result of migration heuristic calculations", "currentlyMigratingCount", currentlyMigratingCount, "totalMigrationCount", migrationCount) - if removeEndpoints[zone] == nil { - removeEndpoints[zone] = types.NewNetworkEndpointSet() + if removeEndpoints[endpointGroupInfo] == nil { + removeEndpoints[endpointGroupInfo] = types.NewNetworkEndpointSet() } for i := 0; i < currentlyMigratingCount; i++ { - endpoint, ok := migrationEndpoints[zone].PopAny() + endpoint, ok := migrationEndpoints[endpointGroupInfo].PopAny() if !ok { break } - removeEndpoints[zone].Insert(endpoint) + removeEndpoints[endpointGroupInfo].Insert(endpoint) } - return zone + return endpointGroupInfo } // Returns true if there are many endpoints waiting to be attached. @@ -314,18 +315,18 @@ func (d *Migrator) tooLongSincePreviousDetach() bool { // modified. The returned value will be two endpoints sets which will contain // the values that were filtered out from the `addEndpoints` and // `removeEndpoints` sets respectively. -func findAndFilterMigrationEndpoints(addEndpoints, removeEndpoints map[string]types.NetworkEndpointSet) (map[string]types.NetworkEndpointSet, map[string]types.NetworkEndpointSet) { - allEndpoints := make(map[string]types.NetworkEndpointSet) - for zone, endpointSet := range addEndpoints { - allEndpoints[zone] = allEndpoints[zone].Union(endpointSet) +func findAndFilterMigrationEndpoints(addEndpoints, removeEndpoints map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) { + allEndpoints := make(map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) + for endpointGroupInfo, endpointSet := range addEndpoints { + allEndpoints[endpointGroupInfo] = allEndpoints[endpointGroupInfo].Union(endpointSet) } - for zone, endpointSet := range removeEndpoints { - allEndpoints[zone] = allEndpoints[zone].Union(endpointSet) + for endpointGroupInfo, endpointSet := range removeEndpoints { + allEndpoints[endpointGroupInfo] = allEndpoints[endpointGroupInfo].Union(endpointSet) } - migrationEndpointsInAddSet := make(map[string]types.NetworkEndpointSet) - migrationEndpointsInRemoveSet := make(map[string]types.NetworkEndpointSet) - for zone, endpointSet := range allEndpoints { + migrationEndpointsInAddSet := make(map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) + migrationEndpointsInRemoveSet := make(map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) + for endpointGroupInfo, endpointSet := range allEndpoints { for endpoint := range endpointSet { if endpoint.IP == "" || endpoint.IPv6 == "" { // Endpoint is not dual-stack so continue. @@ -346,15 +347,15 @@ func findAndFilterMigrationEndpoints(addEndpoints, removeEndpoints map[string]ty isMigrating := false // Check if endpoint is migrating from dual-stack to single-stack. - isMigrating = isMigrating || moveEndpoint(ipv4Only, addEndpoints, migrationEndpointsInAddSet, zone) - isMigrating = isMigrating || moveEndpoint(ipv6Only, addEndpoints, migrationEndpointsInAddSet, zone) + isMigrating = isMigrating || moveEndpoint(ipv4Only, addEndpoints, migrationEndpointsInAddSet, endpointGroupInfo) + isMigrating = isMigrating || moveEndpoint(ipv6Only, addEndpoints, migrationEndpointsInAddSet, endpointGroupInfo) // Check if endpoint is migrating from single-stack to dual-stack. - isMigrating = isMigrating || moveEndpoint(ipv4Only, removeEndpoints, migrationEndpointsInRemoveSet, zone) - isMigrating = isMigrating || moveEndpoint(ipv6Only, removeEndpoints, migrationEndpointsInRemoveSet, zone) + isMigrating = isMigrating || moveEndpoint(ipv4Only, removeEndpoints, migrationEndpointsInRemoveSet, endpointGroupInfo) + isMigrating = isMigrating || moveEndpoint(ipv6Only, removeEndpoints, migrationEndpointsInRemoveSet, endpointGroupInfo) if isMigrating { - moveEndpoint(endpoint, addEndpoints, migrationEndpointsInAddSet, zone) - moveEndpoint(endpoint, removeEndpoints, migrationEndpointsInRemoveSet, zone) + moveEndpoint(endpoint, addEndpoints, migrationEndpointsInAddSet, endpointGroupInfo) + moveEndpoint(endpoint, removeEndpoints, migrationEndpointsInRemoveSet, endpointGroupInfo) } } } @@ -362,26 +363,26 @@ func findAndFilterMigrationEndpoints(addEndpoints, removeEndpoints map[string]ty return migrationEndpointsInAddSet, migrationEndpointsInRemoveSet } -// moveEndpoint deletes endpoint `e` from `source[zone]` and adds it to -// `dest[zone]`. If the move was successful, `true` is returned. A return value +// moveEndpoint deletes endpoint `e` from `source[endpointGroupInfo]` and adds it to +// `dest[endpointGroupInfo]`. If the move was successful, `true` is returned. A return value // of `false` denotes that nothing was moved and no input variable were // modified. -func moveEndpoint(e types.NetworkEndpoint, source, dest map[string]types.NetworkEndpointSet, zone string) bool { +func moveEndpoint(e types.NetworkEndpoint, source, dest map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, endpointGroupInfo negtypes.EndpointGroupInfo) bool { if source == nil || dest == nil { return false } - if source[zone].Has(e) { - source[zone].Delete(e) - if dest[zone] == nil { - dest[zone] = types.NewNetworkEndpointSet() + if source[endpointGroupInfo].Has(e) { + source[endpointGroupInfo].Delete(e) + if dest[endpointGroupInfo] == nil { + dest[endpointGroupInfo] = types.NewNetworkEndpointSet() } - dest[zone].Insert(e) + dest[endpointGroupInfo].Insert(e) return true } return false } -func endpointsCount(endpointSets map[string]types.NetworkEndpointSet) int { +func endpointsCount(endpointSets map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) int { var count int for _, endpointSet := range endpointSets { count += endpointSet.Len() diff --git a/pkg/neg/syncers/dualstack/migrator_test.go b/pkg/neg/syncers/dualstack/migrator_test.go index b4044667d4..fb9296c836 100644 --- a/pkg/neg/syncers/dualstack/migrator_test.go +++ b/pkg/neg/syncers/dualstack/migrator_test.go @@ -15,15 +15,17 @@ import ( clocktesting "k8s.io/utils/clock/testing" ) +const defaultTestSubnet = "default" + func TestFilter(t *testing.T) { testCases := []struct { desc string migrator *Migrator - addEndpoints map[string]types.NetworkEndpointSet - removeEndpoints map[string]types.NetworkEndpointSet - committedEndpoints map[string]types.NetworkEndpointSet - wantAddEndpoints map[string]types.NetworkEndpointSet - wantRemoveEndpoints map[string]types.NetworkEndpointSet + addEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + removeEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + committedEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + wantAddEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + wantRemoveEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet wantMigrationZone bool }{ { @@ -33,30 +35,30 @@ func TestFilter(t *testing.T) { m.Pause() return m }(), - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), }, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "D"}, }...), }, - wantAddEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantAddEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b"}, }...), }, - wantRemoveEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantRemoveEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ // Migration-endpoints were filtered out but no new migration // detachment was started. {IP: "c", IPv6: "C"}, @@ -66,30 +68,30 @@ func TestFilter(t *testing.T) { { desc: "unpaused migrator should filter migration endpoints AND also start detachment", migrator: newMigratorForTest(t, true), - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), }, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "D"}, }...), }, - wantAddEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantAddEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b"}, }...), }, - wantRemoveEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantRemoveEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // Migration detachment started. {IP: "c", IPv6: "C"}, }...), @@ -99,31 +101,31 @@ func TestFilter(t *testing.T) { { desc: "migrator should do nothing if enableDualStack is false", migrator: newMigratorForTest(t, false), - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), }, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "D"}, }...), }, - wantAddEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantAddEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, - wantRemoveEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantRemoveEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), @@ -138,7 +140,7 @@ func TestFilter(t *testing.T) { gotCommittedEndpoints := cloneZoneNetworkEndpointsMap(tc.committedEndpoints) gotMigrationZone := tc.migrator.Filter(gotAddEndpoints, gotRemoveEndpoints, gotCommittedEndpoints) - if tc.wantMigrationZone && gotMigrationZone == "" { + if tc.wantMigrationZone && gotMigrationZone.Zone == "" && gotMigrationZone.Subnet == "" { t.Errorf("Filter() returned empty migrationZone; want non empty migrationZone") } @@ -271,7 +273,7 @@ func TestFilter_FunctionalTest(t *testing.T) { // filteredRemoveEndpoints. // // If shouldAttach is false, no attachments will be done. - doNEGAttachDetach := func(addEndpoints, removeEndpoints, committedEndpoints, filteredAddEndpoints, filteredRemoveEndpoints map[string]types.NetworkEndpointSet, shouldAttach bool) { + doNEGAttachDetach := func(addEndpoints, removeEndpoints, committedEndpoints, filteredAddEndpoints, filteredRemoveEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet, shouldAttach bool) { // Endpoints are detachment by deleting them from the removeEndpoints set. for zone, endpointSet := range filteredRemoveEndpoints { removeEndpoints[zone].Delete(endpointSet.List()...) @@ -293,20 +295,20 @@ func TestFilter_FunctionalTest(t *testing.T) { // Step 1: Generate endpoints to be used as input. ///////////////////////////////////////////////////////////////////////// - addEndpoints := make(map[string]types.NetworkEndpointSet) // Contains dual-stack endpoints which are to be added to the NEG. - removeEndpoints := make(map[string]types.NetworkEndpointSet) // Contains single-stack endpoints which are to be removed from the NEG. - committedEndpoints := make(map[string]types.NetworkEndpointSet) // Initially empty. + addEndpoints := make(map[types.EndpointGroupInfo]types.NetworkEndpointSet) // Contains dual-stack endpoints which are to be added to the NEG. + removeEndpoints := make(map[types.EndpointGroupInfo]types.NetworkEndpointSet) // Contains single-stack endpoints which are to be removed from the NEG. + committedEndpoints := make(map[types.EndpointGroupInfo]types.NetworkEndpointSet) // Initially empty. for i := 0; i < tc.initialNEGEndpointsCount; i++ { - zone := fmt.Sprintf("zone-%v", i%tc.zonesCount) + epGroupInfo := types.EndpointGroupInfo{Zone: fmt.Sprintf("zone-%v", i%tc.zonesCount), Subnet: defaultTestSubnet} ipv4 := fmt.Sprintf("ipv4-%v", 2*i+1) ipv6 := fmt.Sprintf("ipv6-%v", 2*i+2) - if addEndpoints[zone] == nil { - addEndpoints[zone] = types.NewNetworkEndpointSet() - removeEndpoints[zone] = types.NewNetworkEndpointSet() - committedEndpoints[zone] = types.NewNetworkEndpointSet() + if addEndpoints[epGroupInfo] == nil { + addEndpoints[epGroupInfo] = types.NewNetworkEndpointSet() + removeEndpoints[epGroupInfo] = types.NewNetworkEndpointSet() + committedEndpoints[epGroupInfo] = types.NewNetworkEndpointSet() } - addEndpoints[zone].Insert(types.NetworkEndpoint{IP: ipv4, IPv6: ipv6}) - removeEndpoints[zone].Insert(types.NetworkEndpoint{IP: ipv4}) + addEndpoints[epGroupInfo].Insert(types.NetworkEndpoint{IP: ipv4, IPv6: ipv6}) + removeEndpoints[epGroupInfo].Insert(types.NetworkEndpoint{IP: ipv4}) } tc.migrator.errorStateChecker.(*fakeErrorStateChecker).errorState = tc.errorState @@ -323,7 +325,7 @@ func TestFilter_FunctionalTest(t *testing.T) { doNEGAttachDetach(addEndpoints, removeEndpoints, committedEndpoints, filteredAddEndpoints, filteredRemoveEndpoints, tc.attachSucceeds) fakeClock := tc.migrator.clock.(*clocktesting.FakeClock) - if migrationZone != "" { + if migrationZone.Zone != "" || migrationZone.Subnet != "" { // Update the time of the last successful detach. tc.migrator.previousDetach = fakeClock.Now() } @@ -350,16 +352,16 @@ func TestFilter_FunctionalTest(t *testing.T) { } func TestPause(t *testing.T) { - addEndpoints := map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints := map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, {IP: "c", IPv6: "C"}, {IP: "d"}, // migrating }...), } - removeEndpoints := map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints := map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "e", IPv6: "E"}, {IP: "d", IPv6: "D"}, // migrating @@ -372,12 +374,12 @@ func TestPause(t *testing.T) { // starting migration-detachments. clonedAddEndpoints := cloneZoneNetworkEndpointsMap(addEndpoints) clonedRemoveEndpoints := cloneZoneNetworkEndpointsMap(removeEndpoints) - migrator.Filter(clonedAddEndpoints, clonedRemoveEndpoints, map[string]types.NetworkEndpointSet{}) + migrator.Filter(clonedAddEndpoints, clonedRemoveEndpoints, map[types.EndpointGroupInfo]types.NetworkEndpointSet{}) possibleMigrationDetachments := []types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "d", IPv6: "D"}, // migrating } - if !clonedRemoveEndpoints["zone1"].HasAny(possibleMigrationDetachments...) { + if !clonedRemoveEndpoints[types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}].HasAny(possibleMigrationDetachments...) { t.Fatalf("Precondition to verify the behaviour of Pause() not satisfied; Filter() should start migration-detachments; got removeEndpoints=%+v; want non-empty union with %+v", clonedRemoveEndpoints, possibleMigrationDetachments) } @@ -391,9 +393,9 @@ func TestPause(t *testing.T) { // The effect of Pause() is observed by verifying that Filter() does not start // any migration-detachments when paused. - migrator.Filter(addEndpoints, removeEndpoints, map[string]types.NetworkEndpointSet{}) - wantRemoveEndpoints := map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + migrator.Filter(addEndpoints, removeEndpoints, map[types.EndpointGroupInfo]types.NetworkEndpointSet{}) + wantRemoveEndpoints := map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ // Since we don't expect any migration-detachments, this set should be // missing all migration endpoints. {IP: "e", IPv6: "E"}, @@ -523,24 +525,24 @@ func TestContinue_NoInputError_ShouldChangeTimeSincePreviousDetach(t *testing.T) func TestCalculateMigrationEndpointsToDetach(t *testing.T) { testCases := []struct { desc string - addEndpoints map[string]types.NetworkEndpointSet - removeEndpoints map[string]types.NetworkEndpointSet - committedEndpoints map[string]types.NetworkEndpointSet - migrationEndpoints map[string]types.NetworkEndpointSet + addEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + removeEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + committedEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + migrationEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet migrator *Migrator wantCurrentlyMigratingCount int }{ { desc: "less than or equal to 10 (committed + migration) endpoints should only detach 1 at a time", - addEndpoints: map[string]types.NetworkEndpointSet{}, - removeEndpoints: map[string]types.NetworkEndpointSet{}, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, - migrationEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, {IP: "7"}, {IP: "8"}, {IP: "9"}, {IP: "10"}, }...), }, @@ -549,15 +551,15 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { }, { desc: "more than 10 (committed + migration) endpoints can detach more than 1 at a time", - addEndpoints: map[string]types.NetworkEndpointSet{}, - removeEndpoints: map[string]types.NetworkEndpointSet{}, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, - migrationEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, {IP: "7"}, {IP: "8"}, {IP: "9"}, {IP: "10"}, {IP: "11"}, }...), @@ -570,19 +572,19 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { // migration was NOT too long ago, then we will not start any new // detachments since we wait for the pending attaches to complete desc: "many endpoints are waiting to be attached AND previous migration was quite recent", - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{}, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, }...), }, - migrationEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "7"}, }...), }, @@ -598,19 +600,19 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { // migration was too long ago BUT we are in error state, then we will not // start any new detachments since we wait to get out of error state. desc: "many endpoints are waiting to be attached AND previous migration was too long ago BUT in error state", - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{}, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, }...), }, - migrationEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "7"}, }...), }, @@ -627,19 +629,19 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { // want to keep waiting indefinitely for the next detach and we proceed // with the detachments. desc: "many endpoints are waiting to be attached BUT previous migration was too long ago AND not in error state", - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{}, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, }...), }, - migrationEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "7"}, }...), }, @@ -648,22 +650,22 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { }, { desc: "no detachments started since nothing to migrate", - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "2"}, }...), }, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "3"}, {IP: "4"}, {IP: "5"}, {IP: "6"}, }...), }, - migrationEndpoints: map[string]types.NetworkEndpointSet{}, + migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, migrator: newMigratorForTest(t, true), wantCurrentlyMigratingCount: 0, }, @@ -672,21 +674,21 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { // more than the number of endpoints in any single zone, we should not // include endpoints from multiple zones. desc: "endpoints from multiple zones should not be detached at once", - addEndpoints: map[string]types.NetworkEndpointSet{}, - removeEndpoints: map[string]types.NetworkEndpointSet{}, - committedEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "5"}, {IP: "6"}, {IP: "7"}, {IP: "8"}, }...), }, - migrationEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "9"}, {IP: "10"}, }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "11"}, {IP: "12"}, }...), }, @@ -704,8 +706,8 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { migrationZone := tc.migrator.calculateMigrationEndpointsToDetach(tc.addEndpoints, tc.removeEndpoints, tc.committedEndpoints, tc.migrationEndpoints) - if tc.wantCurrentlyMigratingCount > 0 && migrationZone == "" { - t.Fatalf("calculateMigrationEndpointsToDetach(...) returned empty zone which means no migration detachment was started; want %v endpoints to undergo detachment", tc.wantCurrentlyMigratingCount) + if tc.wantCurrentlyMigratingCount > 0 && (migrationZone.Zone == "" && migrationZone.Subnet == "") { + t.Fatalf("calculateMigrationEndpointsToDetach(...) returned empty zone and subnet which means no migration detachment was started; want %v endpoints to undergo detachment", tc.wantCurrentlyMigratingCount) } // Ensure that we didn't modify the addEndpoints and committedEndpoints. @@ -744,84 +746,84 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { func TestFindAndFilterMigrationEndpoints(t *testing.T) { testCases := []struct { name string - addEndpoints map[string]types.NetworkEndpointSet - removeEndpoints map[string]types.NetworkEndpointSet - wantMigrationEndpointsInAddSet map[string]types.NetworkEndpointSet - wantMigrationEndpointsInRemoveSet map[string]types.NetworkEndpointSet + addEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + removeEndpoints map[types.EndpointGroupInfo]types.NetworkEndpointSet + wantMigrationEndpointsInAddSet map[types.EndpointGroupInfo]types.NetworkEndpointSet + wantMigrationEndpointsInRemoveSet map[types.EndpointGroupInfo]types.NetworkEndpointSet }{ { name: "detect multiple migrating endpoints", - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, {IP: "c", IPv6: "C"}, {IP: "d"}, // migrating }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "e", IPv6: "E"}, // migrating }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "f", IPv6: "F"}, {IP: "d", IPv6: "D"}, // migrating }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "E"}, // migrating }...), }, - wantMigrationEndpointsInAddSet: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantMigrationEndpointsInAddSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "d"}, }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "e", IPv6: "E"}, }...), }, - wantMigrationEndpointsInRemoveSet: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantMigrationEndpointsInRemoveSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, {IP: "d", IPv6: "D"}, }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "E"}, }...), }, }, { name: "partial IP change without stack change is not considered migrating", - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", Port: "B"}, }...), }, - wantMigrationEndpointsInAddSet: map[string]types.NetworkEndpointSet{}, - wantMigrationEndpointsInRemoveSet: map[string]types.NetworkEndpointSet{}, + wantMigrationEndpointsInAddSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + wantMigrationEndpointsInRemoveSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, }, { name: "difference in port or node is not considered migrating", - addEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A", Port: "80"}, {IP: "b", Node: "node2"}, }...), }, - removeEndpoints: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", Port: "81"}, {IP: "b", IPv6: "B", Node: "node1"}, }...), }, - wantMigrationEndpointsInAddSet: map[string]types.NetworkEndpointSet{}, - wantMigrationEndpointsInRemoveSet: map[string]types.NetworkEndpointSet{}, + wantMigrationEndpointsInAddSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, + wantMigrationEndpointsInRemoveSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, }, } @@ -841,133 +843,133 @@ func TestFindAndFilterMigrationEndpoints(t *testing.T) { func TestMoveEndpoint(t *testing.T) { testCases := []struct { - name string - endpoint types.NetworkEndpoint - inputSource map[string]types.NetworkEndpointSet - inputDest map[string]types.NetworkEndpointSet - wantSource map[string]types.NetworkEndpointSet - wantDest map[string]types.NetworkEndpointSet - zone string - wantSuccess bool + name string + endpoint types.NetworkEndpoint + inputSource map[types.EndpointGroupInfo]types.NetworkEndpointSet + inputDest map[types.EndpointGroupInfo]types.NetworkEndpointSet + wantSource map[types.EndpointGroupInfo]types.NetworkEndpointSet + wantDest map[types.EndpointGroupInfo]types.NetworkEndpointSet + endpointGroupInfo types.EndpointGroupInfo + wantSuccess bool }{ { name: "completely valid input, shoud successfully move", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, - inputSource: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "b", IPv6: "B"}, }...), }, - inputDest: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - wantSource: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b", IPv6: "B"}, }...), }, - wantDest: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "c", IPv6: "C"}, }...), }, - zone: "zone1", - wantSuccess: true, + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}, + wantSuccess: true, }, { name: "zone does not exist in source", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, - inputSource: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), }, - inputDest: map[string]types.NetworkEndpointSet{ - "zone3": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - wantSource: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), }, - wantDest: map[string]types.NetworkEndpointSet{ - "zone3": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - zone: "zone3", + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3", Subnet: defaultTestSubnet}, }, { name: "zone does not exist in destination, shoud successfully move", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, - inputSource: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "b", IPv6: "B"}, }...), }, - inputDest: map[string]types.NetworkEndpointSet{ - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - wantSource: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b", IPv6: "B"}, }...), }, - wantDest: map[string]types.NetworkEndpointSet{ - "zone1": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), - "zone2": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - zone: "zone1", - wantSuccess: true, + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}, + wantSuccess: true, }, { name: "source is nil", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, - inputDest: map[string]types.NetworkEndpointSet{ - "zone3": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - wantDest: map[string]types.NetworkEndpointSet{ - "zone3": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - zone: "zone3", + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3", Subnet: defaultTestSubnet}, }, { name: "destination is nil", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, - inputSource: map[string]types.NetworkEndpointSet{ - "zone3": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - wantSource: map[string]types.NetworkEndpointSet{ - "zone3": types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - zone: "zone3", + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3", Subnet: defaultTestSubnet}, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - gotSuccess := moveEndpoint(tc.endpoint, tc.inputSource, tc.inputDest, tc.zone) + gotSuccess := moveEndpoint(tc.endpoint, tc.inputSource, tc.inputDest, tc.endpointGroupInfo) if gotSuccess != tc.wantSuccess { t.Errorf("moveEndpoint(%v, ...) = %v, want = %v", tc.endpoint, gotSuccess, tc.wantSuccess) @@ -982,10 +984,10 @@ func TestMoveEndpoint(t *testing.T) { } } -func cloneZoneNetworkEndpointsMap(m map[string]types.NetworkEndpointSet) map[string]types.NetworkEndpointSet { - clone := make(map[string]types.NetworkEndpointSet) - for zone, endpointSet := range m { - clone[zone] = types.NewNetworkEndpointSet(endpointSet.List()...) +func cloneZoneNetworkEndpointsMap(m map[types.EndpointGroupInfo]types.NetworkEndpointSet) map[types.EndpointGroupInfo]types.NetworkEndpointSet { + clone := make(map[types.EndpointGroupInfo]types.NetworkEndpointSet) + for endpointGroupInfo, endpointSet := range m { + clone[endpointGroupInfo] = types.NewNetworkEndpointSet(endpointSet.List()...) } return clone } diff --git a/pkg/neg/syncers/endpoints_calculator.go b/pkg/neg/syncers/endpoints_calculator.go index 6acaf9f476..ae97eb625d 100644 --- a/pkg/neg/syncers/endpoints_calculator.go +++ b/pkg/neg/syncers/endpoints_calculator.go @@ -56,6 +56,7 @@ func NewLocalL4EndpointsCalculator(nodeLister listers.NodeLister, zoneGetter *zo if lbType == negtypes.L4ExternalLB { subsetSize = maxSubsetSizeNetLBLocal } + return &LocalL4EndpointsCalculator{ nodeLister: nodeLister, zoneGetter: zoneGetter, @@ -72,7 +73,7 @@ func (l *LocalL4EndpointsCalculator) Mode() types.EndpointsCalculatorMode { } // CalculateEndpoints determines the endpoints in the NEGs based on the current service endpoints and the current NEGs. -func (l *LocalL4EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsData, currentMap map[string]types.NetworkEndpointSet) (map[string]types.NetworkEndpointSet, types.EndpointPodMap, int, error) { +func (l *LocalL4EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsData, currentMap map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, types.EndpointPodMap, int, error) { // List all nodes where the service endpoints are running. Get a subset of the desired count. zoneNodeMap := make(map[string][]*v1.Node) processedNodes := sets.String{} @@ -106,7 +107,7 @@ func (l *LocalL4EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsDat l.logger.Info("Node not connected to service network", "nodeName", node.Name, "network", l.networkInfo.K8sNetwork) continue } - zone, err := l.zoneGetter.ZoneForNode(node.Name, l.logger) + zone, _, err := l.zoneGetter.ZoneAndSubnetForNode(node.Name, l.logger) if err != nil { l.logger.Error(err, "Unable to find zone for node, skipping", "nodeName", node.Name) metrics.PublishNegControllerErrorCountMetrics(err, true) @@ -121,11 +122,14 @@ func (l *LocalL4EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsDat } // Compute the networkEndpoints, with total endpoints count <= l.subsetSizeLimit l.logger.V(2).Info("Got zoneNodeMap as input for service", "zoneNodeMap", nodeMapToString(zoneNodeMap), "serviceID", l.svcId) + // TODO(sawsa307): Make sure to include logic for subsetting endpoints in non-default subnets. + // Currently we only select endpoints from the default subnet. subsetMap, err := getSubsetPerZone(zoneNodeMap, l.subsetSizeLimit, l.svcId, currentMap, l.logger, l.networkInfo) + return subsetMap, nil, 0, err } -func (l *LocalL4EndpointsCalculator) CalculateEndpointsDegradedMode(eds []types.EndpointsData, currentMap map[string]types.NetworkEndpointSet) (map[string]types.NetworkEndpointSet, types.EndpointPodMap, error) { +func (l *LocalL4EndpointsCalculator) CalculateEndpointsDegradedMode(eds []types.EndpointsData, currentMap map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, types.EndpointPodMap, error) { // this should be the same as CalculateEndpoints for L4 ec subsetMap, podMap, _, err := l.CalculateEndpoints(eds, currentMap) return subsetMap, podMap, err @@ -173,7 +177,7 @@ func (l *ClusterL4EndpointsCalculator) Mode() types.EndpointsCalculatorMode { } // CalculateEndpoints determines the endpoints in the NEGs based on the current service endpoints and the current NEGs. -func (l *ClusterL4EndpointsCalculator) CalculateEndpoints(_ []types.EndpointsData, currentMap map[string]types.NetworkEndpointSet) (map[string]types.NetworkEndpointSet, types.EndpointPodMap, int, error) { +func (l *ClusterL4EndpointsCalculator) CalculateEndpoints(_ []types.EndpointsData, currentMap map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, types.EndpointPodMap, int, error) { // In this mode, any of the cluster nodes can be part of the subset, whether or not a matching pod runs on it. nodes, _ := l.zoneGetter.ListNodes(zonegetter.CandidateAndUnreadyNodesFilter, l.logger) zoneNodeMap := make(map[string][]*v1.Node) @@ -182,7 +186,7 @@ func (l *ClusterL4EndpointsCalculator) CalculateEndpoints(_ []types.EndpointsDat l.logger.Info("Node not connected to service network", "nodeName", node.Name, "network", l.networkInfo.K8sNetwork) continue } - zone, err := l.zoneGetter.ZoneForNode(node.Name, l.logger) + zone, _, err := l.zoneGetter.ZoneAndSubnetForNode(node.Name, l.logger) if err != nil { l.logger.Error(err, "Unable to find zone for node skipping", "nodeName", node.Name) metrics.PublishNegControllerErrorCountMetrics(err, true) @@ -192,11 +196,13 @@ func (l *ClusterL4EndpointsCalculator) CalculateEndpoints(_ []types.EndpointsDat } l.logger.V(2).Info("Got zoneNodeMap as input for service", "zoneNodeMap", nodeMapToString(zoneNodeMap), "serviceID", l.svcId) // Compute the networkEndpoints, with total endpoints <= l.subsetSizeLimit. + // TODO(sawsa307): Make sure to include logic for subsetting endpoints in non-default subnets. + // Currently we only select endpoints from the default subnet. subsetMap, err := getSubsetPerZone(zoneNodeMap, l.subsetSizeLimit, l.svcId, currentMap, l.logger, l.networkInfo) return subsetMap, nil, 0, err } -func (l *ClusterL4EndpointsCalculator) CalculateEndpointsDegradedMode(eps []types.EndpointsData, currentMap map[string]types.NetworkEndpointSet) (map[string]types.NetworkEndpointSet, types.EndpointPodMap, error) { +func (l *ClusterL4EndpointsCalculator) CalculateEndpointsDegradedMode(eps []types.EndpointsData, currentMap map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, types.EndpointPodMap, error) { // this should be the same as CalculateEndpoints for L4 ec subsetMap, podMap, _, err := l.CalculateEndpoints(eps, currentMap) return subsetMap, podMap, err @@ -244,7 +250,7 @@ func (l *L7EndpointsCalculator) Mode() types.EndpointsCalculatorMode { } // CalculateEndpoints determines the endpoints in the NEGs based on the current service endpoints and the current NEGs. -func (l *L7EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsData, _ map[string]types.NetworkEndpointSet) (map[string]types.NetworkEndpointSet, types.EndpointPodMap, int, error) { +func (l *L7EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsData, _ map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, types.EndpointPodMap, int, error) { result, err := toZoneNetworkEndpointMap(eds, l.zoneGetter, l.podLister, l.servicePortName, l.networkEndpointType, l.enableDualStackNEG, l.enableMultiSubnetCluster, l.logger) if err == nil { // If current calculation ends up in error, we trigger and emit metrics in degraded mode. l.syncMetricsCollector.UpdateSyncerEPMetrics(l.syncerKey, result.EPCount, result.EPSCount) @@ -253,7 +259,7 @@ func (l *L7EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsData, _ } // CalculateEndpoints determines the endpoints in the NEGs based on the current service endpoints and the current NEGs. -func (l *L7EndpointsCalculator) CalculateEndpointsDegradedMode(eds []types.EndpointsData, _ map[string]types.NetworkEndpointSet) (map[string]types.NetworkEndpointSet, types.EndpointPodMap, error) { +func (l *L7EndpointsCalculator) CalculateEndpointsDegradedMode(eds []types.EndpointsData, _ map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]types.NetworkEndpointSet, types.EndpointPodMap, error) { result := toZoneNetworkEndpointMapDegradedMode(eds, l.zoneGetter, l.podLister, l.nodeLister, l.serviceLister, l.servicePortName, l.networkEndpointType, l.enableDualStackNEG, l.enableMultiSubnetCluster, l.logger) l.syncMetricsCollector.UpdateSyncerEPMetrics(l.syncerKey, result.EPCount, result.EPSCount) return result.NetworkEndpointSet, result.EndpointPodMap, nil diff --git a/pkg/neg/syncers/endpoints_calculator_test.go b/pkg/neg/syncers/endpoints_calculator_test.go index 8874b55e47..a9a04a971c 100644 --- a/pkg/neg/syncers/endpoints_calculator_test.go +++ b/pkg/neg/syncers/endpoints_calculator_test.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/ingress-gce/pkg/flags" "k8s.io/ingress-gce/pkg/neg/metrics/metricscollector" negtypes "k8s.io/ingress-gce/pkg/neg/types" "k8s.io/ingress-gce/pkg/network" @@ -40,16 +41,18 @@ import ( // TestLocalGetEndpointSet verifies the GetEndpointSet method implemented by the LocalL4EndpointsCalculator. // The L7 implementation is tested in TestToZoneNetworkEndpointMapUtil. func TestLocalGetEndpointSet(t *testing.T) { - t.Parallel() nodeInformer := zonegetter.FakeNodeInformer() zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) zonegetter.PopulateFakeNodeInformer(nodeInformer, false) - defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default"} + defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default", SubnetworkURL: defaultTestSubnetURL} + prevFlag := flags.F.EnableMultiSubnetCluster + defer func() { flags.F.EnableMultiSubnetCluster = prevFlag }() + flags.F.EnableMultiSubnetCluster = false testCases := []struct { desc string endpointsData []negtypes.EndpointsData - wantEndpointSets map[string]negtypes.NetworkEndpointSet + wantEndpointSets map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet networkEndpointType negtypes.NetworkEndpointType nodeLabelsMap map[string]map[string]string nodeAnnotationsMap map[string]map[string]string @@ -61,9 +64,9 @@ func TestLocalGetEndpointSet(t *testing.T) { desc: "default endpoints", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // only 4 out of 6 nodes are picked since there are > 4 endpoints, but they are found only on 4 nodes. - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -73,9 +76,9 @@ func TestLocalGetEndpointSet(t *testing.T) { desc: "default endpoints, all nodes unready", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // only 4 out of 6 nodes are picked since there are > 4 endpoints, but they are found only on 4 nodes. - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -95,9 +98,9 @@ func TestLocalGetEndpointSet(t *testing.T) { }, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, // only 2 out of 6 nodes are picked since there are > 4 endpoints, but they are found only on 4 nodes. 2 out of those 4 are non-candidates. - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), }, networkEndpointType: negtypes.VmIpEndpointType, network: defaultNetwork, @@ -114,7 +117,7 @@ func TestLocalGetEndpointSet(t *testing.T) { { desc: "multinetwork, endpoints only for nodes connected to a matching non-default network", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), - network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other"}, + network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other", SubnetworkURL: defaultTestSubnetURL}, nodeAnnotationsMap: map[string]map[string]string{ testInstance1: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.1")}, testInstance2: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.2")}, @@ -123,11 +126,11 @@ func TestLocalGetEndpointSet(t *testing.T) { //networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, // only 3 out of 6 nodes are picked because only 3 have multi-nic annotation with a matching network name - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "20.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "20.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "20.2.3.3", Node: testInstance3}), }, }, @@ -170,15 +173,18 @@ func nodeInterfacesAnnotation(t *testing.T, network, ip string) string { // TestClusterGetEndpointSet verifies the GetEndpointSet method implemented by the ClusterL4EndpointsCalculator. func TestClusterGetEndpointSet(t *testing.T) { - t.Parallel() nodeInformer := zonegetter.FakeNodeInformer() zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) zonegetter.PopulateFakeNodeInformer(nodeInformer, false) - defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default"} + defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default", SubnetworkURL: defaultTestSubnetURL} + prevFlag := flags.F.EnableMultiSubnetCluster + defer func() { flags.F.EnableMultiSubnetCluster = prevFlag }() + flags.F.EnableMultiSubnetCluster = false + testCases := []struct { desc string endpointsData []negtypes.EndpointsData - wantEndpointSets map[string]negtypes.NetworkEndpointSet + wantEndpointSets map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet networkEndpointType negtypes.NetworkEndpointType nodeLabelsMap map[string]map[string]string nodeAnnotationsMap map[string]map[string]string @@ -190,11 +196,11 @@ func TestClusterGetEndpointSet(t *testing.T) { desc: "default endpoints", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // all nodes are picked since, in this mode, endpoints running do not need to run on the selected node. - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - negtypes.TestZone3: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -204,11 +210,11 @@ func TestClusterGetEndpointSet(t *testing.T) { desc: "default endpoints, all nodes unready", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // all nodes are picked since, in this mode, endpoints running do not need to run on the selected node. - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - negtypes.TestZone3: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -221,11 +227,11 @@ func TestClusterGetEndpointSet(t *testing.T) { desc: "default endpoints, some nodes marked as non-candidates", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // all valid candidate nodes are picked since, in this mode, endpoints running do not need to run on the selected node. - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - negtypes.TestZone3: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -242,11 +248,11 @@ func TestClusterGetEndpointSet(t *testing.T) { // all nodes are picked since, in this mode, endpoints running do not need to run on the selected node. // Even when there are no service endpoints, nodes are selected at random. endpointsData: []negtypes.EndpointsData{}, - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - negtypes.TestZone3: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -255,11 +261,11 @@ func TestClusterGetEndpointSet(t *testing.T) { { desc: "multinetwork endpoints, only for nodes connected to the specified network", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), - wantEndpointSets: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "20.2.3.2", Node: testInstance2}), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "20.2.3.6", Node: testInstance6}), + wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "20.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "20.2.3.6", Node: testInstance6}), }, - network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other"}, + network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other", SubnetworkURL: defaultTestSubnetURL}, nodeAnnotationsMap: map[string]map[string]string{ testInstance1: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.1")}, testInstance2: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.2")}, @@ -292,7 +298,6 @@ func TestClusterGetEndpointSet(t *testing.T) { } func TestValidateEndpoints(t *testing.T) { - t.Parallel() testPortName := "" emptyNamedPort := "" protocolTCP := v1.ProtocolTCP @@ -352,8 +357,8 @@ func TestValidateEndpoints(t *testing.T) { zoneGetterMSC := zonegetter.NewFakeZoneGetter(testContext.NodeInformer, defaultTestSubnetURL, true) L7EndpointsCalculatorMSC := NewL7EndpointsCalculator(zoneGetterMSC, podLister, nodeLister, serviceLister, svcPort, klog.TODO(), testContext.EnableDualStackNEG, metricscollector.FakeSyncerMetrics()) L7EndpointsCalculatorMSC.enableMultiSubnetCluster = true - L4LocalEndpointCalculator := NewLocalL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{}, negtypes.L4InternalLB) - L4ClusterEndpointCalculator := NewClusterL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{}, negtypes.L4InternalLB) + L4LocalEndpointCalculator := NewLocalL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{SubnetworkURL: defaultTestSubnetURL}, negtypes.L4InternalLB) + L4ClusterEndpointCalculator := NewClusterL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{SubnetworkURL: defaultTestSubnetURL}, negtypes.L4InternalLB) l7TestEPS := []*discovery.EndpointSlice{ { @@ -396,12 +401,16 @@ func TestValidateEndpoints(t *testing.T) { return ed, podMap } + prevFlag := flags.F.EnableMultiSubnetCluster + defer func() { flags.F.EnableMultiSubnetCluster = prevFlag }() + flags.F.EnableMultiSubnetCluster = true + testCases := []struct { desc string ec negtypes.NetworkEndpointsCalculator ecMSC negtypes.NetworkEndpointsCalculator testEndpointSlices []*discovery.EndpointSlice - currentMap map[string]negtypes.NetworkEndpointSet + currentMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet // Use mutation to inject error into that we cannot trigger currently. mutation func([]negtypes.EndpointsData, negtypes.EndpointPodMap) ([]negtypes.EndpointsData, negtypes.EndpointPodMap) expect error @@ -634,14 +643,15 @@ func TestValidateEndpoints(t *testing.T) { desc string ecMSC negtypes.NetworkEndpointsCalculator testEndpointSlices []*discovery.EndpointSlice - currentMap map[string]negtypes.NetworkEndpointSet + currentMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet // Use mutation to inject error into that we cannot trigger currently. - mutation func([]negtypes.EndpointsData, negtypes.EndpointPodMap) ([]negtypes.EndpointsData, negtypes.EndpointPodMap) - expect error + mutation func([]negtypes.EndpointsData, negtypes.EndpointPodMap) ([]negtypes.EndpointsData, negtypes.EndpointPodMap) + expectCalculationErr error + expectValidationErr error }{ { desc: "ValidateEndpoints for L7 Endpoint Calculator. Endpoint counts equal, endpointData has an endpoint corresponds to node without PodCIDR", - ecMSC: L7EndpointsCalculator, + ecMSC: L7EndpointsCalculatorMSC, testEndpointSlices: []*discovery.EndpointSlice{ { ObjectMeta: metav1.ObjectMeta{ @@ -687,9 +697,10 @@ func TestValidateEndpoints(t *testing.T) { }, }, }, - mutation: noopMutation, - currentMap: nil, - expect: nil, + mutation: noopMutation, + currentMap: nil, + expectCalculationErr: zonegetter.ErrNodePodCIDRNotSet, + expectValidationErr: negtypes.ErrEPCalculationCountZero, }, { desc: "ValidateEndpoints for L7 Endpoint Calculator. Endpoint counts equal, endpointData has non-default subnet endpoint", @@ -739,9 +750,10 @@ func TestValidateEndpoints(t *testing.T) { }, }, }, - mutation: noopMutation, - currentMap: nil, - expect: nil, + mutation: noopMutation, + currentMap: nil, + expectCalculationErr: nil, + expectValidationErr: nil, }, } @@ -749,12 +761,12 @@ func TestValidateEndpoints(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { endpointData := negtypes.EndpointsDataFromEndpointSlices(tc.testEndpointSlices) _, endpointPodMap, endpointsExcludedInCalculation, err := tc.ecMSC.CalculateEndpoints(endpointData, tc.currentMap) - if err != nil { + if !errors.Is(err, tc.expectCalculationErr) { t.Errorf("With multi-subnet cluster enabled, received error when calculating endpoint: %v", err) } endpointData, endpointPodMap = tc.mutation(endpointData, endpointPodMap) - if got := tc.ecMSC.ValidateEndpoints(endpointData, endpointPodMap, endpointsExcludedInCalculation); !errors.Is(got, tc.expect) { - t.Errorf("With multi-subnet cluster enabled, ValidateEndpoints() = %v, expected %v", got, tc.expect) + if got := tc.ecMSC.ValidateEndpoints(endpointData, endpointPodMap, endpointsExcludedInCalculation); !errors.Is(got, tc.expectValidationErr) { + t.Errorf("With multi-subnet cluster enabled, ValidateEndpoints() = %v, expected %v", got, tc.expectValidationErr) } }) } diff --git a/pkg/neg/syncers/subsets.go b/pkg/neg/syncers/subsets.go index 2fa1baf762..117b560917 100644 --- a/pkg/neg/syncers/subsets.go +++ b/pkg/neg/syncers/subsets.go @@ -163,8 +163,8 @@ func sortZones(nodesPerZone map[string][]*v1.Node) []ZoneInfo { // Since the number of nodes will keep increasing in successive zones due to the sorting, even if fewer nodes were // present in some zones, more nodes will be picked from other nodes, taking the total subset size to the given limit // whenever possible. -func getSubsetPerZone(nodesPerZone map[string][]*v1.Node, totalLimit int, svcID string, currentMap map[string]negtypes.NetworkEndpointSet, logger klog.Logger, networkInfo *network.NetworkInfo) (map[string]negtypes.NetworkEndpointSet, error) { - result := make(map[string]negtypes.NetworkEndpointSet) +func getSubsetPerZone(nodesPerZone map[string][]*v1.Node, totalLimit int, svcID string, currentMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, logger klog.Logger, networkInfo *network.NetworkInfo) (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, error) { + result := make(map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet) var currentList []negtypes.NetworkEndpoint subsetSize := 0 @@ -173,13 +173,21 @@ func getSubsetPerZone(nodesPerZone map[string][]*v1.Node, totalLimit int, svcID // Sort zones in increasing order of node count. zoneList := sortZones(nodesPerZone) + defaultSubnet, err := utils.KeyName(networkInfo.SubnetworkURL) + if err != nil { + logger.Error(err, "Errored getting default subnet from NetworkInfo") + return nil, err + } + for _, zone := range zoneList { // split the limit across the leftover zones. subsetSize = totalLimit / zonesRemaining logger.Info("Picking subset for a zone", "subsetSize", subsetSize, "zone", zone, "svcID", svcID) - result[zone.Name] = negtypes.NewNetworkEndpointSet() + // TODO(sawsa307): Make sure to include logic for subsetting endpoints in non-default subnets. + // Currently we only select endpoints from the default subnet. + result[negtypes.EndpointGroupInfo{Zone: zone.Name, Subnet: defaultSubnet}] = negtypes.NewNetworkEndpointSet() if currentMap != nil { - if zset, ok := currentMap[zone.Name]; ok && zset != nil { + if zset, ok := currentMap[negtypes.EndpointGroupInfo{Zone: zone.Name, Subnet: defaultSubnet}]; ok && zset != nil { currentList = zset.List() } else { currentList = nil @@ -193,7 +201,7 @@ func getSubsetPerZone(nodesPerZone map[string][]*v1.Node, totalLimit int, svcID } else { ip = utils.GetNodePrimaryIP(node, logger) } - result[zone.Name].Insert(negtypes.NetworkEndpoint{Node: node.Name, IP: ip}) + result[negtypes.EndpointGroupInfo{Zone: zone.Name, Subnet: defaultSubnet}].Insert(negtypes.NetworkEndpoint{Node: node.Name, IP: ip}) } totalLimit -= len(subset) zonesRemaining-- diff --git a/pkg/neg/syncers/subsets_test.go b/pkg/neg/syncers/subsets_test.go index 7589481ab3..8af833f1c2 100644 --- a/pkg/neg/syncers/subsets_test.go +++ b/pkg/neg/syncers/subsets_test.go @@ -23,6 +23,7 @@ import ( networkv1 "github.com/GoogleCloudPlatform/gke-networking-api/apis/network/v1" "k8s.io/ingress-gce/pkg/neg/types" + negtypes "k8s.io/ingress-gce/pkg/neg/types" "k8s.io/ingress-gce/pkg/network" "k8s.io/klog/v2" @@ -191,7 +192,7 @@ func TestUnevenNodesInZones(t *testing.T) { }, } for _, tc := range testCases { - subsetMap, err := getSubsetPerZone(tc.nodesMap, tc.subsetLimit, tc.svcKey, nil, klog.TODO(), &network.NetworkInfo{}) + subsetMap, err := getSubsetPerZone(tc.nodesMap, tc.subsetLimit, tc.svcKey, nil, klog.TODO(), &network.NetworkInfo{SubnetworkURL: defaultTestSubnetURL}) if err != nil { t.Errorf("Failed to get subset for test '%s', err %v", tc.description, err) } @@ -223,7 +224,7 @@ func TestGetSubsetPerZoneMultinetwork(t *testing.T) { // expectEmpty indicates that some zones can have empty subsets expectEmpty bool networkInfo network.NetworkInfo - expectedNodesMap map[string]map[string]string + expectedNodesMap map[negtypes.EndpointGroupInfo]map[string]string }{ { description: "Default network, gets primary interface", @@ -233,11 +234,14 @@ func TestGetSubsetPerZoneMultinetwork(t *testing.T) { "zone3": {makeNodeWithNetwork(t, "n3_1", map[string]string{"net1": "172.168.3.1", "net2": "192.168.3.1"})}, }, svcKey: "svc123", + networkInfo: network.NetworkInfo{ + SubnetworkURL: defaultTestSubnetURL, + }, // empty IPs since test can't get the primary IP - expectedNodesMap: map[string]map[string]string{ - "zone1": {"n1_1": "", "n1_2": ""}, - "zone2": {"n2_1": "", "n2_2": ""}, - "zone3": {"n3_1": ""}, + expectedNodesMap: map[negtypes.EndpointGroupInfo]map[string]string{ + {Zone: "zone1", Subnet: defaultTestSubnet}: {"n1_1": "", "n1_2": ""}, + {Zone: "zone2", Subnet: defaultTestSubnet}: {"n2_1": "", "n2_2": ""}, + {Zone: "zone3", Subnet: defaultTestSubnet}: {"n3_1": ""}, }, }, { @@ -249,13 +253,14 @@ func TestGetSubsetPerZoneMultinetwork(t *testing.T) { }, svcKey: "svc123", networkInfo: network.NetworkInfo{ - IsDefault: false, - K8sNetwork: "net1", + IsDefault: false, + K8sNetwork: "net1", + SubnetworkURL: defaultTestSubnetURL, }, - expectedNodesMap: map[string]map[string]string{ - "zone1": {"n1_1": "172.168.1.1", "n1_2": "172.168.1.2"}, - "zone2": {"n2_1": "172.168.2.1", "n2_2": "172.168.2.2"}, - "zone3": {"n3_1": "172.168.3.1"}, + expectedNodesMap: map[negtypes.EndpointGroupInfo]map[string]string{ + {Zone: "zone1", Subnet: defaultTestSubnet}: {"n1_1": "172.168.1.1", "n1_2": "172.168.1.2"}, + {Zone: "zone2", Subnet: defaultTestSubnet}: {"n2_1": "172.168.2.1", "n2_2": "172.168.2.2"}, + {Zone: "zone3", Subnet: defaultTestSubnet}: {"n3_1": "172.168.3.1"}, }, }, } diff --git a/pkg/neg/syncers/transaction.go b/pkg/neg/syncers/transaction.go index 1e109d2251..4212404b29 100644 --- a/pkg/neg/syncers/transaction.go +++ b/pkg/neg/syncers/transaction.go @@ -259,7 +259,13 @@ func (s *transactionSyncer) syncInternalImpl() error { } s.logger.V(2).Info("Sync NEG", "negSyncerKey", s.NegSyncerKey.String(), "endpointsCalculatorMode", s.endpointsCalculator.Mode()) - currentMap, currentPodLabelMap, err := retrieveExistingZoneNetworkEndpointMap(s.NegSyncerKey.NegName, s.zoneGetter, s.cloud, s.NegSyncerKey.GetAPIVersion(), s.endpointsCalculator.Mode(), s.enableDualStackNEG, s.logger) + defaultSubnet, err := utils.KeyName(s.networkInfo.SubnetworkURL) + if err != nil { + s.logger.Error(err, "Errored getting default subnet from NetworkInfo") + return err + } + + currentMap, currentPodLabelMap, err := retrieveExistingZoneNetworkEndpointMap(s.NegSyncerKey.NegName, s.zoneGetter, s.cloud, s.NegSyncerKey.GetAPIVersion(), s.endpointsCalculator.Mode(), s.enableDualStackNEG, defaultSubnet, s.logger) if err != nil { return fmt.Errorf("%w: %w", negtypes.ErrCurrentNegEPNotFound, err) } @@ -270,7 +276,7 @@ func (s *transactionSyncer) syncInternalImpl() error { mergeTransactionIntoZoneEndpointMap(currentMap, s.transactions, s.logger) s.logStats(currentMap, "after in-progress operations have completed, NEG endpoints") - var targetMap map[string]negtypes.NetworkEndpointSet + var targetMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet var endpointPodMap negtypes.EndpointPodMap slices, err := s.endpointSliceLister.ByIndex(endpointslices.EndpointSlicesByServiceIndex, endpointslices.FormatEndpointSlicesServiceKey(s.Namespace, s.Name)) if err != nil { @@ -286,7 +292,7 @@ func (s *transactionSyncer) syncInternalImpl() error { endpointsData := negtypes.EndpointsDataFromEndpointSlices(endpointSlices) targetMap, endpointPodMap, err = s.getEndpointsCalculation(endpointsData, currentMap) - var degradedTargetMap, notInDegraded, onlyInDegraded map[string]negtypes.NetworkEndpointSet + var degradedTargetMap, notInDegraded, onlyInDegraded map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet var degradedPodMap negtypes.EndpointPodMap var degradedModeErr error if s.enableDegradedModeMetrics || s.enableDegradedMode { @@ -378,8 +384,8 @@ func (s *transactionSyncer) syncInternalImpl() error { func (s *transactionSyncer) getEndpointsCalculation( endpointsData []negtypes.EndpointsData, - currentMap map[string]negtypes.NetworkEndpointSet, -) (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap, error) { + currentMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, +) (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap, error) { targetMap, endpointPodMap, endpointsExcludedInCalculation, err := s.endpointsCalculator.CalculateEndpoints(endpointsData, currentMap) if err != nil { return nil, nil, err @@ -470,12 +476,13 @@ func (s *transactionSyncer) ensureNetworkEndpointGroups() error { } // syncNetworkEndpoints spins off go routines to execute NEG operations -func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints map[string]negtypes.NetworkEndpointSet, endpointPodLabelMap labels.EndpointPodLabelMap, migrationZone string) error { - syncFunc := func(endpointMap map[string]negtypes.NetworkEndpointSet, operation transactionOp) error { - for zone, endpointSet := range endpointMap { - zone := zone +func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, endpointPodLabelMap labels.EndpointPodLabelMap, migrationZone negtypes.EndpointGroupInfo) error { + syncFunc := func(endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, operation transactionOp) error { + for endpointGroupInfo, endpointSet := range endpointMap { + zone := endpointGroupInfo.Zone + subnet := endpointGroupInfo.Subnet if endpointSet.Len() == 0 { - s.logger.V(2).Info("0 endpoints in the endpoint list. Skipping operation", "operation", attachOp, "negSyncerKey", s.NegSyncerKey.String(), "zone", zone) + s.logger.V(2).Info("0 endpoints in the endpoint list. Skipping operation", "operation", attachOp, "negSyncerKey", s.NegSyncerKey.String(), "zone", zone, "subnet", subnet) continue } @@ -487,6 +494,7 @@ func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints m transEntry := transactionEntry{ Operation: operation, Zone: zone, + Subnet: subnet, } // Insert networkEndpoint into transaction table @@ -495,15 +503,19 @@ func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints m } if operation == attachOp { + // TODO(sawsa307): Pass in subnet to help distinguish which NEGs needs + // update(in default/non-default subnets). go s.attachNetworkEndpoints(zone, batch) } if operation == detachOp { - if zone == migrationZone { + if zone == migrationZone.Zone && subnet == migrationZone.Subnet { // Prevent any further migration-detachments from starting while one // is already in progress. s.dsMigrator.Pause() } - go s.detachNetworkEndpoints(zone, batch, zone == migrationZone) + // TODO(sawsa307): Pass in subnet to help distinguish which NEGs needs + // update(in default/non-default subnets). + go s.detachNetworkEndpoints(zone, batch, zone == migrationZone.Zone && subnet == migrationZone.Subnet) } } return nil @@ -667,8 +679,8 @@ func (s *transactionSyncer) needCommit() bool { } // commitPods groups the endpoints by zone and signals the readiness reflector to poll pods of the NEG -func (s *transactionSyncer) commitPods(endpointMap map[string]negtypes.NetworkEndpointSet, endpointPodMap negtypes.EndpointPodMap) { - for zone, endpointSet := range endpointMap { +func (s *transactionSyncer) commitPods(endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, endpointPodMap negtypes.EndpointPodMap) { + for endpointGroupInfo, endpointSet := range endpointMap { zoneEndpointMap := negtypes.EndpointPodMap{} for _, endpoint := range endpointSet.List() { podName, ok := endpointPodMap[endpoint] @@ -678,7 +690,9 @@ func (s *transactionSyncer) commitPods(endpointMap map[string]negtypes.NetworkEn } zoneEndpointMap[endpoint] = podName } - s.reflector.CommitPods(s.NegSyncerKey, s.NegSyncerKey.NegName, zone, zoneEndpointMap) + // TODO(sawsa307): Make sure commitPods is called for non-default subnet NEGs. + // Only zone is needed because NEGs from non-default subnet have different names. + s.reflector.CommitPods(s.NegSyncerKey, s.NegSyncerKey.NegName, endpointGroupInfo.Zone, zoneEndpointMap) } } @@ -715,7 +729,7 @@ func (s *transactionSyncer) isZoneChange() bool { } // filterEndpointByTransaction removes the all endpoints from endpoint map if they exists in the transaction table -func filterEndpointByTransaction(endpointMap map[string]negtypes.NetworkEndpointSet, table networkEndpointTransactionTable, logger klog.Logger) { +func filterEndpointByTransaction(endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, table networkEndpointTransactionTable, logger klog.Logger) { for _, endpointSet := range endpointMap { for _, endpoint := range endpointSet.List() { if entry, ok := table.Get(endpoint); ok { @@ -728,7 +742,7 @@ func filterEndpointByTransaction(endpointMap map[string]negtypes.NetworkEndpoint // mergeTransactionIntoZoneEndpointMap merges the ongoing transaction into the endpointMap. // This converts the existing endpointMap to the state when all transactions completed -func mergeTransactionIntoZoneEndpointMap(endpointMap map[string]negtypes.NetworkEndpointSet, transactions networkEndpointTransactionTable, logger klog.Logger) { +func mergeTransactionIntoZoneEndpointMap(endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, transactions networkEndpointTransactionTable, logger klog.Logger) { for _, endpointKey := range transactions.Keys() { entry, ok := transactions.Get(endpointKey) // If called in syncInternal, as the transaction table @@ -736,36 +750,37 @@ func mergeTransactionIntoZoneEndpointMap(endpointMap map[string]negtypes.Network logger.V(2).Info("Transaction entry of key was not found.", "endpointKey", endpointKey) continue } + key := negtypes.EndpointGroupInfo{Zone: entry.Zone, Subnet: entry.Subnet} // Add endpoints in attach transaction if entry.Operation == attachOp { - if _, ok := endpointMap[entry.Zone]; !ok { - endpointMap[entry.Zone] = negtypes.NewNetworkEndpointSet() + if _, ok := endpointMap[key]; !ok { + endpointMap[key] = negtypes.NewNetworkEndpointSet() } - endpointMap[entry.Zone].Insert(endpointKey) + endpointMap[key].Insert(endpointKey) } // Remove endpoints in detach transaction if entry.Operation == detachOp { - if _, ok := endpointMap[entry.Zone]; !ok { + if _, ok := endpointMap[key]; !ok { continue } - endpointMap[entry.Zone].Delete(endpointKey) + endpointMap[key].Delete(endpointKey) } } return } // logStats logs aggregated stats of the input endpointMap -func (s *transactionSyncer) logStats(endpointMap map[string]negtypes.NetworkEndpointSet, desc string) { +func (s *transactionSyncer) logStats(endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, desc string) { var stats []interface{} stats = append(stats, "description", desc) - for zone, endpointSet := range endpointMap { - stats = append(stats, zone, fmt.Sprintf("%d endpoints", endpointSet.Len())) + for endpointGroupInfo, endpointSet := range endpointMap { + stats = append(stats, endpointGroupInfo.Zone, endpointGroupInfo.Subnet, fmt.Sprintf("%d endpoints", endpointSet.Len())) } s.logger.V(3).Info("Stats for NEG", stats...) } // logEndpoints logs individual endpoint in the input endpointMap -func (s *transactionSyncer) logEndpoints(endpointMap map[string]negtypes.NetworkEndpointSet, desc string) { +func (s *transactionSyncer) logEndpoints(endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, desc string) { s.logger.V(3).Info("Endpoints for NEG", "description", desc, "endpointMap", endpointMap) } @@ -869,7 +884,7 @@ func (s *transactionSyncer) computeEPSStaleness(endpointSlices []*discovery.Endp } // computeDegradedModeCorrectness computes degraded mode correctness metrics based on the difference between degraded mode and normal calculation -func computeDegradedModeCorrectness(notInDegraded, onlyInDegraded map[string]negtypes.NetworkEndpointSet, negType string, logger klog.Logger) { +func computeDegradedModeCorrectness(notInDegraded, onlyInDegraded map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negType string, logger klog.Logger) { logger.Info("Exporting degraded mode correctness metrics", "notInDegraded", notInDegraded, "onlyInDegraded", onlyInDegraded) notInDegradedEndpoints := 0 for _, val := range notInDegraded { @@ -1013,7 +1028,7 @@ func findCondition(conditions []negv1beta1.Condition, conditionType string) (neg } // getEndpointPodLabelMap goes through all the endpoints to be attached and fetches the labels from the endpoint pods. -func getEndpointPodLabelMap(endpoints map[string]negtypes.NetworkEndpointSet, endpointPodMap negtypes.EndpointPodMap, podLister cache.Store, lpConfig labels.PodLabelPropagationConfig, recorder record.EventRecorder, logger klog.Logger) labels.EndpointPodLabelMap { +func getEndpointPodLabelMap(endpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, endpointPodMap negtypes.EndpointPodMap, podLister cache.Store, lpConfig labels.PodLabelPropagationConfig, recorder record.EventRecorder, logger klog.Logger) labels.EndpointPodLabelMap { endpointPodLabelMap := labels.EndpointPodLabelMap{} for _, endpointSet := range endpoints { for endpoint := range endpointSet { @@ -1044,7 +1059,7 @@ func getEndpointPodLabelMap(endpoints map[string]negtypes.NetworkEndpointSet, en // publishAnnotationSizeMetrics goes through all the endpoints to be attached // and publish annotation size metrics. -func publishAnnotationSizeMetrics(endpoints map[string]negtypes.NetworkEndpointSet, endpointPodLabelMap labels.EndpointPodLabelMap) { +func publishAnnotationSizeMetrics(endpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, endpointPodLabelMap labels.EndpointPodLabelMap) { for _, endpointSet := range endpoints { for endpoint := range endpointSet { labelMap := endpointPodLabelMap[endpoint] @@ -1054,7 +1069,7 @@ func publishAnnotationSizeMetrics(endpoints map[string]negtypes.NetworkEndpointS } // collectLabelStats calculate the number of endpoints and the number of endpoints with annotations. -func collectLabelStats(currentPodLabelMap, addPodLabelMap labels.EndpointPodLabelMap, targetEndpointMap map[string]negtypes.NetworkEndpointSet) metricscollector.LabelPropagationStats { +func collectLabelStats(currentPodLabelMap, addPodLabelMap labels.EndpointPodLabelMap, targetEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet) metricscollector.LabelPropagationStats { labelPropagationStats := metricscollector.LabelPropagationStats{} for _, endpointSet := range targetEndpointMap { for endpoint := range endpointSet { diff --git a/pkg/neg/syncers/transaction_table.go b/pkg/neg/syncers/transaction_table.go index 2470c9612d..3e6be5474a 100644 --- a/pkg/neg/syncers/transaction_table.go +++ b/pkg/neg/syncers/transaction_table.go @@ -17,8 +17,9 @@ limitations under the License. package syncers import ( - negtypes "k8s.io/ingress-gce/pkg/neg/types" "sync" + + negtypes "k8s.io/ingress-gce/pkg/neg/types" ) type networkEndpointTransactionTable interface { @@ -51,6 +52,8 @@ type transactionEntry struct { Operation transactionOp // Zone represents the zone of the transaction Zone string + // Subnet represents the subnet of the NEG corresponds to this transaction + Subnet string } // transactionTable records ongoing NEG API operation per endpoint diff --git a/pkg/neg/syncers/transaction_table_test.go b/pkg/neg/syncers/transaction_table_test.go index 1f4fbf754f..81c9943c55 100644 --- a/pkg/neg/syncers/transaction_table_test.go +++ b/pkg/neg/syncers/transaction_table_test.go @@ -50,6 +50,7 @@ func TestTransactionTable(t *testing.T) { entry := transactionEntry{ attachOp, fmt.Sprintf("%s%d", zonePrefix, i), + defaultTestSubnet, } table.Put(key, entry) testKeyMap[key] = entry @@ -63,6 +64,7 @@ func TestTransactionTable(t *testing.T) { newEntry := transactionEntry{ detachOp, fmt.Sprintf("%s%d", zonePrefix, i), + defaultTestSubnet, } table.Put(key, newEntry) testKeyMap[key] = newEntry diff --git a/pkg/neg/syncers/transaction_test.go b/pkg/neg/syncers/transaction_test.go index 6456dc6325..0fe2fca5f3 100644 --- a/pkg/neg/syncers/transaction_test.go +++ b/pkg/neg/syncers/transaction_test.go @@ -70,12 +70,16 @@ const ( testInstance6 = "instance6" testUnreadyInstance1 = "unready-instance1" testUnreadyInstance2 = "unready-instance2" + + defaultTestSubnet = "default" ) func TestTransactionSyncNetworkEndpoints(t *testing.T) { t.Parallel() - fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + fakeGCE := gce.NewFakeGCECloud(vals) negtypes.MockNetworkEndpointAPIs(fakeGCE) fakeCloud := negtypes.NewAdapter(fakeGCE) testNegTypes := []negtypes.NetworkEndpointType{ @@ -128,84 +132,84 @@ func TestTransactionSyncNetworkEndpoints(t *testing.T) { testCases := []struct { desc string - addEndpoints map[string]negtypes.NetworkEndpointSet - removeEndpoints map[string]negtypes.NetworkEndpointSet - expectEndpoints map[string]negtypes.NetworkEndpointSet + addEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + removeEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + expectEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet }{ { "empty input", - map[string]negtypes.NetworkEndpointSet{}, - map[string]negtypes.NetworkEndpointSet{}, - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, { "add some endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{}, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, }, { "remove some endpoints", - map[string]negtypes.NetworkEndpointSet{}, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{ - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, }, { "add duplicate endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{}, - map[string]negtypes.NetworkEndpointSet{ - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, }, { "add and remove endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{ - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)).Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), }, }, { "add more endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{}, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), }, }, { "add and remove endpoints in both zones", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, targetPort)), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, targetPort)), }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, targetPort)), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.4.1"), 10, testInstance4, targetPort)), }, }, } @@ -214,7 +218,7 @@ func TestTransactionSyncNetworkEndpoints(t *testing.T) { // TODO(gauravkghildiyal): When the DualStack Migrator is fully // implemented, check if we need to cover scenarios where `migrationZone` // is not empty. - err := transactionSyncer.syncNetworkEndpoints(tc.addEndpoints, tc.removeEndpoints, labels.EndpointPodLabelMap{}, "") + err := transactionSyncer.syncNetworkEndpoints(tc.addEndpoints, tc.removeEndpoints, labels.EndpointPodLabelMap{}, negtypes.EndpointGroupInfo{}) if err != nil { t.Errorf("For case %q, syncNetworkEndpoints() got %v, want nil", tc.desc, err) } @@ -223,8 +227,8 @@ func TestTransactionSyncNetworkEndpoints(t *testing.T) { t.Errorf("For case %q, waitForTransactions() got %v, want nil", tc.desc, err) } - for zone, endpoints := range tc.expectEndpoints { - list, err := fakeCloud.ListNetworkEndpoints(transactionSyncer.NegSyncerKey.NegName, zone, false, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + for endpointGroupInfo, endpoints := range tc.expectEndpoints { + list, err := fakeCloud.ListNetworkEndpoints(transactionSyncer.NegSyncerKey.NegName, endpointGroupInfo.Zone, false, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) if err != nil { t.Errorf("For case %q, ListNetworkEndpoints() got %v, want nil", tc.desc, err) } @@ -239,7 +243,7 @@ func TestTransactionSyncNetworkEndpoints(t *testing.T) { } if !endpoints.Equal(endpointSet) { - t.Errorf("For case %q, in zone %q, negType %q, endpointSets endpoints == %v, but got %v, difference: \n(want - got) = %v\n(got - want) = %v", tc.desc, zone, testNegType, endpoints, endpointSet, endpoints.Difference(endpointSet), endpointSet.Difference(endpoints)) + t.Errorf("For case %q, in zone %q, negType %q, endpointSets endpoints == %v, but got %v, difference: \n(want - got) = %v\n(got - want) = %v", tc.desc, endpointGroupInfo.Zone, testNegType, endpoints, endpointSet, endpoints.Difference(endpointSet), endpointSet.Difference(endpoints)) } } } @@ -270,7 +274,7 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { desc string labelPropagationEnabled bool negType negtypes.NetworkEndpointType - addEndpoints map[string]negtypes.NetworkEndpointSet + addEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet endpointPodLabelMap labels.EndpointPodLabelMap expectedNEAnnotation labels.PodLabelMap }{ @@ -278,7 +282,7 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { "empty input", true, negtypes.VmIpPortEndpointType, - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, labels.EndpointPodLabelMap{}, nil, }, @@ -286,9 +290,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { "add L4 endpoints with label map populated", true, negtypes.VmIpEndpointType, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet1).Union(l4EndpointSet2), - testZone2: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet3).Union(l4EndpointSet4), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet1).Union(l4EndpointSet2), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet3).Union(l4EndpointSet4), }, generateEndpointPodLabelMap( map[string]negtypes.NetworkEndpointSet{ @@ -306,9 +310,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { "add L4 endpoints with empty label map", true, negtypes.VmIpEndpointType, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet1).Union(l4EndpointSet2), - testZone2: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet3).Union(l4EndpointSet4), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet1).Union(l4EndpointSet2), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(l4EndpointSet3).Union(l4EndpointSet4), }, generateEndpointPodLabelMap( map[string]negtypes.NetworkEndpointSet{ @@ -323,9 +327,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { "add L7 endpoints label map populated", true, negtypes.VmIpPortEndpointType, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet1).Union(l7EndpointSet2), - testZone2: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet3).Union(l7EndpointSet4), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet1).Union(l7EndpointSet2), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet3).Union(l7EndpointSet4), }, generateEndpointPodLabelMap( map[string]negtypes.NetworkEndpointSet{ @@ -346,9 +350,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { "add L7 endpoints with empty label map", true, negtypes.VmIpPortEndpointType, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet1).Union(l7EndpointSet2), - testZone2: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet3).Union(l7EndpointSet4), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet1).Union(l7EndpointSet2), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet3).Union(l7EndpointSet4), }, generateEndpointPodLabelMap( map[string]negtypes.NetworkEndpointSet{ @@ -363,9 +367,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { "add L7 endpoints label map populated, but EnableNEGLabelPropagation flag disabled", false, negtypes.VmIpPortEndpointType, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet1).Union(l7EndpointSet2), - testZone2: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet3).Union(l7EndpointSet4), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet1).Union(l7EndpointSet2), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(l7EndpointSet3).Union(l7EndpointSet4), }, generateEndpointPodLabelMap( map[string]negtypes.NetworkEndpointSet{ @@ -383,14 +387,16 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { for _, tc := range testCases { flags.F.EnableNEGLabelPropagation = tc.labelPropagationEnabled - fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + fakeGCE := gce.NewFakeGCECloud(vals) negtypes.MockNetworkEndpointAPIs(fakeGCE) fakeCloud := negtypes.NewAdapter(fakeGCE) _, transactionSyncer := newTestTransactionSyncer(fakeCloud, tc.negType, false) if err := transactionSyncer.ensureNetworkEndpointGroups(); err != nil { t.Errorf("Expect error == nil, but got %v", err) } - err := transactionSyncer.syncNetworkEndpoints(tc.addEndpoints, map[string]negtypes.NetworkEndpointSet{}, tc.endpointPodLabelMap, "") + err := transactionSyncer.syncNetworkEndpoints(tc.addEndpoints, map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, tc.endpointPodLabelMap, negtypes.EndpointGroupInfo{}) if err != nil { t.Errorf("For case %q, syncNetworkEndpoints() got %v, want nil", tc.desc, err) } @@ -398,8 +404,8 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { t.Errorf("For case %q, waitForTransactions() got %v, want nil", tc.desc, err) } - for zone := range tc.addEndpoints { - list, err := fakeCloud.ListNetworkEndpoints(transactionSyncer.NegSyncerKey.NegName, zone, false, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) + for endpointGroupInfo := range tc.addEndpoints { + list, err := fakeCloud.ListNetworkEndpoints(transactionSyncer.NegSyncerKey.NegName, endpointGroupInfo.Zone, false, transactionSyncer.NegSyncerKey.GetAPIVersion(), klog.TODO()) if err != nil { t.Errorf("For case %q, ListNetworkEndpoints() got %v, want nil", tc.desc, err) } @@ -415,7 +421,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { func TestCommitTransaction(t *testing.T) { t.Parallel() - s, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(gce.DefaultTestClusterValues())), negtypes.VmIpPortEndpointType, false) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + s, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(vals)), negtypes.VmIpPortEndpointType, false) // use testSyncer to track the number of Sync got triggered testSyncer := &testSyncer{s.(*syncer), 0} testRetryer := &testRetryHandler{testSyncer, 0} @@ -595,31 +603,31 @@ func TestCommitTransaction(t *testing.T) { func TestMergeTransactionIntoZoneEndpointMap(t *testing.T) { testCases := []struct { desc string - endpointMap map[string]negtypes.NetworkEndpointSet + endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet table func() networkEndpointTransactionTable - expectEndpointMap map[string]negtypes.NetworkEndpointSet + expectEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet }{ { "empty map and transactions", - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, func() networkEndpointTransactionTable { return NewTransactionTable() }, - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, { "empty transactions", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, func() networkEndpointTransactionTable { return NewTransactionTable() }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, }, { "empty map", - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, func() networkEndpointTransactionTable { table := NewTransactionTable() generateTransaction(table, transactionEntry{ @@ -639,16 +647,16 @@ func TestMergeTransactionIntoZoneEndpointMap(t *testing.T) { }, net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") return table }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, }, { "add existing endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, func() networkEndpointTransactionTable { table := NewTransactionTable() @@ -669,16 +677,16 @@ func TestMergeTransactionIntoZoneEndpointMap(t *testing.T) { }, net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") return table }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, }, { "add non-existing endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, func() networkEndpointTransactionTable { table := NewTransactionTable() @@ -699,16 +707,16 @@ func TestMergeTransactionIntoZoneEndpointMap(t *testing.T) { }, net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") return table }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 20, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 20, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 20, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 20, testInstance3, "8080")), }, }, { "remove existing endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, func() networkEndpointTransactionTable { table := NewTransactionTable() @@ -719,16 +727,16 @@ func TestMergeTransactionIntoZoneEndpointMap(t *testing.T) { }, net.ParseIP("1.1.1.1"), 5, testInstance1, "8080") return table }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, }, { "add non-existing endpoints and remove existing endpoints", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, func() networkEndpointTransactionTable { table := NewTransactionTable() @@ -744,9 +752,9 @@ func TestMergeTransactionIntoZoneEndpointMap(t *testing.T) { }, net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") return table }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")).Union(generateEndpointSet(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, }, } @@ -762,19 +770,19 @@ func TestMergeTransactionIntoZoneEndpointMap(t *testing.T) { func TestFilterEndpointByTransaction(t *testing.T) { testCases := []struct { desc string - endpointMap map[string]negtypes.NetworkEndpointSet + endpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet table func() networkEndpointTransactionTable - expectEndpointMap map[string]negtypes.NetworkEndpointSet + expectEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet }{ { "both empty", - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, func() networkEndpointTransactionTable { return NewTransactionTable() }, - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, { "empty map", - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, func() networkEndpointTransactionTable { table := NewTransactionTable() generateTransaction(table, transactionEntry{ @@ -789,30 +797,30 @@ func TestFilterEndpointByTransaction(t *testing.T) { }, net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") return table }, - map[string]negtypes.NetworkEndpointSet{}, + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, { "empty transaction", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, func() networkEndpointTransactionTable { return NewTransactionTable() }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, }, { "empty transaction", - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, func() networkEndpointTransactionTable { return NewTransactionTable() }, - map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")), - testZone2: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), + map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.1.6"), 5, testInstance1, "8080")), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(generateEndpointSet(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080")), }, }, } @@ -829,27 +837,29 @@ func TestFilterEndpointByTransaction(t *testing.T) { func TestCommitPods(t *testing.T) { t.Parallel() - _, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(gce.DefaultTestClusterValues())), negtypes.VmIpPortEndpointType, false) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + _, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(vals)), negtypes.VmIpPortEndpointType, false) reflector := &testReflector{} transactionSyncer.reflector = reflector for _, tc := range []struct { desc string - input func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) + input func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) expectOutput func() map[string]negtypes.EndpointPodMap }{ { desc: "empty input", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { return nil, nil }, expectOutput: func() map[string]negtypes.EndpointPodMap { return map[string]negtypes.EndpointPodMap{} }, }, { desc: "10 endpoints from 1 instance in 1 zone", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { endpointSet, endpointMap := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080") - return map[string]negtypes.NetworkEndpointSet{testZone1: endpointSet}, endpointMap + return map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{{Zone: testZone1}: endpointSet}, endpointMap }, expectOutput: func() map[string]negtypes.EndpointPodMap { _, endpointMap := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080") @@ -858,23 +868,23 @@ func TestCommitPods(t *testing.T) { }, { desc: "40 endpoints from 4 instances in 2 zone", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { - retSet := map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet(), - testZone2: negtypes.NewNetworkEndpointSet(), + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + retSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet(), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet(), } retMap := negtypes.EndpointPodMap{} endpointSet, endpointMap := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) retMap = unionEndpointMap(retMap, endpointMap) endpointSet, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) retMap = unionEndpointMap(retMap, endpointMap) endpointSet, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) retMap = unionEndpointMap(retMap, endpointMap) endpointSet, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.4.1"), 10, testInstance4, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) retMap = unionEndpointMap(retMap, endpointMap) return retSet, retMap }, @@ -896,30 +906,30 @@ func TestCommitPods(t *testing.T) { }, { desc: "40 endpoints from 4 instances in 2 zone, but half of the endpoints does not have corresponding pod mapping", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { - retSet := map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet(), - testZone2: negtypes.NewNetworkEndpointSet(), + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + retSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet(), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet(), } retMap := negtypes.EndpointPodMap{} endpointSet, _ := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) _, endpointMap := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 5, testInstance1, "8080") retMap = unionEndpointMap(retMap, endpointMap) endpointSet, _ = generateEndpointSetAndMap(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) _, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.2.1"), 5, testInstance2, "8080") retMap = unionEndpointMap(retMap, endpointMap) endpointSet, _ = generateEndpointSetAndMap(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) _, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.3.1"), 5, testInstance3, "8080") retMap = unionEndpointMap(retMap, endpointMap) endpointSet, _ = generateEndpointSetAndMap(net.ParseIP("1.1.4.1"), 10, testInstance4, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) _, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.4.1"), 5, testInstance4, "8080") retMap = unionEndpointMap(retMap, endpointMap) return retSet, retMap @@ -942,30 +952,30 @@ func TestCommitPods(t *testing.T) { }, { desc: "40 endpoints from 4 instances in 2 zone, and more endpoints are in pod mapping", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { - retSet := map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet(), - testZone2: negtypes.NewNetworkEndpointSet(), + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + retSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet(), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet(), } retMap := negtypes.EndpointPodMap{} endpointSet, _ := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) _, endpointMap := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 15, testInstance1, "8080") retMap = unionEndpointMap(retMap, endpointMap) endpointSet, _ = generateEndpointSetAndMap(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) _, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.2.1"), 15, testInstance2, "8080") retMap = unionEndpointMap(retMap, endpointMap) endpointSet, _ = generateEndpointSetAndMap(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) _, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.3.1"), 15, testInstance3, "8080") retMap = unionEndpointMap(retMap, endpointMap) endpointSet, _ = generateEndpointSetAndMap(net.ParseIP("1.1.4.1"), 10, testInstance4, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) _, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.4.1"), 15, testInstance4, "8080") retMap = unionEndpointMap(retMap, endpointMap) return retSet, retMap @@ -988,22 +998,22 @@ func TestCommitPods(t *testing.T) { }, { desc: "40 endpoints from 4 instances in 2 zone, but some nodes do not have endpoint pod mapping", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { - retSet := map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet(), - testZone2: negtypes.NewNetworkEndpointSet(), + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + retSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet(), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet(), } retMap := negtypes.EndpointPodMap{} endpointSet, endpointMap := generateEndpointSetAndMap(net.ParseIP("1.1.1.1"), 10, testInstance1, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) retMap = unionEndpointMap(retMap, endpointMap) endpointSet, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.2.1"), 10, testInstance2, "8080") - retSet[testZone1] = retSet[testZone1].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone1}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone1}].Union(endpointSet) endpointSet, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.3.1"), 10, testInstance3, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) retMap = unionEndpointMap(retMap, endpointMap) endpointSet, endpointMap = generateEndpointSetAndMap(net.ParseIP("1.1.4.1"), 10, testInstance4, "8080") - retSet[testZone2] = retSet[testZone2].Union(endpointSet) + retSet[negtypes.EndpointGroupInfo{Zone: testZone2}] = retSet[negtypes.EndpointGroupInfo{Zone: testZone2}].Union(endpointSet) return retSet, retMap }, expectOutput: func() map[string]negtypes.EndpointPodMap { @@ -1036,7 +1046,8 @@ func TestCommitPods(t *testing.T) { func TestTransactionSyncerWithNegCR(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) + testSubnetwork := defaultTestSubnetURL + fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) testNegType := negtypes.VmIpPortEndpointType @@ -1332,9 +1343,11 @@ func TestTransactionSyncerWithNegCR(t *testing.T) { // zone has the expected State. func TestUpdateInitStatusWithMultiSubnetCluster(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) testNegType := negtypes.VmIpPortEndpointType flags.F.EnableMultiSubnetClusterPhase1 = true + defer func() { + flags.F.EnableMultiSubnetClusterPhase1 = false + }() // Active zones: zone1, zone2. // Inactive zones: zone3 @@ -1364,9 +1377,7 @@ func TestUpdateInitStatusWithMultiSubnetCluster(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - t.Parallel() - - fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) + fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(defaultTestSubnetURL, testNetwork) _, syncer := newTestTransactionSyncer(fakeCloud, testNegType, false) svcNegClient := syncer.svcNegClient @@ -1453,7 +1464,6 @@ func TestUpdateInitStatusWithMultiSubnetCluster(t *testing.T) { func TestUpdateStatus(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) testNegType := negtypes.VmIpPortEndpointType testNegRefs := []negv1beta1.NegObjectReference{ { @@ -1524,7 +1534,7 @@ func TestUpdateStatus(t *testing.T) { for _, syncErr := range []error{nil, fmt.Errorf("error")} { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) + fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(defaultTestSubnetURL, testNetwork) _, syncer := newTestTransactionSyncer(fakeCloud, testNegType, false) svcNegClient := syncer.svcNegClient syncer.needInit = false @@ -1582,7 +1592,7 @@ func TestUpdateStatus(t *testing.T) { func TestIsZoneChange(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) + testSubnetwork := defaultTestSubnetURL fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) testNegType := negtypes.VmIpPortEndpointType @@ -1722,7 +1732,7 @@ func TestUnknownNodes(t *testing.T) { zonegetter.PopulateFakeNodeInformer(nodeInformer, false) zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) + testSubnetwork := defaultTestSubnetURL fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) testIP1 := "10.100.1.1" @@ -1732,20 +1742,20 @@ func TestUnknownNodes(t *testing.T) { testEndpointSlices := getDefaultEndpointSlices() testEndpointSlices[0].Endpoints[0].NodeName = utilpointer.StringPtr("unknown-node") - testEndpointMap := map[string]*composite.NetworkEndpoint{ - negtypes.TestZone1: &composite.NetworkEndpoint{ + testEndpointMap := map[negtypes.EndpointGroupInfo]*composite.NetworkEndpoint{ + {Zone: negtypes.TestZone1}: &composite.NetworkEndpoint{ Instance: negtypes.TestInstance1, IpAddress: testIP1, Port: testPort, }, - negtypes.TestZone2: &composite.NetworkEndpoint{ + {Zone: negtypes.TestZone2}: &composite.NetworkEndpoint{ Instance: negtypes.TestInstance3, IpAddress: testIP2, Port: testPort, }, - negtypes.TestZone4: &composite.NetworkEndpoint{ + {Zone: negtypes.TestZone4}: &composite.NetworkEndpoint{ Instance: negtypes.TestUpgradeInstance1, IpAddress: testIP3, Port: testPort, @@ -1754,10 +1764,10 @@ func TestUnknownNodes(t *testing.T) { // Create initial NetworkEndpointGroups in cloud var objRefs []negv1beta1.NegObjectReference - for zone, endpoint := range testEndpointMap { - fakeCloud.CreateNetworkEndpointGroup(&composite.NetworkEndpointGroup{Name: testNegName, Version: meta.VersionGA}, zone, klog.TODO()) - fakeCloud.AttachNetworkEndpoints(testNegName, zone, []*composite.NetworkEndpoint{endpoint}, meta.VersionGA, klog.TODO()) - neg, err := fakeCloud.GetNetworkEndpointGroup(testNegName, zone, meta.VersionGA, klog.TODO()) + for endpointGroupInfo, endpoint := range testEndpointMap { + fakeCloud.CreateNetworkEndpointGroup(&composite.NetworkEndpointGroup{Name: testNegName, Version: meta.VersionGA}, endpointGroupInfo.Zone, klog.TODO()) + fakeCloud.AttachNetworkEndpoints(testNegName, endpointGroupInfo.Zone, []*composite.NetworkEndpoint{endpoint}, meta.VersionGA, klog.TODO()) + neg, err := fakeCloud.GetNetworkEndpointGroup(testNegName, endpointGroupInfo.Zone, meta.VersionGA, klog.TODO()) if err != nil { t.Fatalf("failed to get neg from fake cloud: %s", err) } @@ -1790,19 +1800,19 @@ func TestUnknownNodes(t *testing.T) { } // Check that unknown zone did not cause endpoints to be removed - out, _, err := retrieveExistingZoneNetworkEndpointMap(testNegName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, klog.TODO()) + out, _, err := retrieveExistingZoneNetworkEndpointMap(testNegName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, "", klog.TODO()) if err != nil { t.Errorf("errored retrieving existing network endpoints") } - expectedEndpoints := map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpoints := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: testIP1, Node: negtypes.TestInstance1, Port: strconv.Itoa(int(testPort))}, ), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: testIP2, Node: negtypes.TestInstance3, Port: strconv.Itoa(int(testPort))}, ), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: testIP3, Node: negtypes.TestUpgradeInstance1, Port: strconv.Itoa(int(testPort))}, ), } @@ -1814,11 +1824,12 @@ func TestUnknownNodes(t *testing.T) { // TestEnableDegradedMode verifies if DegradedMode has been correctly enabled for L7 endpoint calculator func TestEnableDegradedMode(t *testing.T) { - t.Parallel() nodeInformer := zonegetter.FakeNodeInformer() zonegetter.PopulateFakeNodeInformer(nodeInformer, false) zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) - fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + fakeGCE := gce.NewFakeGCECloud(vals) negtypes.MockNetworkEndpointAPIs(fakeGCE) fakeCloud := negtypes.NewAdapter(fakeGCE) mockGCE := fakeGCE.Compute().(*cloud.MockGCE) @@ -1864,39 +1875,39 @@ func TestEnableDegradedMode(t *testing.T) { }, } - initialEndpoints := map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + initialEndpoints := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"), ), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.2||instance3||80"), ), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.2.1||upgrade-instance1||80"), ), } - updateSucceedEndpoints := map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + updateSucceedEndpoints := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.2||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.2.1||instance2||80"), networkEndpointFromEncodedEndpoint("10.100.1.3||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.4||instance1||80")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.3.1||instance3||80")), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet(), } - updateFailedEndpoints := map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + updateFailedEndpoints := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.2||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.2.1||instance2||80"), networkEndpointFromEncodedEndpoint("10.100.1.3||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.4||instance1||80")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet(), } testCases := []struct { @@ -1904,7 +1915,7 @@ func TestEnableDegradedMode(t *testing.T) { modify func(ts *transactionSyncer) negName string // to distinguish endpoints in differnt NEGs testEndpointSlices []*discovery.EndpointSlice - expectedEndpoints map[string]negtypes.NetworkEndpointSet + expectedEndpoints map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expectedInErrorState bool expectErr error }{ @@ -2087,7 +2098,7 @@ func TestEnableDegradedMode(t *testing.T) { (s.syncer.(*syncer)).stopped = false tc.modify(s) - out, _, err := retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, klog.TODO()) + out, _, err := retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, "", klog.TODO()) if err != nil { t.Errorf("errored retrieving existing network endpoints") } @@ -2100,7 +2111,7 @@ func TestEnableDegradedMode(t *testing.T) { t.Errorf("syncInternal returned %v, expected %v", err, tc.expectErr) } err = wait.PollImmediate(time.Second, 3*time.Second, func() (bool, error) { - out, _, err = retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, klog.TODO()) + out, _, err = retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, "", klog.TODO()) if err != nil { return false, nil } @@ -2245,13 +2256,13 @@ func TestGetEndpointPodLabelMap(t *testing.T) { for _, tc := range []struct { desc string - input func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) + input func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) expect func() labels.EndpointPodLabelMap }{ { desc: "empty inputs", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { - return map[string]negtypes.NetworkEndpointSet{}, negtypes.EndpointPodMap{} + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + return map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, negtypes.EndpointPodMap{} }, expect: func() labels.EndpointPodLabelMap { return labels.EndpointPodLabelMap{} @@ -2259,10 +2270,10 @@ func TestGetEndpointPodLabelMap(t *testing.T) { }, { desc: "Add endpoints in diferent zones", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { - retSet := map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(endpointSet1), - testZone2: negtypes.NewNetworkEndpointSet().Union(endpointSet2), + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + retSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(endpointSet1), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(endpointSet2), } retMap := negtypes.EndpointPodMap{} retMap = unionEndpointMap(retMap, endpointPodMap1) @@ -2282,10 +2293,10 @@ func TestGetEndpointPodLabelMap(t *testing.T) { }, { desc: "Add endpoints in diferent zones with pod not found", - input: func() (map[string]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { - retSet := map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet().Union(endpointSet1), - testZone2: negtypes.NewNetworkEndpointSet().Union(endpointSet2).Union(endpointSet3), + input: func() (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, negtypes.EndpointPodMap) { + retSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet().Union(endpointSet1), + {Zone: testZone2}: negtypes.NewNetworkEndpointSet().Union(endpointSet2).Union(endpointSet3), } retMap := negtypes.EndpointPodMap{} retMap = unionEndpointMap(retMap, endpointPodMap1) @@ -2331,14 +2342,14 @@ func TestCollectLabelStats(t *testing.T) { desc string curLabelMap labels.EndpointPodLabelMap addLabelMap labels.EndpointPodLabelMap - targetEndpointMap map[string]negtypes.NetworkEndpointSet + targetEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expect metricscollector.LabelPropagationStats }{ { desc: "Empty inputs", curLabelMap: labels.EndpointPodLabelMap{}, addLabelMap: labels.EndpointPodLabelMap{}, - targetEndpointMap: map[string]negtypes.NetworkEndpointSet{}, + targetEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, expect: metricscollector.LabelPropagationStats{ EndpointsWithAnnotation: 0, NumberOfEndpoints: 0, @@ -2352,8 +2363,8 @@ func TestCollectLabelStats(t *testing.T) { }, }, addLabelMap: labels.EndpointPodLabelMap{}, - targetEndpointMap: map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet( + targetEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), @@ -2375,12 +2386,12 @@ func TestCollectLabelStats(t *testing.T) { "foo": "bar", }, }, - targetEndpointMap: map[string]negtypes.NetworkEndpointSet{ - testZone1: negtypes.NewNetworkEndpointSet( + targetEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone1}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - testZone2: negtypes.NewNetworkEndpointSet( + {Zone: testZone2}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), @@ -2398,8 +2409,8 @@ func TestCollectLabelStats(t *testing.T) { "foo": "bar", }, }, - targetEndpointMap: map[string]negtypes.NetworkEndpointSet{ - testZone2: negtypes.NewNetworkEndpointSet( + targetEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: testZone2}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), @@ -2457,7 +2468,7 @@ func newTestTransactionSyncer(fakeGCE negtypes.NetworkEndpointGroupCloud, negTyp testContext.SvcNegInformer.GetIndexer(), reflector, GetEndpointsCalculator(testContext.PodInformer.GetIndexer(), testContext.NodeInformer.GetIndexer(), testContext.ServiceInformer.GetIndexer(), - fakeZoneGetter, svcPort, mode, klog.TODO(), testContext.EnableDualStackNEG, metricscollector.FakeSyncerMetrics(), &network.NetworkInfo{IsDefault: true}, negtypes.L4InternalLB), + fakeZoneGetter, svcPort, mode, klog.TODO(), testContext.EnableDualStackNEG, metricscollector.FakeSyncerMetrics(), &network.NetworkInfo{IsDefault: true, SubnetworkURL: defaultTestSubnetURL}, negtypes.L4InternalLB), string(kubeSystemUID), testContext.SvcNegClient, metricscollector.FakeSyncerMetrics(), diff --git a/pkg/neg/syncers/utils.go b/pkg/neg/syncers/utils.go index 7e16c66b64..d6a12cac4b 100644 --- a/pkg/neg/syncers/utils.go +++ b/pkg/neg/syncers/utils.go @@ -89,20 +89,20 @@ func calculateDifference(targetMap, currentMap map[string]sets.String) (map[stri } // calculateNetworkEndpointDifference determines what endpoints needs to be added and removed in order to move current state to target state. -func calculateNetworkEndpointDifference(targetMap, currentMap map[string]negtypes.NetworkEndpointSet) (map[string]negtypes.NetworkEndpointSet, map[string]negtypes.NetworkEndpointSet) { - addSet := map[string]negtypes.NetworkEndpointSet{} - removeSet := map[string]negtypes.NetworkEndpointSet{} - for zone, endpointSet := range targetMap { - diff := endpointSet.Difference(currentMap[zone]) +func calculateNetworkEndpointDifference(targetMap, currentMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet) (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet) { + addSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{} + removeSet := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{} + for endpointGroupInfo, endpointSet := range targetMap { + diff := endpointSet.Difference(currentMap[endpointGroupInfo]) if len(diff) > 0 { - addSet[zone] = diff + addSet[endpointGroupInfo] = diff } } - for zone, endpointSet := range currentMap { - diff := endpointSet.Difference(targetMap[zone]) + for endpointGroupInfo, endpointSet := range currentMap { + diff := endpointSet.Difference(targetMap[endpointGroupInfo]) if len(diff) > 0 { - removeSet[zone] = diff + removeSet[endpointGroupInfo] = diff } } return addSet, removeSet @@ -236,7 +236,7 @@ func ensureNetworkEndpointGroup(svcNamespace, svcName, negName, zone, negService } type ZoneNetworkEndpointMapResult struct { - NetworkEndpointSet map[string]negtypes.NetworkEndpointSet + NetworkEndpointSet map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet EndpointPodMap negtypes.EndpointPodMap EPCount negtypes.StateCountMap EPSCount negtypes.StateCountMap @@ -244,7 +244,7 @@ type ZoneNetworkEndpointMapResult struct { // toZoneNetworkEndpointMap translates addresses in endpoints object into zone and endpoints map, and also return the count for duplicated endpoints func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter *zonegetter.ZoneGetter, podLister cache.Indexer, servicePortName string, networkEndpointType negtypes.NetworkEndpointType, enableDualStackNEG, enableMultiSubnetCluster bool, logger klog.Logger) (ZoneNetworkEndpointMapResult, error) { - zoneNetworkEndpointMap := map[string]negtypes.NetworkEndpointSet{} + zoneNetworkEndpointMap := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{} networkEndpointPodMap := negtypes.EndpointPodMap{} ipsForPod := ipsForPod(eds) globalEPCount := make(negtypes.StateCountMap) @@ -288,16 +288,16 @@ func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter *zonegett continue } globalEPCount[negtypes.Total] += 1 - zone, _, getZoneErr := getEndpointZone(endpointAddress, zoneGetter, logger) - if getZoneErr != nil { - metrics.PublishNegControllerErrorCountMetrics(getZoneErr, true) - if enableMultiSubnetCluster && errors.Is(getZoneErr, zonegetter.ErrNodeNotInDefaultSubnet) { - epLogger.Error(getZoneErr, "Detected endpoint not from default subnet. Skipping") + epGroupInfo, _, getEpGroupInfoErr := getEndpointZoneSubnet(endpointAddress, zoneGetter, logger) + if getEpGroupInfoErr != nil { + metrics.PublishNegControllerErrorCountMetrics(getEpGroupInfoErr, true) + if enableMultiSubnetCluster && errors.Is(getEpGroupInfoErr, zonegetter.ErrNodeNotInDefaultSubnet) { + epLogger.Error(getEpGroupInfoErr, "Detected endpoint not from default subnet. Skipping") localEPCount[negtypes.NodeInNonDefaultSubnet]++ continue } - epLogger.Error(getZoneErr, "Detected unexpected error when getting zone for endpoint") - return ZoneNetworkEndpointMapResult{}, fmt.Errorf("unexpected error when getting zone for endpoint %q in endpoint slice %s/%s: %w", endpointAddress.Addresses, ed.Meta.Namespace, ed.Meta.Name, getZoneErr) + epLogger.Error(getEpGroupInfoErr, "Detected unexpected error when getting zone for endpoint") + return ZoneNetworkEndpointMapResult{}, fmt.Errorf("unexpected error when getting zone for endpoint %q in endpoint slice %s/%s: %w", endpointAddress.Addresses, ed.Meta.Namespace, ed.Meta.Name, getEpGroupInfoErr) } _, _, getPodErr := getEndpointPod(endpointAddress, podLister) @@ -311,8 +311,8 @@ func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter *zonegett epLogger.V(2).Info("Endpoint does not have an associated pod. Skipping") continue } - if zoneNetworkEndpointMap[zone] == nil { - zoneNetworkEndpointMap[zone] = negtypes.NewNetworkEndpointSet() + if zoneNetworkEndpointMap[epGroupInfo] == nil { + zoneNetworkEndpointMap[epGroupInfo] = negtypes.NewNetworkEndpointSet() } podIPs := ipsForPod[types.NamespacedName{Namespace: endpointAddress.TargetRef.Namespace, Name: endpointAddress.TargetRef.Name}] @@ -331,7 +331,7 @@ func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter *zonegett // Non-GCP network endpoints don't have associated nodes. networkEndpoint.Node = "" } - zoneNetworkEndpointMap[zone].Insert(networkEndpoint) + zoneNetworkEndpointMap[epGroupInfo].Insert(networkEndpoint) // if existing name is alphabetically lower than current one, continue and don't replace if existingPod, contains := networkEndpointPodMap[networkEndpoint]; contains { @@ -373,30 +373,30 @@ func mergeWithGlobalCounts(localEPCount, globalEPCount, globalEPSCount negtypes. } } -// getEndpointZone use an endpoint's nodeName to get its corresponding zone -func getEndpointZone(endpointAddress negtypes.AddressData, zoneGetter *zonegetter.ZoneGetter, logger klog.Logger) (string, negtypes.StateCountMap, error) { +// getEndpointZoneSubnet use an endpoint's nodeName to get its corresponding zone and subnet +func getEndpointZoneSubnet(endpointAddress negtypes.AddressData, zoneGetter *zonegetter.ZoneGetter, logger klog.Logger) (negtypes.EndpointGroupInfo, negtypes.StateCountMap, error) { count := make(negtypes.StateCountMap) if endpointAddress.NodeName == nil || len(*endpointAddress.NodeName) == 0 { count[negtypes.NodeMissing]++ count[negtypes.ZoneMissing]++ - return "", count, negtypes.ErrEPNodeMissing + return negtypes.EndpointGroupInfo{}, count, negtypes.ErrEPNodeMissing } - zone, err := zoneGetter.ZoneForNode(*endpointAddress.NodeName, logger) + zone, subnet, err := zoneGetter.ZoneAndSubnetForNode(*endpointAddress.NodeName, logger) // Fail to get the node object. if errors.Is(err, zonegetter.ErrNodeNotFound) { count[negtypes.NodeNotFound]++ - return zone, count, fmt.Errorf("%w: %v", negtypes.ErrEPNodeNotFound, err) + return negtypes.EndpointGroupInfo{}, count, fmt.Errorf("%w: %v", negtypes.ErrEPNodeNotFound, err) } if errors.Is(err, zonegetter.ErrNodePodCIDRNotSet) { count[negtypes.NodePodCIDRNotSet]++ - return zone, count, fmt.Errorf("%w: %w", negtypes.ErrEPNodePodCIDRNotSet, err) + return negtypes.EndpointGroupInfo{}, count, fmt.Errorf("%w: %w", negtypes.ErrEPNodePodCIDRNotSet, err) } // providerID missing in node or zone information missing in providerID. if errors.Is(err, zonegetter.ErrProviderIDNotFound) || errors.Is(err, zonegetter.ErrSplitProviderID) { count[negtypes.ZoneMissing]++ - return zone, count, fmt.Errorf("%w: zone is missing for node %v", negtypes.ErrEPZoneMissing, *endpointAddress.NodeName) + return negtypes.EndpointGroupInfo{}, count, fmt.Errorf("%w: zone is missing for node %v", negtypes.ErrEPZoneMissing, *endpointAddress.NodeName) } - return zone, count, err + return negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet}, count, err } // getEndpointPod use an endpoint's pod name and namespace to get its corresponding pod object @@ -423,7 +423,7 @@ func getEndpointPod(endpointAddress negtypes.AddressData, podLister cache.Indexe // toZoneNetworkEndpointMap translates addresses in endpoints object into zone and endpoints map, and also return the count for duplicated endpoints // we will not raise error in degraded mode for misconfigured endpoints, instead they will be filtered directly func toZoneNetworkEndpointMapDegradedMode(eds []negtypes.EndpointsData, zoneGetter *zonegetter.ZoneGetter, podLister, nodeLister, serviceLister cache.Indexer, servicePortName string, networkEndpointType negtypes.NetworkEndpointType, enableDualStackNEG, enableMultiSubnetCluster bool, logger klog.Logger) ZoneNetworkEndpointMapResult { - zoneNetworkEndpointMap := map[string]negtypes.NetworkEndpointSet{} + zoneNetworkEndpointMap := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{} networkEndpointPodMap := negtypes.EndpointPodMap{} ipsForPod := ipsForPod(eds) globalEPCount := make(negtypes.StateCountMap) @@ -470,20 +470,21 @@ func toZoneNetworkEndpointMapDegradedMode(eds []negtypes.EndpointsData, zoneGett localEPCount[negtypes.NodeMissing]++ continue } - zone, getZoneErr := zoneGetter.ZoneForNode(nodeName, logger) - if getZoneErr != nil { - metrics.PublishNegControllerErrorCountMetrics(getZoneErr, true) - if enableMultiSubnetCluster && errors.Is(getZoneErr, zonegetter.ErrNodeNotInDefaultSubnet) { - epLogger.Error(getZoneErr, "Detected endpoint not from default subnet. Skipping", "nodeName", nodeName) + zone, subnet, err := zoneGetter.ZoneAndSubnetForNode(nodeName, logger) + if err != nil { + metrics.PublishNegControllerErrorCountMetrics(err, true) + if enableMultiSubnetCluster && errors.Is(err, zonegetter.ErrNodeNotInDefaultSubnet) { + epLogger.Error(err, "Detected endpoint not from default subnet. Skipping", "nodeName", nodeName) localEPCount[negtypes.NodeInNonDefaultSubnet]++ continue } - epLogger.Error(getZoneErr, "Endpoint's corresponding node does not have valid zone information, skipping", "nodeName", nodeName) + epLogger.Error(err, "Endpoint's corresponding node does not have valid zone information, skipping", "nodeName", nodeName) localEPCount[negtypes.NodeNotFound]++ continue } - if zoneNetworkEndpointMap[zone] == nil { - zoneNetworkEndpointMap[zone] = negtypes.NewNetworkEndpointSet() + epGroupInfo := negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet} + if zoneNetworkEndpointMap[epGroupInfo] == nil { + zoneNetworkEndpointMap[epGroupInfo] = negtypes.NewNetworkEndpointSet() } podIPs := ipsForPod[types.NamespacedName{Namespace: endpointAddress.TargetRef.Namespace, Name: endpointAddress.TargetRef.Name}] @@ -526,7 +527,7 @@ func toZoneNetworkEndpointMapDegradedMode(eds []negtypes.EndpointsData, zoneGett // Non-GCP network endpoints don't have associated nodes. networkEndpoint.Node = "" } - zoneNetworkEndpointMap[zone].Insert(networkEndpoint) + zoneNetworkEndpointMap[epGroupInfo].Insert(networkEndpoint) // if existing name is alphabetically lower than current one, continue and don't replace if existingPod, contains := networkEndpointPodMap[networkEndpoint]; contains { @@ -694,8 +695,9 @@ func podBelongsToService(pod *apiv1.Pod, service *apiv1.Service) error { return nil } -// retrieveExistingZoneNetworkEndpointMap lists existing network endpoints in the neg and return the zone and endpoints map -func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegetter.ZoneGetter, cloud negtypes.NetworkEndpointGroupCloud, version meta.Version, mode negtypes.EndpointsCalculatorMode, enableDualStackNEG bool, logger klog.Logger) (map[string]negtypes.NetworkEndpointSet, labels.EndpointPodLabelMap, error) { +// retrieveExistingZoneNetworkEndpointMap lists existing network endpoints in the neg and return the zone and endpoints map. +// TODO(sawsa307): Make sure to include endpoints from non-default NEGs after syncers create non-default subnet NEGs. +func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegetter.ZoneGetter, cloud negtypes.NetworkEndpointGroupCloud, version meta.Version, mode negtypes.EndpointsCalculatorMode, enableDualStackNEG bool, subnet string, logger klog.Logger) (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, labels.EndpointPodLabelMap, error) { // Include zones that have non-candidate nodes currently. It is possible that NEGs were created in those zones previously and the endpoints now became non-candidates. // Endpoints in those NEGs now need to be removed. This mostly applies to VM_IP_NEGs where the endpoints are nodes. zones, err := zoneGetter.ListZones(zonegetter.AllNodesFilter, logger) @@ -709,7 +711,7 @@ func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegett } candidateZonesMap := sets.NewString(candidateNodeZones...) - zoneNetworkEndpointMap := map[string]negtypes.NetworkEndpointSet{} + zoneNetworkEndpointMap := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{} endpointPodLabelMap := labels.EndpointPodLabelMap{} for _, zone := range zones { networkEndpointsWithHealthStatus, err := cloud.ListNetworkEndpoints(negName, zone, false, version, logger) @@ -723,7 +725,7 @@ func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegett } return nil, nil, fmt.Errorf("Failed to lookup NEG in zone %q, candidate zones %v, err - %w", zone, candidateZonesMap, err) } - zoneNetworkEndpointMap[zone] = negtypes.NewNetworkEndpointSet() + zoneNetworkEndpointMap[negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet}] = negtypes.NewNetworkEndpointSet() for _, ne := range networkEndpointsWithHealthStatus { newNE := negtypes.NetworkEndpoint{IP: ne.NetworkEndpoint.IpAddress, Node: ne.NetworkEndpoint.Instance} if ne.NetworkEndpoint.Port != 0 { @@ -732,7 +734,7 @@ func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegett if enableDualStackNEG { newNE.IPv6 = ne.NetworkEndpoint.Ipv6Address } - zoneNetworkEndpointMap[zone].Insert(newNE) + zoneNetworkEndpointMap[negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet}].Insert(newNE) endpointPodLabelMap[newNE] = ne.NetworkEndpoint.Annotations } } diff --git a/pkg/neg/syncers/utils_test.go b/pkg/neg/syncers/utils_test.go index 9a06746312..f232adaf31 100644 --- a/pkg/neg/syncers/utils_test.go +++ b/pkg/neg/syncers/utils_test.go @@ -188,109 +188,109 @@ func TestNetworkEndpointCalculateDifference(t *testing.T) { t.Parallel() testCases := []struct { - targetSet map[string]negtypes.NetworkEndpointSet - currentSet map[string]negtypes.NetworkEndpointSet - addSet map[string]negtypes.NetworkEndpointSet - removeSet map[string]negtypes.NetworkEndpointSet + targetSet map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + currentSet map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + addSet map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet + removeSet map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet }{ // unchanged { - targetSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, - currentSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, - addSet: map[string]negtypes.NetworkEndpointSet{}, - removeSet: map[string]negtypes.NetworkEndpointSet{}, + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, // unchanged { - targetSet: map[string]negtypes.NetworkEndpointSet{}, - currentSet: map[string]negtypes.NetworkEndpointSet{}, - addSet: map[string]negtypes.NetworkEndpointSet{}, - removeSet: map[string]negtypes.NetworkEndpointSet{}, + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, // add in one zone { - targetSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, - currentSet: map[string]negtypes.NetworkEndpointSet{}, - addSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, - removeSet: map[string]negtypes.NetworkEndpointSet{}, + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, // add in 2 zones { - targetSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), }, - currentSet: map[string]negtypes.NetworkEndpointSet{}, - addSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), }, - removeSet: map[string]negtypes.NetworkEndpointSet{}, + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, }, // remove in one zone { - targetSet: map[string]negtypes.NetworkEndpointSet{}, - currentSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, - addSet: map[string]negtypes.NetworkEndpointSet{}, - removeSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, }, // remove in 2 zones { - targetSet: map[string]negtypes.NetworkEndpointSet{}, - currentSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), }, - addSet: map[string]negtypes.NetworkEndpointSet{}, - removeSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("e"), genNetworkEndpoint("f"), genNetworkEndpoint("g")), }, }, // add and delete in one zone { - targetSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, - currentSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("b"), genNetworkEndpoint("c"), genNetworkEndpoint("d")), + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("b"), genNetworkEndpoint("c"), genNetworkEndpoint("d")), }, - addSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a")), + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a")), }, - removeSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("d")), + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("d")), }, }, // add and delete in 2 zones { - targetSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + targetSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a"), genNetworkEndpoint("b"), genNetworkEndpoint("c")), }, - currentSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("b"), genNetworkEndpoint("c"), genNetworkEndpoint("d")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("b"), genNetworkEndpoint("c"), genNetworkEndpoint("d")), + currentSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("b"), genNetworkEndpoint("c"), genNetworkEndpoint("d")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("b"), genNetworkEndpoint("c"), genNetworkEndpoint("d")), }, - addSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a")), + addSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("a")), }, - removeSet: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("d")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("d")), + removeSet: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("d")), + {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(genNetworkEndpoint("d")), }, }, } @@ -502,7 +502,7 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { testCases := []struct { desc string portName string - wantZoneNetworkEndpointMap map[string]negtypes.NetworkEndpointSet + wantZoneNetworkEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet wantNetworkEndpointPodMap negtypes.EndpointPodMap networkEndpointType negtypes.NetworkEndpointType enableDualStackNEG bool @@ -510,22 +510,22 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { { desc: "target port does not exist", portName: "non-exists", - wantZoneNetworkEndpointMap: map[string]negtypes.NetworkEndpointSet{}, + wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, wantNetworkEndpointPodMap: negtypes.EndpointPodMap{}, networkEndpointType: negtypes.VmIpPortEndpointType, }, { desc: "default service port name", portName: "", - wantZoneNetworkEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.1.1", Node: "instance1", Port: "80"}, {IP: "10.100.1.2", Node: "instance1", Port: "80"}, {IP: "10.100.1.3", Node: "instance1", Port: "80"}, {IP: "10.100.1.4", Node: "instance1", Port: "80"}, {IP: "10.100.2.1", Node: "instance2", Port: "80"}, }...), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.3.1", Node: "instance3", Port: "80"}, }...), }, @@ -542,11 +542,11 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { { desc: "explicitly named service port", portName: testNamedPort, - wantZoneNetworkEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.2.2", Node: "instance2", Port: "81"}, }...), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.4.1", Node: "instance4", Port: "81"}, {IP: "10.100.3.2", Node: "instance3", Port: "8081"}, {IP: "10.100.4.2", Node: "instance4", Port: "8081"}, @@ -567,11 +567,11 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { { desc: "dual stack enabled with explicitly named service ports", portName: testNamedPort, - wantZoneNetworkEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.2.2", Node: "instance2", Port: "81"}, }...), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.4.1", Node: "instance4", Port: "81"}, {IP: "10.100.3.2", IPv6: "a:b::1", Node: "instance3", Port: "8081"}, {IP: "10.100.4.2", IPv6: "a:b::2", Node: "instance4", Port: "8081"}, @@ -593,15 +593,15 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { { desc: "non GCP network endpoints", portName: "", - wantZoneNetworkEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.1.1", Port: "80"}, {IP: "10.100.1.2", Port: "80"}, {IP: "10.100.1.3", Port: "80"}, {IP: "10.100.1.4", Port: "80"}, {IP: "10.100.2.1", Port: "80"}, }...), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.3.1", Port: "80"}, }...), }, @@ -774,7 +774,7 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { desc string mutate func(cloud negtypes.NetworkEndpointGroupCloud) mode negtypes.EndpointsCalculatorMode - expect map[string]negtypes.NetworkEndpointSet + expect map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expectAnnotationMap labels.EndpointPodLabelMap expectErr bool }{ @@ -803,10 +803,10 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { cloud.CreateNetworkEndpointGroup(&composite.NetworkEndpointGroup{Name: testNegName, Version: meta.VersionGA}, negtypes.TestZone2, klog.TODO()) cloud.CreateNetworkEndpointGroup(&composite.NetworkEndpointGroup{Name: testNegName, Version: meta.VersionGA}, negtypes.TestZone4, klog.TODO()) }, - expect: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet(), + expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{}, expectErr: false, @@ -825,10 +825,10 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, }, meta.VersionGA, klog.TODO()) }, - expect: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet(endpoint1), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet(), + expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(endpoint1), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{ endpoint1: labels.PodLabelMap{ @@ -851,13 +851,13 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, }, meta.VersionGA, klog.TODO()) }, - expect: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet(), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{ endpoint1: labels.PodLabelMap{ @@ -891,16 +891,16 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, }, meta.VersionGA, klog.TODO()) }, - expect: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{ endpoint1: labels.PodLabelMap{ @@ -934,16 +934,16 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, }, meta.VersionGA, klog.TODO()) }, - expect: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint6, endpoint7, ), @@ -977,16 +977,16 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, }, meta.VersionGA, klog.TODO()) }, - expect: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint6, endpoint7, ), @@ -1023,20 +1023,20 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, // set mode to L4 since this scenario applies more to VM_IP NEGs. mode: negtypes.L4LocalMode, - expect: map[string]negtypes.NetworkEndpointSet{ + expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ // NEGs in zone1, zone2 and zone4 are created from previous test case. - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - negtypes.TestZone3: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint5, ), - negtypes.TestZone4: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint6, endpoint7, ), @@ -1072,7 +1072,7 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { for _, tc := range testCases { tc.mutate(negCloud) // tc.mode of "" will result in the default node predicate being selected, which is ok for this test. - endpointSets, annotationMap, err := retrieveExistingZoneNetworkEndpointMap(negName, zoneGetter, negCloud, meta.VersionGA, tc.mode, false, klog.TODO()) + endpointSets, annotationMap, err := retrieveExistingZoneNetworkEndpointMap(negName, zoneGetter, negCloud, meta.VersionGA, tc.mode, false, defaultTestSubnet, klog.TODO()) if tc.expectErr { if err == nil { @@ -1640,7 +1640,7 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { desc string testEndpointSlices []*discovery.EndpointSlice portName string - expectedEndpointMap map[string]negtypes.NetworkEndpointSet + expectedEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expectedPodMap negtypes.EndpointPodMap networkEndpointType negtypes.NetworkEndpointType }{ @@ -1648,7 +1648,7 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { desc: "non exist target port", testEndpointSlices: getDefaultEndpointSlices(), portName: testNonExistPort, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{}, + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{}, expectedPodMap: negtypes.EndpointPodMap{}, networkEndpointType: negtypes.VmIpPortEndpointType, }, @@ -1656,14 +1656,14 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { desc: "empty named port", testEndpointSlices: getDefaultEndpointSlices(), portName: testEmptyNamedPort, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.2||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.2.1||instance2||80"), networkEndpointFromEncodedEndpoint("10.100.1.3||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.4||instance1||80")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.3.1||instance3||80")), }, expectedPodMap: negtypes.EndpointPodMap{ @@ -1680,10 +1680,10 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { desc: "named target port", testEndpointSlices: getDefaultEndpointSlices(), portName: testNamedPort, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.2.2||instance2||81")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.4.1||instance4||81"), networkEndpointFromEncodedEndpoint("10.100.3.2||instance3||8081"), networkEndpointFromEncodedEndpoint("10.100.4.2||instance4||8081"), @@ -1704,14 +1704,14 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { desc: "Non-GCP network endpoints", testEndpointSlices: getDefaultEndpointSlices(), portName: testEmptyNamedPort, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.1||||80"), networkEndpointFromEncodedEndpoint("10.100.1.2||||80"), networkEndpointFromEncodedEndpoint("10.100.2.1||||80"), networkEndpointFromEncodedEndpoint("10.100.1.3||||80"), networkEndpointFromEncodedEndpoint("10.100.1.4||||80")), - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.3.1||||80")), }, expectedPodMap: negtypes.EndpointPodMap{ @@ -1757,8 +1757,8 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { }, }, portName: testEmptyNamedPort, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.10.0.1||instance1||80"), ), }, @@ -1785,8 +1785,6 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { // and toZoneNetworkEndpointMapDegradedMode return the correct endpoints and // correct type of error with the supplied invalid endpoint information. func TestValidateEndpointFields(t *testing.T) { - t.Parallel() - emptyNamedPort := "" emptyNodeName := "" port80 := int32(80) @@ -1843,8 +1841,8 @@ func TestValidateEndpointFields(t *testing.T) { }) // endpointMap and podMap contain all correct endpoints. - endpointMap := map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + endpointMap := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), @@ -1862,8 +1860,8 @@ func TestValidateEndpointFields(t *testing.T) { // // In degraded mode, we should exclude the invalid endpoint for non-coverable cases(pod invalid or empty zone). // We always inject to first endpoint, so the result only contain the second endpoint. - endpointMapExcluded := map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + endpointMapExcluded := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), } @@ -1874,11 +1872,11 @@ func TestValidateEndpointFields(t *testing.T) { testCases := []struct { desc string testEndpointSlices []*discovery.EndpointSlice - expectedEndpointMap map[string]negtypes.NetworkEndpointSet + expectedEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expectedPodMap negtypes.EndpointPodMap expectErr error expectErrorState bool - expectedEndpointMapDegradedMode map[string]negtypes.NetworkEndpointSet + expectedEndpointMapDegradedMode map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expectedPodMapDegradedMode negtypes.EndpointPodMap }{ { @@ -2253,8 +2251,8 @@ func TestValidateEndpointFields(t *testing.T) { }, }, }, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.2.2", Node: instance2, Port: "80"}, @@ -2363,8 +2361,8 @@ func TestValidateEndpointFields(t *testing.T) { }, }, }, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.3.2", IPv6: "a:b::1", Node: instance3, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.4.2", IPv6: "a:b::2", Node: instance4, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.4.4", IPv6: "a:b::4", Node: instance4, Port: "80"}, @@ -2377,8 +2375,8 @@ func TestValidateEndpointFields(t *testing.T) { }, expectErr: nil, expectErrorState: false, - expectedEndpointMapDegradedMode: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone2: negtypes.NewNetworkEndpointSet( + expectedEndpointMapDegradedMode: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.3.2", IPv6: "a:b::1", Node: instance3, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.4.2", IPv6: "a:b::2", Node: instance4, Port: "80"}, ), @@ -2544,8 +2542,8 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }, }) - endpointMap := map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + endpointMap := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), @@ -2558,11 +2556,11 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { testCases := []struct { desc string testEndpointSlices []*discovery.EndpointSlice - expectedEndpointMap map[string]negtypes.NetworkEndpointSet + expectedEndpointMap map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expectedPodMap negtypes.EndpointPodMap expectErr error expectErrorState bool - expectedEndpointMapDegradedMode map[string]negtypes.NetworkEndpointSet + expectedEndpointMapDegradedMode map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet expectedPodMapDegradedMode negtypes.EndpointPodMap }{ { @@ -2612,12 +2610,12 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }, }, }, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - negtypes.TestZone5: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone5, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.1.1", Node: defaultSubnetLabelInstance, Port: "80"}, ), }, @@ -2628,12 +2626,12 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }, expectErr: nil, expectErrorState: false, - expectedEndpointMapDegradedMode: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMapDegradedMode: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - negtypes.TestZone5: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone5, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.1.1", Node: defaultSubnetLabelInstance, Port: "80"}, ), }, @@ -2690,12 +2688,12 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }, }, }, - expectedEndpointMap: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - negtypes.TestZone6: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone6, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.2.1", Node: emptySubnetLabelInstance, Port: "80"}, ), }, @@ -2706,12 +2704,12 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }, expectErr: nil, expectErrorState: false, - expectedEndpointMapDegradedMode: map[string]negtypes.NetworkEndpointSet{ - negtypes.TestZone1: negtypes.NewNetworkEndpointSet( + expectedEndpointMapDegradedMode: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - negtypes.TestZone6: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone6, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.2.1", Node: emptySubnetLabelInstance, Port: "80"}, ), }, diff --git a/pkg/neg/types/interfaces.go b/pkg/neg/types/interfaces.go index 89576e1eb6..6b0e60bf86 100644 --- a/pkg/neg/types/interfaces.go +++ b/pkg/neg/types/interfaces.go @@ -81,11 +81,13 @@ type NegSyncerManager interface { } type NetworkEndpointsCalculator interface { - // CalculateEndpoints computes the NEG endpoints based on service endpoints and the current NEG state and returns a - // map of zone name to network endpoint set - CalculateEndpoints(eds []EndpointsData, currentMap map[string]NetworkEndpointSet) (map[string]NetworkEndpointSet, EndpointPodMap, int, error) + // CalculateEndpoints computes the NEG endpoints based on service endpoints + // and the current NEG state and returns a map of EndpointGroupInfo to + // network endpoint set. EndpointGroupInfo contains the zone and subnet of + // this NEG. + CalculateEndpoints(eds []EndpointsData, currentMap map[EndpointGroupInfo]NetworkEndpointSet) (map[EndpointGroupInfo]NetworkEndpointSet, EndpointPodMap, int, error) // CalculateEndpointsDegradedMode computes the NEG endpoints using degraded mode calculation - CalculateEndpointsDegradedMode(eds []EndpointsData, currentMap map[string]NetworkEndpointSet) (map[string]NetworkEndpointSet, EndpointPodMap, error) + CalculateEndpointsDegradedMode(eds []EndpointsData, currentMap map[EndpointGroupInfo]NetworkEndpointSet) (map[EndpointGroupInfo]NetworkEndpointSet, EndpointPodMap, error) // Mode indicates the mode that the EndpointsCalculator is operating in. Mode() EndpointsCalculatorMode // ValidateEndpoints validates the NEG endpoint information is correct diff --git a/pkg/neg/types/types.go b/pkg/neg/types/types.go index 630811589c..2e1725b16a 100644 --- a/pkg/neg/types/types.go +++ b/pkg/neg/types/types.go @@ -412,3 +412,8 @@ func NegInfoFromNegRef(negRef negv1beta1.NegObjectReference) (NegInfo, error) { } return NegInfo{Name: resourceID.Key.Name, Zone: resourceID.Key.Zone}, nil } + +type EndpointGroupInfo struct { + Zone string + Subnet string +} diff --git a/pkg/utils/zonegetter/zone_getter.go b/pkg/utils/zonegetter/zone_getter.go index 1900baf723..3e594e6026 100644 --- a/pkg/utils/zonegetter/zone_getter.go +++ b/pkg/utils/zonegetter/zone_getter.go @@ -78,41 +78,48 @@ type ZoneGetter struct { defaultSubnetURL string } -// ZoneForNode returns the zone for a given node by looking up providerID. -func (z *ZoneGetter) ZoneForNode(name string, logger klog.Logger) (string, error) { +// ZoneAndSubnetForNode returns the zone and subnet for a given node by looking up providerID. +func (z *ZoneGetter) ZoneAndSubnetForNode(name string, logger klog.Logger) (string, string, error) { // Return the single stored zone if the zoneGetter is in non-gcp mode. + // In non-gcp mode, the subnet will be empty, so it matches the behavior + // for non-gcp NEGs since their SubnetworkURL is empty. if z.mode == NonGCP { logger.Info("ZoneGetter in non-gcp mode, return the single stored zone", "zone", z.singleStoredZone) - return z.singleStoredZone, nil + return z.singleStoredZone, "", nil } nodeLogger := logger.WithValues("nodeName", name) node, err := listers.NewNodeLister(z.nodeLister).Get(name) if err != nil { nodeLogger.Error(err, "Failed to get node") - return "", fmt.Errorf("%w: failed to get node %s", ErrNodeNotFound, name) - } - if z.onlyIncludeDefaultSubnetNodes { - isInDefaultSubnet, err := isNodeInDefaultSubnet(node, z.defaultSubnetURL, nodeLogger) - if err != nil { - nodeLogger.Error(err, "Failed to verify if the node is in default subnet.") - return "", err - } - if !isInDefaultSubnet { - nodeLogger.Error(ErrNodeNotInDefaultSubnet, "Node is invalid since it is not in the default subnet") - return "", ErrNodeNotInDefaultSubnet - } + return "", "", fmt.Errorf("%w: failed to get node %s", ErrNodeNotFound, name) } if node.Spec.ProviderID == "" { nodeLogger.Error(ErrProviderIDNotFound, "Node does not have providerID") - return "", ErrProviderIDNotFound + return "", "", ErrProviderIDNotFound } zone, err := getZone(node) if err != nil { nodeLogger.Error(err, "Failed to get zone from the providerID") - return "", err + return "", "", err + } + + subnet, err := getSubnet(node, z.defaultSubnetURL) + if err != nil { + nodeLogger.Error(err, "Failed to get subnet from node's LabelNodeSubnet") + return "", "", err } - return zone, nil + if z.onlyIncludeDefaultSubnetNodes { + defaultSubnet, err := utils.KeyName(z.defaultSubnetURL) + if err != nil { + nodeLogger.Error(err, "Failed to extract default subnet information from URL", "defaultSubnetURL", z.defaultSubnetURL) + return "", "", err + } + if subnet != defaultSubnet { + return "", "", ErrNodeNotInDefaultSubnet + } + } + return zone, subnet, nil } // ListNodes returns a list of nodes that satisfy the given node filtering mode. @@ -291,22 +298,11 @@ func (z *ZoneGetter) IsDefaultSubnetNode(nodeName string, logger klog.Logger) (b // existing nodes, they will not have label and can only be in the default // subnet. func isNodeInDefaultSubnet(node *api_v1.Node, defaultSubnetURL string, nodeLogger klog.Logger) (bool, error) { - if node.Spec.PodCIDR == "" { - nodeLogger.Info("Node does not have PodCIDR set") - return false, ErrNodePodCIDRNotSet - } - - nodeSubnet, exist := node.Labels[utils.LabelNodeSubnet] - if !exist { - nodeLogger.Info("Node does not have subnet label key, assumed to be in the default subnet", "subnet label key", utils.LabelNodeSubnet) - return true, nil - } - if nodeSubnet == "" { - nodeLogger.Info("Node has empty value for subnet label key, assumed to be in the default subnet", "subnet label key", utils.LabelNodeSubnet) - return true, nil + nodeSubnet, err := getSubnet(node, defaultSubnetURL) + if err != nil { + nodeLogger.Error(err, "Failed to get node subnet", "nodeName", node.Name) + return false, err } - nodeLogger.Info("Node has an non-empty value for subnet label key", "subnet label key", utils.LabelNodeSubnet, "subnet label value", nodeSubnet) - defaultSubnet, err := utils.KeyName(defaultSubnetURL) if err != nil { nodeLogger.Error(err, "Failed to extract default subnet information from URL", "defaultSubnetURL", defaultSubnetURL) @@ -331,6 +327,26 @@ func getZone(node *api_v1.Node) (string, error) { return matches[2], nil } +// getSubnet gets subnet information from node's LabelNodeSubnet. +// If a node doesn't have this label, or the label value is empty, it means +// this node is in the defaultSubnet and we will parse the subnet from the +// defaultSubnetURL. +func getSubnet(node *api_v1.Node, defaultSubnetURL string) (string, error) { + if node.Spec.PodCIDR == "" { + return "", ErrNodePodCIDRNotSet + } + + nodeSubnet, exist := node.Labels[utils.LabelNodeSubnet] + if exist && nodeSubnet != "" { + return nodeSubnet, nil + } + defaultSubnet, err := utils.KeyName(defaultSubnetURL) + if err != nil { + return "", err + } + return defaultSubnet, nil +} + // NewNonGCPZoneGetter initialize a ZoneGetter in Non-GCP mode. func NewNonGCPZoneGetter(zone string) *ZoneGetter { return &ZoneGetter{ @@ -344,7 +360,7 @@ func NewZoneGetter(nodeInformer cache.SharedIndexInformer, defaultSubnetURL stri return &ZoneGetter{ mode: GCP, nodeLister: nodeInformer.GetIndexer(), - onlyIncludeDefaultSubnetNodes: flags.F.EnableMultiSubnetCluster, + onlyIncludeDefaultSubnetNodes: flags.F.EnableMultiSubnetCluster && !flags.F.EnableMultiSubnetClusterPhase1, defaultSubnetURL: defaultSubnetURL, } } diff --git a/pkg/utils/zonegetter/zone_getter_test.go b/pkg/utils/zonegetter/zone_getter_test.go index 4a6edd5867..0c9471a2f3 100644 --- a/pkg/utils/zonegetter/zone_getter_test.go +++ b/pkg/utils/zonegetter/zone_getter_test.go @@ -26,6 +26,7 @@ import ( api_v1 "k8s.io/api/core/v1" apiv1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/ingress-gce/pkg/flags" "k8s.io/ingress-gce/pkg/utils" "k8s.io/klog/v2" ) @@ -199,62 +200,80 @@ func TestListNodesMultipleSubnets(t *testing.T) { } } -func TestZoneForNode(t *testing.T) { - t.Parallel() - +func TestZoneAndSubnetForNode(t *testing.T) { nodeInformer := FakeNodeInformer() PopulateFakeNodeInformer(nodeInformer, false) zoneGetter := NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) testCases := []struct { - desc string - nodeName string - expectZone string - expectErr error + desc string + nodeName string + expectZone string + expectedSubnet string + expectErr error }{ { - desc: "Node not found", - nodeName: "fooNode", - expectZone: "", - expectErr: ErrNodeNotFound, + desc: "Node not found", + nodeName: "fooNode", + expectZone: "", + expectedSubnet: "", + expectErr: ErrNodeNotFound, }, { - desc: "Node with valid provider ID", - nodeName: "instance1", - expectZone: "zone1", - expectErr: nil, + desc: "Node with valid provider ID", + nodeName: "instance1", + expectZone: "zone1", + expectedSubnet: defaultTestSubnet, + expectErr: nil, }, { - desc: "Node with invalid provider ID", - nodeName: "instance-invalid-providerID", - expectZone: "", - expectErr: ErrSplitProviderID, + desc: "Node with invalid provider ID", + nodeName: "instance-invalid-providerID", + expectZone: "", + expectedSubnet: "", + expectErr: ErrSplitProviderID, }, { - desc: "Node with no provider ID", - nodeName: "instance-empty-providerID", - expectZone: "", - expectErr: ErrProviderIDNotFound, + desc: "Node with no provider ID", + nodeName: "instance-empty-providerID", + expectZone: "", + expectedSubnet: "", + expectErr: ErrProviderIDNotFound, }, { - desc: "Node with empty zone in providerID", - nodeName: "instance-empty-zone-providerID", - expectZone: "", - expectErr: ErrSplitProviderID, + desc: "Node with empty zone in providerID", + nodeName: "instance-empty-zone-providerID", + expectZone: "", + expectedSubnet: "", + expectErr: ErrSplitProviderID, }, } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { for _, enableMultiSubnetCluster := range []bool{true, false} { zoneGetter.onlyIncludeDefaultSubnetNodes = enableMultiSubnetCluster - zone, err := zoneGetter.ZoneForNode(tc.nodeName, klog.TODO()) + zone, _, err := zoneGetter.ZoneAndSubnetForNode(tc.nodeName, klog.TODO()) if zone != tc.expectZone { t.Errorf("For test case %q with onlyIncludeDefaultSubnetNodes = %v , got zone: %s, want: %s,", tc.desc, enableMultiSubnetCluster, zone, tc.expectZone) } + if !errors.Is(err, tc.expectErr) { t.Errorf("For test case %q with onlyIncludeDefaultSubnetNodes = %v, got error: %s, want: %s,", tc.desc, enableMultiSubnetCluster, err, tc.expectErr) } + } + for _, enableMultiSubnetClusterPhase1 := range []bool{true, false} { + flags.F.EnableMultiSubnetClusterPhase1 = enableMultiSubnetClusterPhase1 + zone, subnet, err := zoneGetter.ZoneAndSubnetForNode(tc.nodeName, klog.TODO()) + if zone != tc.expectZone { + t.Errorf("For test case %q with EnableMultiSubnetClusterPhase1 = %v , got zone: %s, want: %s,", tc.desc, enableMultiSubnetClusterPhase1, zone, tc.expectZone) + } + if enableMultiSubnetClusterPhase1 && subnet != tc.expectedSubnet { + t.Errorf("For test case %q with EnableMultiSubnetClusterPhase1 = %v , got subnet: %s, want: %s,", tc.desc, enableMultiSubnetClusterPhase1, subnet, tc.expectedSubnet) + } + if !errors.Is(err, tc.expectErr) { + t.Errorf("For test case %q with EnableMultiSubnetClusterPhase1 = %v, got error: %s, want: %s,", tc.desc, enableMultiSubnetClusterPhase1, err, tc.expectErr) + } } }) } @@ -300,7 +319,7 @@ func TestZoneForNodeMultipleSubnets(t *testing.T) { } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - zone, err := zoneGetter.ZoneForNode(tc.nodeName, klog.TODO()) + zone, _, err := zoneGetter.ZoneAndSubnetForNode(tc.nodeName, klog.TODO()) if zone != tc.expectZone { t.Errorf("For test case %q with multi-subnet cluster enabled, got zone: %s, want: %s,", tc.desc, zone, tc.expectZone) } @@ -369,8 +388,76 @@ func TestGetZone(t *testing.T) { } } +func TestGetSubnet(t *testing.T) { + for _, tc := range []struct { + desc string + node apiv1.Node + expectSubnet string + expectErr error + }{ + { + desc: "Node with subnet label value as defaultSubnet", + node: apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + utils.LabelNodeSubnet: defaultTestSubnet, + }, + }, + Spec: apiv1.NodeSpec{ + PodCIDR: "10.100.1.0/24", + PodCIDRs: []string{"10.100.1.0/24"}, + }, + }, + expectSubnet: defaultTestSubnet, + expectErr: nil, + }, + { + desc: "Node with subnet label value as empty string", + node: apiv1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + utils.LabelNodeSubnet: "", + }, + }, + Spec: apiv1.NodeSpec{ + PodCIDR: "10.100.1.0/24", + PodCIDRs: []string{"10.100.1.0/24"}, + }, + }, + expectSubnet: defaultTestSubnet, + expectErr: nil, + }, + { + desc: "Node without subnet label", + node: apiv1.Node{ + Spec: apiv1.NodeSpec{ + PodCIDR: "10.100.1.0/24", + PodCIDRs: []string{"10.100.1.0/24"}, + }, + }, + expectSubnet: defaultTestSubnet, + expectErr: nil, + }, + { + desc: "Node without PodCIDR", + node: apiv1.Node{}, + expectSubnet: "", + expectErr: ErrNodePodCIDRNotSet, + }, + } { + subnet, err := getSubnet(&tc.node, defaultTestSubnetURL) + if subnet != tc.expectSubnet { + t.Errorf("For test case %q, got subnet: %s, want: %s,", tc.desc, subnet, tc.expectSubnet) + } + if !errors.Is(err, tc.expectErr) { + t.Errorf("For test case %q, got error: %s, want: %s,", tc.desc, err, tc.expectErr) + } + } +} + func TestNonGCPZoneGetter(t *testing.T) { zone := "foo" + subnet := "" zoneGetter := NewNonGCPZoneGetter(zone) ret, err := zoneGetter.ListZones(AllNodesFilter, klog.TODO()) if err != nil { @@ -382,7 +469,7 @@ func TestNonGCPZoneGetter(t *testing.T) { } validateGetZoneForNode := func(node string) { - retZone, err := zoneGetter.ZoneForNode(node, klog.TODO()) + retZone, retSubnet, err := zoneGetter.ZoneAndSubnetForNode(node, klog.TODO()) if err != nil { t.Errorf("expect err = nil, but got %v", err) } @@ -390,6 +477,10 @@ func TestNonGCPZoneGetter(t *testing.T) { if retZone != zone { t.Errorf("expect zone = %q, but got %q", zone, retZone) } + + if retSubnet != subnet { + t.Errorf("expect subnet = %q, but got %q", subnet, retSubnet) + } } validateGetZoneForNode("foo-node") validateGetZoneForNode("bar-node")