diff --git a/pkg/instancegroups/manager.go b/pkg/instancegroups/manager.go index 7576dfca93..92652ddd6e 100644 --- a/pkg/instancegroups/manager.go +++ b/pkg/instancegroups/manager.go @@ -240,7 +240,7 @@ func (m *manager) List(logger klog.Logger) ([]string, error) { func (m *manager) splitNodesByZone(names []string, logger klog.Logger) map[string][]string { nodesByZone := map[string][]string{} for _, name := range names { - zone, err := m.ZoneGetter.ZoneForNode(name, logger) + zone, _, err := m.ZoneGetter.ZoneAndSubnetForNode(name, logger) if err != nil { logger.Error(err, "Failed to get zones for instance node, skipping", "name", name) continue diff --git a/pkg/l4lb/l4netlbcontroller_test.go b/pkg/l4lb/l4netlbcontroller_test.go index 795862a2db..c0e09337c3 100644 --- a/pkg/l4lb/l4netlbcontroller_test.go +++ b/pkg/l4lb/l4netlbcontroller_test.go @@ -74,6 +74,8 @@ const ( shortSessionAffinityIdleTimeout = int32(20) // 20 sec could be used for regular Session Affinity longSessionAffinityIdleTimeout = int32(2 * 60) // 2 min or 120 sec for Strong Session Affinity + + defaultTestSubnetURL = "https://www.googleapis.com/compute/v1/projects/proj/regions/us-central1/subnetworks/default" ) var ( @@ -323,8 +325,13 @@ func buildContext(vals gce.TestClusterValues) *ingctx.ControllerContext { } func newL4NetLBServiceController() *L4NetLBController { - stopCh := make(chan struct{}) vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + return createL4NetLBServiceController(vals) +} + +func createL4NetLBServiceController(vals gce.TestClusterValues) *L4NetLBController { + stopCh := make(chan struct{}) ctx := buildContext(vals) nodes, err := test.CreateAndInsertNodes(ctx.Cloud, []string{"instance-1", "instance-2"}, vals.ZoneName) if err != nil { @@ -1663,7 +1670,7 @@ func TestDualStackServiceNeedsUpdate(t *testing.T) { t.Run(tc.desc, func(t *testing.T) { t.Parallel() - controller := newL4NetLBServiceController() + controller := createL4NetLBServiceController(gce.DefaultTestClusterValues()) controller.enableDualStack = true oldSvc := test.NewL4NetLBRBSService(8080) oldSvc.Spec.IPFamilies = tc.initialIPFamilies @@ -1862,7 +1869,7 @@ func TestCreateDeleteDualStackNetLBService(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - controller := newL4NetLBServiceController() + controller := createL4NetLBServiceController(gce.DefaultTestClusterValues()) controller.enableDualStack = true svc := test.NewL4NetLBRBSService(8080) svc.Spec.IPFamilies = tc.ipFamilies @@ -1907,7 +1914,7 @@ func TestCreateDeleteDualStackNetLBService(t *testing.T) { } func TestProcessDualStackNetLBServiceOnUserError(t *testing.T) { t.Parallel() - controller := newL4NetLBServiceController() + controller := createL4NetLBServiceController(gce.DefaultTestClusterValues()) controller.enableDualStack = true svc := test.NewL4NetLBRBSService(8080) svc.Spec.IPFamilies = []v1.IPFamily{v1.IPv6Protocol, v1.IPv4Protocol} diff --git a/pkg/neg/manager_test.go b/pkg/neg/manager_test.go index e8f28079ca..bd71a484cf 100644 --- a/pkg/neg/manager_test.go +++ b/pkg/neg/manager_test.go @@ -81,6 +81,7 @@ const ( labelValue2 = "v2" negName1 = "neg1" + defaultTestSubnet = "default" defaultTestSubnetURL = "https://www.googleapis.com/compute/v1/projects/proj/regions/us-central1/subnetworks/default" ) diff --git a/pkg/neg/syncers/dualstack/migrator_test.go b/pkg/neg/syncers/dualstack/migrator_test.go index 6e43d91574..fb9296c836 100644 --- a/pkg/neg/syncers/dualstack/migrator_test.go +++ b/pkg/neg/syncers/dualstack/migrator_test.go @@ -15,6 +15,8 @@ import ( clocktesting "k8s.io/utils/clock/testing" ) +const defaultTestSubnet = "default" + func TestFilter(t *testing.T) { testCases := []struct { desc string @@ -34,29 +36,29 @@ func TestFilter(t *testing.T) { return m }(), addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), }, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "D"}, }...), }, wantAddEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b"}, }...), }, wantRemoveEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ // Migration-endpoints were filtered out but no new migration // detachment was started. {IP: "c", IPv6: "C"}, @@ -67,29 +69,29 @@ func TestFilter(t *testing.T) { desc: "unpaused migrator should filter migration endpoints AND also start detachment", migrator: newMigratorForTest(t, true), addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), }, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "D"}, }...), }, wantAddEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b"}, }...), }, wantRemoveEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // Migration detachment started. {IP: "c", IPv6: "C"}, }...), @@ -100,30 +102,30 @@ func TestFilter(t *testing.T) { desc: "migrator should do nothing if enableDualStack is false", migrator: newMigratorForTest(t, false), addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), }, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "D"}, }...), }, wantAddEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, }...), }, wantRemoveEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "c", IPv6: "C"}, }...), @@ -297,16 +299,16 @@ func TestFilter_FunctionalTest(t *testing.T) { removeEndpoints := make(map[types.EndpointGroupInfo]types.NetworkEndpointSet) // Contains single-stack endpoints which are to be removed from the NEG. committedEndpoints := make(map[types.EndpointGroupInfo]types.NetworkEndpointSet) // Initially empty. for i := 0; i < tc.initialNEGEndpointsCount; i++ { - zone := types.EndpointGroupInfo{Zone: fmt.Sprintf("zone-%v", i%tc.zonesCount)} + epGroupInfo := types.EndpointGroupInfo{Zone: fmt.Sprintf("zone-%v", i%tc.zonesCount), Subnet: defaultTestSubnet} ipv4 := fmt.Sprintf("ipv4-%v", 2*i+1) ipv6 := fmt.Sprintf("ipv6-%v", 2*i+2) - if addEndpoints[zone] == nil { - addEndpoints[zone] = types.NewNetworkEndpointSet() - removeEndpoints[zone] = types.NewNetworkEndpointSet() - committedEndpoints[zone] = types.NewNetworkEndpointSet() + if addEndpoints[epGroupInfo] == nil { + addEndpoints[epGroupInfo] = types.NewNetworkEndpointSet() + removeEndpoints[epGroupInfo] = types.NewNetworkEndpointSet() + committedEndpoints[epGroupInfo] = types.NewNetworkEndpointSet() } - addEndpoints[zone].Insert(types.NetworkEndpoint{IP: ipv4, IPv6: ipv6}) - removeEndpoints[zone].Insert(types.NetworkEndpoint{IP: ipv4}) + addEndpoints[epGroupInfo].Insert(types.NetworkEndpoint{IP: ipv4, IPv6: ipv6}) + removeEndpoints[epGroupInfo].Insert(types.NetworkEndpoint{IP: ipv4}) } tc.migrator.errorStateChecker.(*fakeErrorStateChecker).errorState = tc.errorState @@ -351,7 +353,7 @@ func TestFilter_FunctionalTest(t *testing.T) { func TestPause(t *testing.T) { addEndpoints := map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, {IP: "c", IPv6: "C"}, @@ -359,7 +361,7 @@ func TestPause(t *testing.T) { }...), } removeEndpoints := map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "e", IPv6: "E"}, {IP: "d", IPv6: "D"}, // migrating @@ -377,7 +379,7 @@ func TestPause(t *testing.T) { {IP: "a"}, // migrating {IP: "d", IPv6: "D"}, // migrating } - if !clonedRemoveEndpoints[types.EndpointGroupInfo{Zone: "zone1"}].HasAny(possibleMigrationDetachments...) { + if !clonedRemoveEndpoints[types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}].HasAny(possibleMigrationDetachments...) { t.Fatalf("Precondition to verify the behaviour of Pause() not satisfied; Filter() should start migration-detachments; got removeEndpoints=%+v; want non-empty union with %+v", clonedRemoveEndpoints, possibleMigrationDetachments) } @@ -393,7 +395,7 @@ func TestPause(t *testing.T) { // any migration-detachments when paused. migrator.Filter(addEndpoints, removeEndpoints, map[types.EndpointGroupInfo]types.NetworkEndpointSet{}) wantRemoveEndpoints := map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - types.EndpointGroupInfo{Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ // Since we don't expect any migration-detachments, this set should be // missing all migration endpoints. {IP: "e", IPv6: "E"}, @@ -535,12 +537,12 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, {IP: "7"}, {IP: "8"}, {IP: "9"}, {IP: "10"}, }...), }, @@ -552,12 +554,12 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, {IP: "7"}, {IP: "8"}, {IP: "9"}, {IP: "10"}, {IP: "11"}, }...), @@ -571,18 +573,18 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { // detachments since we wait for the pending attaches to complete desc: "many endpoints are waiting to be attached AND previous migration was quite recent", addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, }...), }, migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "7"}, }...), }, @@ -599,18 +601,18 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { // start any new detachments since we wait to get out of error state. desc: "many endpoints are waiting to be attached AND previous migration was too long ago BUT in error state", addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, }...), }, migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "7"}, }...), }, @@ -628,18 +630,18 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { // with the detachments. desc: "many endpoints are waiting to be attached BUT previous migration was too long ago AND not in error state", addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, {IP: "5"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "6"}, }...), }, migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "7"}, }...), }, @@ -649,17 +651,17 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { { desc: "no detachments started since nothing to migrate", addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "2"}, }...), }, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "3"}, {IP: "4"}, {IP: "5"}, {IP: "6"}, }...), }, @@ -675,18 +677,18 @@ func TestCalculateMigrationEndpointsToDetach(t *testing.T) { addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{}, committedEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "1"}, {IP: "2"}, {IP: "3"}, {IP: "4"}, }...), - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "5"}, {IP: "6"}, {IP: "7"}, {IP: "8"}, }...), }, migrationEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "9"}, {IP: "10"}, }...), - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "11"}, {IP: "12"}, }...), }, @@ -752,41 +754,41 @@ func TestFindAndFilterMigrationEndpoints(t *testing.T) { { name: "detect multiple migrating endpoints", addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, // migrating {IP: "b"}, {IP: "c", IPv6: "C"}, {IP: "d"}, // migrating }...), - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "e", IPv6: "E"}, // migrating }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, // migrating {IP: "f", IPv6: "F"}, {IP: "d", IPv6: "D"}, // migrating }...), - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "E"}, // migrating }...), }, wantMigrationEndpointsInAddSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "d"}, }...), - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "e", IPv6: "E"}, }...), }, wantMigrationEndpointsInRemoveSet: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a"}, {IP: "d", IPv6: "D"}, }...), - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IPv6: "E"}, }...), }, @@ -794,12 +796,12 @@ func TestFindAndFilterMigrationEndpoints(t *testing.T) { { name: "partial IP change without stack change is not considered migrating", addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", Port: "B"}, }...), }, @@ -809,13 +811,13 @@ func TestFindAndFilterMigrationEndpoints(t *testing.T) { { name: "difference in port or node is not considered migrating", addEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A", Port: "80"}, {IP: "b", Node: "node2"}, }...), }, removeEndpoints: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", Port: "81"}, {IP: "b", IPv6: "B", Node: "node1"}, }...), @@ -854,114 +856,114 @@ func TestMoveEndpoint(t *testing.T) { name: "completely valid input, shoud successfully move", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "b", IPv6: "B"}, }...), }, inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b", IPv6: "B"}, }...), }, wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "c", IPv6: "C"}, }...), }, - endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone1"}, + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}, wantSuccess: true, }, { name: "zone does not exist in source", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), }, inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone3"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), }, wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone3"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3"}, + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3", Subnet: defaultTestSubnet}, }, { name: "zone does not exist in destination, shoud successfully move", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, {IP: "b", IPv6: "B"}, }...), }, inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "b", IPv6: "B"}, }...), }, wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone1"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone1", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "a", IPv6: "A"}, }...), - {Zone: "zone2"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone2", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone1"}, + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone1", Subnet: defaultTestSubnet}, wantSuccess: true, }, { name: "source is nil", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, inputDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone3"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, wantDest: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone3"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3"}, + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3", Subnet: defaultTestSubnet}, }, { name: "destination is nil", endpoint: types.NetworkEndpoint{IP: "a", IPv6: "A"}, inputSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone3"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, wantSource: map[types.EndpointGroupInfo]types.NetworkEndpointSet{ - {Zone: "zone3"}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ + {Zone: "zone3", Subnet: defaultTestSubnet}: types.NewNetworkEndpointSet([]types.NetworkEndpoint{ {IP: "c", IPv6: "C"}, }...), }, - endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3"}, + endpointGroupInfo: types.EndpointGroupInfo{Zone: "zone3", Subnet: defaultTestSubnet}, }, } diff --git a/pkg/neg/syncers/endpoints_calculator.go b/pkg/neg/syncers/endpoints_calculator.go index 0354809fca..ae97eb625d 100644 --- a/pkg/neg/syncers/endpoints_calculator.go +++ b/pkg/neg/syncers/endpoints_calculator.go @@ -56,6 +56,7 @@ func NewLocalL4EndpointsCalculator(nodeLister listers.NodeLister, zoneGetter *zo if lbType == negtypes.L4ExternalLB { subsetSize = maxSubsetSizeNetLBLocal } + return &LocalL4EndpointsCalculator{ nodeLister: nodeLister, zoneGetter: zoneGetter, @@ -106,7 +107,7 @@ func (l *LocalL4EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsDat l.logger.Info("Node not connected to service network", "nodeName", node.Name, "network", l.networkInfo.K8sNetwork) continue } - zone, err := l.zoneGetter.ZoneForNode(node.Name, l.logger) + zone, _, err := l.zoneGetter.ZoneAndSubnetForNode(node.Name, l.logger) if err != nil { l.logger.Error(err, "Unable to find zone for node, skipping", "nodeName", node.Name) metrics.PublishNegControllerErrorCountMetrics(err, true) @@ -121,7 +122,10 @@ func (l *LocalL4EndpointsCalculator) CalculateEndpoints(eds []types.EndpointsDat } // Compute the networkEndpoints, with total endpoints count <= l.subsetSizeLimit l.logger.V(2).Info("Got zoneNodeMap as input for service", "zoneNodeMap", nodeMapToString(zoneNodeMap), "serviceID", l.svcId) + // TODO(sawsa307): Make sure to include logic for subsetting endpoints in non-default subnets. + // Currently we only select endpoints from the default subnet. subsetMap, err := getSubsetPerZone(zoneNodeMap, l.subsetSizeLimit, l.svcId, currentMap, l.logger, l.networkInfo) + return subsetMap, nil, 0, err } @@ -182,7 +186,7 @@ func (l *ClusterL4EndpointsCalculator) CalculateEndpoints(_ []types.EndpointsDat l.logger.Info("Node not connected to service network", "nodeName", node.Name, "network", l.networkInfo.K8sNetwork) continue } - zone, err := l.zoneGetter.ZoneForNode(node.Name, l.logger) + zone, _, err := l.zoneGetter.ZoneAndSubnetForNode(node.Name, l.logger) if err != nil { l.logger.Error(err, "Unable to find zone for node skipping", "nodeName", node.Name) metrics.PublishNegControllerErrorCountMetrics(err, true) @@ -192,6 +196,8 @@ func (l *ClusterL4EndpointsCalculator) CalculateEndpoints(_ []types.EndpointsDat } l.logger.V(2).Info("Got zoneNodeMap as input for service", "zoneNodeMap", nodeMapToString(zoneNodeMap), "serviceID", l.svcId) // Compute the networkEndpoints, with total endpoints <= l.subsetSizeLimit. + // TODO(sawsa307): Make sure to include logic for subsetting endpoints in non-default subnets. + // Currently we only select endpoints from the default subnet. subsetMap, err := getSubsetPerZone(zoneNodeMap, l.subsetSizeLimit, l.svcId, currentMap, l.logger, l.networkInfo) return subsetMap, nil, 0, err } diff --git a/pkg/neg/syncers/endpoints_calculator_test.go b/pkg/neg/syncers/endpoints_calculator_test.go index 8cafe31044..a9a04a971c 100644 --- a/pkg/neg/syncers/endpoints_calculator_test.go +++ b/pkg/neg/syncers/endpoints_calculator_test.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" + "k8s.io/ingress-gce/pkg/flags" "k8s.io/ingress-gce/pkg/neg/metrics/metricscollector" negtypes "k8s.io/ingress-gce/pkg/neg/types" "k8s.io/ingress-gce/pkg/network" @@ -40,11 +41,13 @@ import ( // TestLocalGetEndpointSet verifies the GetEndpointSet method implemented by the LocalL4EndpointsCalculator. // The L7 implementation is tested in TestToZoneNetworkEndpointMapUtil. func TestLocalGetEndpointSet(t *testing.T) { - t.Parallel() nodeInformer := zonegetter.FakeNodeInformer() zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) zonegetter.PopulateFakeNodeInformer(nodeInformer, false) - defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default"} + defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default", SubnetworkURL: defaultTestSubnetURL} + prevFlag := flags.F.EnableMultiSubnetCluster + defer func() { flags.F.EnableMultiSubnetCluster = prevFlag }() + flags.F.EnableMultiSubnetCluster = false testCases := []struct { desc string @@ -62,8 +65,8 @@ func TestLocalGetEndpointSet(t *testing.T) { endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // only 4 out of 6 nodes are picked since there are > 4 endpoints, but they are found only on 4 nodes. wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -74,8 +77,8 @@ func TestLocalGetEndpointSet(t *testing.T) { endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // only 4 out of 6 nodes are picked since there are > 4 endpoints, but they are found only on 4 nodes. wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -96,8 +99,8 @@ func TestLocalGetEndpointSet(t *testing.T) { nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, // only 2 out of 6 nodes are picked since there are > 4 endpoints, but they are found only on 4 nodes. 2 out of those 4 are non-candidates. wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}), }, networkEndpointType: negtypes.VmIpEndpointType, network: defaultNetwork, @@ -114,7 +117,7 @@ func TestLocalGetEndpointSet(t *testing.T) { { desc: "multinetwork, endpoints only for nodes connected to a matching non-default network", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), - network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other"}, + network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other", SubnetworkURL: defaultTestSubnetURL}, nodeAnnotationsMap: map[string]map[string]string{ testInstance1: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.1")}, testInstance2: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.2")}, @@ -124,10 +127,10 @@ func TestLocalGetEndpointSet(t *testing.T) { nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, // only 3 out of 6 nodes are picked because only 3 have multi-nic annotation with a matching network name wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "20.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "20.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "20.2.3.3", Node: testInstance3}), }, }, @@ -170,11 +173,14 @@ func nodeInterfacesAnnotation(t *testing.T, network, ip string) string { // TestClusterGetEndpointSet verifies the GetEndpointSet method implemented by the ClusterL4EndpointsCalculator. func TestClusterGetEndpointSet(t *testing.T) { - t.Parallel() nodeInformer := zonegetter.FakeNodeInformer() zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) zonegetter.PopulateFakeNodeInformer(nodeInformer, false) - defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default"} + defaultNetwork := network.NetworkInfo{IsDefault: true, K8sNetwork: "default", SubnetworkURL: defaultTestSubnetURL} + prevFlag := flags.F.EnableMultiSubnetCluster + defer func() { flags.F.EnableMultiSubnetCluster = prevFlag }() + flags.F.EnableMultiSubnetCluster = false + testCases := []struct { desc string endpointsData []negtypes.EndpointsData @@ -191,10 +197,10 @@ func TestClusterGetEndpointSet(t *testing.T) { endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // all nodes are picked since, in this mode, endpoints running do not need to run on the selected node. wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - {Zone: negtypes.TestZone3}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -205,10 +211,10 @@ func TestClusterGetEndpointSet(t *testing.T) { endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // all nodes are picked since, in this mode, endpoints running do not need to run on the selected node. wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - {Zone: negtypes.TestZone3}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -222,10 +228,10 @@ func TestClusterGetEndpointSet(t *testing.T) { endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), // all valid candidate nodes are picked since, in this mode, endpoints running do not need to run on the selected node. wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - {Zone: negtypes.TestZone3}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -243,10 +249,10 @@ func TestClusterGetEndpointSet(t *testing.T) { // Even when there are no service endpoints, nodes are selected at random. endpointsData: []negtypes.EndpointsData{}, wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "1.2.3.4", Node: testInstance4}, negtypes.NetworkEndpoint{IP: "1.2.3.5", Node: testInstance5}, negtypes.NetworkEndpoint{IP: "1.2.3.6", Node: testInstance6}), - {Zone: negtypes.TestZone3}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "1.2.3.7", Node: testUnreadyInstance1}, negtypes.NetworkEndpoint{IP: "1.2.3.8", Node: testUnreadyInstance2}), }, networkEndpointType: negtypes.VmIpEndpointType, nodeNames: []string{testInstance1, testInstance2, testInstance3, testInstance4, testInstance5, testInstance6}, @@ -256,10 +262,10 @@ func TestClusterGetEndpointSet(t *testing.T) { desc: "multinetwork endpoints, only for nodes connected to the specified network", endpointsData: negtypes.EndpointsDataFromEndpointSlices(getDefaultEndpointSlices()), wantEndpointSets: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "20.2.3.2", Node: testInstance2}), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "20.2.3.6", Node: testInstance6}), + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.1", Node: testInstance1}, negtypes.NetworkEndpoint{IP: "20.2.3.2", Node: testInstance2}), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(negtypes.NetworkEndpoint{IP: "20.2.3.3", Node: testInstance3}, negtypes.NetworkEndpoint{IP: "20.2.3.6", Node: testInstance6}), }, - network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other"}, + network: network.NetworkInfo{IsDefault: false, K8sNetwork: "other", SubnetworkURL: defaultTestSubnetURL}, nodeAnnotationsMap: map[string]map[string]string{ testInstance1: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.1")}, testInstance2: {networkv1.NorthInterfacesAnnotationKey: nodeInterfacesAnnotation(t, "other", "20.2.3.2")}, @@ -292,7 +298,6 @@ func TestClusterGetEndpointSet(t *testing.T) { } func TestValidateEndpoints(t *testing.T) { - t.Parallel() testPortName := "" emptyNamedPort := "" protocolTCP := v1.ProtocolTCP @@ -352,8 +357,8 @@ func TestValidateEndpoints(t *testing.T) { zoneGetterMSC := zonegetter.NewFakeZoneGetter(testContext.NodeInformer, defaultTestSubnetURL, true) L7EndpointsCalculatorMSC := NewL7EndpointsCalculator(zoneGetterMSC, podLister, nodeLister, serviceLister, svcPort, klog.TODO(), testContext.EnableDualStackNEG, metricscollector.FakeSyncerMetrics()) L7EndpointsCalculatorMSC.enableMultiSubnetCluster = true - L4LocalEndpointCalculator := NewLocalL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{}, negtypes.L4InternalLB) - L4ClusterEndpointCalculator := NewClusterL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{}, negtypes.L4InternalLB) + L4LocalEndpointCalculator := NewLocalL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{SubnetworkURL: defaultTestSubnetURL}, negtypes.L4InternalLB) + L4ClusterEndpointCalculator := NewClusterL4EndpointsCalculator(listers.NewNodeLister(nodeLister), zoneGetter, fmt.Sprintf("%s/%s", testServiceName, testServiceNamespace), klog.TODO(), &network.NetworkInfo{SubnetworkURL: defaultTestSubnetURL}, negtypes.L4InternalLB) l7TestEPS := []*discovery.EndpointSlice{ { @@ -396,6 +401,10 @@ func TestValidateEndpoints(t *testing.T) { return ed, podMap } + prevFlag := flags.F.EnableMultiSubnetCluster + defer func() { flags.F.EnableMultiSubnetCluster = prevFlag }() + flags.F.EnableMultiSubnetCluster = true + testCases := []struct { desc string ec negtypes.NetworkEndpointsCalculator diff --git a/pkg/neg/syncers/subsets.go b/pkg/neg/syncers/subsets.go index 3a6d52bb2b..117b560917 100644 --- a/pkg/neg/syncers/subsets.go +++ b/pkg/neg/syncers/subsets.go @@ -173,13 +173,21 @@ func getSubsetPerZone(nodesPerZone map[string][]*v1.Node, totalLimit int, svcID // Sort zones in increasing order of node count. zoneList := sortZones(nodesPerZone) + defaultSubnet, err := utils.KeyName(networkInfo.SubnetworkURL) + if err != nil { + logger.Error(err, "Errored getting default subnet from NetworkInfo") + return nil, err + } + for _, zone := range zoneList { // split the limit across the leftover zones. subsetSize = totalLimit / zonesRemaining logger.Info("Picking subset for a zone", "subsetSize", subsetSize, "zone", zone, "svcID", svcID) - result[negtypes.EndpointGroupInfo{Zone: zone.Name}] = negtypes.NewNetworkEndpointSet() + // TODO(sawsa307): Make sure to include logic for subsetting endpoints in non-default subnets. + // Currently we only select endpoints from the default subnet. + result[negtypes.EndpointGroupInfo{Zone: zone.Name, Subnet: defaultSubnet}] = negtypes.NewNetworkEndpointSet() if currentMap != nil { - if zset, ok := currentMap[negtypes.EndpointGroupInfo{Zone: zone.Name}]; ok && zset != nil { + if zset, ok := currentMap[negtypes.EndpointGroupInfo{Zone: zone.Name, Subnet: defaultSubnet}]; ok && zset != nil { currentList = zset.List() } else { currentList = nil @@ -193,7 +201,7 @@ func getSubsetPerZone(nodesPerZone map[string][]*v1.Node, totalLimit int, svcID } else { ip = utils.GetNodePrimaryIP(node, logger) } - result[negtypes.EndpointGroupInfo{Zone: zone.Name}].Insert(negtypes.NetworkEndpoint{Node: node.Name, IP: ip}) + result[negtypes.EndpointGroupInfo{Zone: zone.Name, Subnet: defaultSubnet}].Insert(negtypes.NetworkEndpoint{Node: node.Name, IP: ip}) } totalLimit -= len(subset) zonesRemaining-- diff --git a/pkg/neg/syncers/subsets_test.go b/pkg/neg/syncers/subsets_test.go index e1a56bdb20..8af833f1c2 100644 --- a/pkg/neg/syncers/subsets_test.go +++ b/pkg/neg/syncers/subsets_test.go @@ -192,7 +192,7 @@ func TestUnevenNodesInZones(t *testing.T) { }, } for _, tc := range testCases { - subsetMap, err := getSubsetPerZone(tc.nodesMap, tc.subsetLimit, tc.svcKey, nil, klog.TODO(), &network.NetworkInfo{}) + subsetMap, err := getSubsetPerZone(tc.nodesMap, tc.subsetLimit, tc.svcKey, nil, klog.TODO(), &network.NetworkInfo{SubnetworkURL: defaultTestSubnetURL}) if err != nil { t.Errorf("Failed to get subset for test '%s', err %v", tc.description, err) } @@ -234,11 +234,14 @@ func TestGetSubsetPerZoneMultinetwork(t *testing.T) { "zone3": {makeNodeWithNetwork(t, "n3_1", map[string]string{"net1": "172.168.3.1", "net2": "192.168.3.1"})}, }, svcKey: "svc123", + networkInfo: network.NetworkInfo{ + SubnetworkURL: defaultTestSubnetURL, + }, // empty IPs since test can't get the primary IP expectedNodesMap: map[negtypes.EndpointGroupInfo]map[string]string{ - {Zone: "zone1"}: {"n1_1": "", "n1_2": ""}, - {Zone: "zone2"}: {"n2_1": "", "n2_2": ""}, - {Zone: "zone3"}: {"n3_1": ""}, + {Zone: "zone1", Subnet: defaultTestSubnet}: {"n1_1": "", "n1_2": ""}, + {Zone: "zone2", Subnet: defaultTestSubnet}: {"n2_1": "", "n2_2": ""}, + {Zone: "zone3", Subnet: defaultTestSubnet}: {"n3_1": ""}, }, }, { @@ -250,13 +253,14 @@ func TestGetSubsetPerZoneMultinetwork(t *testing.T) { }, svcKey: "svc123", networkInfo: network.NetworkInfo{ - IsDefault: false, - K8sNetwork: "net1", + IsDefault: false, + K8sNetwork: "net1", + SubnetworkURL: defaultTestSubnetURL, }, expectedNodesMap: map[negtypes.EndpointGroupInfo]map[string]string{ - {Zone: "zone1"}: {"n1_1": "172.168.1.1", "n1_2": "172.168.1.2"}, - {Zone: "zone2"}: {"n2_1": "172.168.2.1", "n2_2": "172.168.2.2"}, - {Zone: "zone3"}: {"n3_1": "172.168.3.1"}, + {Zone: "zone1", Subnet: defaultTestSubnet}: {"n1_1": "172.168.1.1", "n1_2": "172.168.1.2"}, + {Zone: "zone2", Subnet: defaultTestSubnet}: {"n2_1": "172.168.2.1", "n2_2": "172.168.2.2"}, + {Zone: "zone3", Subnet: defaultTestSubnet}: {"n3_1": "172.168.3.1"}, }, }, } diff --git a/pkg/neg/syncers/transaction.go b/pkg/neg/syncers/transaction.go index 56c1fafacd..4212404b29 100644 --- a/pkg/neg/syncers/transaction.go +++ b/pkg/neg/syncers/transaction.go @@ -259,7 +259,13 @@ func (s *transactionSyncer) syncInternalImpl() error { } s.logger.V(2).Info("Sync NEG", "negSyncerKey", s.NegSyncerKey.String(), "endpointsCalculatorMode", s.endpointsCalculator.Mode()) - currentMap, currentPodLabelMap, err := retrieveExistingZoneNetworkEndpointMap(s.NegSyncerKey.NegName, s.zoneGetter, s.cloud, s.NegSyncerKey.GetAPIVersion(), s.endpointsCalculator.Mode(), s.enableDualStackNEG, s.logger) + defaultSubnet, err := utils.KeyName(s.networkInfo.SubnetworkURL) + if err != nil { + s.logger.Error(err, "Errored getting default subnet from NetworkInfo") + return err + } + + currentMap, currentPodLabelMap, err := retrieveExistingZoneNetworkEndpointMap(s.NegSyncerKey.NegName, s.zoneGetter, s.cloud, s.NegSyncerKey.GetAPIVersion(), s.endpointsCalculator.Mode(), s.enableDualStackNEG, defaultSubnet, s.logger) if err != nil { return fmt.Errorf("%w: %w", negtypes.ErrCurrentNegEPNotFound, err) } @@ -497,6 +503,8 @@ func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints m } if operation == attachOp { + // TODO(sawsa307): Pass in subnet to help distinguish which NEGs needs + // update(in default/non-default subnets). go s.attachNetworkEndpoints(zone, batch) } if operation == detachOp { @@ -505,6 +513,8 @@ func (s *transactionSyncer) syncNetworkEndpoints(addEndpoints, removeEndpoints m // is already in progress. s.dsMigrator.Pause() } + // TODO(sawsa307): Pass in subnet to help distinguish which NEGs needs + // update(in default/non-default subnets). go s.detachNetworkEndpoints(zone, batch, zone == migrationZone.Zone && subnet == migrationZone.Subnet) } } diff --git a/pkg/neg/syncers/transaction_table_test.go b/pkg/neg/syncers/transaction_table_test.go index cdac426f38..81c9943c55 100644 --- a/pkg/neg/syncers/transaction_table_test.go +++ b/pkg/neg/syncers/transaction_table_test.go @@ -50,7 +50,7 @@ func TestTransactionTable(t *testing.T) { entry := transactionEntry{ attachOp, fmt.Sprintf("%s%d", zonePrefix, i), - "", + defaultTestSubnet, } table.Put(key, entry) testKeyMap[key] = entry @@ -64,7 +64,7 @@ func TestTransactionTable(t *testing.T) { newEntry := transactionEntry{ detachOp, fmt.Sprintf("%s%d", zonePrefix, i), - "", + defaultTestSubnet, } table.Put(key, newEntry) testKeyMap[key] = newEntry diff --git a/pkg/neg/syncers/transaction_test.go b/pkg/neg/syncers/transaction_test.go index 523c5201d1..0fe2fca5f3 100644 --- a/pkg/neg/syncers/transaction_test.go +++ b/pkg/neg/syncers/transaction_test.go @@ -70,12 +70,16 @@ const ( testInstance6 = "instance6" testUnreadyInstance1 = "unready-instance1" testUnreadyInstance2 = "unready-instance2" + + defaultTestSubnet = "default" ) func TestTransactionSyncNetworkEndpoints(t *testing.T) { t.Parallel() - fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + fakeGCE := gce.NewFakeGCECloud(vals) negtypes.MockNetworkEndpointAPIs(fakeGCE) fakeCloud := negtypes.NewAdapter(fakeGCE) testNegTypes := []negtypes.NetworkEndpointType{ @@ -383,7 +387,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { for _, tc := range testCases { flags.F.EnableNEGLabelPropagation = tc.labelPropagationEnabled - fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + fakeGCE := gce.NewFakeGCECloud(vals) negtypes.MockNetworkEndpointAPIs(fakeGCE) fakeCloud := negtypes.NewAdapter(fakeGCE) _, transactionSyncer := newTestTransactionSyncer(fakeCloud, tc.negType, false) @@ -415,7 +421,9 @@ func TestSyncNetworkEndpointLabel(t *testing.T) { func TestCommitTransaction(t *testing.T) { t.Parallel() - s, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(gce.DefaultTestClusterValues())), negtypes.VmIpPortEndpointType, false) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + s, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(vals)), negtypes.VmIpPortEndpointType, false) // use testSyncer to track the number of Sync got triggered testSyncer := &testSyncer{s.(*syncer), 0} testRetryer := &testRetryHandler{testSyncer, 0} @@ -829,7 +837,9 @@ func TestFilterEndpointByTransaction(t *testing.T) { func TestCommitPods(t *testing.T) { t.Parallel() - _, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(gce.DefaultTestClusterValues())), negtypes.VmIpPortEndpointType, false) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + _, transactionSyncer := newTestTransactionSyncer(negtypes.NewAdapter(gce.NewFakeGCECloud(vals)), negtypes.VmIpPortEndpointType, false) reflector := &testReflector{} transactionSyncer.reflector = reflector @@ -1036,7 +1046,8 @@ func TestCommitPods(t *testing.T) { func TestTransactionSyncerWithNegCR(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) + testSubnetwork := defaultTestSubnetURL + fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) testNegType := negtypes.VmIpPortEndpointType @@ -1332,9 +1343,11 @@ func TestTransactionSyncerWithNegCR(t *testing.T) { // zone has the expected State. func TestUpdateInitStatusWithMultiSubnetCluster(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) testNegType := negtypes.VmIpPortEndpointType flags.F.EnableMultiSubnetClusterPhase1 = true + defer func() { + flags.F.EnableMultiSubnetClusterPhase1 = false + }() // Active zones: zone1, zone2. // Inactive zones: zone3 @@ -1364,9 +1377,7 @@ func TestUpdateInitStatusWithMultiSubnetCluster(t *testing.T) { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - t.Parallel() - - fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) + fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(defaultTestSubnetURL, testNetwork) _, syncer := newTestTransactionSyncer(fakeCloud, testNegType, false) svcNegClient := syncer.svcNegClient @@ -1453,7 +1464,6 @@ func TestUpdateInitStatusWithMultiSubnetCluster(t *testing.T) { func TestUpdateStatus(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) testNegType := negtypes.VmIpPortEndpointType testNegRefs := []negv1beta1.NegObjectReference{ { @@ -1524,7 +1534,7 @@ func TestUpdateStatus(t *testing.T) { for _, syncErr := range []error{nil, fmt.Errorf("error")} { for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) + fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(defaultTestSubnetURL, testNetwork) _, syncer := newTestTransactionSyncer(fakeCloud, testNegType, false) svcNegClient := syncer.svcNegClient syncer.needInit = false @@ -1582,7 +1592,7 @@ func TestUpdateStatus(t *testing.T) { func TestIsZoneChange(t *testing.T) { testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) + testSubnetwork := defaultTestSubnetURL fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) testNegType := negtypes.VmIpPortEndpointType @@ -1722,7 +1732,7 @@ func TestUnknownNodes(t *testing.T) { zonegetter.PopulateFakeNodeInformer(nodeInformer, false) zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) testNetwork := cloud.ResourcePath("network", &meta.Key{Name: "test-network"}) - testSubnetwork := cloud.ResourcePath("subnetwork", &meta.Key{Name: "test-subnetwork"}) + testSubnetwork := defaultTestSubnetURL fakeCloud := negtypes.NewFakeNetworkEndpointGroupCloud(testSubnetwork, testNetwork) testIP1 := "10.100.1.1" @@ -1790,7 +1800,7 @@ func TestUnknownNodes(t *testing.T) { } // Check that unknown zone did not cause endpoints to be removed - out, _, err := retrieveExistingZoneNetworkEndpointMap(testNegName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, klog.TODO()) + out, _, err := retrieveExistingZoneNetworkEndpointMap(testNegName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, "", klog.TODO()) if err != nil { t.Errorf("errored retrieving existing network endpoints") } @@ -1814,11 +1824,12 @@ func TestUnknownNodes(t *testing.T) { // TestEnableDegradedMode verifies if DegradedMode has been correctly enabled for L7 endpoint calculator func TestEnableDegradedMode(t *testing.T) { - t.Parallel() nodeInformer := zonegetter.FakeNodeInformer() zonegetter.PopulateFakeNodeInformer(nodeInformer, false) zoneGetter := zonegetter.NewFakeZoneGetter(nodeInformer, defaultTestSubnetURL, false) - fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + vals := gce.DefaultTestClusterValues() + vals.SubnetworkURL = defaultTestSubnetURL + fakeGCE := gce.NewFakeGCECloud(vals) negtypes.MockNetworkEndpointAPIs(fakeGCE) fakeCloud := negtypes.NewAdapter(fakeGCE) mockGCE := fakeGCE.Compute().(*cloud.MockGCE) @@ -2087,7 +2098,7 @@ func TestEnableDegradedMode(t *testing.T) { (s.syncer.(*syncer)).stopped = false tc.modify(s) - out, _, err := retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, klog.TODO()) + out, _, err := retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, "", klog.TODO()) if err != nil { t.Errorf("errored retrieving existing network endpoints") } @@ -2100,7 +2111,7 @@ func TestEnableDegradedMode(t *testing.T) { t.Errorf("syncInternal returned %v, expected %v", err, tc.expectErr) } err = wait.PollImmediate(time.Second, 3*time.Second, func() (bool, error) { - out, _, err = retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, klog.TODO()) + out, _, err = retrieveExistingZoneNetworkEndpointMap(tc.negName, zoneGetter, fakeCloud, meta.VersionGA, negtypes.L7Mode, false, "", klog.TODO()) if err != nil { return false, nil } @@ -2457,7 +2468,7 @@ func newTestTransactionSyncer(fakeGCE negtypes.NetworkEndpointGroupCloud, negTyp testContext.SvcNegInformer.GetIndexer(), reflector, GetEndpointsCalculator(testContext.PodInformer.GetIndexer(), testContext.NodeInformer.GetIndexer(), testContext.ServiceInformer.GetIndexer(), - fakeZoneGetter, svcPort, mode, klog.TODO(), testContext.EnableDualStackNEG, metricscollector.FakeSyncerMetrics(), &network.NetworkInfo{IsDefault: true}, negtypes.L4InternalLB), + fakeZoneGetter, svcPort, mode, klog.TODO(), testContext.EnableDualStackNEG, metricscollector.FakeSyncerMetrics(), &network.NetworkInfo{IsDefault: true, SubnetworkURL: defaultTestSubnetURL}, negtypes.L4InternalLB), string(kubeSystemUID), testContext.SvcNegClient, metricscollector.FakeSyncerMetrics(), diff --git a/pkg/neg/syncers/utils.go b/pkg/neg/syncers/utils.go index ca432de8bb..d6a12cac4b 100644 --- a/pkg/neg/syncers/utils.go +++ b/pkg/neg/syncers/utils.go @@ -288,19 +288,18 @@ func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter *zonegett continue } globalEPCount[negtypes.Total] += 1 - zone, _, getZoneErr := getEndpointZone(endpointAddress, zoneGetter, logger) - if getZoneErr != nil { - metrics.PublishNegControllerErrorCountMetrics(getZoneErr, true) - if enableMultiSubnetCluster && errors.Is(getZoneErr, zonegetter.ErrNodeNotInDefaultSubnet) { - epLogger.Error(getZoneErr, "Detected endpoint not from default subnet. Skipping") + epGroupInfo, _, getEpGroupInfoErr := getEndpointZoneSubnet(endpointAddress, zoneGetter, logger) + if getEpGroupInfoErr != nil { + metrics.PublishNegControllerErrorCountMetrics(getEpGroupInfoErr, true) + if enableMultiSubnetCluster && errors.Is(getEpGroupInfoErr, zonegetter.ErrNodeNotInDefaultSubnet) { + epLogger.Error(getEpGroupInfoErr, "Detected endpoint not from default subnet. Skipping") localEPCount[negtypes.NodeInNonDefaultSubnet]++ continue } - epLogger.Error(getZoneErr, "Detected unexpected error when getting zone for endpoint") - return ZoneNetworkEndpointMapResult{}, fmt.Errorf("unexpected error when getting zone for endpoint %q in endpoint slice %s/%s: %w", endpointAddress.Addresses, ed.Meta.Namespace, ed.Meta.Name, getZoneErr) + epLogger.Error(getEpGroupInfoErr, "Detected unexpected error when getting zone for endpoint") + return ZoneNetworkEndpointMapResult{}, fmt.Errorf("unexpected error when getting zone for endpoint %q in endpoint slice %s/%s: %w", endpointAddress.Addresses, ed.Meta.Namespace, ed.Meta.Name, getEpGroupInfoErr) } - epLocation := negtypes.EndpointGroupInfo{Zone: zone} _, _, getPodErr := getEndpointPod(endpointAddress, podLister) if getPodErr != nil { metrics.PublishNegControllerErrorCountMetrics(getPodErr, true) @@ -312,8 +311,8 @@ func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter *zonegett epLogger.V(2).Info("Endpoint does not have an associated pod. Skipping") continue } - if zoneNetworkEndpointMap[epLocation] == nil { - zoneNetworkEndpointMap[epLocation] = negtypes.NewNetworkEndpointSet() + if zoneNetworkEndpointMap[epGroupInfo] == nil { + zoneNetworkEndpointMap[epGroupInfo] = negtypes.NewNetworkEndpointSet() } podIPs := ipsForPod[types.NamespacedName{Namespace: endpointAddress.TargetRef.Namespace, Name: endpointAddress.TargetRef.Name}] @@ -332,7 +331,7 @@ func toZoneNetworkEndpointMap(eds []negtypes.EndpointsData, zoneGetter *zonegett // Non-GCP network endpoints don't have associated nodes. networkEndpoint.Node = "" } - zoneNetworkEndpointMap[epLocation].Insert(networkEndpoint) + zoneNetworkEndpointMap[epGroupInfo].Insert(networkEndpoint) // if existing name is alphabetically lower than current one, continue and don't replace if existingPod, contains := networkEndpointPodMap[networkEndpoint]; contains { @@ -374,30 +373,30 @@ func mergeWithGlobalCounts(localEPCount, globalEPCount, globalEPSCount negtypes. } } -// getEndpointZone use an endpoint's nodeName to get its corresponding zone -func getEndpointZone(endpointAddress negtypes.AddressData, zoneGetter *zonegetter.ZoneGetter, logger klog.Logger) (string, negtypes.StateCountMap, error) { +// getEndpointZoneSubnet use an endpoint's nodeName to get its corresponding zone and subnet +func getEndpointZoneSubnet(endpointAddress negtypes.AddressData, zoneGetter *zonegetter.ZoneGetter, logger klog.Logger) (negtypes.EndpointGroupInfo, negtypes.StateCountMap, error) { count := make(negtypes.StateCountMap) if endpointAddress.NodeName == nil || len(*endpointAddress.NodeName) == 0 { count[negtypes.NodeMissing]++ count[negtypes.ZoneMissing]++ - return "", count, negtypes.ErrEPNodeMissing + return negtypes.EndpointGroupInfo{}, count, negtypes.ErrEPNodeMissing } - zone, err := zoneGetter.ZoneForNode(*endpointAddress.NodeName, logger) + zone, subnet, err := zoneGetter.ZoneAndSubnetForNode(*endpointAddress.NodeName, logger) // Fail to get the node object. if errors.Is(err, zonegetter.ErrNodeNotFound) { count[negtypes.NodeNotFound]++ - return zone, count, fmt.Errorf("%w: %v", negtypes.ErrEPNodeNotFound, err) + return negtypes.EndpointGroupInfo{}, count, fmt.Errorf("%w: %v", negtypes.ErrEPNodeNotFound, err) } if errors.Is(err, zonegetter.ErrNodePodCIDRNotSet) { count[negtypes.NodePodCIDRNotSet]++ - return zone, count, fmt.Errorf("%w: %w", negtypes.ErrEPNodePodCIDRNotSet, err) + return negtypes.EndpointGroupInfo{}, count, fmt.Errorf("%w: %w", negtypes.ErrEPNodePodCIDRNotSet, err) } // providerID missing in node or zone information missing in providerID. if errors.Is(err, zonegetter.ErrProviderIDNotFound) || errors.Is(err, zonegetter.ErrSplitProviderID) { count[negtypes.ZoneMissing]++ - return zone, count, fmt.Errorf("%w: zone is missing for node %v", negtypes.ErrEPZoneMissing, *endpointAddress.NodeName) + return negtypes.EndpointGroupInfo{}, count, fmt.Errorf("%w: zone is missing for node %v", negtypes.ErrEPZoneMissing, *endpointAddress.NodeName) } - return zone, count, err + return negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet}, count, err } // getEndpointPod use an endpoint's pod name and namespace to get its corresponding pod object @@ -471,21 +470,21 @@ func toZoneNetworkEndpointMapDegradedMode(eds []negtypes.EndpointsData, zoneGett localEPCount[negtypes.NodeMissing]++ continue } - zone, getZoneErr := zoneGetter.ZoneForNode(nodeName, logger) - if getZoneErr != nil { - metrics.PublishNegControllerErrorCountMetrics(getZoneErr, true) - if enableMultiSubnetCluster && errors.Is(getZoneErr, zonegetter.ErrNodeNotInDefaultSubnet) { - epLogger.Error(getZoneErr, "Detected endpoint not from default subnet. Skipping", "nodeName", nodeName) + zone, subnet, err := zoneGetter.ZoneAndSubnetForNode(nodeName, logger) + if err != nil { + metrics.PublishNegControllerErrorCountMetrics(err, true) + if enableMultiSubnetCluster && errors.Is(err, zonegetter.ErrNodeNotInDefaultSubnet) { + epLogger.Error(err, "Detected endpoint not from default subnet. Skipping", "nodeName", nodeName) localEPCount[negtypes.NodeInNonDefaultSubnet]++ continue } - epLogger.Error(getZoneErr, "Endpoint's corresponding node does not have valid zone information, skipping", "nodeName", nodeName) + epLogger.Error(err, "Endpoint's corresponding node does not have valid zone information, skipping", "nodeName", nodeName) localEPCount[negtypes.NodeNotFound]++ continue } - epLocation := negtypes.EndpointGroupInfo{Zone: zone} - if zoneNetworkEndpointMap[epLocation] == nil { - zoneNetworkEndpointMap[epLocation] = negtypes.NewNetworkEndpointSet() + epGroupInfo := negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet} + if zoneNetworkEndpointMap[epGroupInfo] == nil { + zoneNetworkEndpointMap[epGroupInfo] = negtypes.NewNetworkEndpointSet() } podIPs := ipsForPod[types.NamespacedName{Namespace: endpointAddress.TargetRef.Namespace, Name: endpointAddress.TargetRef.Name}] @@ -528,7 +527,7 @@ func toZoneNetworkEndpointMapDegradedMode(eds []negtypes.EndpointsData, zoneGett // Non-GCP network endpoints don't have associated nodes. networkEndpoint.Node = "" } - zoneNetworkEndpointMap[epLocation].Insert(networkEndpoint) + zoneNetworkEndpointMap[epGroupInfo].Insert(networkEndpoint) // if existing name is alphabetically lower than current one, continue and don't replace if existingPod, contains := networkEndpointPodMap[networkEndpoint]; contains { @@ -696,8 +695,9 @@ func podBelongsToService(pod *apiv1.Pod, service *apiv1.Service) error { return nil } -// retrieveExistingZoneNetworkEndpointMap lists existing network endpoints in the neg and return the zone and endpoints map -func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegetter.ZoneGetter, cloud negtypes.NetworkEndpointGroupCloud, version meta.Version, mode negtypes.EndpointsCalculatorMode, enableDualStackNEG bool, logger klog.Logger) (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, labels.EndpointPodLabelMap, error) { +// retrieveExistingZoneNetworkEndpointMap lists existing network endpoints in the neg and return the zone and endpoints map. +// TODO(sawsa307): Make sure to include endpoints from non-default NEGs after syncers create non-default subnet NEGs. +func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegetter.ZoneGetter, cloud negtypes.NetworkEndpointGroupCloud, version meta.Version, mode negtypes.EndpointsCalculatorMode, enableDualStackNEG bool, subnet string, logger klog.Logger) (map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet, labels.EndpointPodLabelMap, error) { // Include zones that have non-candidate nodes currently. It is possible that NEGs were created in those zones previously and the endpoints now became non-candidates. // Endpoints in those NEGs now need to be removed. This mostly applies to VM_IP_NEGs where the endpoints are nodes. zones, err := zoneGetter.ListZones(zonegetter.AllNodesFilter, logger) @@ -725,7 +725,7 @@ func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegett } return nil, nil, fmt.Errorf("Failed to lookup NEG in zone %q, candidate zones %v, err - %w", zone, candidateZonesMap, err) } - zoneNetworkEndpointMap[negtypes.EndpointGroupInfo{Zone: zone}] = negtypes.NewNetworkEndpointSet() + zoneNetworkEndpointMap[negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet}] = negtypes.NewNetworkEndpointSet() for _, ne := range networkEndpointsWithHealthStatus { newNE := negtypes.NetworkEndpoint{IP: ne.NetworkEndpoint.IpAddress, Node: ne.NetworkEndpoint.Instance} if ne.NetworkEndpoint.Port != 0 { @@ -734,7 +734,7 @@ func retrieveExistingZoneNetworkEndpointMap(negName string, zoneGetter *zonegett if enableDualStackNEG { newNE.IPv6 = ne.NetworkEndpoint.Ipv6Address } - zoneNetworkEndpointMap[negtypes.EndpointGroupInfo{Zone: zone}].Insert(newNE) + zoneNetworkEndpointMap[negtypes.EndpointGroupInfo{Zone: zone, Subnet: subnet}].Insert(newNE) endpointPodLabelMap[newNE] = ne.NetworkEndpoint.Annotations } } diff --git a/pkg/neg/syncers/utils_test.go b/pkg/neg/syncers/utils_test.go index c6996cdab4..f232adaf31 100644 --- a/pkg/neg/syncers/utils_test.go +++ b/pkg/neg/syncers/utils_test.go @@ -518,14 +518,14 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { desc: "default service port name", portName: "", wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.1.1", Node: "instance1", Port: "80"}, {IP: "10.100.1.2", Node: "instance1", Port: "80"}, {IP: "10.100.1.3", Node: "instance1", Port: "80"}, {IP: "10.100.1.4", Node: "instance1", Port: "80"}, {IP: "10.100.2.1", Node: "instance2", Port: "80"}, }...), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.3.1", Node: "instance3", Port: "80"}, }...), }, @@ -543,10 +543,10 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { desc: "explicitly named service port", portName: testNamedPort, wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.2.2", Node: "instance2", Port: "81"}, }...), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.4.1", Node: "instance4", Port: "81"}, {IP: "10.100.3.2", Node: "instance3", Port: "8081"}, {IP: "10.100.4.2", Node: "instance4", Port: "8081"}, @@ -568,10 +568,10 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { desc: "dual stack enabled with explicitly named service ports", portName: testNamedPort, wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.2.2", Node: "instance2", Port: "81"}, }...), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.4.1", Node: "instance4", Port: "81"}, {IP: "10.100.3.2", IPv6: "a:b::1", Node: "instance3", Port: "8081"}, {IP: "10.100.4.2", IPv6: "a:b::2", Node: "instance4", Port: "8081"}, @@ -594,14 +594,14 @@ func TestToZoneNetworkEndpointMap(t *testing.T) { desc: "non GCP network endpoints", portName: "", wantZoneNetworkEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.1.1", Port: "80"}, {IP: "10.100.1.2", Port: "80"}, {IP: "10.100.1.3", Port: "80"}, {IP: "10.100.1.4", Port: "80"}, {IP: "10.100.2.1", Port: "80"}, }...), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet([]negtypes.NetworkEndpoint{ {IP: "10.100.3.1", Port: "80"}, }...), }, @@ -804,9 +804,9 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { cloud.CreateNetworkEndpointGroup(&composite.NetworkEndpointGroup{Name: testNegName, Version: meta.VersionGA}, negtypes.TestZone4, klog.TODO()) }, expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(), - {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{}, expectErr: false, @@ -826,9 +826,9 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, meta.VersionGA, klog.TODO()) }, expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet(endpoint1), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(), - {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(endpoint1), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{ endpoint1: labels.PodLabelMap{ @@ -852,12 +852,12 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, meta.VersionGA, klog.TODO()) }, expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet(), - {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{ endpoint1: labels.PodLabelMap{ @@ -892,15 +892,15 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, meta.VersionGA, klog.TODO()) }, expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet(), + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet(), }, expectAnnotationMap: labels.EndpointPodLabelMap{ endpoint1: labels.PodLabelMap{ @@ -935,15 +935,15 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, meta.VersionGA, klog.TODO()) }, expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint6, endpoint7, ), @@ -978,15 +978,15 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { }, meta.VersionGA, klog.TODO()) }, expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint6, endpoint7, ), @@ -1025,18 +1025,18 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { mode: negtypes.L4LocalMode, expect: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ // NEGs in zone1, zone2 and zone4 are created from previous test case. - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint1, endpoint2, ), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint3, endpoint4, ), - {Zone: negtypes.TestZone3}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone3, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint5, ), - {Zone: negtypes.TestZone4}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone4, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( endpoint6, endpoint7, ), @@ -1072,7 +1072,7 @@ func TestRetrieveExistingZoneNetworkEndpointMap(t *testing.T) { for _, tc := range testCases { tc.mutate(negCloud) // tc.mode of "" will result in the default node predicate being selected, which is ok for this test. - endpointSets, annotationMap, err := retrieveExistingZoneNetworkEndpointMap(negName, zoneGetter, negCloud, meta.VersionGA, tc.mode, false, klog.TODO()) + endpointSets, annotationMap, err := retrieveExistingZoneNetworkEndpointMap(negName, zoneGetter, negCloud, meta.VersionGA, tc.mode, false, defaultTestSubnet, klog.TODO()) if tc.expectErr { if err == nil { @@ -1657,13 +1657,13 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { testEndpointSlices: getDefaultEndpointSlices(), portName: testEmptyNamedPort, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.1||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.2||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.2.1||instance2||80"), networkEndpointFromEncodedEndpoint("10.100.1.3||instance1||80"), networkEndpointFromEncodedEndpoint("10.100.1.4||instance1||80")), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.3.1||instance3||80")), }, expectedPodMap: negtypes.EndpointPodMap{ @@ -1681,9 +1681,9 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { testEndpointSlices: getDefaultEndpointSlices(), portName: testNamedPort, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.2.2||instance2||81")), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.4.1||instance4||81"), networkEndpointFromEncodedEndpoint("10.100.3.2||instance3||8081"), networkEndpointFromEncodedEndpoint("10.100.4.2||instance4||8081"), @@ -1705,13 +1705,13 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { testEndpointSlices: getDefaultEndpointSlices(), portName: testEmptyNamedPort, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.1.1||||80"), networkEndpointFromEncodedEndpoint("10.100.1.2||||80"), networkEndpointFromEncodedEndpoint("10.100.2.1||||80"), networkEndpointFromEncodedEndpoint("10.100.1.3||||80"), networkEndpointFromEncodedEndpoint("10.100.1.4||||80")), - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.100.3.1||||80")), }, expectedPodMap: negtypes.EndpointPodMap{ @@ -1758,7 +1758,7 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { }, portName: testEmptyNamedPort, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( networkEndpointFromEncodedEndpoint("10.10.0.1||instance1||80"), ), }, @@ -1785,8 +1785,6 @@ func TestToZoneNetworkEndpointMapDegradedMode(t *testing.T) { // and toZoneNetworkEndpointMapDegradedMode return the correct endpoints and // correct type of error with the supplied invalid endpoint information. func TestValidateEndpointFields(t *testing.T) { - t.Parallel() - emptyNamedPort := "" emptyNodeName := "" port80 := int32(80) @@ -1844,7 +1842,7 @@ func TestValidateEndpointFields(t *testing.T) { // endpointMap and podMap contain all correct endpoints. endpointMap := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), @@ -1863,7 +1861,7 @@ func TestValidateEndpointFields(t *testing.T) { // In degraded mode, we should exclude the invalid endpoint for non-coverable cases(pod invalid or empty zone). // We always inject to first endpoint, so the result only contain the second endpoint. endpointMapExcluded := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), } @@ -2254,7 +2252,7 @@ func TestValidateEndpointFields(t *testing.T) { }, }, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.2.2", Node: instance2, Port: "80"}, @@ -2364,7 +2362,7 @@ func TestValidateEndpointFields(t *testing.T) { }, }, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.3.2", IPv6: "a:b::1", Node: instance3, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.4.2", IPv6: "a:b::2", Node: instance4, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.4.4", IPv6: "a:b::4", Node: instance4, Port: "80"}, @@ -2378,7 +2376,7 @@ func TestValidateEndpointFields(t *testing.T) { expectErr: nil, expectErrorState: false, expectedEndpointMapDegradedMode: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone2}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone2, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.3.2", IPv6: "a:b::1", Node: instance3, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.4.2", IPv6: "a:b::2", Node: instance4, Port: "80"}, ), @@ -2545,7 +2543,7 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }) endpointMap := map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), @@ -2613,11 +2611,11 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }, }, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - {Zone: negtypes.TestZone5}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone5, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.1.1", Node: defaultSubnetLabelInstance, Port: "80"}, ), }, @@ -2629,11 +2627,11 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { expectErr: nil, expectErrorState: false, expectedEndpointMapDegradedMode: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - {Zone: negtypes.TestZone5}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone5, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.1.1", Node: defaultSubnetLabelInstance, Port: "80"}, ), }, @@ -2691,11 +2689,11 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { }, }, expectedEndpointMap: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - {Zone: negtypes.TestZone6}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone6, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.2.1", Node: emptySubnetLabelInstance, Port: "80"}, ), }, @@ -2707,11 +2705,11 @@ func TestValidateEndpointFieldsMultipleSubnets(t *testing.T) { expectErr: nil, expectErrorState: false, expectedEndpointMapDegradedMode: map[negtypes.EndpointGroupInfo]negtypes.NetworkEndpointSet{ - {Zone: negtypes.TestZone1}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone1, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.100.1.1", Node: instance1, Port: "80"}, negtypes.NetworkEndpoint{IP: "10.100.1.2", Node: instance1, Port: "80"}, ), - {Zone: negtypes.TestZone6}: negtypes.NewNetworkEndpointSet( + {Zone: negtypes.TestZone6, Subnet: defaultTestSubnet}: negtypes.NewNetworkEndpointSet( negtypes.NetworkEndpoint{IP: "10.101.2.1", Node: emptySubnetLabelInstance, Port: "80"}, ), },