From 11c30125730267c4ec3c1ebd7e0fc106d069ea1e Mon Sep 17 00:00:00 2001 From: Lubron Date: Sun, 23 Jun 2024 23:04:25 -0700 Subject: [PATCH] Add option to exclude end ips (#155) * Update doc to reflect using cidr will exclude network address and broadcast address by default Signed-off-by: Lubron Zhan --- README.md | 30 ++++- pkg/config/config.go | 37 ++++++ pkg/ipam/addressbuilder.go | 22 +++- pkg/ipam/ipam.go | 23 ++-- pkg/ipam/ipam_test.go | 176 ++++++++++++++++++++----- pkg/provider/loadBalancer.go | 51 +++---- pkg/provider/loadBalancer_test.go | 7 +- pkg/provider/loadbalancerclass_test.go | 45 ++++--- pkg/provider/provider.go | 6 - 9 files changed, 281 insertions(+), 116 deletions(-) create mode 100644 pkg/config/config.go diff --git a/README.md b/README.md index f8f5069..c3f0bf6 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ The `kube-vip-cloud-provider` will only implement the `loadBalancer` functionali - Support loadbalancerClass `kube-vip.io/kube-vip-class` - Support assigning multiple services on single VIP (IPv4 only, optional) - Support specifying service interface per namespace or at global level +- Support excluding first and last ip from cidr ## Installing the `kube-vip-cloud-provider` @@ -146,7 +147,7 @@ If users only want kube-vip-cloud-provider to allocate ip for specific set of se When enabled, kube-vip-cloud-provider tries to assign services to already used VIPs if the ports of the services do not overlap. -If you want to enable VIP-sharing between services, you can set `allow-shared`-`namespace` to true. It follows the same rules as +If you want to enable VIP-sharing between services, you can set `allow-share`-`namespace` to true. It follows the same rules as the configuration for global and namespace pools. Example-object with sharing enabled for namespace `development`: @@ -164,7 +165,7 @@ data: allow-share-development: true ``` -### Namespace pool +### Specify namespace scoped service interface Kube-vip 0.8.0 supports `kube-vip.io/serviceInterface` annotation on service type LB. Now user can specify a ip range/cidr at namespace level, we would assume these ips within a namespace should share the same interface, then we support specifying interface per namespace level by @@ -183,7 +184,30 @@ data: interface-default: eth5 ``` -`interface-global` could be used to specify all ips would use this ip address. If there is no interface specified for a namespace, it will fall back to this `interface-global`. But this is usually not needed since kube-vip has `vip_servicesinterface` for user to define default interface for service type LB. +`interface-global` could be used to specify all services under all namespace would use this ip interface. If there is no interface specified for a namespace, it will fall back to this `interface-global`. But this is usually not needed since kube-vip has `vip_servicesinterface` for user to define default interface for service type LB. + + +## Exclude first and last ip from cidr + +By default, when specifying cidr-, all ips within that cidr will be allocated to service type lb. But in some case that +the first IP might be used for network ip and last ip might be used for broadcast ip, and user might no want to alloacte those to a service type LB. In +this case, the user could specify skip-end-ips-in-cidr: true in the configmap to skip the first and last ip for cidr. + +Example object: +``` +$ kubectl get configmap -n kube-system kubevip -o yaml + +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubevip + namespace: kube-system +data: + cidr-default: 192.168.0.200/29 + skip-end-ips-in-cidr: true +``` + +In this case, only ips `192.168.0.201-192.168.0.206` will be allocated to service, `192.168.0.200` and `192.168.0.207` are excluded. ## Debugging diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..823f4ba --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,37 @@ +package config + +import v1 "k8s.io/api/core/v1" + +const ( + // ConfigMapSearchOrderKey is the key in the ConfigMap that defines whether IPs are allocated from the beginning or from the end. + ConfigMapSearchOrderKey = "search-order" + + // ConfigMapSkipStartIPsKey is the key in the ConfigMap that has the IPs to skip at the start and end of the CIDR + ConfigMapSkipEndIPsKey = "skip-end-ips-in-cidr" + + // ConfigMapServiceInterfacePrefix is prefix of the key in the ConfigMap for specifying the service interface for that namespace + ConfigMapServiceInterfacePrefix = "interface" +) + +// KubevipLBConfig defines the configuration for the kube-vip load balancer in the kubevip configMap +// TODO: move all config into here so that it can be easily accessed and processed +type KubevipLBConfig struct { + ReturnIPInDescOrder bool + SkipEndIPsInCIDR bool +} + +// GetKubevipLBConfig returns the KubevipLBConfig from the ConfigMap +func GetKubevipLBConfig(cm *v1.ConfigMap) *KubevipLBConfig { + c := &KubevipLBConfig{} + if searchOrder, ok := cm.Data[ConfigMapSearchOrderKey]; ok { + if searchOrder == "desc" { + c.ReturnIPInDescOrder = true + } + } + if skip, ok := cm.Data[ConfigMapSkipEndIPsKey]; ok { + if skip == "true" { + c.SkipEndIPsInCIDR = true + } + } + return c +} diff --git a/pkg/ipam/addressbuilder.go b/pkg/ipam/addressbuilder.go index 544cd0a..f9a836c 100644 --- a/pkg/ipam/addressbuilder.go +++ b/pkg/ipam/addressbuilder.go @@ -5,6 +5,7 @@ import ( "net/netip" "strings" + "github.com/kube-vip/kube-vip-cloud-provider/pkg/config" "go4.org/netipx" ) @@ -30,7 +31,7 @@ func parseCidrs(cidr string) (*netipx.IPSet, error) { // buildHostsFromCidr - Builds a IPSet constructed from the cidr and filters out // the broadcast IP and network IP for IPv4 networks -func buildHostsFromCidr(cidr string) (*netipx.IPSet, error) { +func buildHostsFromCidr(cidr string, kubevipLBConfig *config.KubevipLBConfig) (*netipx.IPSet, error) { unfilteredSet, err := parseCidrs(cidr) if err != nil { return nil, err @@ -38,22 +39,31 @@ func buildHostsFromCidr(cidr string) (*netipx.IPSet, error) { builder := &netipx.IPSetBuilder{} for _, prefix := range unfilteredSet.Prefixes() { - if prefix.IsSingleIP() { - builder.Add(prefix.Addr()) - continue - } + // If the prefix is IPv6 address, add it to the builder directly if !prefix.Addr().Is4() { builder.AddPrefix(prefix) continue } + + // Only skip the end IPs if skip-end-ips-in-cidr in configMap is set to true. + if prefix.IsSingleIP() && kubevipLBConfig != nil && kubevipLBConfig.SkipEndIPsInCIDR { + builder.Add(prefix.Addr()) + continue + } + if r := netipx.RangeOfPrefix(prefix); r.IsValid() { if prefix.Bits() == 31 { // rfc3021 Using 31-Bit Prefixes on IPv4 Point-to-Point Links builder.AddRange(netipx.IPRangeFrom(r.From(), r.To())) continue } + + from, to := r.From(), r.To() // For 192.168.0.200/23, 192.168.0.206 is the BroadcastIP, and 192.168.0.201 is the NetworkID - builder.AddRange(netipx.IPRangeFrom(r.From().Next(), r.To().Prev())) + if kubevipLBConfig != nil && kubevipLBConfig.SkipEndIPsInCIDR { + from, to = from.Next(), to.Prev() + } + builder.AddRange(netipx.IPRangeFrom(from, to)) } } return builder.IPSet() diff --git a/pkg/ipam/ipam.go b/pkg/ipam/ipam.go index 3cb9803..bd94fdf 100644 --- a/pkg/ipam/ipam.go +++ b/pkg/ipam/ipam.go @@ -5,6 +5,7 @@ import ( "fmt" "net/netip" + "github.com/kube-vip/kube-vip-cloud-provider/pkg/config" "go4.org/netipx" "k8s.io/klog" ) @@ -41,7 +42,7 @@ type ipManager struct { } // FindAvailableHostFromRange - will look through the cidr and the address Manager and find a free address (if possible) -func FindAvailableHostFromRange(namespace, ipRange string, inUseIPSet *netipx.IPSet, descOrder bool) (string, error) { +func FindAvailableHostFromRange(namespace, ipRange string, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig) (string, error) { // Look through namespaces and update one if it exists for x := range Manager { if Manager[x].namespace == namespace { @@ -58,7 +59,7 @@ func FindAvailableHostFromRange(namespace, ipRange string, inUseIPSet *netipx.IP Manager[x].ipRange = ipRange } - addr, err := FindFreeAddress(Manager[x].poolIPSet, inUseIPSet, descOrder) + addr, err := FindFreeAddress(Manager[x].poolIPSet, inUseIPSet, kubevipLBConfig) if err != nil { return "", &OutOfIPsError{namespace: namespace, pool: ipRange, isCidr: false} } @@ -79,7 +80,7 @@ func FindAvailableHostFromRange(namespace, ipRange string, inUseIPSet *netipx.IP Manager = append(Manager, newManager) - addr, err := FindFreeAddress(poolIPSet, inUseIPSet, descOrder) + addr, err := FindFreeAddress(poolIPSet, inUseIPSet, kubevipLBConfig) if err != nil { return "", &OutOfIPsError{namespace: namespace, pool: ipRange, isCidr: false} } @@ -87,30 +88,28 @@ func FindAvailableHostFromRange(namespace, ipRange string, inUseIPSet *netipx.IP } // FindAvailableHostFromCidr - will look through the cidr and the address Manager and find a free address (if possible) -func FindAvailableHostFromCidr(namespace, cidr string, inUseIPSet *netipx.IPSet, descOrder bool) (string, error) { +func FindAvailableHostFromCidr(namespace, cidr string, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig) (string, error) { // Look through namespaces and update one if it exists for x := range Manager { if Manager[x].namespace == namespace { // Check that the address range is the same if Manager[x].cidr != cidr { // If not rebuild the available hosts - poolIPSet, err := buildHostsFromCidr(cidr) + poolIPSet, err := buildHostsFromCidr(cidr, kubevipLBConfig) if err != nil { return "", err } Manager[x].poolIPSet = poolIPSet Manager[x].cidr = cidr - } - addr, err := FindFreeAddress(Manager[x].poolIPSet, inUseIPSet, descOrder) + addr, err := FindFreeAddress(Manager[x].poolIPSet, inUseIPSet, kubevipLBConfig) if err != nil { return "", &OutOfIPsError{namespace: namespace, pool: cidr, isCidr: true} } return addr.String(), nil - } } - poolIPSet, err := buildHostsFromCidr(cidr) + poolIPSet, err := buildHostsFromCidr(cidr, kubevipLBConfig) if err != nil { return "", err } @@ -122,7 +121,7 @@ func FindAvailableHostFromCidr(namespace, cidr string, inUseIPSet *netipx.IPSet, } Manager = append(Manager, newManager) - addr, err := FindFreeAddress(poolIPSet, inUseIPSet, descOrder) + addr, err := FindFreeAddress(poolIPSet, inUseIPSet, kubevipLBConfig) if err != nil { return "", &OutOfIPsError{namespace: namespace, pool: cidr, isCidr: true} } @@ -152,8 +151,8 @@ func FindAvailableHostFromCidr(namespace, cidr string, inUseIPSet *netipx.IPSet, // FindFreeAddress returns the next free IP Address in a range based on a set of existing addresses. // It will skip assumed gateway ip or broadcast ip for IPv4 address -func FindFreeAddress(poolIPSet *netipx.IPSet, inUseIPSet *netipx.IPSet, descOrder bool) (netip.Addr, error) { - if descOrder { +func FindFreeAddress(poolIPSet *netipx.IPSet, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig) (netip.Addr, error) { + if kubevipLBConfig != nil && kubevipLBConfig.ReturnIPInDescOrder { ipranges := poolIPSet.Ranges() for i := range len(ipranges) { iprange := ipranges[len(ipranges)-1-i] diff --git a/pkg/ipam/ipam_test.go b/pkg/ipam/ipam_test.go index f83ca17..8e83e1e 100644 --- a/pkg/ipam/ipam_test.go +++ b/pkg/ipam/ipam_test.go @@ -4,6 +4,7 @@ import ( "net/netip" "testing" + "github.com/kube-vip/kube-vip-cloud-provider/pkg/config" "go4.org/netipx" ) @@ -122,7 +123,8 @@ func Test_buildHostsFromRange(t *testing.T) { func Test_buildHostsFromCidr(t *testing.T) { type args struct { - cidr string + cidr string + kvlbc *config.KubevipLBConfig } tests := []struct { name string @@ -130,18 +132,71 @@ func Test_buildHostsFromCidr(t *testing.T) { want []string wantErr bool }{ + { + name: "single entry, /32, 1 address", + args: args{ + cidr: "192.168.0.200/32", + }, + want: []string{"192.168.0.200"}, + wantErr: false, + }, + { + name: "single entry, /32, 1 address, if skipEndIPsInCIDR is set", + args: args{ + cidr: "192.168.0.200/32", + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, + }, + want: []string{"192.168.0.200"}, + wantErr: false, + }, { name: "single entry, 4 address", args: args{ - "192.168.0.200/30", + cidr: "192.168.0.200/30", + }, + want: []string{"192.168.0.200", "192.168.0.201", "192.168.0.202", "192.168.0.203"}, + wantErr: false, + }, + { + name: "single entry, 2 address, if skipEndIPsInCIDR is set", + args: args{ + cidr: "192.168.0.200/30", + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, }, want: []string{"192.168.0.201", "192.168.0.202"}, wantErr: false, }, { - name: "dual entry, overlap address", + name: "single entry, /31, 2 address, if skipEndIPsInCIDR is set", + args: args{ + cidr: "192.168.0.200/31", + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, + }, + want: []string{"192.168.0.200", "192.168.0.201"}, + wantErr: false, + }, + { + name: "single entry, /31, 2 address, if skipEndIPsInCIDR is set", + args: args{ + cidr: "192.168.0.200/31", + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, + }, + want: []string{"192.168.0.200", "192.168.0.201"}, + wantErr: false, + }, + { + name: "dual entry, overlap address, return 8 addresses", + args: args{ + cidr: "192.168.0.200/30,192.168.0.200/29", + }, + want: []string{"192.168.0.200", "192.168.0.201", "192.168.0.202", "192.168.0.203", "192.168.0.204", "192.168.0.205", "192.168.0.206", "192.168.0.207"}, + wantErr: false, + }, + { + name: "dual entry, overlap addressm return 6 addresses, if skipEndIPsInCIDR is set", args: args{ - "192.168.0.200/30,192.168.0.200/29", + cidr: "192.168.0.200/30,192.168.0.200/29", + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, }, want: []string{"192.168.0.201", "192.168.0.202", "192.168.0.203", "192.168.0.204", "192.168.0.205", "192.168.0.206"}, wantErr: false, @@ -149,7 +204,7 @@ func Test_buildHostsFromCidr(t *testing.T) { { name: "ipv6, two ips", args: args{ - "fe80::10/127", + cidr: "fe80::10/127", }, want: []string{"fe80::10", "fe80::11"}, wantErr: false, @@ -157,7 +212,7 @@ func Test_buildHostsFromCidr(t *testing.T) { { name: "ipv6, two cidrs", args: args{ - "fe80::10/127,fe80::fe/127", + cidr: "fe80::10/127,fe80::fe/127", }, want: []string{"fe80::10", "fe80::11", "fe80::fe", "fe80::ff"}, wantErr: false, @@ -165,7 +220,7 @@ func Test_buildHostsFromCidr(t *testing.T) { { name: "ipv6, two cidrs with overlap", args: args{ - "fe80::10/126,fe80::12/127", + cidr: "fe80::10/126,fe80::12/127", }, want: []string{"fe80::10", "fe80::11", "fe80::12", "fe80::13"}, wantErr: false, @@ -173,7 +228,7 @@ func Test_buildHostsFromCidr(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := buildHostsFromCidr(tt.args.cidr) + got, err := buildHostsFromCidr(tt.args.cidr, tt.args.kvlbc) if (err != nil) != tt.wantErr { t.Errorf("buildHostsFromCidr() error = %v, wantErr %v", err, tt.wantErr) return @@ -416,7 +471,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "192.168.0.10", }, { - name: "simple range, revert", + name: "simple range, reverse order", args: args{ namespace: "default", ipRange: "192.168.0.10-192.168.0.10", @@ -435,7 +490,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "192.168.0.10", }, { - name: "single range, three addresses, revert", + name: "single range, three addresses, reverse order", args: args{ namespace: "default2", ipRange: "192.168.0.10-192.168.0.12", @@ -454,7 +509,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "192.168.1.1", }, { - name: "single range, across third octet, revert", + name: "single range, across third octet, reverse order", args: args{ namespace: "default2", ipRange: "192.168.0.253-192.168.1.2", @@ -473,7 +528,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "192.168.0.11", }, { - name: "two ranges, four addresses, revert", + name: "two ranges, four addresses, reverse order", args: args{ namespace: "default2", ipRange: "192.168.0.10-192.168.0.11,192.168.1.20-192.168.1.22", @@ -492,7 +547,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "fe80::13", }, { - name: "ipv6, simple range, revert", + name: "ipv6, simple range, reverse order", args: args{ namespace: "default", ipRange: "fe80::13-fe80::14", @@ -511,7 +566,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "fe80::13", }, { - name: "ipv6, single range, three addresses, revert", + name: "ipv6, single range, three addresses, reverse order", args: args{ namespace: "default2", ipRange: "fe80::13-fe80::15", @@ -530,7 +585,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "fe80::1:0", }, { - name: "ipv6, single range, across third octet, revert", + name: "ipv6, single range, across third octet, reverse order", args: args{ namespace: "default2", ipRange: "fe80::ffff-fe80::1:3", @@ -549,7 +604,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { want: "fe81::20", }, { - name: "ipv6, two ranges, 5 addresses, revert", + name: "ipv6, two ranges, 5 addresses, reverse order", args: args{ namespace: "default2", ipRange: "fe80::10-fe80::12,fe81::20-fe81::21", @@ -577,7 +632,7 @@ func TestFindAvailableHostFromRange(t *testing.T) { return } - got, err := FindAvailableHostFromRange(tt.args.namespace, tt.args.ipRange, s, tt.args.descOrder) + got, err := FindAvailableHostFromRange(tt.args.namespace, tt.args.ipRange, s, &config.KubevipLBConfig{ReturnIPInDescOrder: tt.args.descOrder}) if (err != nil) != tt.wantErr { t.Errorf("FindAvailableHostFromRange() error = %v, wantErr %v", err, tt.wantErr) return @@ -594,7 +649,7 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { namespace string cidr string existingServices []string - descOrder bool + kvlbc *config.KubevipLBConfig } tests := []struct { name string @@ -603,21 +658,41 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { wantErr bool }{ { - name: "single entry, two address", + name: "single entry, 4 addresses, allocate the first one", + args: args{ + namespace: "default", + cidr: "192.168.0.200/30", + existingServices: []string{}, + }, + want: "192.168.0.200", + }, + { + name: "single entry, 4 addresses, allocate second address if SkipEndIPsInCIDR is set", args: args{ namespace: "default", cidr: "192.168.0.200/30", existingServices: []string{}, + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, }, want: "192.168.0.201", }, { - name: "single entry, two address, revert", + name: "single entry, two address, reverse ip order", args: args{ namespace: "default", cidr: "192.168.0.200/30", existingServices: []string{}, - descOrder: true, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true}, + }, + want: "192.168.0.203", + }, + { + name: "single entry, two address, reverse ip order, allocate second last address if SkipEndIPsInCIDR is set", + args: args{ + namespace: "default", + cidr: "192.168.0.200/30", + existingServices: []string{}, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true, SkipEndIPsInCIDR: true}, }, want: "192.168.0.202", }, @@ -631,12 +706,12 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { want: "192.168.0.1", }, { - name: "simple cidr, cidr contains .0 and .255, revert", + name: "simple cidr, cidr contains .0 and .255, reverse order", args: args{ namespace: "default2", cidr: "192.168.0.10/24", existingServices: []string{}, - descOrder: true, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true}, }, want: "192.168.0.254", }, @@ -650,12 +725,12 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { wantErr: true, }, { - name: "no ip available, revert", + name: "no ip available, reverse order", args: args{ namespace: "default2", cidr: "192.168.0.255/30", existingServices: []string{"192.168.0.254", "192.168.0.252", "192.168.0.253"}, - descOrder: true, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true}, }, wantErr: true, }, @@ -666,15 +741,35 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { cidr: "192.168.0.200/30,192.168.0.200/29", existingServices: []string{"192.168.0.201", "192.168.0.202"}, }, + want: "192.168.0.200", + }, + { + name: "dual entry, overlap address, set SkipEndIPsInCIDR, pick next available address after first one", + args: args{ + namespace: "default2", + cidr: "192.168.0.200/30,192.168.0.200/29", + existingServices: []string{"192.168.0.201", "192.168.0.202"}, + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, + }, want: "192.168.0.203", }, { - name: "dual entry, overlap address, revert", + name: "dual entry, overlap address, reverse order, pick next available address from last", args: args{ namespace: "default2", cidr: "192.168.0.200/30,192.168.0.200/29", existingServices: []string{"192.168.0.201", "192.168.0.202"}, - descOrder: true, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true}, + }, + want: "192.168.0.207", + }, + { + name: "dual entry, overlap address, reverse order, set SkipEndIPsInCIDR, pick next available address before last", + args: args{ + namespace: "default2", + cidr: "192.168.0.200/30,192.168.0.200/29", + existingServices: []string{"192.168.0.201", "192.168.0.202"}, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true, SkipEndIPsInCIDR: true}, }, want: "192.168.0.206", }, @@ -688,12 +783,22 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { want: "2001::49fe", }, { - name: "ipv6, single entry, two address, revert", + name: "ipv6, single entry, two address, set SkipEndIPsInCIDR no effect", args: args{ namespace: "default", cidr: "2001::49fe/127", existingServices: []string{}, - descOrder: true, + kvlbc: &config.KubevipLBConfig{SkipEndIPsInCIDR: true}, + }, + want: "2001::49fe", + }, + { + name: "ipv6, single entry, two address, reverse order", + args: args{ + namespace: "default", + cidr: "2001::49fe/127", + existingServices: []string{}, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true}, }, want: "2001::49ff", }, @@ -707,12 +812,12 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { wantErr: true, }, { - name: "ipv6, no ip available, revert", + name: "ipv6, no ip available, reverse order", args: args{ namespace: "default2", cidr: "2001::49fe/127", existingServices: []string{"2001::49fe", "2001::49ff"}, - descOrder: true, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true}, }, wantErr: true, }, @@ -726,12 +831,12 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { want: "2001::12", }, { - name: "ipv6, dual entry, overlap address, revert", + name: "ipv6, dual entry, overlap address, reverse order", args: args{ namespace: "default2", cidr: "2001::10/126,2001::12/127", existingServices: []string{"2001::10", "2001::11"}, - descOrder: true, + kvlbc: &config.KubevipLBConfig{ReturnIPInDescOrder: true}, }, want: "2001::13", }, @@ -753,8 +858,7 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { t.Errorf("FindAvailableHostFromCIDR() error = %v", err) return } - - got, err := FindAvailableHostFromCidr(tt.args.namespace, tt.args.cidr, s, tt.args.descOrder) + got, err := FindAvailableHostFromCidr(tt.args.namespace, tt.args.cidr, s, tt.args.kvlbc) if (err != nil) != tt.wantErr { t.Errorf("FindAvailableHostFromCIDR() error = %v, wantErr %v", err, tt.wantErr) return @@ -762,6 +866,8 @@ func TestFindAvailableHostFromCIDR(t *testing.T) { if got != tt.want { t.Errorf("FindAvailableHostFromCIDR() = %v, want %v", got, tt.want) } + // clean up the ipManager so it doesn't impact other test + Manager = []ipManager{} }) } } diff --git a/pkg/provider/loadBalancer.go b/pkg/provider/loadBalancer.go index 2af8364..04febdc 100644 --- a/pkg/provider/loadBalancer.go +++ b/pkg/provider/loadBalancer.go @@ -7,17 +7,17 @@ import ( "strconv" "strings" - "k8s.io/utils/set" - - "github.com/kube-vip/kube-vip-cloud-provider/pkg/ipam" "go4.org/netipx" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" "k8s.io/client-go/util/retry" cloudprovider "k8s.io/cloud-provider" - "k8s.io/klog" + "k8s.io/utils/set" + + "github.com/kube-vip/kube-vip-cloud-provider/pkg/config" + "github.com/kube-vip/kube-vip-cloud-provider/pkg/ipam" ) const ( @@ -268,7 +268,7 @@ func syncLoadBalancer(ctx context.Context, kubeClient kubernetes.Interface, serv return nil, err } - descOrder := getSearchOrder(controllerCM) + kubevipLBConfig := config.GetKubevipLBConfig(controllerCM) preferredIpv4ServiceIP := "" @@ -277,7 +277,7 @@ func syncLoadBalancer(ctx context.Context, kubeClient kubernetes.Interface, serv } // If allowedShare is true but no IP could be shared, or allowedShare is false, switch to use IPAM lookup - loadBalancerIPs, err := discoverVIPs(service.Namespace, pool, preferredIpv4ServiceIP, inUseSet, descOrder, service.Spec.IPFamilyPolicy, service.Spec.IPFamilies) + loadBalancerIPs, err := discoverVIPs(service.Namespace, pool, preferredIpv4ServiceIP, inUseSet, kubevipLBConfig, service.Spec.IPFamilyPolicy, service.Spec.IPFamilies) if err != nil { return nil, err } @@ -422,7 +422,7 @@ func discoverSharedVIPs(service *v1.Service, servicePortMap map[string]*set.Set[ return "" } -func discoverVIPsSingleStack(namespace, ipv4Pool, ipv6Pool string, preferredIpv4ServiceIP string, inUseIPSet *netipx.IPSet, descOrder bool, +func discoverVIPsSingleStack(namespace, ipv4Pool, ipv6Pool string, preferredIpv4ServiceIP string, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig, ipFamilies []v1.IPFamily) (vips string, err error) { ipPool := ipv4Pool @@ -439,11 +439,11 @@ func discoverVIPsSingleStack(namespace, ipv4Pool, ipv6Pool string, preferredIpv4 if ipPool == ipv4Pool && len(preferredIpv4ServiceIP) > 0 { return preferredIpv4ServiceIP, nil } - return discoverAddress(namespace, ipPool, inUseIPSet, descOrder) + return discoverAddress(namespace, ipPool, inUseIPSet, kubevipLBConfig) } -func discoverFromPool(namespace, pool, preferredIpv4ServiceIP, ipv4Pool string, inUseIPSet *netipx.IPSet, descOrder bool, vipList *[]string) (poolError, err error) { +func discoverFromPool(namespace, pool, preferredIpv4ServiceIP, ipv4Pool string, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig, vipList *[]string) (poolError, err error) { if len(pool) == 0 { return nil, nil } @@ -452,7 +452,7 @@ func discoverFromPool(namespace, pool, preferredIpv4ServiceIP, ipv4Pool string, if pool == ipv4Pool && len(preferredIpv4ServiceIP) > 0 { vip = preferredIpv4ServiceIP } else { - vip, err = discoverAddress(namespace, pool, inUseIPSet, descOrder) + vip, err = discoverAddress(namespace, pool, inUseIPSet, kubevipLBConfig) } if err == nil { @@ -465,7 +465,7 @@ func discoverFromPool(namespace, pool, preferredIpv4ServiceIP, ipv4Pool string, return nil, err } -func discoverVIPsDualStack(namespace, ipv4Pool, ipv6Pool string, preferredIpv4ServiceIP string, inUseIPSet *netipx.IPSet, descOrder bool, +func discoverVIPsDualStack(namespace, ipv4Pool, ipv6Pool string, preferredIpv4ServiceIP string, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig, ipFamilyPolicy *v1.IPFamilyPolicy, ipFamilies []v1.IPFamily) (vips string, err error) { var vipList []string @@ -490,14 +490,14 @@ func discoverVIPsDualStack(namespace, ipv4Pool, ipv6Pool string, preferredIpv4Se var primaryPoolErr, secondaryPoolErr error if len(primaryPool) > 0 { - primaryPoolErr, err = discoverFromPool(namespace, primaryPool, preferredIpv4ServiceIP, ipv4Pool, inUseIPSet, descOrder, &vipList) + primaryPoolErr, err = discoverFromPool(namespace, primaryPool, preferredIpv4ServiceIP, ipv4Pool, inUseIPSet, kubevipLBConfig, &vipList) if err != nil { return "", err } } if len(secondaryPool) > 0 { - secondaryPoolErr, err = discoverFromPool(namespace, secondaryPool, preferredIpv4ServiceIP, ipv4Pool, inUseIPSet, descOrder, &vipList) + secondaryPoolErr, err = discoverFromPool(namespace, secondaryPool, preferredIpv4ServiceIP, ipv4Pool, inUseIPSet, kubevipLBConfig, &vipList) if err != nil { return "", err } @@ -524,7 +524,7 @@ func discoverVIPsDualStack(namespace, ipv4Pool, ipv6Pool string, preferredIpv4Se } func discoverVIPs( - namespace, pool, preferredIpv4ServiceIP string, inUseIPSet *netipx.IPSet, descOrder bool, + namespace, pool, preferredIpv4ServiceIP string, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig, ipFamilyPolicy *v1.IPFamilyPolicy, ipFamilies []v1.IPFamily, ) (vips string, err error) { var ipv4Pool, ipv6Pool string @@ -545,23 +545,23 @@ func discoverVIPs( } if ipFamilyPolicy == nil || *ipFamilyPolicy == v1.IPFamilyPolicySingleStack { - return discoverVIPsSingleStack(namespace, ipv4Pool, ipv6Pool, preferredIpv4ServiceIP, inUseIPSet, descOrder, ipFamilies) + return discoverVIPsSingleStack(namespace, ipv4Pool, ipv6Pool, preferredIpv4ServiceIP, inUseIPSet, kubevipLBConfig, ipFamilies) } - return discoverVIPsDualStack(namespace, ipv4Pool, ipv6Pool, preferredIpv4ServiceIP, inUseIPSet, descOrder, ipFamilyPolicy, ipFamilies) + return discoverVIPsDualStack(namespace, ipv4Pool, ipv6Pool, preferredIpv4ServiceIP, inUseIPSet, kubevipLBConfig, ipFamilyPolicy, ipFamilies) } -func discoverAddress(namespace, pool string, inUseIPSet *netipx.IPSet, descOrder bool) (vip string, err error) { +func discoverAddress(namespace, pool string, inUseIPSet *netipx.IPSet, kubevipLBConfig *config.KubevipLBConfig) (vip string, err error) { // Check if DHCP is required if pool == "0.0.0.0/32" { vip = "0.0.0.0" // Check if ip pool contains a cidr, if not assume it is a range } else if strings.Contains(pool, "/") { - vip, err = ipam.FindAvailableHostFromCidr(namespace, pool, inUseIPSet, descOrder) + vip, err = ipam.FindAvailableHostFromCidr(namespace, pool, inUseIPSet, kubevipLBConfig) if err != nil { return "", err } } else { - vip, err = ipam.FindAvailableHostFromRange(namespace, pool, inUseIPSet, descOrder) + vip, err = ipam.FindAvailableHostFromRange(namespace, pool, inUseIPSet, kubevipLBConfig) if err != nil { return "", err } @@ -574,15 +574,6 @@ func getKubevipImplementationLabel() string { return fmt.Sprintf("%s=%s", ImplementationLabelKey, ImplementationLabelValue) } -func getSearchOrder(cm *v1.ConfigMap) (descOrder bool) { - if searchOrder, ok := cm.Data[ConfigMapSearchOrderKey]; ok { - if searchOrder == "desc" { - return true - } - } - return false -} - func renderErrors(errs ...error) string { s := strings.Builder{} for _, err := range errs { @@ -596,11 +587,11 @@ func renderErrors(errs ...error) string { // found interface of that service from configmap. // if not found, return "" func discoverInterface(cm *v1.ConfigMap, svcNS string) string { - if interfaceName, ok := cm.Data[fmt.Sprintf("%s-%s", ConfigMapServiceInterfacePrefix, svcNS)]; ok { + if interfaceName, ok := cm.Data[fmt.Sprintf("%s-%s", config.ConfigMapServiceInterfacePrefix, svcNS)]; ok { return interfaceName } // fall back to global interface - if interfaceName, ok := cm.Data[fmt.Sprintf("%s-global", ConfigMapServiceInterfacePrefix)]; ok { + if interfaceName, ok := cm.Data[fmt.Sprintf("%s-global", config.ConfigMapServiceInterfacePrefix)]; ok { return interfaceName } diff --git a/pkg/provider/loadBalancer_test.go b/pkg/provider/loadBalancer_test.go index d3e9994..b14710d 100644 --- a/pkg/provider/loadBalancer_test.go +++ b/pkg/provider/loadBalancer_test.go @@ -5,6 +5,7 @@ import ( "net/netip" "testing" + "github.com/kube-vip/kube-vip-cloud-provider/pkg/config" "github.com/stretchr/testify/assert" "go4.org/netipx" v1 "k8s.io/api/core/v1" @@ -224,7 +225,7 @@ func Test_DiscoveryAddressCIDR(t *testing.T) { return } - gotString, err := discoverAddress(tt.args.namespace, tt.args.pool, s, false) + gotString, err := discoverAddress(tt.args.namespace, tt.args.pool, s, &config.KubevipLBConfig{}) if (err != nil) != tt.wantErr { t.Errorf("discoverAddress() error: %v, expected: %v", err, tt.wantErr) return @@ -308,7 +309,7 @@ func Test_DiscoveryAddressRange(t *testing.T) { return } - gotString, err := discoverAddress(tt.args.namespace, tt.args.pool, s, false) + gotString, err := discoverAddress(tt.args.namespace, tt.args.pool, s, &config.KubevipLBConfig{}) if (err != nil) != tt.wantErr { t.Errorf("discoverAddress() error: %v, expected: %v", err, tt.wantErr) return @@ -692,7 +693,7 @@ func Test_discoverVIPs(t *testing.T) { return } - gotString, err := discoverVIPs("discover-vips-test-ns", tt.args.pool, tt.args.preferredIpv4ServiceIP, s, false, tt.args.ipFamilyPolicy, tt.args.ipFamilies) + gotString, err := discoverVIPs("discover-vips-test-ns", tt.args.pool, tt.args.preferredIpv4ServiceIP, s, &config.KubevipLBConfig{}, tt.args.ipFamilyPolicy, tt.args.ipFamilies) if (err != nil) != tt.wantErr { t.Errorf("discoverVIP() error: %v, expected: %v", err, tt.wantErr) return diff --git a/pkg/provider/loadbalancerclass_test.go b/pkg/provider/loadbalancerclass_test.go index 1177e76..fab53de 100644 --- a/pkg/provider/loadbalancerclass_test.go +++ b/pkg/provider/loadbalancerclass_test.go @@ -80,12 +80,6 @@ func TestSyncLoadBalancerIfNeeded(t *testing.T) { expectNumOfUpdate: 1, expectNumOfPatch: 1, }, - { - desc: "service specifies incorrect loadBalancerClass", - service: tu.NewService("with-external-balancer", tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), - expectNumOfUpdate: 1, - expectNumOfPatch: 1, - }, { desc: "service that needs cleanup", service: tu.NewService("basic-service2", tu.TweakAddLBIngress("8.8.8.8"), tu.TweakAddFinalizers(servicehelper.LoadBalancerCleanupFinalizer), tu.TweakAddDeletionTimestamp(time.Now()), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), @@ -148,7 +142,7 @@ func newSmallIPPoolConfigMap() *corev1.ConfigMap { Namespace: KubeVipClientConfigNamespace, }, Data: map[string]string{ - "cidr-global": "10.0.0.0/30,2001::0/48", + "cidr-global": "10.0.0.2/31,2001::0/48", "allow-share-global": "true", }, } @@ -166,28 +160,21 @@ func TestSyncLoadBalancerIfNeededWithMultipleIpUse(t *testing.T) { { desc: "udp service that wants LB", service: tu.NewService("udp-service", tu.TweakDualStack(), tu.TweakAddPorts(corev1.ProtocolUDP, 123, 123), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), - expectIP: "10.0.0.1,2001::", + expectIP: "10.0.0.2,2001::", expectNumOfUpdate: 1, expectNumOfPatch: 1, }, { desc: "tcp service that wants LB", service: tu.NewService("basic-service1", tu.TweakDualStack(), tu.TweakAddPorts(corev1.ProtocolTCP, 345, 345), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), - expectIP: "10.0.0.1,2001::1", + expectIP: "10.0.0.2,2001::1", expectNumOfUpdate: 1, expectNumOfPatch: 1, }, { desc: "sctp service that wants LB", service: tu.NewService("sctp-service", tu.TweakAddPorts(corev1.ProtocolSCTP, 1234, 1234), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), - expectIP: "10.0.0.1", - expectNumOfUpdate: 1, - expectNumOfPatch: 1, - }, - { - desc: "service specifies incorrect loadBalancerClass", - service: tu.NewService("with-external-balancer", tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), - expectIP: "10.0.0.1", + expectIP: "10.0.0.2", expectNumOfUpdate: 1, expectNumOfPatch: 1, }, @@ -203,17 +190,33 @@ func TestSyncLoadBalancerIfNeededWithMultipleIpUse(t *testing.T) { expectNumOfUpdate: 1, }, { - desc: "another tcp service that wants LB", - service: tu.NewService("basic-service4", tu.TweakAddPorts(corev1.ProtocolTCP, 80, 80), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), - expectNumOfUpdate: 0, + desc: "now there is not enough ip, another tcp service that wants LB, but still could share ip with existing service", + service: tu.NewService("basic-service4", tu.TweakAddPorts(corev1.ProtocolTCP, 8080, 8080), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), + expectNumOfUpdate: 1, expectNumOfPatch: 1, + expectIP: "10.0.0.2", expectError: true, }, + { + desc: "another service who wants same port, get a new IP address", + service: tu.NewService("basic-service5", tu.TweakAddPorts(corev1.ProtocolTCP, 80, 80), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), + expectNumOfUpdate: 1, + expectNumOfPatch: 1, + expectIP: "10.0.0.3", + expectError: true, + }, + { + desc: "another service who wants same port, no ip left, get no ip", + service: tu.NewService("basic-service6", tu.TweakAddPorts(corev1.ProtocolTCP, 80, 80), tu.TweakAddLBClass(ptr.To(LoadbalancerClass))), + expectNumOfPatch: 1, + expectError: true, + }, } // create ip pool for service to use client := fake.NewSimpleClientset() ctx := context.Background() + // This pool has 2 ipv4 addresses and 8 ipv6 address cm := newSmallIPPoolConfigMap() if _, err := client.CoreV1().ConfigMaps(cm.Namespace).Create(ctx, cm, metav1.CreateOptions{}); err != nil { t.Errorf("Failed to prepare configmap %s for testing: %v", cm.Name, err) @@ -256,7 +259,7 @@ func TestSyncLoadBalancerIfNeededWithMultipleIpUse(t *testing.T) { t.Errorf("expect %d patches, got %d patches.", tc.expectNumOfPatch, patchNum) } if lbIP != tc.expectIP { - t.Errorf("expect %s LbIP, got %s.", tc.expectIP, lbIP) + t.Errorf("expect '%s' LbIP, got '%s'.", tc.expectIP, lbIP) } }) } diff --git a/pkg/provider/provider.go b/pkg/provider/provider.go index 11b8418..5b3f1fa 100644 --- a/pkg/provider/provider.go +++ b/pkg/provider/provider.go @@ -39,12 +39,6 @@ const ( // EnableLoadbalancerClassEnvKey environment key for enabling loadbalancerclass. EnableLoadbalancerClassEnvKey = "KUBEVIP_ENABLE_LOADBALANCERCLASS" - - // ConfigMapSearchOrderKey is the key in the ConfigMap that specifying the search order of finding IPs - ConfigMapSearchOrderKey = "search-order" - - // ConfigMapServiceInterfacePrefix is prefix of the key in the ConfigMap for specifying the service interface for that namespace - ConfigMapServiceInterfacePrefix = "interface" ) func init() {