Add reverse proxy support

Problem K8GB reads IP addresses from `Ingress.Status.LoadBalancer.Ingress` or from `Service.Status.LoadBalancer.Ingress` for ingress configured with Kubernetes Ingress and Istio Virtual Service, respectively. The IP addresses exposed by these resources are the IP addresses exposed by the Kubernetes Cluster. However, in some setups the clients do not route their traffic to these IP addresses because the cluster is behind a reverse proxy. Solution To support this setup, K8GB should expose DNS records with the IP address of the reverse proxy. Since the address is unknown to the cluster the K8GB administrator must provide it via configuration. This PR adds to K8GB the capability to read IP address from an annotation `k8gb.io/external-ips` on the GSLB resource. Example ``` apiVersion: k8gb.absa.oss/v1beta1 kind: Gslb metadata: labels: app: ingress annotations: k8gb.io/external-ips: "185.199.110.153" ``` Fixes #1275 Signed-off-by: Andre Baptista Aguas <[email protected]>
k8gb-io · Oct 13, 2024 · 64327c1 · 64327c1
1 parent a1419d8
commit 64327c1
Show file tree

Hide file tree

Showing 10 changed files with 83 additions and 18 deletions.
diff --git a/Makefile b/Makefile
@@ -44,7 +44,7 @@ HELM_ARGS ?=
 K8GB_COREDNS_IP ?= kubectl get svc k8gb-coredns -n k8gb -o custom-columns='IP:spec.clusterIP' --no-headers
 LOG_FORMAT ?= simple
 LOG_LEVEL ?= debug
-CONTROLLER_GEN_VERSION  ?= v0.15.0
+CONTROLLER_GEN_VERSION  ?= v0.16.4
 GOLIC_VERSION  ?= v0.7.2
 GOLANGCI_VERSION ?= v1.59.1
 POD_NAMESPACE ?= k8gb

diff --git a/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml b/chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml
@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.15.0
+    controller-gen.kubebuilder.io/version: v0.16.4
   name: gslbs.k8gb.absa.oss
 spec:
   group: k8gb.absa.oss
@@ -111,6 +111,7 @@ spec:
                                 format: int32
                                 type: integer
                             type: object
+                            x-kubernetes-map-type: atomic
                         required:
                         - name
                         type: object
@@ -150,8 +151,8 @@ spec:
                             these may change in the future.\nIncoming requests are
                             matched against the host before the\nIngressRuleValue.
                             If the host is unspecified, the Ingress routes all\ntraffic
-                            based on the specified IngressRuleValue.\n\n\nHost can
-                            be \"precise\" which is a domain name without the terminating
+                            based on the specified IngressRuleValue.\n\nHost can be
+                            \"precise\" which is a domain name without the terminating
                             dot of\na network host (e.g. \"foo.bar.com\") or \"wildcard\",
                             which is a domain name\nprefixed with a single wildcard
                             label (e.g. \"*.foo.com\").\nThe wildcard character '*'
@@ -240,6 +241,7 @@ spec:
                                                 format: int32
                                                 type: integer
                                             type: object
+                                            x-kubernetes-map-type: atomic
                                         required:
                                         - name
                                         type: object

diff --git a/controllers/depresolver/testdata/invalid_omitempty_empty.yaml b/controllers/depresolver/testdata/invalid_omitempty_empty.yaml
@@ -37,5 +37,3 @@ spec:
     type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too
     splitBrainThresholdSeconds:
     weight:
-
-
diff --git a/controllers/gslb_controller_reconciliation.go b/controllers/gslb_controller_reconciliation.go
@@ -170,7 +170,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	}
 	gslb.Status.Servers = servers
 
-	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Config.EdgeDNSServers)
+	loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb.Annotations, r.Config.EdgeDNSServers)
 	if err != nil {
 		m.IncrementError(gslb)
 		return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err))

diff --git a/controllers/mocks/refresolver_mock.go b/controllers/mocks/refresolver_mock.go
diff --git a/controllers/refresolver/ingress/ingress.go b/controllers/refresolver/ingress/ingress.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strings"
 
 	k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
 	"github.com/k8gb-io/k8gb/controllers/logging"
@@ -35,6 +36,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	ingress *netv1.Ingress
 }
@@ -158,9 +164,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return strings.Split(ingressIPsFromAnnotation, ","), nil
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.ingress.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

diff --git a/controllers/refresolver/ingress/ingress_test.go b/controllers/refresolver/ingress/ingress_test.go
@@ -97,24 +97,46 @@ func TestGetServers(t *testing.T) {
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
 		name        string
+		annotations map[string]string
 		ingressYaml string
 		expectedIPs []string
 	}{
 		{
 			name:        "no exposed IPs",
+			annotations: map[string]string{},
 			ingressYaml: "./testdata/ingress_no_ips.yaml",
 			expectedIPs: []string{},
 		},
 		{
 			name:        "single exposed IP",
+			annotations: map[string]string{},
 			ingressYaml: "../testdata/ingress_referenced.yaml",
 			expectedIPs: []string{"10.0.0.1"},
 		},
 		{
 			name:        "multiple exposed IPs",
+			annotations: map[string]string{},
 			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
 			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
 		},
+		{
+			name:        "annotation with no exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs: []string{""},
+		},
+		{
+			name:        "annotation with single exposed IP",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153"},
+		},
+		{
+			name:        "annotation with multiple exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
+			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -125,7 +147,7 @@ func TestGetGslbExposedIPs(t *testing.T) {
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
+			IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{})
 			assert.NoError(t, err)
 
 			// assert

diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go
@@ -37,6 +37,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	virtualService *istio.VirtualService
 	lbService      *corev1.Service
@@ -191,9 +196,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 }
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
+		return strings.Split(ingressIPsFromAnnotation, ","), nil
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.lbService.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go
@@ -112,24 +112,46 @@ func TestGetServers(t *testing.T) {
 func TestGetGslbExposedIPs(t *testing.T) {
 	var tests = []struct {
 		name        string
+		annotations map[string]string
 		serviceYaml string
 		expectedIPs []string
 	}{
 		{
 			name:        "no exposed IPs",
 			serviceYaml: "./testdata/istio_service_no_ips.yaml",
+			annotations: map[string]string{},
 			expectedIPs: []string{},
 		},
 		{
 			name:        "single exposed IP",
+			annotations: map[string]string{},
 			serviceYaml: "../testdata/istio_service.yaml",
 			expectedIPs: []string{"10.0.0.1"},
 		},
 		{
 			name:        "multiple exposed IPs",
+			annotations: map[string]string{},
 			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
 			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
 		},
+		{
+			name:        "annotation with no exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""},
+			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs: []string{""},
+		},
+		{
+			name:        "annotation with single exposed IP",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
+			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153"},
+		},
+		{
+			name:        "annotation with multiple exposed IPs",
+			annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
+			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
@@ -140,7 +162,7 @@ func TestGetGslbExposedIPs(t *testing.T) {
 			}
 
 			// act
-			IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
+			IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{})
 			assert.NoError(t, err)
 
 			// assert

diff --git a/controllers/refresolver/refresolver.go b/controllers/refresolver/refresolver.go
@@ -34,7 +34,7 @@ type GslbReferenceResolver interface {
 	// GetServers retrieves GSLB the server configuration
 	GetServers() ([]*k8gbv1beta1.Server, error)
 	// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
-	GetGslbExposedIPs(utils.DNSList) ([]string, error)
+	GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error)
 }
 
 // New creates a new GSLBReferenceResolver