Add reverse proxy support

Problem K8GB reads IP addresses from `Ingress.Status.LoadBalancer.Ingress` or from `Service.Status.LoadBalancer.Ingress` for ingress configured with Kubernetes Ingress and Istio Virtual Service, respectively. The IP addresses exposed by these resources are the IP addresses exposed by the Kubernetes Cluster. However, in some setups the clients do not route their traffic to these IP addresses because the cluster is behind a reverse proxy. Solution To support this setup, K8GB should expose DNS records with the IP address of the reverse proxy. Since the address is unknown to the cluster the K8GB administrator must provide it via configuration. This PR adds to K8GB the capability to read IP address from an annotation `k8gb.io/external-ips` on Ingress and Service resources. Examples ``` apiVersion: networking.k8s.io/v1 kind: Ingress metadata: labels: app: ingress annotations: k8gb.io/external-ips: "185.199.110.153" ``` ``` apiVersion: v1 kind: Service metadata: name: istio-ingressgateway namespace: istio-ingress labels: app: istio-ingressgateway annotations: k8gb.io/external-ips: "185.199.110.153,185.199.109.153" ``` Fixes k8gb-io#1275 Signed-off-by: Andre Baptista Aguas <[email protected]>
abaguas · Aug 25, 2024 · 8274656 · 8274656
1 parent fa96d9b
commit 8274656
Show file tree

Hide file tree

Showing 10 changed files with 200 additions and 2 deletions.
diff --git a/controllers/refresolver/ingress/ingress.go b/controllers/refresolver/ingress/ingress.go
@@ -22,6 +22,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strings"
 
 	k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
 	"github.com/k8gb-io/k8gb/controllers/logging"
@@ -35,6 +36,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	ingress *netv1.Ingress
 }
@@ -158,8 +164,13 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
 func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := rr.ingress.Annotations[externalIPsAnnotation]; ok {
+		return strings.Split(ingressIPsFromAnnotation, ","), nil
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.ingress.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

diff --git a/controllers/refresolver/ingress/ingress_test.go b/controllers/refresolver/ingress/ingress_test.go
@@ -115,6 +115,21 @@ func TestGetGslbExposedIPs(t *testing.T) {
 			ingressYaml: "./testdata/ingress_multiple_ips.yaml",
 			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
 		},
+		{
+			name:        "annotation with no exposed IPs",
+			ingressYaml: "./testdata/ingress_annotation_no_ips.yaml",
+			expectedIPs: []string{""},
+		},
+		{
+			name:        "annotation with single exposed IP",
+			ingressYaml: "./testdata/ingress_annotation_single_ip.yaml",
+			expectedIPs: []string{"185.199.110.153"},
+		},
+		{
+			name:        "annotation with multiple exposed IPs",
+			ingressYaml: "./testdata/ingress_annotation_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

diff --git a/controllers/refresolver/ingress/testdata/ingress_annotation_multiple_ips.yaml b/controllers/refresolver/ingress/testdata/ingress_annotation_multiple_ips.yaml
@@ -0,0 +1,27 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  labels:
+    app: ingress-referenced
+  annotations:
+    k8gb.io/exposed-ip-addresses: "185.199.110.153,185.199.109.153"
+  name: ingress-referenced
+  namespace: test-gslb
+  resourceVersion: "999"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: ingress-referenced.cloud.example.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: referenced
+                port:
+                  name: http
+status:
+  loadBalancer:
+    ingress:
+      - ip: 10.0.0.1
diff --git a/controllers/refresolver/ingress/testdata/ingress_annotation_no_ips.yaml b/controllers/refresolver/ingress/testdata/ingress_annotation_no_ips.yaml
@@ -0,0 +1,27 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  labels:
+    app: ingress-referenced
+  annotations:
+    k8gb.io/exposed-ip-addresses: ""
+  name: ingress-referenced
+  namespace: test-gslb
+  resourceVersion: "999"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: ingress-referenced.cloud.example.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: referenced
+                port:
+                  name: http
+status:
+  loadBalancer:
+    ingress:
+      - ip: 10.0.0.1
diff --git a/controllers/refresolver/ingress/testdata/ingress_annotation_single_ip.yaml b/controllers/refresolver/ingress/testdata/ingress_annotation_single_ip.yaml
@@ -0,0 +1,27 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  labels:
+    app: ingress-referenced
+  annotations:
+    k8gb.io/exposed-ip-addresses: "185.199.110.153"
+  name: ingress-referenced
+  namespace: test-gslb
+  resourceVersion: "999"
+spec:
+  ingressClassName: nginx
+  rules:
+    - host: ingress-referenced.cloud.example.com
+      http:
+        paths:
+          - path: /
+            pathType: Prefix
+            backend:
+              service:
+                name: referenced
+                port:
+                  name: http
+status:
+  loadBalancer:
+    ingress:
+      - ip: 10.0.0.1
diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice.go
@@ -38,6 +38,11 @@ import (
 
 var log = logging.Logger()
 
+const (
+	// comma separated list of external IP addresses
+	externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
+)
+
 type ReferenceResolver struct {
 	virtualService *istio.VirtualService
 	lbService      *corev1.Service
@@ -199,8 +204,13 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
 
 // GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
 func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
-	gslbIngressIPs := []string{}
+	// fetch the IP addresses of the reverse proxy from an annotation if it exists
+	if ingressIPsFromAnnotation, ok := rr.lbService.Annotations[externalIPsAnnotation]; ok {
+		return strings.Split(ingressIPsFromAnnotation, ","), nil
+	}
 
+	// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
+	gslbIngressIPs := []string{}
 	for _, ip := range rr.lbService.Status.LoadBalancer.Ingress {
 		if len(ip.IP) > 0 {
 			gslbIngressIPs = append(gslbIngressIPs, ip.IP)

diff --git a/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go b/controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go
@@ -130,6 +130,21 @@ func TestGetGslbExposedIPs(t *testing.T) {
 			serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
 			expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
 		},
+		{
+			name:        "annotation with no exposed IPs",
+			serviceYaml: "./testdata/istio_service_annotation_no_ips.yaml",
+			expectedIPs: []string{""},
+		},
+		{
+			name:        "annotation with single exposed IP",
+			serviceYaml: "./testdata/istio_service_annotation_single_ip.yaml",
+			expectedIPs: []string{"185.199.110.153"},
+		},
+		{
+			name:        "annotation with multiple exposed IPs",
+			serviceYaml: "./testdata/istio_service_annotation_multiple_ips.yaml",
+			expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
+		},
 	}
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {

diff --git a/...llers/refresolver/istiovirtualservice/testdata/istio_service_annotation_multiple_ips.yaml b/...llers/refresolver/istiovirtualservice/testdata/istio_service_annotation_multiple_ips.yaml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: istio-ingressgateway
+  namespace: istio-ingress
+  labels:
+    app: istio-ingressgateway
+  annotations:
+    k8gb.io/exposed-ip-addresses: "185.199.110.153,185.199.109.153"
+spec:
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: istio-ingressgateway
+  type: LoadBalancer
+status:
+  loadBalancer:
+    ingress:
+      - ip: 10.0.0.1
diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_no_ips.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_no_ips.yaml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: istio-ingressgateway
+  namespace: istio-ingress
+  labels:
+    app: istio-ingressgateway
+  annotations:
+    k8gb.io/exposed-ip-addresses: ""
+spec:
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: istio-ingressgateway
+  type: LoadBalancer
+status:
+  loadBalancer:
+    ingress:
+      - ip: 10.0.0.1
diff --git a/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_single_ip.yaml b/controllers/refresolver/istiovirtualservice/testdata/istio_service_annotation_single_ip.yaml
@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: istio-ingressgateway
+  namespace: istio-ingress
+  labels:
+    app: istio-ingressgateway
+  annotations:
+    k8gb.io/exposed-ip-addresses: "185.199.110.153"
+spec:
+  ports:
+    - name: http
+      port: 80
+      protocol: TCP
+      targetPort: 8080
+  selector:
+    app: istio-ingressgateway
+  type: LoadBalancer
+status:
+  loadBalancer:
+    ingress:
+      - ip: 10.0.0.1