Skip to content

Commit

Permalink
Add reverse proxy support
Browse files Browse the repository at this point in the history 
Problem
K8GB reads IP addresses from `Ingress.Status.LoadBalancer.Ingress` or from `Service.Status.LoadBalancer.Ingress` for ingress configured with Kubernetes Ingress and Istio Virtual Service, respectively.
The IP addresses exposed by these resources are the IP addresses exposed by the Kubernetes Cluster. However, in some setups the clients do not route their traffic to these IP addresses because the cluster is behind a reverse proxy.

Solution
To support this setup, K8GB should expose DNS records with the IP address of the reverse proxy. Since the address is unknown to the cluster the K8GB administrator must provide it via configuration. This PR adds to K8GB the capability to read IP address from an annotation `k8gb.io/external-ips` on the GSLB resource.

Examples
```
apiVersion: k8gb.absa.oss/v1beta1
kind: Gslb
metadata:
  labels:
    app: ingress
  annotations:
    k8gb.io/external-ips: "185.199.110.153"
```

Fixes #1275

Signed-off-by: Andre Baptista Aguas <[email protected]>
  • Loading branch information
@abaguas
abaguas committed Oct 13, 2024
1 parent a1419d8 commit af3314d
Show file tree
Hide file tree
Showing 13 changed files with 164 additions and 18 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ HELM_ARGS ?=
K8GB_COREDNS_IP ?= kubectl get svc k8gb-coredns -n k8gb -o custom-columns='IP:spec.clusterIP' --no-headers
LOG_FORMAT ?= simple
LOG_LEVEL ?= debug
CONTROLLER_GEN_VERSION ?= v0.15.0
CONTROLLER_GEN_VERSION ?= v0.16.4
GOLIC_VERSION ?= v0.7.2
GOLANGCI_VERSION ?= v1.59.1
POD_NAMESPACE ?= k8gb
Expand Down
8 changes: 5 additions & 3 deletions chart/k8gb/crd/k8gb.absa.oss_gslbs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.15.0
controller-gen.kubebuilder.io/version: v0.16.4
name: gslbs.k8gb.absa.oss
spec:
group: k8gb.absa.oss
Expand Down Expand Up @@ -111,6 +111,7 @@ spec:
format: int32
type: integer
type: object
x-kubernetes-map-type: atomic
required:
- name
type: object
Expand Down Expand Up @@ -150,8 +151,8 @@ spec:
these may change in the future.\nIncoming requests are
matched against the host before the\nIngressRuleValue.
If the host is unspecified, the Ingress routes all\ntraffic
based on the specified IngressRuleValue.\n\n\nHost can
be \"precise\" which is a domain name without the terminating
based on the specified IngressRuleValue.\n\nHost can be
\"precise\" which is a domain name without the terminating
dot of\na network host (e.g. \"foo.bar.com\") or \"wildcard\",
which is a domain name\nprefixed with a single wildcard
label (e.g. \"*.foo.com\").\nThe wildcard character '*'
Expand Down Expand Up @@ -240,6 +241,7 @@ spec:
format: int32
type: integer
type: object
x-kubernetes-map-type: atomic
required:
- name
type: object
Expand Down
2 changes: 0 additions & 2 deletions controllers/depresolver/testdata/invalid_omitempty_empty.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,5 +37,3 @@ spec:
type: roundRobin # Use a round robin load balancing strategy, when deciding which downstream clusters to route clients too
splitBrainThresholdSeconds:
weight:


2 changes: 1 addition & 1 deletion controllers/gslb_controller_reconciliation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
}
gslb.Status.Servers = servers

loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(r.Config.EdgeDNSServers)
loadBalancerExposedIPs, err := refResolver.GetGslbExposedIPs(gslb.Annotations, r.Config.EdgeDNSServers)
if err != nil {
m.IncrementError(gslb)
return result.RequeueError(fmt.Errorf("getting load balancer exposed IPs (%s)", err))
Expand Down
8 changes: 4 additions & 4 deletions controllers/mocks/refresolver_mock.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 13 additions & 2 deletions controllers/refresolver/ingress/ingress.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"context"
"fmt"
"reflect"
"strings"

k8gbv1beta1 "github.com/k8gb-io/k8gb/api/v1beta1"
"github.com/k8gb-io/k8gb/controllers/logging"
Expand All @@ -35,6 +36,11 @@ import (

var log = logging.Logger()

const (
// comma separated list of external IP addresses
externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
)

type ReferenceResolver struct {
ingress *netv1.Ingress
}
Expand Down Expand Up @@ -158,9 +164,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
}

// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
gslbIngressIPs := []string{}
func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
// fetch the IP addresses of the reverse proxy from an annotation if it exists
if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
return strings.Split(ingressIPsFromAnnotation, ","), nil
}

// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
gslbIngressIPs := []string{}
for _, ip := range rr.ingress.Status.LoadBalancer.Ingress {
if len(ip.IP) > 0 {
gslbIngressIPs = append(gslbIngressIPs, ip.IP)
Expand Down
24 changes: 23 additions & 1 deletion controllers/refresolver/ingress/ingress_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,24 +97,46 @@ func TestGetServers(t *testing.T) {
func TestGetGslbExposedIPs(t *testing.T) {
var tests = []struct {
name string
annotations map[string]string
ingressYaml string
expectedIPs []string
}{
{
name: "no exposed IPs",
annotations: map[string]string{},
ingressYaml: "./testdata/ingress_no_ips.yaml",
expectedIPs: []string{},
},
{
name: "single exposed IP",
annotations: map[string]string{},
ingressYaml: "../testdata/ingress_referenced.yaml",
expectedIPs: []string{"10.0.0.1"},
},
{
name: "multiple exposed IPs",
annotations: map[string]string{},
ingressYaml: "./testdata/ingress_multiple_ips.yaml",
expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
},
{
name: "annotation with no exposed IPs",
annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""},
ingressYaml: "./testdata/ingress_multiple_ips.yaml",
expectedIPs: []string{""},
},
{
name: "annotation with single exposed IP",
annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
ingressYaml: "./testdata/ingress_multiple_ips.yaml",
expectedIPs: []string{"185.199.110.153"},
},
{
name: "annotation with multiple exposed IPs",
annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
ingressYaml: "./testdata/ingress_multiple_ips.yaml",
expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
Expand All @@ -125,7 +147,7 @@ func TestGetGslbExposedIPs(t *testing.T) {
}

// act
IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{})
assert.NoError(t, err)

// assert
Expand Down
27 changes: 27 additions & 0 deletions controllers/refresolver/ingress/testdata/ingress_annotation_multiple_ips.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
labels:
app: ingress-referenced
annotations:
k8gb.io/exposed-ip-addresses: "185.199.110.153,185.199.109.153"
name: ingress-referenced
namespace: test-gslb
resourceVersion: "999"
spec:
ingressClassName: nginx
rules:
- host: ingress-referenced.cloud.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: referenced
port:
name: http
status:
loadBalancer:
ingress:
- ip: 10.0.0.1
27 changes: 27 additions & 0 deletions controllers/refresolver/ingress/testdata/ingress_annotation_no_ips.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
labels:
app: ingress-referenced
annotations:
k8gb.io/exposed-ip-addresses: ""
name: ingress-referenced
namespace: test-gslb
resourceVersion: "999"
spec:
ingressClassName: nginx
rules:
- host: ingress-referenced.cloud.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: referenced
port:
name: http
status:
loadBalancer:
ingress:
- ip: 10.0.0.1
27 changes: 27 additions & 0 deletions controllers/refresolver/ingress/testdata/ingress_annotation_single_ip.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
labels:
app: ingress-referenced
annotations:
k8gb.io/exposed-ip-addresses: "185.199.110.153"
name: ingress-referenced
namespace: test-gslb
resourceVersion: "999"
spec:
ingressClassName: nginx
rules:
- host: ingress-referenced.cloud.example.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: referenced
port:
name: http
status:
loadBalancer:
ingress:
- ip: 10.0.0.1
14 changes: 12 additions & 2 deletions controllers/refresolver/istiovirtualservice/istiovirtualservice.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,11 @@ import (

var log = logging.Logger()

const (
// comma separated list of external IP addresses
externalIPsAnnotation = "k8gb.io/exposed-ip-addresses"
)

type ReferenceResolver struct {
virtualService *istio.VirtualService
lbService *corev1.Service
Expand Down Expand Up @@ -191,9 +196,14 @@ func (rr *ReferenceResolver) GetServers() ([]*k8gbv1beta1.Server, error) {
}

// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
func (rr *ReferenceResolver) GetGslbExposedIPs(edgeDNSServers utils.DNSList) ([]string, error) {
gslbIngressIPs := []string{}
func (rr *ReferenceResolver) GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error) {
// fetch the IP addresses of the reverse proxy from an annotation if it exists
if ingressIPsFromAnnotation, ok := gslbAnnotations[externalIPsAnnotation]; ok {
return strings.Split(ingressIPsFromAnnotation, ","), nil
}

// if there is no annotation -> fetch the IP addresses from the Status of the Ingress resource
gslbIngressIPs := []string{}
for _, ip := range rr.lbService.Status.LoadBalancer.Ingress {
if len(ip.IP) > 0 {
gslbIngressIPs = append(gslbIngressIPs, ip.IP)
Expand Down
24 changes: 23 additions & 1 deletion controllers/refresolver/istiovirtualservice/istiovirtualservice_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,24 +112,46 @@ func TestGetServers(t *testing.T) {
func TestGetGslbExposedIPs(t *testing.T) {
var tests = []struct {
name string
annotations map[string]string
serviceYaml string
expectedIPs []string
}{
{
name: "no exposed IPs",
serviceYaml: "./testdata/istio_service_no_ips.yaml",
annotations: map[string]string{},
expectedIPs: []string{},
},
{
name: "single exposed IP",
annotations: map[string]string{},
serviceYaml: "../testdata/istio_service.yaml",
expectedIPs: []string{"10.0.0.1"},
},
{
name: "multiple exposed IPs",
annotations: map[string]string{},
serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
expectedIPs: []string{"10.0.0.1", "10.0.0.2"},
},
{
name: "annotation with no exposed IPs",
annotations: map[string]string{"k8gb.io/exposed-ip-addresses": ""},
serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
expectedIPs: []string{""},
},
{
name: "annotation with single exposed IP",
annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153"},
serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
expectedIPs: []string{"185.199.110.153"},
},
{
name: "annotation with multiple exposed IPs",
annotations: map[string]string{"k8gb.io/exposed-ip-addresses": "185.199.110.153,185.199.109.153"},
serviceYaml: "./testdata/istio_service_multiple_ips.yaml",
expectedIPs: []string{"185.199.110.153", "185.199.109.153"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
Expand All @@ -140,7 +162,7 @@ func TestGetGslbExposedIPs(t *testing.T) {
}

// act
IPs, err := resolver.GetGslbExposedIPs([]utils.DNSServer{})
IPs, err := resolver.GetGslbExposedIPs(test.annotations, []utils.DNSServer{})
assert.NoError(t, err)

// assert
Expand Down
2 changes: 1 addition & 1 deletion controllers/refresolver/refresolver.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ type GslbReferenceResolver interface {
// GetServers retrieves GSLB the server configuration
GetServers() ([]*k8gbv1beta1.Server, error)
// GetGslbExposedIPs retrieves the load balancer IP address of the GSLB
GetGslbExposedIPs(utils.DNSList) ([]string, error)
GetGslbExposedIPs(gslbAnnotations map[string]string, edgeDNSServers utils.DNSList) ([]string, error)
}

// New creates a new GSLBReferenceResolver
Expand Down

0 comments on commit af3314d

Please sign in to comment.