feat(discovery) enable gateway discovery in db mode (#4828)

Support --kong-admin-svc in DB mode. Choose only a single endpoint from the Service at random and send updates to it. Track updates applied to one endpoint as applied to all endpoints in DB mode. --------- Co-authored-by: Patryk Małek <[email protected]> Co-authored-by: Grzegorz Burzyński <[email protected]>
Kong · Nov 2, 2023 · c39e31e · c39e31e
1 parent ffdb3db
commit c39e31e
Show file tree

Hide file tree

Showing 20 changed files with 3,111 additions and 79 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -249,6 +249,10 @@ Adding a new version? You'll need three changes:
 - KIC now specifies its `UserAgent` when communicating with kube-apiserver
   as `kong-ingress-controller/${VERSION}` where version is the version of KIC.
   [#5019](https://github.com/Kong/kubernetes-ingress-controller/pull/5019)
+- Allow Gateway Discovery with database backed Kong. KIC will send Kong 
+  configurations to one of the backend pods of the service specified by the
+  flag `--kong-admin-svc` if Kong gateway is DB backed.
+  [#4828](https://github.com/Kong/kubernetes-ingress-controller/pull/4828)
 
 [KIC Annotations reference]: https://docs.konghq.com/kubernetes-ingress-controller/latest/references/annotations/
 [KIC CRDs reference]: https://docs.konghq.com/kubernetes-ingress-controller/latest/references/custom-resources/

diff --git a/config/variants/multi-gw-postgres/gateway_admin_service.yaml b/config/variants/multi-gw-postgres/gateway_admin_service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kong-admin
+  namespace: kong
+spec:
+  clusterIP: "None"
+  selector:
+    app: proxy-kong
+  ports:
+  - name: admin
+    port: 8444
+    targetPort: 8444
+    protocol: TCP
diff --git a/config/variants/multi-gw-postgres/gateway_deployment.yaml b/config/variants/multi-gw-postgres/gateway_deployment.yaml
@@ -0,0 +1,113 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: proxy-kong
+  name: proxy-kong
+  namespace: kong
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: proxy-kong
+  template:
+    metadata:
+      annotations:
+        traffic.sidecar.istio.io/includeInboundPorts: ""
+        kuma.io/gateway: enabled
+        kuma.io/service-account-token-volume: kong-serviceaccount-token
+      labels:
+        app: proxy-kong
+    spec:
+      serviceAccountName: kong-serviceaccount
+      automountServiceAccountToken: false
+      volumes:
+      - name: kong-serviceaccount-token
+        projected:
+          sources:
+            - serviceAccountToken:
+                expirationSeconds: 3607
+                path: token
+            - configMap:
+                name: kube-root-ca.crt
+                items:
+                  - key: ca.crt
+                    path: ca.crt
+            - downwardAPI:
+                items:
+                  - fieldRef:
+                      apiVersion: v1
+                      fieldPath: metadata.namespace
+                    path: namespace
+      containers:
+      - name: proxy
+        image: kong-placeholder:placeholder # This is replaced by the config/image.yaml component
+        env:
+          # servers
+        - name: KONG_PROXY_LISTEN
+          value: 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384
+        - name: KONG_PORT_MAPS
+          value: "80:8000, 443:8443"
+        - name: KONG_ADMIN_LISTEN
+          value: 0.0.0.0:8444 http2 ssl reuseport backlog=16384
+        - name: KONG_STATUS_LISTEN
+          value: 0.0.0.0:8100
+          # DB
+        - name: KONG_DATABASE
+          value: "postgres"
+        - name: KONG_PG_HOST
+          value: postgres
+        - name: KONG_PG_PASSWORD
+          value: kong
+          # runtime tweaks
+        - name: KONG_NGINX_WORKER_PROCESSES
+          value: "2"
+        - name: KONG_KIC
+          value: "on"
+          # logging
+        - name: KONG_ADMIN_ACCESS_LOG
+          value: /dev/stdout
+        - name: KONG_ADMIN_ERROR_LOG
+          value: /dev/stderr
+        # - name: KONG_PROXY_ACCESS_LOG
+        # - value: /dev/stdout
+        - name: KONG_PROXY_ERROR_LOG
+          value: /dev/stderr
+        # router mode in 3.0.0. use `traditional` here for full compatibility.
+        - name: KONG_ROUTER_FLAVOR
+          value: traditional
+        lifecycle:
+          preStop:
+            exec:
+              command: [ "/bin/bash", "-c", "kong quit" ]
+        ports:
+        - name: proxy
+          containerPort: 8000
+          protocol: TCP
+        - name: proxy-ssl
+          containerPort: 8443
+          protocol: TCP
+        - name: metrics
+          containerPort: 8100
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            path: /status
+            port: 8100
+            scheme: HTTP
+          initialDelaySeconds: 5
+          timeoutSeconds: 1
+          periodSeconds: 10
+          successThreshold: 1
+          failureThreshold: 3
+        readinessProbe:
+          httpGet:
+            path: /status/ready
+            port: 8100
+            scheme: HTTP
+          initialDelaySeconds: 5
+          timeoutSeconds: 1
+          periodSeconds: 10
+          successThreshold: 1
+          failureThreshold: 3
diff --git a/config/variants/multi-gw-postgres/gateway_service_patch.yaml b/config/variants/multi-gw-postgres/gateway_service_patch.yaml
@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kong-proxy
+  namespace: kong
+spec:
+  selector:
+    app: proxy-kong
diff --git a/config/variants/multi-gw-postgres/kustomization.yaml b/config/variants/multi-gw-postgres/kustomization.yaml
@@ -0,0 +1,22 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: kong
+
+resources:
+- ../postgres
+- gateway_admin_service.yaml
+- gateway_deployment.yaml
+
+components:
+  - ../../image/oss
+
+patches:
+- path: manager_multi_gateway_patch.yaml
+- path: gateway_service_patch.yaml
+- target:
+    group: apps
+    version: v1
+    kind: Deployment
+    name: ingress-kong
+  path: ./remove_proxy_container.yaml
diff --git a/config/variants/multi-gw-postgres/manager_multi_gateway_patch.yaml b/config/variants/multi-gw-postgres/manager_multi_gateway_patch.yaml
@@ -0,0 +1,17 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: ingress-kong
+  name: ingress-kong
+  namespace: kong
+spec:
+  template:
+    spec:
+      containers:
+      - name: ingress-controller
+        env:
+        - name: CONTROLLER_KONG_ADMIN_SVC
+          value: kong/kong-admin
+        - name: CONTROLLER_KONG_ADMIN_URL
+          $patch: delete
diff --git a/config/variants/multi-gw-postgres/remove_proxy_container.yaml b/config/variants/multi-gw-postgres/remove_proxy_container.yaml
@@ -0,0 +1,2 @@
+- op: remove
+  path: "/spec/template/spec/containers/1"
diff --git a/deploy/single/all-in-one-postgres-multiple-gateways.yaml b/deploy/single/all-in-one-postgres-multiple-gateways.yaml
diff --git a/internal/clients/manager.go b/internal/clients/manager.go
@@ -13,6 +13,7 @@ import (
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/adminapi"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util"
 	"github.com/kong/kubernetes-ingress-controller/v3/internal/util/clock"
+	dataplaneutil "github.com/kong/kubernetes-ingress-controller/v3/internal/util/dataplane"
 )
 
 // DefaultReadinessReconciliationInterval is the interval at which the manager will run readiness reconciliation loop.
@@ -29,6 +30,7 @@ type ClientFactory interface {
 type AdminAPIClientsProvider interface {
 	KonnectClient() *adminapi.KonnectClient
 	GatewayClients() []*adminapi.Client
+	GatewayClientsToConfigure() []*adminapi.Client
 }
 
 // Ticker is an interface that allows to control a ticker.
@@ -48,6 +50,8 @@ type AdminAPIClientsManager struct {
 	discoveredAdminAPIsNotifyChan    chan []adminapi.DiscoveredAdminAPI
 	gatewayClientsChangesSubscribers []chan struct{}
 
+	dbMode string
+
 	ctx                   context.Context
 	onceNotifyLoopRunning sync.Once
 	runningChan           chan struct{}
@@ -85,6 +89,12 @@ func WithReadinessReconciliationTicker(ticker Ticker) AdminAPIClientsManagerOpti
 	}
 }
 
+// WithDBMode allows to set the DBMode of the Kong gateway instances behind the admin API service.
+func (c *AdminAPIClientsManager) WithDBMode(dbMode string) *AdminAPIClientsManager {
+	c.dbMode = dbMode
+	return c
+}
+
 func NewAdminAPIClientsManager(
 	ctx context.Context,
 	logger logr.Logger,
@@ -173,6 +183,29 @@ func (c *AdminAPIClientsManager) GatewayClients() []*adminapi.Client {
 	return lo.Values(c.readyGatewayClients)
 }
 
+// GatewayClientsToConfigure returns the gateway clients which need to be configured with the new configuration.
+// In DBLess mode, it returns ALL gateway clients
+// because we need to update configurations of each gateway instance.
+// In DB-backed mode, it returns ONE random gateway client
+// because we only need to send configurations to one gateway instance
+// while others will be synced using the DB.
+func (c *AdminAPIClientsManager) GatewayClientsToConfigure() []*adminapi.Client {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	readyGatewayClients := lo.Values(c.readyGatewayClients)
+	// With DB-less mode, we should send the configuration to ALL gateway instances.
+	if dataplaneutil.IsDBLessMode(c.dbMode) {
+		return readyGatewayClients
+	}
+	// When a gateway is DB-backed, we return a random client
+	// since KIC only needs to send requests to one instance.
+	// If there are no ready gateway clients, we return an empty list.
+	if len(readyGatewayClients) == 0 {
+		return []*adminapi.Client{}
+	}
+	return readyGatewayClients[:1]
+}
+
 func (c *AdminAPIClientsManager) GatewayClientsCount() int {
 	c.lock.RLock()
 	defer c.lock.RUnlock()

diff --git a/internal/clients/manager_test.go b/internal/clients/manager_test.go
@@ -158,7 +158,12 @@ func TestAdminAPIClientsManager_OnNotifyClientsAreUpdatedAccordingly(t *testing.
 }
 
 func TestNewAdminAPIClientsManager_NoInitialClientsDisallowed(t *testing.T) {
-	_, err := clients.NewAdminAPIClientsManager(context.Background(), zapr.NewLogger(zap.NewNop()), nil, &mockReadinessChecker{})
+	_, err := clients.NewAdminAPIClientsManager(
+		context.Background(),
+		zapr.NewLogger(zap.NewNop()),
+		nil,
+		&mockReadinessChecker{},
+	)
 	require.ErrorContains(t, err, "at least one initial client must be provided")
 }
 
@@ -196,6 +201,7 @@ func TestAdminAPIClientsManager_Clients(t *testing.T) {
 	require.NoError(t, err)
 	require.Len(t, m.GatewayClients(), 1, "expecting one initial client")
 	require.Equal(t, m.GatewayClientsCount(), 1, "expecting one initial client")
+	require.Len(t, m.GatewayClientsToConfigure(), 1, "Expecting one initial client")
 
 	konnectTestClient := &adminapi.KonnectClient{}
 	m.SetKonnectClient(konnectTestClient)
@@ -204,6 +210,35 @@ func TestAdminAPIClientsManager_Clients(t *testing.T) {
 	require.Equal(t, konnectTestClient, m.KonnectClient(), "konnect client should be returned from KonnectClient")
 }
 
+func TestAdminAPIClientsManager_Clients_DBMode(t *testing.T) {
+	testClient, err := adminapi.NewTestClient("localhost:8080")
+	require.NoError(t, err)
+	testClient2, err := adminapi.NewTestClient("localhost:8081")
+	require.NoError(t, err)
+	initialClients := []*adminapi.Client{testClient, testClient2}
+	require.NoError(t, err)
+
+	m, err := clients.NewAdminAPIClientsManager(
+		context.Background(),
+		zapr.NewLogger(zap.NewNop()),
+		initialClients,
+		&mockReadinessChecker{},
+	)
+	require.NoError(t, err)
+	m = m.WithDBMode("postgres")
+
+	clients := m.GatewayClients()
+	require.Len(t, clients, 2, "Expecting 2 clients returned with DB mode")
+
+	configureClients := m.GatewayClientsToConfigure()
+	require.Len(t, configureClients, 1, "Expecting 1 client to configure")
+	require.Truef(t, lo.ContainsBy(initialClients, func(c *adminapi.Client) bool {
+		return c.BaseRootURL() == configureClients[0].BaseRootURL()
+	}), "Client's address %s should be in initial clients")
+
+	require.Equal(t, m.GatewayClientsCount(), 2, "Expecting 2 initial clients")
+}
+
 func TestAdminAPIClientsManager_SubscribeToGatewayClientsChanges(t *testing.T) {
 	t.Parallel()
 
@@ -212,7 +247,12 @@ func TestAdminAPIClientsManager_SubscribeToGatewayClientsChanges(t *testing.T) {
 	require.NoError(t, err)
 
 	ctx, cancel := context.WithCancel(context.Background())
-	m, err := clients.NewAdminAPIClientsManager(ctx, zapr.NewLogger(zap.NewNop()), []*adminapi.Client{testClient}, readinessChecker)
+	m, err := clients.NewAdminAPIClientsManager(
+		ctx,
+		zapr.NewLogger(zap.NewNop()),
+		[]*adminapi.Client{testClient},
+		readinessChecker)
+
 	require.NoError(t, err)
 
 	t.Run("no notify loop running should return false when subscribing", func(t *testing.T) {

diff --git a/internal/dataplane/kong_client.go b/internal/dataplane/kong_client.go
@@ -454,15 +454,38 @@ func (c *KongClient) sendOutToGatewayClients(
 	ctx context.Context, s *kongstate.KongState, config sendconfig.Config,
 ) ([]string, error) {
 	gatewayClients := c.clientsProvider.GatewayClients()
-	c.logger.V(util.DebugLevel).Info("sending configuration to gateway clients", "count", len(gatewayClients))
-	shas, err := iter.MapErr(gatewayClients, func(client **adminapi.Client) (string, error) {
+	if len(gatewayClients) == 0 {
+		c.logger.Error(
+			errors.New("no ready gateway clients"),
+			"Could not send configuration to gateways",
+		)
+		// Should not store the configuration in last valid config because the configuration is not validated on Kong gateway.
+		return c.SHAs, nil
+	}
+
+	gatewayClientsToConfigure := c.clientsProvider.GatewayClientsToConfigure()
+	configureGatewayClientURLs := lo.Map(gatewayClientsToConfigure, func(cl *adminapi.Client, _ int) string { return cl.BaseRootURL() })
+	c.logger.V(util.DebugLevel).Info("Sending configuration to gateway clients", "urls", configureGatewayClientURLs)
+
+	shas, err := iter.MapErr(gatewayClientsToConfigure, func(client **adminapi.Client) (string, error) {
 		return c.sendToClient(ctx, *client, s, config)
 	})
 	if err != nil {
 		return nil, err
 	}
-	previousSHAs := c.SHAs
 
+	// After a successful configuration update in DB mode,
+	// since only ONE gateway client is chosen to send requests and store SHA of latest configurations,
+	// we should propagate the SHA from the chosen client to other clients
+	// as well as they will pick the configuration from the shared database.
+	if dataplaneutil.DBBacked(c.dbmode) &&
+		len(gatewayClients) > 1 {
+		for _, client := range gatewayClients {
+			client.SetLastConfigSHA([]byte(shas[0]))
+		}
+	}
+
+	previousSHAs := c.SHAs
 	sort.Strings(shas)
 	c.SHAs = shas