Add support for TLS Passthrough using TLSRoutes (nginx#2356)

Problem: As a user of NKG, I want to enable TLS Passthrough for my application's endpoints, so that I can achieve end-to-end encryption for my incoming traffic, and so that I do not have to manage certificates at the Gateway. Solution: Allow users to configure TLS Passthrough for their apps using TLSRoute. Adds basic support for TLSRoute. Cross-namespace routing via ReferenceGrants, traffic splitting, and TLS termination use case will be added in a future release. Note that the stream conf volume are always enabled in the deployment.yaml because our nginx conf reads from it. If the file did not exist then nginx will error.
miledxz · Aug 9, 2024 · fc6331b · fc6331b
1 parent 5d2928d
commit fc6331b
Show file tree

Hide file tree

Showing 65 changed files with 4,678 additions and 406 deletions.
diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml
@@ -75,6 +75,7 @@ jobs:
         run: |
           ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
           ngf_tag=${{ steps.ngf-meta.outputs.version }}
+          if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
           make generate-static-deployment PLUS_ENABLED=${{ inputs.image == 'plus' && 'true' || 'false' }} PREFIX=${ngf_prefix} TAG=${ngf_tag}
         working-directory: ./tests
 
@@ -146,6 +147,7 @@ jobs:
 
       - name: Run conformance tests
         run: |
+          if [ ${{ inputs.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
           make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} NGF_VERSION=${{ github.ref_name }} CLUSTER_NAME=${{ github.run_id }}
           core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
           extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')

diff --git a/charts/nginx-gateway-fabric/templates/clusterrole.yaml b/charts/nginx-gateway-fabric/templates/clusterrole.yaml
@@ -72,6 +72,7 @@ rules:
   - grpcroutes
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies
+  - tlsroutes
 {{- end }}
   verbs:
   - list
@@ -85,6 +86,7 @@ rules:
   - grpcroutes/status
 {{- if .Values.nginxGateway.gwAPIExperimentalFeatures.enable }}
   - backendtlspolicies/status
+  - tlsroutes/status
 {{- end }}
   verbs:
   - update

diff --git a/charts/nginx-gateway-fabric/templates/deployment.yaml b/charts/nginx-gateway-fabric/templates/deployment.yaml
@@ -129,6 +129,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -166,6 +168,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -200,6 +204,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/config/tests/static-deployment.yaml b/config/tests/static-deployment.yaml
@@ -72,6 +72,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -102,6 +104,8 @@ spec:
         volumeMounts:
         - name: nginx-conf
           mountPath: /etc/nginx/conf.d
+        - name: nginx-stream-conf
+          mountPath: /etc/nginx/stream-conf.d
         - name: module-includes
           mountPath: /etc/nginx/module-includes
         - name: nginx-secrets
@@ -121,6 +125,8 @@ spec:
       volumes:
       - name: nginx-conf
         emptyDir: {}
+      - name: nginx-stream-conf
+        emptyDir: {}
       - name: module-includes
         emptyDir: {}
       - name: nginx-secrets

diff --git a/deploy/aws-nlb/deploy.yaml b/deploy/aws-nlb/deploy.yaml
@@ -246,6 +246,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -276,6 +278,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -295,6 +299,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/azure/deploy.yaml b/deploy/azure/deploy.yaml
@@ -243,6 +243,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -273,6 +275,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -294,6 +298,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/default/deploy.yaml b/deploy/default/deploy.yaml
@@ -243,6 +243,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -273,6 +275,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -292,6 +296,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/experimental-nginx-plus/deploy.yaml b/deploy/experimental-nginx-plus/deploy.yaml
@@ -82,6 +82,7 @@ rules:
   - referencegrants
   - grpcroutes
   - backendtlspolicies
+  - tlsroutes
   verbs:
   - list
   - watch
@@ -93,6 +94,7 @@ rules:
   - gatewayclasses/status
   - grpcroutes/status
   - backendtlspolicies/status
+  - tlsroutes/status
   verbs:
   - update
 - apiGroups:
@@ -256,6 +258,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -286,6 +290,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -305,6 +311,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/experimental/deploy.yaml b/deploy/experimental/deploy.yaml
@@ -74,6 +74,7 @@ rules:
   - referencegrants
   - grpcroutes
   - backendtlspolicies
+  - tlsroutes
   verbs:
   - list
   - watch
@@ -85,6 +86,7 @@ rules:
   - gatewayclasses/status
   - grpcroutes/status
   - backendtlspolicies/status
+  - tlsroutes/status
   verbs:
   - update
 - apiGroups:
@@ -247,6 +249,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -277,6 +281,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -296,6 +302,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/nginx-plus/deploy.yaml b/deploy/nginx-plus/deploy.yaml
@@ -254,6 +254,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -284,6 +286,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -303,6 +307,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/nodeport/deploy.yaml b/deploy/nodeport/deploy.yaml
@@ -243,6 +243,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -273,6 +275,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -292,6 +296,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/deploy/openshift/deploy.yaml b/deploy/openshift/deploy.yaml
@@ -251,6 +251,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -281,6 +283,8 @@ spec:
         volumeMounts:
         - mountPath: /etc/nginx/conf.d
           name: nginx-conf
+        - mountPath: /etc/nginx/stream-conf.d
+          name: nginx-stream-conf
         - mountPath: /etc/nginx/module-includes
           name: module-includes
         - mountPath: /etc/nginx/secrets
@@ -300,6 +304,8 @@ spec:
       volumes:
       - emptyDir: {}
         name: nginx-conf
+      - emptyDir: {}
+        name: nginx-stream-conf
       - emptyDir: {}
         name: module-includes
       - emptyDir: {}

diff --git a/internal/framework/gatewayclass/validate.go b/internal/framework/gatewayclass/validate.go
@@ -23,6 +23,7 @@ var gatewayCRDs = map[string]apiVersion{
 	"referencegrants.gateway.networking.k8s.io":    {},
 	"backendtlspolicies.gateway.networking.k8s.io": {},
 	"grpcroutes.gateway.networking.k8s.io":         {},
+	"tlsroutes.gateway.networking.k8s.io":          {},
 }
 
 type apiVersion struct {

diff --git a/internal/framework/kinds/kinds.go b/internal/framework/kinds/kinds.go
@@ -19,6 +19,8 @@ const (
 	HTTPRoute = "HTTPRoute"
 	// GRPCRoute is the GRPCRoute kind.
 	GRPCRoute = "GRPCRoute"
+	// TLSRoute is the TLSRoute kind.
+	TLSRoute = "TLSRoute"
 )
 
 // NGINX Gateway Fabric kinds.

diff --git a/internal/mode/static/handler.go b/internal/mode/static/handler.go
@@ -246,6 +246,7 @@ func (h *eventHandlerImpl) updateStatuses(ctx context.Context, logger logr.Logge
 		gcReqs = status.PrepareGatewayClassRequests(graph.GatewayClass, graph.IgnoredGatewayClasses, transitionTime)
 	}
 	routeReqs := status.PrepareRouteRequests(
+		graph.L4Routes,
 		graph.Routes,
 		transitionTime,
 		h.latestReloadResult,

diff --git a/internal/mode/static/manager.go b/internal/mode/static/manager.go
@@ -31,6 +31,7 @@ import (
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
 	k8spredicate "sigs.k8s.io/controller-runtime/pkg/predicate"
 	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
+	gatewayv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gatewayv1alpha3 "sigs.k8s.io/gateway-api/apis/v1alpha3"
 	gatewayv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1"
 
@@ -73,6 +74,7 @@ func init() {
 	utilruntime.Must(gatewayv1beta1.Install(scheme))
 	utilruntime.Must(gatewayv1.Install(scheme))
 	utilruntime.Must(gatewayv1alpha3.Install(scheme))
+	utilruntime.Must(gatewayv1alpha2.Install(scheme))
 	utilruntime.Must(apiv1.AddToScheme(scheme))
 	utilruntime.Must(discoveryV1.AddToScheme(scheme))
 	utilruntime.Must(ngfAPI.AddToScheme(scheme))
@@ -489,6 +491,12 @@ func registerControllers(
 				// https://github.com/nginxinc/nginx-gateway-fabric/issues/1545
 				objectType: &apiv1.ConfigMap{},
 			},
+			{
+				objectType: &gatewayv1alpha2.TLSRoute{},
+				options: []controller.Option{
+					controller.WithK8sPredicate(k8spredicate.GenerationChangedPredicate{}),
+				},
+			},
 		}
 		controllerRegCfgs = append(controllerRegCfgs, gwExpFeatures...)
 	}
@@ -663,6 +671,7 @@ func prepareFirstEventBatchPreparerArgs(
 			objectLists,
 			&gatewayv1alpha3.BackendTLSPolicyList{},
 			&apiv1.ConfigMapList{},
+			&gatewayv1alpha2.TLSRouteList{},
 		)
 	}