Dynamically secure RouteGroups using Kubernetes TLS Secrets (#2814)

* Add TLS spec to RouteGroup CRD * Add RouteGroup implementation and tests for securing TLS using secrets Signed-off-by: Ricardo Herrera <[email protected]>
zalando · Jan 9, 2024 · cec514c · cec514c
1 parent a7f049c
commit cec514c
Show file tree

Hide file tree

Showing 42 changed files with 805 additions and 174 deletions.
diff --git a/dataclients/kubernetes/definitions/routegroups.go b/dataclients/kubernetes/definitions/routegroups.go
@@ -59,6 +59,10 @@ type RouteGroupSpec struct {
 	// Routes specifies the list of route based on path, method
 	// and predicates.
 	Routes []*RouteSpec `json:"routes,omitempty"`
+
+	// TLS specifies the list of Kubernetes TLS secrets to
+	// be used to terminate the TLS connection
+	TLS []*RouteTLSSpec `json:"tls,omitempty"`
 }
 
 // SkipperBackend is the type safe version of skipperBackendParser
@@ -155,6 +159,16 @@ type RouteSpec struct {
 	Methods []string `json:"methods,omitempty"`
 }
 
+type RouteTLSSpec struct {
+	// Hosts specifies the list of hosts included in the
+	// TLS certificate
+	Hosts []string `json:"hosts,omitempty"`
+
+	// SecretName specifies the Kubernetes TLS secret to be
+	// used to terminate the TLS SNI connection
+	SecretName string `json:"secretName,omitempty"`
+}
+
 func backendsWithDuplicateName(name string) error {
 	return fmt.Errorf("backends with duplicate name: %s", name)
 }

diff --git a/dataclients/kubernetes/deploy/apply/routegroups_crd.yaml b/dataclients/kubernetes/deploy/apply/routegroups_crd.yaml
@@ -1,5 +1,3 @@
-# This is a copy of https://github.com/szuecs/routegroup-client/blob/master/zalando.org_routegroups.yaml
-# DO NOT EDIT.
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
@@ -216,6 +214,31 @@ spec:
                   type: object
                 minItems: 1
                 type: array
+              tls:
+                description: TLS defines which Kubernetes secret will be used to terminate
+                  the connection based on the matching hostnames
+                items:
+                  properties:
+                    hosts:
+                      description: TLS hosts specify the list of hosts included in
+                        the TLS secret. The values in this list must match the host
+                        name(s) used for the RouteGroup in order to terminate TLS
+                        for the host(s).
+                      items:
+                        pattern: "^[a-z0-9]([-a-z0-9]*[a-z0-9])?([.][a-z0-9]([-a-z0-9]*[a-z0-9])?)*$"
+                        type: string
+                      minItems: 1
+                      type: array
+                    secretName:
+                      description: SecretName is the name of the secret used to terminate
+                        TLS traffic. Secret should reside in the same namespace as
+                        the RouteGroup.
+                      type: string
+                  required:
+                  - hosts
+                  - secretName
+                  type: object
+                type: array
             required:
             - backends
             type: object

diff --git a/dataclients/kubernetes/ingress.go b/dataclients/kubernetes/ingress.go
@@ -337,27 +337,6 @@ func hasCatchAllRoutes(routes []*eskip.Route) bool {
 	return false
 }
 
-// addHostTLSCert adds a TLS certificate to the certificate registry per host when the referenced
-// secret is found and is a valid TLS secret.
-func addHostTLSCert(ic *ingressContext, hosts []string, secretID *definitions.ResourceID) {
-	secret, ok := ic.state.secrets[*secretID]
-	if !ok {
-		ic.logger.Errorf("Failed to find secret %s in namespace %s", secretID.Name, secretID.Namespace)
-		return
-	}
-	cert, err := generateTLSCertFromSecret(secret)
-	if err != nil {
-		ic.logger.Errorf("Failed to generate TLS certificate from secret: %v", err)
-		return
-	}
-	for _, host := range hosts {
-		err := ic.certificateRegistry.ConfigureCertificate(host, cert)
-		if err != nil {
-			ic.logger.Errorf("Failed to configure certificate: %v", err)
-		}
-	}
-}
-
 // convert logs if an invalid found, but proceeds with the valid ones.
 // Reporting failures in Ingress status is not possible, because
 // Ingress status field only supports IP and Hostname as string.

diff --git a/dataclients/kubernetes/ingressv1.go b/dataclients/kubernetes/ingressv1.go
@@ -293,15 +293,31 @@ func (ing *ingress) addSpecRuleV1(ic *ingressContext, ru *definitions.RuleV1) er
 // addSpecIngressTLSV1 is used to add TLS Certificates from Ingress resources. Certificates will be added
 // only if the Ingress rule host matches a host in TLS config
 func (ing *ingress) addSpecIngressTLSV1(ic *ingressContext, ingtls *definitions.TLSV1) {
+	ingressHosts := definitions.GetHostsFromIngressRulesV1(ic.ingressV1)
+
 	// Hosts in the tls section need to explicitly match the host in the rules section.
-	hostlist := compareStringList(ingtls.Hosts, definitions.GetHostsFromIngressRulesV1(ic.ingressV1))
+	hostlist := compareStringList(ingtls.Hosts, ingressHosts)
 	if len(hostlist) == 0 {
-		ic.logger.Infof("No matching tls hosts found")
+		ic.logger.Errorf("No matching tls hosts found - tls hosts: %s, ingress hosts: %s", ingtls.Hosts, ingressHosts)
+		return
+	} else if len(hostlist) != len(ingtls.Hosts) {
+		ic.logger.Infof("Hosts in TLS and Ingress don't match: tls hosts: %s, ingress hosts: %s", ingtls.Hosts, definitions.GetHostsFromIngressRulesV1(ic.ingressV1))
+	}
+
+	// Skip adding certs to registry since no certs defined
+	if ingtls.SecretName == "" {
+		ic.logger.Debugf("No tls secret defined for hosts - %s", ingtls.Hosts)
 		return
 	}
+
 	// Secrets should always reside in same namespace as the Ingress
-	secretID := &definitions.ResourceID{Name: ingtls.SecretName, Namespace: ic.ingressV1.Metadata.Namespace}
-	addHostTLSCert(ic, hostlist, secretID)
+	secretID := definitions.ResourceID{Name: ingtls.SecretName, Namespace: ic.ingressV1.Metadata.Namespace}
+	secret, ok := ic.state.secrets[secretID]
+	if !ok {
+		ic.logger.Errorf("Failed to find secret %s in namespace %s", secretID.Name, secretID.Namespace)
+		return
+	}
+	addTLSCertToRegistry(ic.certificateRegistry, ic.logger, hostlist, secret)
 }
 
 // converts the default backend if any

diff --git a/dataclients/kubernetes/kube.go b/dataclients/kubernetes/kube.go
@@ -422,7 +422,7 @@ func (c *Client) loadAndConvert() ([]*eskip.Route, error) {
 		return nil, err
 	}
 
-	rg, err := c.routeGroups.convert(state, defaultFilters, loggingEnabled)
+	rg, err := c.routeGroups.convert(state, defaultFilters, loggingEnabled, c.ClusterClient.certificateRegistry)
 	if err != nil {
 		return nil, err
 	}
@@ -595,3 +595,19 @@ func compareStringList(a, b []string) []string {
 	}
 	return c
 }
+
+// addTLSCertToRegistry adds a TLS certificate to the certificate registry per host using the provided
+// Kubernetes TLS secret
+func addTLSCertToRegistry(cr *certregistry.CertRegistry, logger *logger, hosts []string, secret *secret) {
+	cert, err := generateTLSCertFromSecret(secret)
+	if err != nil {
+		logger.Errorf("Failed to generate TLS certificate from secret: %v", err)
+		return
+	}
+	for _, host := range hosts {
+		err := cr.ConfigureCertificate(host, cert)
+		if err != nil {
+			logger.Errorf("Failed to configure certificate: %v", err)
+		}
+	}
+}
diff --git a/dataclients/kubernetes/routegroup.go b/dataclients/kubernetes/routegroup.go
@@ -8,6 +8,7 @@ import (
 	"github.com/zalando/skipper/dataclients/kubernetes/definitions"
 	"github.com/zalando/skipper/eskip"
 	"github.com/zalando/skipper/loadbalancer"
+	"github.com/zalando/skipper/secrets/certregistry"
 )
 
 const backendNameTracingTagName = "skipper.backend_name"
@@ -39,6 +40,7 @@ type routeGroupContext struct {
 	provideHTTPSRedirect         bool
 	calculateTraffic             func([]*definitions.BackendReference) map[string]backendTraffic
 	defaultLoadBalancerAlgorithm string
+	certificateRegistry          *certregistry.CertRegistry
 }
 
 type routeContext struct {
@@ -485,7 +487,36 @@ func splitHosts(hosts []string, domains []string) ([]string, []string) {
 	return internalHosts, externalHosts
 }
 
-func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled bool) ([]*eskip.Route, error) {
+// addRouteGroupTLS compares the RouteGroup host list and the RouteGroup TLS host list
+// and adds the TLS secret to the registry if a match is found.
+func (r *routeGroups) addRouteGroupTLS(ctx *routeGroupContext, tls *definitions.RouteTLSSpec) {
+	// Host in the tls section need to explicitly match the host in the RouteGroup
+	hostlist := compareStringList(tls.Hosts, ctx.routeGroup.Spec.UniqueHosts())
+	if len(hostlist) == 0 {
+		ctx.logger.Errorf("No matching tls hosts found - tls hosts: %s, routegroup hosts: %s", tls.Hosts, ctx.routeGroup.Spec.UniqueHosts())
+		return
+	} else if len(hostlist) != len(tls.Hosts) {
+		ctx.logger.Infof("Hosts in TLS and RouteGroup don't match: tls hosts: %s, routegroup hosts: %s", tls.Hosts, ctx.routeGroup.Spec.UniqueHosts())
+	}
+
+	// Skip adding certs to registry since no certs defined
+	if tls.SecretName == "" {
+		ctx.logger.Debugf("No tls secret defined for hosts - %s", tls.Hosts)
+		return
+	}
+
+	// Secrets should always reside in the same namespace as the RouteGroup
+	secretID := definitions.ResourceID{Name: tls.SecretName, Namespace: ctx.routeGroup.Metadata.Namespace}
+	secret, ok := ctx.state.secrets[secretID]
+	if !ok {
+		ctx.logger.Errorf("Failed to find secret %s in namespace %s", secretID.Name, secretID.Namespace)
+		return
+	}
+	addTLSCertToRegistry(ctx.certificateRegistry, ctx.logger, hostlist, secret)
+
+}
+
+func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled bool, cr *certregistry.CertRegistry) ([]*eskip.Route, error) {
 	var rs []*eskip.Route
 	redirect := createRedirectInfo(r.options.ProvideHTTPSRedirect, r.options.HTTPSRedirectCode)
 
@@ -537,6 +568,7 @@ func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled
 				allowedExternalNames:         r.options.AllowedExternalNames,
 				calculateTraffic:             getBackendTrafficCalculator[*definitions.BackendReference](r.options.BackendTrafficAlgorithm),
 				defaultLoadBalancerAlgorithm: r.options.DefaultLoadBalancerAlgorithm,
+				certificateRegistry:          cr,
 			}
 
 			ri, err := transformRouteGroup(ctx)
@@ -553,6 +585,12 @@ func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled
 				ri = append(ri, catchAll...)
 			}
 
+			if ctx.certificateRegistry != nil {
+				for _, ctxTls := range rg.Spec.TLS {
+					r.addRouteGroupTLS(ctx, ctxTls)
+				}
+			}
+
 			rs = append(rs, ri...)
 		}
 
@@ -572,6 +610,7 @@ func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled
 				allowedExternalNames:         r.options.AllowedExternalNames,
 				calculateTraffic:             getBackendTrafficCalculator[*definitions.BackendReference](r.options.BackendTrafficAlgorithm),
 				defaultLoadBalancerAlgorithm: r.options.DefaultLoadBalancerAlgorithm,
+				certificateRegistry:          cr,
 			}
 
 			internalRi, err := transformRouteGroup(internalCtx)
@@ -591,6 +630,12 @@ func (r *routeGroups) convert(s *clusterState, df defaultFilters, loggingEnabled
 
 			applyEastWestRangePredicates(internalRi, r.options.KubernetesEastWestRangePredicates)
 
+			if internalCtx.certificateRegistry != nil {
+				for _, ctxTls := range rg.Spec.TLS {
+					r.addRouteGroupTLS(internalCtx, ctxTls)
+				}
+			}
+
 			rs = append(rs, internalRi...)
 		}
 	}

diff --git a/dataclients/kubernetes/routegroups_test.go b/dataclients/kubernetes/routegroups_test.go
@@ -57,3 +57,7 @@ func TestRouteGroupExternalName(t *testing.T) {
 func TestRouteGroupDefaultLoadBalancerAlgorithm(t *testing.T) {
 	kubernetestest.FixturesToTest(t, "testdata/routegroups/loadbalancer-algorithm")
 }
+
+func TestRouteGroupTLS(t *testing.T) {
+	kubernetestest.FixturesToTest(t, "testdata/routegroups/tls")
+}
diff --git a/dataclients/kubernetes/testdata/ingressV1/tls/tls-host-mismatch.kube b/dataclients/kubernetes/testdata/ingressV1/tls/tls-host-mismatch.kube
@@ -0,0 +1 @@
+kubernetes-enable-tls: true
diff --git a/dataclients/kubernetes/testdata/ingressV1/tls/tls-host-mismatch.log b/dataclients/kubernetes/testdata/ingressV1/tls/tls-host-mismatch.log
@@ -0,0 +1,2 @@
+level=info msg="Hosts in TLS and Ingress don't match: tls hosts: \[example.org bar.org\], ingress hosts: \[example.org\]" kind=Ingress name=myapp-ingress ns=default
+level=info msg="adding certificate to registry - example.org"
diff --git a/dataclients/kubernetes/testdata/ingressV1/tls/tls-host-mismatch.yaml b/dataclients/kubernetes/testdata/ingressV1/tls/tls-host-mismatch.yaml
@@ -0,0 +1,69 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: myapp-deployment
+  name: myapp-service
+spec:
+  clusterIP: 10.3.190.1
+  ports:
+    - name: this-is-my-service-port-name
+      port: 8080
+      protocol: TCP
+      targetPort: my-port
+  selector:
+    app: myapp
+  type: ClusterIP
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  labels:
+    app: myapp
+  name: myapp-ingress
+  namespace: default
+spec:
+  tls:
+  - secretName: myapp-secret
+    hosts:
+    - example.org
+    - bar.org
+  rules:
+    - host: example.org
+      http:
+        paths:
+          - backend:
+              service:
+                name: myapp-service
+                port:
+                  number: 8080
+            pathType: ImplementationSpecific
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  labels:
+    app: myapp-deployment
+  name: myapp-service
+subsets:
+  - addresses:
+      - ip: 10.3.0.3
+        targetRef:
+          kind: Pod
+          name: myapp-deployment-6786bf95fd-fnqnq
+    ports:
+      - name: this-is-my-service-port-name
+        port: 80
+        protocol: TCP
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: myapp-secret
+  namespace: default
+type: kubernetes.io/tls
+data:
+  tls.crt: |
+        LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUZhekNDQTFPZ0F3SUJBZ0lVZm9WZWI1Y3Y2alZlOC9ZQWFVaGVJejJCSXBNd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1JURUxNQWtHQTFVRUJoTUNRVlV4RXpBUkJnTlZCQWdNQ2xOdmJXVXRVM1JoZEdVeElUQWZCZ05WQkFvTQpHRWx1ZEdWeWJtVjBJRmRwWkdkcGRITWdVSFI1SUV4MFpEQWVGdzB5TWpBek1UQXlNREU0TURSYUZ3MHlNekF6Ck1UQXlNREU0TURSYU1FVXhDekFKQmdOVkJBWVRBa0ZWTVJNd0VRWURWUVFJREFwVGIyMWxMVk4wWVhSbE1TRXcKSHdZRFZRUUtEQmhKYm5SbGNtNWxkQ0JYYVdSbmFYUnpJRkIwZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQgpBUVVBQTRJQ0R3QXdnZ0lLQW9JQ0FRRE10ZGpVUUhzUGJDamNQMjJORmpKN3NzOXJYOVEydmloVVpLN2cvbGF4Cm1hMnpmelV6QitKUUNhdTlFZkRQUVpqVC91NWVGL29vaUtqbGI1Q25USEZMbG52eFd6N1pKN0hWYzAzTnZhWEUKUk54VmdPMXNCbkxSME9URTZRampBYW9lU29RSnFDMEI2em5KdTlNaVdUeVVsY2xWNHVocmZERGZUK1hUcHNrVgpLV2pnMG9ORCtFN01zMTlsRTRwMVYrV0dPWVRub0E1a3pvS2Z4aGpIN3R6SWZwbXFTWGJ2RGxkOWNacUJGbEtHCjBGeWFxK2pUS1lkRWtRL2xQekxJQWtaWW5NVVJDcklJYkZ3WHpuV3VqYzNQYVNnQkdCSmowVFlsN1Z6SExzM2sKL2dCdzd4N1FZZk41SGFZckhYN0FqMlQ4b2Z2a2xhcUxDRDFUaS9nM3dNVzgyR0NJVW52OW13Z3F5TFJoSHBscgpiUmV0a3BSU0JueDlOMWtac2VkaFVoR2FuS3I5MVhrajBySlRTcnBNUkZsSjdVWTlzTEMwenRwVGtMTjlvMnZpCmxrVFd0S3dKTzZXTzBsOTNDSitsWXhUcndNU3RLT0JDN2tVOXdhcVN3REQ0MHJtS1c3VTE0TlVhcTVIYlVISTYKMmY3UzBxVlRhb255VUN6OUVhVmJLaTV0SDFuVkJ1SWwzUDFRQS9RMkh1ZWo5TTU5YWlaT0lDVnhmQmJUbUNIeApyeG5pai83MWlDbTcrMWllNUxIMTI4Y2krNW1nRTQvditPZFZwM0RucVkycmFNeUFJMFBJNGVNVFlmc2tRWVFkCjlCRU8rVkI4ZGdtcnlSR2dPaHNSWFFieXFNOERYTXc4S1BrS2IxTjBwR2l1NXVNRVFKaHd4S3N6T0JGWGtyRG8KQndJREFRQUJvMU13VVRBZEJnTlZIUTRFRmdRVUVXTTh0WjRsVng0MlFMeE8xSE03dGRCZUMvOHdId1lEVlIwagpCQmd3Rm9BVUVXTTh0WjRsVng0MlFMeE8xSE03dGRCZUMvOHdEd1lEVlIwVEFRSC9CQVV3QXdFQi96QU5CZ2txCmhraUc5dzBCQVFzRkFBT0NBZ0VBd3UwN2doaHRyUkNMZ0JuNENHbE5vVkxLRkx2SjQ3T21GUUc1eVp5MTEvdzkKaC9oblJMeDVCbk8wb1lZMmw4M0Z3OUozUWVIaThDTk84Ujc4NTRmRk56WGMxSEFZa1RWc1VZbE9wOW4wTTYwSgpKb3MyUFA5TldNTUpCcFg3Q3JQYUZRRjJoU09hb1NqaXZ1dVoxRnQrZVZEY3FWMjM0VFkrK25hYzBRc053RTYrCnpMai91TGFEM0xWUXh6Y2RuNWMrYVpSNjV0K3I0Q0RsWW1MMGVJY2RTeXF5UUtUWDlMaE1lVXQ3RUxJdEpnVkgKZDdSaDRuRU8rRDVhVEszNkZNSk9TM0VUL0Y1RksrT2QzVmgwMW9RTTJwR0dqQ3A3d2dMeWxNNTVaMWhsTnVXMQo4YWp2eHJDNWVVd2RkaTA4WWFBQjlpR1VRLzRmeUFmQkNkNjJZVVRXSUhib1NMKy84MEpySFZIQnhTaWZ0NmRFClI0SVBtbmtoakovOFcvK1g1WThvOFdrVUF3Zm00QWpOL3ZOZGUxWm1NSVFsSEZhQVRuWUJEaXBaOElSUnBndFEKcnc4d044U0NOV0plZHppdlVoYjdXdUdHbndCcDZ1Wjg1TDUzblN3SFBBS2Y3eGNhTXROVnpuZ1VJaXU1bm9PNwpZSFFzcG1xRVhQQzQ4NERmMHhTUWllUUhTWGxPUVFXS285QTQ5ZXM1NnV2ZGw0c0pTbW9uUTZMblgzV21sUGFGCmdxQSs2ZXZHanVqQmppaTBybncvUWpxY3NteHNtWU84alpGY21pZWpJL3AxUE1OalBpRWJjZGhrNFNrdDlxdnkKcDhvTXVLVzFLNHFpRUp0R1VOT3hkUEt2b1V1MWllQUtuY0FtdUhxdWNHWDBva2JrdmZjT0tYQjFoRi9kK1gwPQotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
+  tls.key: |
+        LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUpSQUlCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQ1M0d2dna3FBZ0VBQW9JQ0FRRE10ZGpVUUhzUGJDamMKUDIyTkZqSjdzczlyWDlRMnZpaFVaSzdnL2xheG1hMnpmelV6QitKUUNhdTlFZkRQUVpqVC91NWVGL29vaUtqbApiNUNuVEhGTGxudnhXejdaSjdIVmMwM052YVhFUk54VmdPMXNCbkxSME9URTZRampBYW9lU29RSnFDMEI2em5KCnU5TWlXVHlVbGNsVjR1aHJmRERmVCtYVHBza1ZLV2pnMG9ORCtFN01zMTlsRTRwMVYrV0dPWVRub0E1a3pvS2YKeGhqSDd0eklmcG1xU1hidkRsZDljWnFCRmxLRzBGeWFxK2pUS1lkRWtRL2xQekxJQWtaWW5NVVJDcklJYkZ3WAp6bld1amMzUGFTZ0JHQkpqMFRZbDdWekhMczNrL2dCdzd4N1FZZk41SGFZckhYN0FqMlQ4b2Z2a2xhcUxDRDFUCmkvZzN3TVc4MkdDSVVudjltd2dxeUxSaEhwbHJiUmV0a3BSU0JueDlOMWtac2VkaFVoR2FuS3I5MVhrajBySlQKU3JwTVJGbEo3VVk5c0xDMHp0cFRrTE45bzJ2aWxrVFd0S3dKTzZXTzBsOTNDSitsWXhUcndNU3RLT0JDN2tVOQp3YXFTd0RENDBybUtXN1UxNE5VYXE1SGJVSEk2MmY3UzBxVlRhb255VUN6OUVhVmJLaTV0SDFuVkJ1SWwzUDFRCkEvUTJIdWVqOU01OWFpWk9JQ1Z4ZkJiVG1DSHhyeG5pai83MWlDbTcrMWllNUxIMTI4Y2krNW1nRTQvditPZFYKcDNEbnFZMnJhTXlBSTBQSTRlTVRZZnNrUVlRZDlCRU8rVkI4ZGdtcnlSR2dPaHNSWFFieXFNOERYTXc4S1BrSwpiMU4wcEdpdTV1TUVRSmh3eEtzek9CRlhrckRvQndJREFRQUJBb0lDQVFDZ1M4enI5MG5sZTdaTE1NZWg4TDI3Ckt2dE1ndzl6aGxlaUxlemFkWTZCSjJ0aTRMdFJxRnpJZTZvbE5RVXg1Wlc4ZXlWQVBOcEFIekxSNWhpSlNFeDIKK2ZFM3YxRnBUYkh0Q0lybURoamRwV1k2OWVmejdPQy85eWtNSDhZN3E3UUFZQzBnT3JaemlEUUtDYTk4ZUEvOQo4WVJrWW5mSW9zaktOSkFzdWE4L2lOdDlJSnAxQU4wRFlYblRkZ2UzZHdwZG5uQzV4eFUrVG83dWVYb3lKSkp4CjFPQ1MvVS9LUlpxd3VlSllMcGlVeEZlZkxmbjBUOUtDR0cwdi85ck56eW95ZlIrN0JucitLWXU3Z0ltYUU2UVQKUTY1MW5OblptNXNnKzhyb2xYZllWaVVXU2NnQk9KSWtSdGsxYmJVeEo1ZldVeWtoaHlpeUVkT2p0amk1djVPVgpzN0o5V1dzcW9qYUdacG8vTUc0VllsRzA4N0VYR1c3ZUt2U2JWL1VCTHEzcEZtUEVYUjFGVUZyL0xFT2M0MmZnCmEvOGgyQ2M2N2p6Z1EzbHpFY0VzSzZJMnZud3FESzlXR1hBd2hXZG9zUG1QVzFKQ2JOMkhXcUxHTHdJTnZCbG0KcnJQUjFQN1Rhc2ZHZE1jaER4QVdKUkJXblNUdzJyZUo0YVp5dlk5Tk1hdHdodHBvQnJrR3pGVjNYZEV6SEZxVwpjaHJDdnZKSURQdzBqaUN2akdSdnRLYWZ2WWJ5dktvaFNTQTNpMjhtYlB1TzBuSEdacEJ6OEZMTzA3d0Fmd1ZyCkdWWWNxSGdVRk5BYUp1Qld4WndMR0NGeU9sWUJadVNRL0ZONjJZQ091Vm1SRVpTWmtqekFuczhqNVBBdnBSUFgKVFJsM3dxakdrV3BlSHg3clM1OXdVUUtDQVFFQTdsZk5NdFc3b1dvNC90b0dvaE1LK3RXOWphN3JPd08wZ0VNdQo5OGV6WitOakJsRGdoOVRkTk1JMFZqekNZeGxIQUM0MnZxcHZ1TG1rSk40Rm8xY0lDaVFjS01FYitxOGRMUzFZClhXRlRKdFdMY1poUXFUampCNWlRZ1QvZFpBS3dHT3ZSQlVuRi9id0VKeEhTLzZBYk1YaWs4bVlpWWNZZkI3bTUKMkZBcUZHUmdhQjJOMXBiMmduU3hDRW12UjhHbk1Bam9MY2x3S0FHYUtwMVFmK1hCL25wVlpFVXpRL3hxSkZrWgppa21YUDZ5ekpZbXF6UnRGVHhTZDVqQ3hVMTBDMDVSREEraFUydTd0VGU4SW5sTmtpYWhud0xLOHlBUkg0Z201CjlXRmZ0KzBCQ1BDVkFHTk00NjdmcEt5VmwzODdsdG5pSkF0RG1oYXJ6Z2dNeVBidXN3S0NBUUVBMitBeStsVTIKMkFURHN2Wmp6OEc0alI3RnN4MUhYK1pmbTdtTVhKajVYMWhVekduRHZzcngwSE16Z0Zxbk9pYTlrNXpBaFhNSQpwZnl5RitPR0RVb3c3eXJzWGtTeWZDWTYyUVBSRkd2Z01qMUhoYUUwQzlkSWVXZmNlOTVlUUFVZ0FsVjlDRkQrCjlaYmtUMzZPZy9VTUVWU0dkdUhFVVFiRW52Qnp2Zm9UOVJkcEQ3eWd2Sk03T0RFR1A2Z1djNkVXOUcrWXpJNG8Ka2k4SEVRbDJRa1N2cTJkVVBjSzNoZVFvbHJCWERUWk9QeS8vaG1jbkpnbEMvUW1nSUdyWHhQN0F3TjkrU2hYSAo4d1RMcVd1cll1eWpQdkkraXlLR0tHaTk2NmhwNW5JVDdjcllFOStJWlpQQWJrbFRXUVg0SGNrYzhtMTFMZkJQCjAxT2NZMit4clp1TFhRS0NBUUVBcEIzVFR3aTdQVWVPWFhZMWtRNTV4Z1M5bER6NC90YnJTRko2bWVWcDFNUlAKWUg3NlRLMjNiK2UxOEJmQVprcDJpRnBLR2ZuMEdnZkNUaHlQVjB6TFhXaEY5NDRaUFFHdG5ua1YycDcwaWM0TApTYm51K01jU0ZSM3Bpd1kxNVBLdzNVZ3IwbTlkSlAxOUFvWVVleTU4NnhDK3k2YW1VQnNETE9lblg1cTdqdlViCktUWUlmOVhOZ2tEbDBlWWpDczcrMTJXYXNrUjl0UjU5VUpDb2FKa0Zmcnd2NW01OEFYbGlnUXJWT2xLNEVnRlMKRGl3QWIyRXkxV1JGNGNadnBBNXNydEh0WDFod2JaeU56TmNtWVJiZEtLak1ZSFR5NXV1RHI5S3d1SlZIT3JlNAp1YlluYzIvczl1NW9VdFQvNEtTY25LQUZSbnAzSHpnekx4aDk3VGVUWHdLQ0FRRUF3TTBRaG5oUWRnMS9lUjhhCm1LTzY2MnZQV2VkVG5lRUpkeWkxenNDSThyVW03blBUcENxYTdma0djUWVNMmEzODBFSkVnd0JDMWlJR0hISnoKS3BZaTRLV1h6SFdhdU1oaEU4aUgvc3MxTlhpTWpiMjBRS25QTUQ0RmxVeUJBc3c3ckRCQVNobVQ1OUFmZFNGNQpZSFp3MVlWenZ5enJFMDNHL2NQRkNoSU9pL3l5TUkxcnVNKzF2dWttSEkyTTJtbW9Fa0VGRUdHYmE4djIrMVo3CnIxSkJaQ0JnT3lQUi80TDRvR0lTZzFCYVBvZ2RIVUs0amw3U3NjVk45djhaSXZGc0hmUWI3bVM1QnZ6dWhTb1gKaDlBT3VYUjdxVTlscW10bUZnMkFod1VET3FHQzViSStEU3dKTWV2MFBQekIrNFJOY0xyUVpLN3pvRkFSc3hQUApEbmQxTlFLQ0FRQXlJM0xCOUQxUlg1NkhEYmI4WTZobk1wM2xTTmU3aWxpNy8zMjNCczB2QTZ6R0hod2o4ajlnCjVHcEJSd2ZXdCszUkRvZlQrL1h3K1MxTmdUTVJxWkw3M3V3VjdIWnhaTkE4NU5TUDk2bG5EbmpXN3NzTFhtY0wKRFRNQy83UVBaOU1xYmt4SXNDV0wwaml2bSs0a0UrSStqWXlHdkx0a2hlU1NHOU5pOGM1SzlwdjF1Z3ZVOUpwSAo0QlRSblhCd2JOUnVMZ3JpcHJwZjJKS2dYQmlPWEFTZUVmbzMvSTRETjZISWxwUFdoRDkwb0MyaWhvU3JrL2lVCmFMSnVCUmhncDJBelFYMWlBMDcxczNJR3ZlUERnN1RYd2Y4ZDM2bFhnVWNTTEJ0N2FmVzNoOGordEVCZ1FQTEwKVk9seXgzYzVNdVpaZDhHWXBKNGUwUDFJYWJzN2ZwZEEKLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=
diff --git a/dataclients/kubernetes/testdata/ingressV1/tls/tls-invalid-secret.yaml b/dataclients/kubernetes/testdata/ingressV1/tls/tls-invalid-secret.yaml
@@ -1,27 +1,3 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: myapp-deployment
-  labels:
-    app: myapp
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: myapp
-  template:
-    metadata:
-      labels:
-        app: myapp
-    spec:
-      containers:
-        - name: myapp
-          image: myapp:v1
-          ports:
-            - containerPort: 80
-              name: my-port
-              protocol: TCP
----
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/dataclients/kubernetes/testdata/ingressV1/tls/tls-invalid-tls.yaml b/dataclients/kubernetes/testdata/ingressV1/tls/tls-invalid-tls.yaml
@@ -1,27 +1,3 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: myapp-deployment
-  labels:
-    app: myapp
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: myapp
-  template:
-    metadata:
-      labels:
-        app: myapp
-    spec:
-      containers:
-        - name: myapp
-          image: myapp:v1
-          ports:
-            - containerPort: 80
-              name: my-port
-              protocol: TCP
----
 apiVersion: v1
 kind: Service
 metadata: