NET-10576 - consul-k8s - configure acceptance test to configure CoreDNS the way a user would

acceptance/tests/consul-dns/consul_dns_partitions_test.go

@@ -32,12 +32,13 @@ const defaultPartition = "default"
 const secondaryPartition = "secondary"
 const defaultNamespace = "default"
 
-// Test that Sync Catalog works in a default and ACLsEnabled installations for partitions
-// DNS queries for services across partitions. It validates:
-// - returning the local partition's service when tenancy is not included in the question.
+// TestConsulDNSProxy_WithPartitionsAndCatalogSync verifies DNS queries for services across partitions
+// when DNS proxy is enabled. It configures CoreDNS to use configure consul domain queries to
+// be forwarded to the Consul DNS Proxy.  The test validates:
+// - returning the local partition's service when tenancy is not included in the DNS question.
 // - properly not resolving DNS for unexported services when ACLs are enabled.
 // - properly resolving DNS for exported services when ACLs are enabled.
-func TestConsulDNS_WithPartitionsAndCatalogSync(t *testing.T) {
+func TestConsulDNSProxy_WithPartitionsAndCatalogSync(t *testing.T) {
 	env := suite.Environment()
 	cfg := suite.Config()
 
@@ -71,6 +72,9 @@ func TestConsulDNS_WithPartitionsAndCatalogSync(t *testing.T) {
 				primaryClusterContext, secondaryClusterContext, c, secondaryPartition,
 				defaultPartition)
 
+			updateCoreDNSWithConsulDomain(t, primaryClusterContext, releaseName, true)
+			updateCoreDNSWithConsulDomain(t, secondaryClusterContext, releaseName, true)
+
 			podLabelSelector := "app=static-server"
 			serviceRequestWithNoPartition := fmt.Sprintf("%s.service.consul", staticServerName)
 			serviceRequestInDefaultPartition := fmt.Sprintf("%s.service.%s.ap.consul", staticServerName, defaultPartition)
@@ -102,7 +106,7 @@ func TestConsulDNS_WithPartitionsAndCatalogSync(t *testing.T) {
 			require.Equal(t, 1, len(serviceInSecondary))
 			require.Equal(t, []string{"k8s"}, serviceInSecondary[0].ServiceTags)
 
-			logger.Log(t, "verify static-server.service.secondary.ap.consul from the secondary partition.")
+			logger.Log(t, "- verify static-server.service.secondary.ap.consul from the secondary partition.")
 			verifyDNS(t, releaseName, true, staticServerNamespace, secondaryClusterContext, secondaryClusterContext,
 				podLabelSelector, serviceRequestInSecondaryPartition,
 				true, dnsUtilsPodIndex)

acceptance/tests/consul-dns/consul_dns_test.go

@@ -7,9 +7,11 @@ import (
 	"context"
 	"fmt"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/environment"
+	"os"
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
@@ -20,6 +22,10 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+// TestConsulDNS configures CoreDNS to use configure consul domain queries to
+// be forwarded to the Consul DNS Service or the Consul DNS Proxy depending on
+// the test case.  The test validates that the DNS queries are resolved when querying
+// for .consul services in secure and non-secure modes.
 func TestConsulDNS(t *testing.T) {
 	cfg := suite.Config()
 	if cfg.EnableCNI {
@@ -52,29 +58,53 @@ func TestConsulDNS(t *testing.T) {
 				"global.acls.manageSystemACLs": strconv.FormatBool(c.secure),
 				"dns.proxy.enabled":            strconv.FormatBool(c.enableDNSProxy),
 			}
+
 			cluster := consul.NewHelmCluster(t, helmValues, ctx, suite.Config(), releaseName)
 			cluster.Create(t)
 
-			contextNamespace := ctx.KubectlOptions(t).Namespace
-
-			verifyDNS(t, releaseName, c.enableDNSProxy, contextNamespace, ctx, ctx, "app=consul,component=server",
+			updateCoreDNSWithConsulDomain(t, ctx, releaseName, c.enableDNSProxy)
+			verifyDNS(t, releaseName, c.enableDNSProxy, ctx.KubectlOptions(t).Namespace, ctx, ctx, "app=consul,component=server",
 				"consul.service.consul", true, 0)
-
 		})
 	}
 }
 
-func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespace string, requestingCtx, svcContext environment.TestContext,
-	podLabelSelector, svcName string, shouldResolveDNSRecord bool, dnsUtilsPodIndex int) {
-	logger.Log(t, "get the in cluster dns service or proxy.")
-	dnsSvcName := fmt.Sprintf("%s-consul-dns", releaseName)
-	if enableDNSProxy {
-		dnsSvcName += "-proxy"
+func updateCoreDNSWithConsulDomain(t *testing.T, ctx environment.TestContext, releaseName string, enableDNSProxy bool) {
+	updateCoreDNSFile(t, ctx, releaseName, enableDNSProxy, "coredns-custom.yaml")
+	updateCoreDNS(t, ctx, "coredns-custom.yaml")
+	time.Sleep(5 * time.Second)
+	t.Cleanup(func() {
+		updateCoreDNS(t, ctx, "coredns-original.yaml")
+		time.Sleep(5 * time.Second)
+	})
+}
+
+func updateCoreDNSFile(t *testing.T, ctx environment.TestContext, releaseName string,
+	enableDNSProxy bool, dnsFileName string) {
+	dnsIP, err := getDNSServiceClusterIP(t, ctx, releaseName, enableDNSProxy)
+	require.NoError(t, err)
+
+	input, err := os.ReadFile("coredns-template.yaml")
+	require.NoError(t, err)
+	newContents := strings.Replace(string(input), "{{CONSUL_DNS_IP}}", dnsIP, -1)
+	err = os.WriteFile(dnsFileName, []byte(newContents), os.FileMode(0644))
+	require.NoError(t, err)
+}
+
+func updateCoreDNS(t *testing.T, ctx environment.TestContext, coreDNSConfigFile string) {
+	coreDNSCommand := []string{
+		"replace", "-n", "kube-system", "-f", coreDNSConfigFile,
 	}
-	dnsService, err := requestingCtx.KubernetesClient(t).CoreV1().Services(requestingCtx.KubectlOptions(t).Namespace).Get(context.Background(), dnsSvcName, metav1.GetOptions{})
+	logs, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), coreDNSCommand...)
+	require.NoError(t, err)
+	require.Contains(t, logs, "configmap/coredns replaced")
+	restartCoreDNSCommand := []string{"rollout", "restart", "deployment/coredns", "-n", "kube-system"}
+	_, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), restartCoreDNSCommand...)
 	require.NoError(t, err)
-	dnsIP := dnsService.Spec.ClusterIP
+}
 
+func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespace string, requestingCtx, svcContext environment.TestContext,
+	podLabelSelector, svcName string, shouldResolveDNSRecord bool, dnsUtilsPodIndex int) {
 	podList, err := svcContext.KubernetesClient(t).CoreV1().Pods(svcNamespace).List(context.Background(), metav1.ListOptions{
 		LabelSelector: podLabelSelector,
 	})
@@ -88,7 +118,7 @@ func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespa
 	logger.Log(t, "launch a pod to test the dns resolution.")
 	dnsUtilsPod := fmt.Sprintf("%s-dns-utils-pod-%d", releaseName, dnsUtilsPodIndex)
 	dnsTestPodArgs := []string{
-		"run", "-it", dnsUtilsPod, "--restart", "Never", "--image", "anubhavmishra/tiny-tools", "--", "dig", fmt.Sprintf("@%s", dnsSvcName), svcName,
+		"run", "-it", dnsUtilsPod, "--restart", "Never", "--image", "anubhavmishra/tiny-tools", "--", "dig", svcName,
 	}
 
 	helpers.Cleanup(t, suite.Config().NoCleanupOnFailure, suite.Config().NoCleanup, func() {
@@ -117,22 +147,33 @@ func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespa
 		// We assert on the existence of the ANSWER SECTION, The consul-server IPs being present in the ANSWER SECTION and the the DNS IP mentioned in the SERVER: field
 
 		logger.Log(t, "verify the DNS results.")
-		require.Contains(r, logs, fmt.Sprintf("SERVER: %s", dnsIP))
 		// strip logs of tabs, newlines and spaces to make it easier to assert on the content when there is a DNS match
 		strippedLogs := strings.Replace(logs, "\t", "", -1)
 		strippedLogs = strings.Replace(strippedLogs, "\n", "", -1)
 		strippedLogs = strings.Replace(strippedLogs, " ", "", -1)
 		for _, ip := range servicePodIPs {
-			aRecordPattern := "%s.0INA%s"
+			aRecordPattern := "%s.5INA%s"
+			aRecord := fmt.Sprintf(aRecordPattern, svcName, ip)
 			if shouldResolveDNSRecord {
 				require.Contains(r, logs, "ANSWER SECTION:")
-				require.Contains(r, strippedLogs, fmt.Sprintf(aRecordPattern, svcName, ip))
+				require.Contains(r, strippedLogs, aRecord)
 			} else {
 				require.NotContains(r, logs, "ANSWER SECTION:")
-				require.NotContains(r, strippedLogs, fmt.Sprintf(aRecordPattern, svcName, ip))
+				require.NotContains(r, strippedLogs, aRecord)
 				require.Contains(r, logs, "status: NXDOMAIN")
-				require.Contains(r, logs, "AUTHORITY SECTION:\nconsul.\t\t\t0\tIN\tSOA\tns.consul. hostmaster.consul.")
+				require.Contains(r, logs, "AUTHORITY SECTION:\nconsul.\t\t\t5\tIN\tSOA\tns.consul. hostmaster.consul.")
 			}
 		}
 	})
 }
+
+func getDNSServiceClusterIP(t *testing.T, requestingCtx environment.TestContext, releaseName string, enableDNSProxy bool) (string, error) {
+	logger.Log(t, "get the in cluster dns service or proxy.")
+	dnsSvcName := fmt.Sprintf("%s-consul-dns", releaseName)
+	if enableDNSProxy {
+		dnsSvcName += "-proxy"
+	}
+	dnsService, err := requestingCtx.KubernetesClient(t).CoreV1().Services(requestingCtx.KubectlOptions(t).Namespace).Get(context.Background(), dnsSvcName, metav1.GetOptions{})
+	require.NoError(t, err)
+	return dnsService.Spec.ClusterIP, err
+}

acceptance/tests/consul-dns/coredns-original.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+           lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+           pods insecure
+           fallthrough in-addr.arpa ip6.arpa
+           ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+           max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system

acceptance/tests/consul-dns/coredns-template.yaml

@@ -0,0 +1,32 @@
+apiVersion: v1
+data:
+  Corefile: |
+    .:53 {
+        errors
+        health {
+           lameduck 5s
+        }
+        ready
+        kubernetes cluster.local in-addr.arpa ip6.arpa {
+           pods insecure
+           fallthrough in-addr.arpa ip6.arpa
+           ttl 30
+        }
+        prometheus :9153
+        forward . /etc/resolv.conf {
+           max_concurrent 1000
+        }
+        cache 30
+        loop
+        reload
+        loadbalance
+    }
+    consul:53 {
+        errors
+        cache 30
+        forward . {{CONSUL_DNS_IP}}
+    }
+kind: ConfigMap
+metadata:
+  name: coredns
+  namespace: kube-system