From ee72ea406060a9f7c65776a5cdb8439450dfe405 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 25 Jul 2024 17:05:36 -0600 Subject: [PATCH 1/3] NET-10576 - consul-k8s - configure acceptance test to configure CoreDNS the way a user would --- .../consul-dns/consul_dns_partitions_test.go | 12 ++- .../tests/consul-dns/consul_dns_test.go | 74 ++++++++++++++----- .../tests/consul-dns/coredns-original.yaml | 27 +++++++ .../tests/consul-dns/coredns-template.yaml | 32 ++++++++ 4 files changed, 124 insertions(+), 21 deletions(-) create mode 100644 acceptance/tests/consul-dns/coredns-original.yaml create mode 100644 acceptance/tests/consul-dns/coredns-template.yaml diff --git a/acceptance/tests/consul-dns/consul_dns_partitions_test.go b/acceptance/tests/consul-dns/consul_dns_partitions_test.go index 61e552c7d2..624378ac78 100644 --- a/acceptance/tests/consul-dns/consul_dns_partitions_test.go +++ b/acceptance/tests/consul-dns/consul_dns_partitions_test.go @@ -32,12 +32,13 @@ const defaultPartition = "default" const secondaryPartition = "secondary" const defaultNamespace = "default" -// Test that Sync Catalog works in a default and ACLsEnabled installations for partitions -// DNS queries for services across partitions. It validates: -// - returning the local partition's service when tenancy is not included in the question. +// TestConsulDNSProxy_WithPartitionsAndCatalogSync verifies DNS queries for services across partitions +// when DNS proxy is enabled. It configures CoreDNS to use configure consul domain queries to +// be forwarded to the Consul DNS Proxy. The test validates: +// - returning the local partition's service when tenancy is not included in the DNS question. // - properly not resolving DNS for unexported services when ACLs are enabled. // - properly resolving DNS for exported services when ACLs are enabled. -func TestConsulDNS_WithPartitionsAndCatalogSync(t *testing.T) { +func TestConsulDNSProxy_WithPartitionsAndCatalogSync(t *testing.T) { env := suite.Environment() cfg := suite.Config() @@ -71,6 +72,9 @@ func TestConsulDNS_WithPartitionsAndCatalogSync(t *testing.T) { primaryClusterContext, secondaryClusterContext, c, secondaryPartition, defaultPartition) + updateCoreDNSWithConsulDomain(t, primaryClusterContext, releaseName, true) + updateCoreDNSWithConsulDomain(t, secondaryClusterContext, releaseName, true) + podLabelSelector := "app=static-server" serviceRequestWithNoPartition := fmt.Sprintf("%s.service.consul", staticServerName) serviceRequestInDefaultPartition := fmt.Sprintf("%s.service.%s.ap.consul", staticServerName, defaultPartition) diff --git a/acceptance/tests/consul-dns/consul_dns_test.go b/acceptance/tests/consul-dns/consul_dns_test.go index ab86b26e19..d13de17ed0 100644 --- a/acceptance/tests/consul-dns/consul_dns_test.go +++ b/acceptance/tests/consul-dns/consul_dns_test.go @@ -7,9 +7,11 @@ import ( "context" "fmt" "github.com/hashicorp/consul-k8s/acceptance/framework/environment" + "os" "strconv" "strings" "testing" + "time" "github.com/hashicorp/consul-k8s/acceptance/framework/consul" "github.com/hashicorp/consul-k8s/acceptance/framework/helpers" @@ -20,6 +22,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// TestConsulDNS configures CoreDNS to use configure consul domain queries to +// be forwarded to the Consul DNS Service or the Consul DNS Proxy depending on +// the test case. The test validates that the DNS queries are resolved when querying +// for .consul services in secure and non-secure modes. func TestConsulDNS(t *testing.T) { cfg := suite.Config() if cfg.EnableCNI { @@ -52,29 +58,52 @@ func TestConsulDNS(t *testing.T) { "global.acls.manageSystemACLs": strconv.FormatBool(c.secure), "dns.proxy.enabled": strconv.FormatBool(c.enableDNSProxy), } + cluster := consul.NewHelmCluster(t, helmValues, ctx, suite.Config(), releaseName) cluster.Create(t) - contextNamespace := ctx.KubectlOptions(t).Namespace - - verifyDNS(t, releaseName, c.enableDNSProxy, contextNamespace, ctx, ctx, "app=consul,component=server", + updateCoreDNSWithConsulDomain(t, ctx, releaseName, c.enableDNSProxy) + verifyDNS(t, releaseName, c.enableDNSProxy, ctx.KubectlOptions(t).Namespace, ctx, ctx, "app=consul,component=server", "consul.service.consul", true, 0) - }) } } -func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespace string, requestingCtx, svcContext environment.TestContext, - podLabelSelector, svcName string, shouldResolveDNSRecord bool, dnsUtilsPodIndex int) { - logger.Log(t, "get the in cluster dns service or proxy.") - dnsSvcName := fmt.Sprintf("%s-consul-dns", releaseName) - if enableDNSProxy { - dnsSvcName += "-proxy" +func updateCoreDNSWithConsulDomain(t *testing.T, ctx environment.TestContext, releaseName string, enableDNSProxy bool) { + updateCoreDNSFile(t, ctx, releaseName, enableDNSProxy, "coredns-custom.yaml") + updateCoreDNS(t, ctx, "coredns-custom.yaml") + time.Sleep(5 * time.Second) + t.Cleanup(func() { + updateCoreDNS(t, ctx, "coredns-original.yaml") + time.Sleep(5 * time.Second) + }) +} + +func updateCoreDNSFile(t *testing.T, ctx environment.TestContext, releaseName string, + enableDNSProxy bool, dnsFileName string) { + dnsIP, err := getDNSServiceClusterIP(t, ctx, releaseName, enableDNSProxy) + require.NoError(t, err) + + input, err := os.ReadFile("coredns-template.yaml") + require.NoError(t, err) + newContents := strings.Replace(string(input), "{{CONSUL_DNS_IP}}", dnsIP, -1) + err = os.WriteFile(dnsFileName, []byte(newContents), os.FileMode(0644)) +} + +func updateCoreDNS(t *testing.T, ctx environment.TestContext, coreDNSConfigFile string) { + coreDNSCommand := []string{ + "replace", "-n", "kube-system", "-f", coreDNSConfigFile, } - dnsService, err := requestingCtx.KubernetesClient(t).CoreV1().Services(requestingCtx.KubectlOptions(t).Namespace).Get(context.Background(), dnsSvcName, metav1.GetOptions{}) + logs, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), coreDNSCommand...) require.NoError(t, err) - dnsIP := dnsService.Spec.ClusterIP + require.Contains(t, logs, "configmap/coredns replaced") + restartCoreDNSCommand := []string{"rollout", "restart", "deployment/coredns", "-n", "kube-system"} + logs, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), restartCoreDNSCommand...) + require.NoError(t, err) +} +func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespace string, requestingCtx, svcContext environment.TestContext, + podLabelSelector, svcName string, shouldResolveDNSRecord bool, dnsUtilsPodIndex int) { podList, err := svcContext.KubernetesClient(t).CoreV1().Pods(svcNamespace).List(context.Background(), metav1.ListOptions{ LabelSelector: podLabelSelector, }) @@ -88,7 +117,7 @@ func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespa logger.Log(t, "launch a pod to test the dns resolution.") dnsUtilsPod := fmt.Sprintf("%s-dns-utils-pod-%d", releaseName, dnsUtilsPodIndex) dnsTestPodArgs := []string{ - "run", "-it", dnsUtilsPod, "--restart", "Never", "--image", "anubhavmishra/tiny-tools", "--", "dig", fmt.Sprintf("@%s", dnsSvcName), svcName, + "run", "-it", dnsUtilsPod, "--restart", "Never", "--image", "anubhavmishra/tiny-tools", "--", "dig", svcName, } helpers.Cleanup(t, suite.Config().NoCleanupOnFailure, suite.Config().NoCleanup, func() { @@ -117,22 +146,33 @@ func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespa // We assert on the existence of the ANSWER SECTION, The consul-server IPs being present in the ANSWER SECTION and the the DNS IP mentioned in the SERVER: field logger.Log(t, "verify the DNS results.") - require.Contains(r, logs, fmt.Sprintf("SERVER: %s", dnsIP)) // strip logs of tabs, newlines and spaces to make it easier to assert on the content when there is a DNS match strippedLogs := strings.Replace(logs, "\t", "", -1) strippedLogs = strings.Replace(strippedLogs, "\n", "", -1) strippedLogs = strings.Replace(strippedLogs, " ", "", -1) for _, ip := range servicePodIPs { - aRecordPattern := "%s.0INA%s" + aRecordPattern := "%s.5INA%s" + aRecord := fmt.Sprintf(aRecordPattern, svcName, ip) if shouldResolveDNSRecord { require.Contains(r, logs, "ANSWER SECTION:") - require.Contains(r, strippedLogs, fmt.Sprintf(aRecordPattern, svcName, ip)) + require.Contains(r, strippedLogs, aRecord) } else { require.NotContains(r, logs, "ANSWER SECTION:") - require.NotContains(r, strippedLogs, fmt.Sprintf(aRecordPattern, svcName, ip)) + require.NotContains(r, strippedLogs, aRecord) require.Contains(r, logs, "status: NXDOMAIN") require.Contains(r, logs, "AUTHORITY SECTION:\nconsul.\t\t\t0\tIN\tSOA\tns.consul. hostmaster.consul.") } } }) } + +func getDNSServiceClusterIP(t *testing.T, requestingCtx environment.TestContext, releaseName string, enableDNSProxy bool) (string, error) { + logger.Log(t, "get the in cluster dns service or proxy.") + dnsSvcName := fmt.Sprintf("%s-consul-dns", releaseName) + if enableDNSProxy { + dnsSvcName += "-proxy" + } + dnsService, err := requestingCtx.KubernetesClient(t).CoreV1().Services(requestingCtx.KubectlOptions(t).Namespace).Get(context.Background(), dnsSvcName, metav1.GetOptions{}) + require.NoError(t, err) + return dnsService.Spec.ClusterIP, err +} diff --git a/acceptance/tests/consul-dns/coredns-original.yaml b/acceptance/tests/consul-dns/coredns-original.yaml new file mode 100644 index 0000000000..ba59c03471 --- /dev/null +++ b/acceptance/tests/consul-dns/coredns-original.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system \ No newline at end of file diff --git a/acceptance/tests/consul-dns/coredns-template.yaml b/acceptance/tests/consul-dns/coredns-template.yaml new file mode 100644 index 0000000000..1a00674704 --- /dev/null +++ b/acceptance/tests/consul-dns/coredns-template.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +data: + Corefile: | + .:53 { + errors + health { + lameduck 5s + } + ready + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + ttl 30 + } + prometheus :9153 + forward . /etc/resolv.conf { + max_concurrent 1000 + } + cache 30 + loop + reload + loadbalance + } + consul:53 { + errors + cache 30 + forward . {{CONSUL_DNS_IP}} + } +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system \ No newline at end of file From 14ffdff40414195debc2f5fe81d1a9e386e8ee58 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 25 Jul 2024 17:24:18 -0600 Subject: [PATCH 2/3] fix linting --- acceptance/tests/consul-dns/consul_dns_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/acceptance/tests/consul-dns/consul_dns_test.go b/acceptance/tests/consul-dns/consul_dns_test.go index d13de17ed0..5c2b890cad 100644 --- a/acceptance/tests/consul-dns/consul_dns_test.go +++ b/acceptance/tests/consul-dns/consul_dns_test.go @@ -88,6 +88,7 @@ func updateCoreDNSFile(t *testing.T, ctx environment.TestContext, releaseName st require.NoError(t, err) newContents := strings.Replace(string(input), "{{CONSUL_DNS_IP}}", dnsIP, -1) err = os.WriteFile(dnsFileName, []byte(newContents), os.FileMode(0644)) + require.NoError(t, err) } func updateCoreDNS(t *testing.T, ctx environment.TestContext, coreDNSConfigFile string) { @@ -98,7 +99,7 @@ func updateCoreDNS(t *testing.T, ctx environment.TestContext, coreDNSConfigFile require.NoError(t, err) require.Contains(t, logs, "configmap/coredns replaced") restartCoreDNSCommand := []string{"rollout", "restart", "deployment/coredns", "-n", "kube-system"} - logs, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), restartCoreDNSCommand...) + _, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), restartCoreDNSCommand...) require.NoError(t, err) } From c539d8be789a87c2e99ae55096264c506dd69656 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 25 Jul 2024 18:15:52 -0600 Subject: [PATCH 3/3] fix how tests compare to DNS response --- acceptance/tests/consul-dns/consul_dns_partitions_test.go | 2 +- acceptance/tests/consul-dns/consul_dns_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/tests/consul-dns/consul_dns_partitions_test.go b/acceptance/tests/consul-dns/consul_dns_partitions_test.go index 624378ac78..b98d7160e1 100644 --- a/acceptance/tests/consul-dns/consul_dns_partitions_test.go +++ b/acceptance/tests/consul-dns/consul_dns_partitions_test.go @@ -106,7 +106,7 @@ func TestConsulDNSProxy_WithPartitionsAndCatalogSync(t *testing.T) { require.Equal(t, 1, len(serviceInSecondary)) require.Equal(t, []string{"k8s"}, serviceInSecondary[0].ServiceTags) - logger.Log(t, "verify static-server.service.secondary.ap.consul from the secondary partition.") + logger.Log(t, "- verify static-server.service.secondary.ap.consul from the secondary partition.") verifyDNS(t, releaseName, true, staticServerNamespace, secondaryClusterContext, secondaryClusterContext, podLabelSelector, serviceRequestInSecondaryPartition, true, dnsUtilsPodIndex) diff --git a/acceptance/tests/consul-dns/consul_dns_test.go b/acceptance/tests/consul-dns/consul_dns_test.go index 5c2b890cad..7872a652ed 100644 --- a/acceptance/tests/consul-dns/consul_dns_test.go +++ b/acceptance/tests/consul-dns/consul_dns_test.go @@ -161,7 +161,7 @@ func verifyDNS(t *testing.T, releaseName string, enableDNSProxy bool, svcNamespa require.NotContains(r, logs, "ANSWER SECTION:") require.NotContains(r, strippedLogs, aRecord) require.Contains(r, logs, "status: NXDOMAIN") - require.Contains(r, logs, "AUTHORITY SECTION:\nconsul.\t\t\t0\tIN\tSOA\tns.consul. hostmaster.consul.") + require.Contains(r, logs, "AUTHORITY SECTION:\nconsul.\t\t\t5\tIN\tSOA\tns.consul. hostmaster.consul.") } } })