accomodate ipv4 and ipv6 and change annotation patch function

Signed-off-by: Hemant <[email protected]>

test/e2e/fqdn_dns_cache_test.go

@@ -15,10 +15,12 @@
 package e2e
 
 import (
+	"bytes"
 	"fmt"
 	"strconv"
 	"strings"
 	"testing"
+	"text/template"
 	"time"
 
 	"github.com/stretchr/testify/assert"
@@ -92,30 +94,17 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) {
 
 	setDnsServerAddressInAntrea(t, data, customDnsService.Spec.ClusterIP)
 
-	var agnHostPodIps []*PodIPs
 	podCount := 2
+	agnHostPodIps := make([]*PodIPs, podCount)
 	for i := 0; i < podCount; i++ {
-		podIp := createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue})
-		agnHostPodIps = append(agnHostPodIps, podIp)
-	}
-	//domainMapping holds whether the IP is mapped to Domain or not.
-	domainMapping := make(map[string]bool)
-
-	// pick an IP to be added in config
-	var ipForDnsConfig string
-	for idx, podIp := range agnHostPodIps {
-		ipStr := podIp.IPv4.String()
-		domainMapping[ipStr] = false
-		//pick last IP for config
-		if idx == len(agnHostPodIps)-1 {
-			ipForDnsConfig = ipStr
-		}
+		agnHostPodIps[i] = createHttpAgnhostPod(t, data, randName(agnHostPodNamePreFix), map[string]string{agnHostLabelKey: agnHostLabelValue})
 	}
 
-	dnsConfigData := createDnsConfig(ipForDnsConfig)
+	ipv4, ipv6 := extractIPs(agnHostPodIps[0])
+
+	dnsConfigData := createDnsConfig(t, ipv4, ipv6)
 	customDnsConfigMapObject, err := data.CreateConfigMap(data.testNamespace, customDnsConfigName, dnsConfigData, nil, false)
 	require.NoError(t, err, "failed to create custom dns ConfigMap: %v", err)
-	domainMapping[ipForDnsConfig] = true
 
 	createDnsPod(t, data)
 
@@ -149,30 +138,35 @@ func TestFQDNPolicyWithCachedDNS(t *testing.T) {
 
 	t.Logf("received ip using dig for test fqdn %+v ", fqdnIp)
 
-	var newIP string
-	for ip, mapped := range domainMapping {
-		if ip != fqdnIp && mapped == false {
-			newIP = ip
-		}
+	ipv4, ipv6 = extractIPs(agnHostPodIps[1])
+	if ipv6 != "" {
+		t.Logf("New IPs to update to DNS | ipv4 : %v, ipv6 : %+v", ipv4, ipv6)
+	} else {
+		t.Logf("New IPs to update to DNS | ipv4 : %v", ipv4)
 	}
 
-	t.Logf("New IP to update to DNS %v", newIP)
-
 	// Update the custom DNS configMap
-	UpdatedCustomDNSconfig := createDnsConfig(newIP)
+	UpdatedCustomDNSconfig := createDnsConfig(t, ipv4, ipv6)
 
 	customDnsConfigMapObject.Data = UpdatedCustomDNSconfig
 	err = data.UpdateConfigMap(customDnsConfigMapObject)
 	require.NoError(t, err, "failed to update configmap with new IP : %v", err)
-	t.Logf("successfully updated dns configMap with new IP : %+v", newIP)
+	if ipv6 != "" {
+		t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v, ipv6 :%+v", ipv4, ipv6)
+	} else {
+		t.Logf("successfully updated dns configMap with new IPs | ipv4 : %+v", ipv4)
+	}
 
-	require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to update custom dns pod annotation.")
+	require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to update custom dns pod annotation.")
 
 	defer setDnsServerAddressInAntrea(t, data, "")
 
 	assert.Eventually(t, func() bool {
 		t.Logf("trying to curl the existing cached IP of the domain  %v", fqdnIp)
 		err = curlFqdn(toolBoxPodName, toolboxContainerName, fqdnIp)
+		if err != nil {
+			t.Logf("The test failed because of error  %+v", err)
+		}
 		return assert.Error(t, err)
 	}, eventualWaitTime, 1*time.Second)
 
@@ -185,31 +179,59 @@ func setDnsServerAddressInAntrea(t *testing.T, data *TestData, dnsServiceIP stri
 	err := data.mutateAntreaConfigMap(nil, agentChanges, false, true)
 	require.NoError(t, err, "Error when setting up customDNS server IP in Antrea configmap : %v", err)
 
-	t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP)
+	if dnsServiceIP == "" {
+		t.Logf("removing dns server IP from antrea agent as part of teardown")
+	} else {
+		t.Logf("dns server value set to %+v in antrea \n", dnsServiceIP)
+	}
 
 }
 
-func createDnsConfig(ipForConfig string) map[string]string {
-	config := fmt.Sprintf(`lfx.test:53 {
-    errors
-    health
-    hosts {
-        %s %s
-        no_reverse
-        pods verified
-        ttl 10
-    }
-    loop
-    reload
-}`, ipForConfig, testFullyQualifiedDomainName)
-
+func createDnsConfig(t *testing.T, ipv4Address, ipv6Address string) map[string]string {
+	configMapData, _ := generateCoreFile(t, ipv4Address, ipv6Address)
 	configData := map[string]string{
-		"Corefile": config,
+		"Corefile": configMapData,
 	}
 
 	return configData
 }
 
+func generateCoreFile(t *testing.T, ipv4Address, ipv6Address string) (string, error) {
+	const coreFileTemplate = `lfx.test:53 {
+        errors
+        log
+        health
+        hosts {
+            {{ if .IPv4 }}{{ .IPv4 }} {{ $.FQDN }}{{ end }}
+            {{ if .IPv6 }}{{ .IPv6 }} {{ $.FQDN }}{{ end }}
+            no_reverse
+            pods verified
+            ttl 10
+        }
+        loop
+        reload
+    }`
+
+	data := struct {
+		IPv4 string
+		IPv6 string
+		FQDN string
+	}{
+		IPv4: ipv4Address,
+		IPv6: ipv6Address,
+		FQDN: testFullyQualifiedDomainName,
+	}
+
+	tmpl, err := template.New("configMapData").Parse(coreFileTemplate)
+	require.NoError(t, err, "error while creating template ", err)
+
+	var output bytes.Buffer
+	err = tmpl.Execute(&output, data)
+	require.NoError(t, err, "error while executing config map template", err)
+
+	return strings.TrimSpace(output.String()), nil
+}
+
 func createDnsPod(t *testing.T, data *TestData) {
 	volume := []corev1.Volume{
 		{
@@ -247,7 +269,7 @@ func createDnsPod(t *testing.T, data *TestData) {
 
 	require.NoError(t, pb.Create(data))
 	require.NoError(t, data.podWaitForRunning(defaultTimeout, customDnsPodName, data.testNamespace))
-	require.NoError(t, data.patchPodAnnotation(data.testNamespace, customDnsPodName, nil), "failed to annotate the custom dns pod.")
+	require.NoError(t, data.setPodAnnotationToRandomValue(data.testNamespace, customDnsPodName, randomPatchAnnotationKey), "failed to annotate the custom dns pod.")
 
 }
 
@@ -310,7 +332,19 @@ func createHttpAgnhostPod(t *testing.T, data *TestData, podName string, agnLabel
 	}
 
 	require.NoError(t, NewPodBuilder(podName, data.testNamespace, agnhostImage).WithArgs(args).WithPorts(ports).WithLabels(agnLabels).Create(data))
-	podIP, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace)
+	podIPs, err := data.podWaitForIPs(defaultTimeout, podName, data.testNamespace)
 	require.NoError(t, err)
-	return podIP
+	return podIPs
+}
+
+func extractIPs(podIPs *PodIPs) (string, string) {
+	var ipv4, ipv6 string
+	res := podIPs.AsSlice()
+	if len(res) > 0 {
+		ipv4 = res[0].String()
+		if len(res) == 2 {
+			ipv6 = res[1].String()
+		}
+	}
+	return ipv4, ipv6
 }

test/e2e/framework.go

@@ -141,9 +141,9 @@ const (
 
 	statefulSetRestartAnnotationKey = "antrea-e2e/restartedAt"
 	randomPatchAnnotationKey        = "test.antrea.io/random-value"
-
-	iperfPort    = 5201
-	iperfSvcPort = 9999
+	annotationValueLen              = 8
+	iperfPort                       = 5201
+	iperfSvcPort                    = 9999
 )
 
 type ClusterNode struct {
@@ -3230,17 +3230,14 @@ func (data *TestData) runDNSQuery(
 	useTCP bool,
 	dnsServiceIP string) (net.IP, error) {
 
-	digCmd := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr)
+	digCmdStr := fmt.Sprintf("dig "+"@"+dnsServiceIP+" +short %s", dstAddr)
 	if useTCP {
-		digCmd += " +tcp"
-	}
-	cmd := []string{
-		"/bin/sh",
-		"-c",
-		digCmd,
+		digCmdStr += " +tcp"
 	}
-	fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(cmd, " "))
-	stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, cmd)
+
+	digCmd := strings.Fields(digCmdStr)
+	fmt.Printf("Running: kubectl exec %s -c %s -n %s -- %s", podName, containerName, podNamespace, strings.Join(digCmd, " "))
+	stdout, stderr, err := data.RunCommandFromPod(podNamespace, podName, containerName, digCmd)
 	if err != nil {
 		return nil, fmt.Errorf("error when running dig command in Pod '%s': %v - stdout: %s - stderr: %s", podName, err, stdout, stderr)
 	}
@@ -3253,27 +3250,19 @@ func (data *TestData) runDNSQuery(
 	}
 }
 
-// patchPodAnnotation Patches an annotation for a given pod in a given namespace. If no annotation is
-// provided as a parameter then it generates a random string value for a predefined key and adds it as annotation.
-// The patchType used is MergePatchType since we intend to make updates to annotations in pods instead of updating fields in complex k8s
-// resources like deployments.
+// patchPodAnnotation Patches a pod with given map of keys and values.
 func (data *TestData) patchPodAnnotation(namespace, podName string, annotation map[string]string) error {
-	numberOfRunes := 8
 	annotationPatch := map[string]interface{}{
 		"metadata": map[string]interface{}{
 			"annotations": make(map[string]string),
 		},
 	}
 
 	if annotation == nil {
-		annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[randomPatchAnnotationKey] = randSeq(numberOfRunes)
+		return fmt.Errorf("no annotations were provided")
 	} else {
 		for key, value := range annotation {
-			if key == randomPatchAnnotationKey {
-				annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = randSeq(numberOfRunes)
-			} else {
-				annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value
-			}
+			annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[key] = value
 		}
 	}
 
@@ -3292,3 +3281,30 @@ func (data *TestData) patchPodAnnotation(namespace, podName string, annotation m
 	log.Infof("Successfully patched pod %s in namespace %s with provided annotation", podName, namespace)
 	return nil
 }
+
+// setPodAnnotationToRandomValue Patches a pod by adding an annotation with a specified key and a randomly generated string as the value.
+func (data *TestData) setPodAnnotationToRandomValue(namespace, podName, annotationKey string) error {
+	annotationPatch := map[string]interface{}{
+		"metadata": map[string]interface{}{
+			"annotations": make(map[string]string),
+		},
+	}
+
+	annotationPatch["metadata"].(map[string]interface{})["annotations"].(map[string]string)[annotationKey] = randSeq(annotationValueLen)
+
+	patchData, err := json.Marshal(annotationPatch)
+	if err != nil {
+		log.Infof("Error marshalling annotation: %+v", err)
+		return err
+	}
+
+	_, err = data.clientset.CoreV1().Pods(namespace).Patch(context.TODO(), podName, types.MergePatchType, patchData, metav1.PatchOptions{})
+	if err != nil {
+		log.Infof("Error patching pod %s in namespace %s with annotations. Error: %+v", podName, namespace, err)
+		return err
+	}
+
+	log.Infof("Successfully patched pod %s in namespace %s with a random value annotation", podName, namespace)
+	return nil
+
+}

test/e2e/k8s_util.go

@@ -103,7 +103,6 @@ type TestNamespaceMeta struct {
 // GetPodByLabel returns a Pod with the matching Namespace and "pod" label if it's found.
 // If the pod is not found, GetPodByLabel returns "ErrPodNotFound".
 func (k *KubernetesUtils) GetPodByLabel(ns string, name string) (*v1.Pod, error) {
-	//TODO: Query why this is hard coded as "pod"
 	pods, err := k.getPodsUncached(ns, "pod", name)
 	if err != nil {
 		return nil, fmt.Errorf("unable to get Pod in Namespace %s with label pod=%s: %w", ns, name, err)
@@ -668,20 +667,24 @@ func (data *TestData) UpdateConfigMap(configMap *v1.ConfigMap) error {
 	return err
 }
 
-func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string]byte, immutable bool) (*v1.ConfigMap, error) {
+func (data *TestData) CreateConfigMap(namespace, name string, configData map[string]string, binaryData map[string][]byte, immutable bool) (*v1.ConfigMap, error) {
 	configMap := &v1.ConfigMap{
 		TypeMeta: metav1.TypeMeta{},
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
 		},
-		Immutable:  nil,
+		Immutable:  &immutable,
 		Data:       configData,
-		BinaryData: nil,
+		BinaryData: binaryData,
 	}
 
-	configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.Background(), configMap, metav1.CreateOptions{})
-	return configMapObject, err
+	configMapObject, err := data.clientset.CoreV1().ConfigMaps(namespace).Create(context.TODO(), configMap, metav1.CreateOptions{})
+	if err != nil {
+		log.Infof("Unable to create configMap : %s", err)
+		return nil, err
+	}
+	return configMapObject, nil
 }
 
 // DeleteService is a convenience function for deleting a Service by Namespace and name.