test: add atm e2e tests (#231)

.github/workflows/e2e-tests.yml

@@ -107,6 +107,7 @@ jobs:
           AZURE_SUBSCRIPTION_ID: ${{ secrets.E2E_AZURE_SUBSCRIPTION_ID }}
           AZURE_NETWORK_SETTING: ${{ matrix.network-setting }}
           AZURE_RESOURCE_GROUP: ${{ env.AZURE_RESOURCE_GROUP }}
+          ENABLE_TRAFFIC_MANAGER: ${{ matrix.enable-traffic-manager }}
       - name: Cleanup e2e
         if: always()
         run: |

test/README.md

@@ -21,6 +21,7 @@ export AZURE_SUBSCRIPTION_ID=<YOUR-SUBSCRIPTION-ID>
 # Available values for AZURE_NETWORK_SETTING are shared-vnet, dynamic-ip-allocation and peered-vnet,
 # and detailed explanations for each network setting are provided in the scripts under folder "test/scripts".
 export AZURE_NETWORK_SETTING=shared-vnet
+export ENABLE_TRAFFIC_MANAGER=true
 ```
 
 Run Makefile Target to setup e2e environment:

test/common/trafficmanager/validator/backend.go

@@ -13,6 +13,7 @@ import (
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"github.com/onsi/gomega"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -28,6 +29,12 @@ var (
 		cmpopts.SortSlices(func(s1, s2 fleetnetv1alpha1.TrafficManagerEndpointStatus) bool {
 			return s1.Cluster.Cluster < s2.Cluster.Cluster
 		}),
+		cmpConditionOptions,
+	}
+
+	cmpTrafficManagerStatusByIgnoringEndpointName = cmp.Options{
+		cmpConditionOptions,
+		cmpopts.IgnoreFields(fleetnetv1alpha1.TrafficManagerEndpointStatus{}, "Name"), // ignore the generated endpoint name
 	}
 )
 
@@ -45,6 +52,71 @@ func IsTrafficManagerBackendFinalizerAdded(ctx context.Context, k8sClient client
 	}, timeout, interval).Should(gomega.Succeed(), "Failed to add finalizer to trafficManagerBackend %s", name)
 }
 
+// ValidateTrafficManagerBackendIfAcceptedAndIgnoringEndpointName validates the trafficManagerBackend object if it is accepted
+// while ignoring the generated endpoint name.
+func ValidateTrafficManagerBackendIfAcceptedAndIgnoringEndpointName(ctx context.Context, k8sClient client.Client, backendName types.NamespacedName, wantEndpoints []fleetnetv1alpha1.TrafficManagerEndpointStatus) fleetnetv1alpha1.TrafficManagerBackendStatus {
+	var wantStatus fleetnetv1alpha1.TrafficManagerBackendStatus
+	if len(wantEndpoints) == 0 {
+		wantStatus = fleetnetv1alpha1.TrafficManagerBackendStatus{
+			Conditions: []metav1.Condition{
+				{
+					Status: metav1.ConditionFalse,
+					Type:   string(fleetnetv1alpha1.TrafficManagerBackendConditionAccepted),
+					Reason: string(fleetnetv1alpha1.TrafficManagerBackendReasonInvalid),
+				},
+			},
+		}
+	} else {
+		wantStatus = fleetnetv1alpha1.TrafficManagerBackendStatus{
+			Conditions: []metav1.Condition{
+				{
+					Status: metav1.ConditionTrue,
+					Type:   string(fleetnetv1alpha1.TrafficManagerBackendConditionAccepted),
+					Reason: string(fleetnetv1alpha1.TrafficManagerBackendReasonAccepted),
+				},
+			},
+			Endpoints: wantEndpoints,
+		}
+	}
+
+	gomega.Eventually(func() error {
+		backend := &fleetnetv1alpha1.TrafficManagerBackend{}
+		if err := k8sClient.Get(ctx, backendName, backend); err != nil {
+			return err
+		}
+
+		if diff := cmp.Diff(
+			backend.Status,
+			wantStatus,
+			cmpTrafficManagerStatusByIgnoringEndpointName,
+		); diff != "" {
+			return fmt.Errorf("trafficManagerBackend status diff (-got, +want): %s", diff)
+		}
+		return nil
+	}, timeout, interval).Should(gomega.Succeed(), "Get() trafficManagerBackend status mismatch")
+	return wantStatus
+}
+
+// ValidateTrafficManagerBackendStatusAndIgnoringEndpointNameConsistently validates the trafficManagerBackend status consistently
+// while ignoring the generated endpoint name.
+func ValidateTrafficManagerBackendStatusAndIgnoringEndpointNameConsistently(ctx context.Context, k8sClient client.Client, backendName types.NamespacedName, want fleetnetv1alpha1.TrafficManagerBackendStatus) {
+	key := types.NamespacedName{Name: backendName.Name, Namespace: backendName.Namespace}
+	backend := &fleetnetv1alpha1.TrafficManagerBackend{}
+	gomega.Consistently(func() error {
+		if err := k8sClient.Get(ctx, key, backend); err != nil {
+			return err
+		}
+		if diff := cmp.Diff(
+			backend.Status,
+			want,
+			cmpTrafficManagerStatusByIgnoringEndpointName,
+		); diff != "" {
+			return fmt.Errorf("trafficManagerBackend status diff (-got, +want): %s", diff)
+		}
+		return nil
+	}, duration, interval).Should(gomega.Succeed(), "Get() trafficManagerBackend status mismatch")
+}
+
 // ValidateTrafficManagerBackend validates the trafficManagerBackend object.
 func ValidateTrafficManagerBackend(ctx context.Context, k8sClient client.Client, want *fleetnetv1alpha1.TrafficManagerBackend) {
 	key := types.NamespacedName{Name: want.Name, Namespace: want.Namespace}

test/common/trafficmanager/validator/profile.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1"
@@ -32,13 +33,16 @@ var (
 	commonCmpOptions = cmp.Options{
 		cmpopts.IgnoreFields(metav1.ObjectMeta{}, "ResourceVersion", "UID", "CreationTimestamp", "ManagedFields", "Generation"),
 		cmpopts.IgnoreFields(metav1.OwnerReference{}, "UID"),
-		cmpopts.IgnoreFields(metav1.Condition{}, "Message", "LastTransitionTime", "ObservedGeneration"),
 		cmpopts.SortSlices(func(c1, c2 metav1.Condition) bool {
 			return c1.Type < c2.Type
 		}),
 	}
+	cmpConditionOptions = cmp.Options{
+		cmpopts.IgnoreFields(metav1.Condition{}, "Message", "LastTransitionTime", "ObservedGeneration"),
+	}
 	cmpTrafficManagerProfileOptions = cmp.Options{
 		commonCmpOptions,
+		cmpConditionOptions,
 		cmpopts.IgnoreFields(fleetnetv1alpha1.TrafficManagerProfile{}, "TypeMeta"),
 	}
 )
@@ -58,6 +62,38 @@ func ValidateTrafficManagerProfile(ctx context.Context, k8sClient client.Client,
 	}, timeout, interval).Should(gomega.Succeed(), "Get() trafficManagerProfile mismatch")
 }
 
+// ValidateIfTrafficManagerProfileIsProgrammed validates the trafficManagerProfile is programmed and returns the DNSName.
+func ValidateIfTrafficManagerProfileIsProgrammed(ctx context.Context, k8sClient client.Client, profileName types.NamespacedName) string {
+	wantDNSName := fmt.Sprintf("%s-%s.trafficmanager.net", profileName.Namespace, profileName.Name)
+	wantStatus := fleetnetv1alpha1.TrafficManagerProfileStatus{
+		DNSName: ptr.To(wantDNSName),
+		Conditions: []metav1.Condition{
+			{
+				Status: metav1.ConditionTrue,
+				Type:   string(fleetnetv1alpha1.TrafficManagerProfileConditionProgrammed),
+				Reason: string(fleetnetv1alpha1.TrafficManagerProfileReasonProgrammed),
+			},
+		},
+	}
+
+	gomega.Eventually(func() error {
+		profile := &fleetnetv1alpha1.TrafficManagerProfile{}
+		if err := k8sClient.Get(ctx, profileName, profile); err != nil {
+			return err
+		}
+
+		if diff := cmp.Diff(
+			profile.Status,
+			wantStatus,
+			cmpConditionOptions,
+		); diff != "" {
+			return fmt.Errorf("trafficManagerProfile status diff (-got, +want): %s", diff)
+		}
+		return nil
+	}, timeout, interval).Should(gomega.Succeed(), "Get() trafficManagerProfile status mismatch")
+	return wantDNSName
+}
+
 // IsTrafficManagerProfileDeleted validates whether the profile is deleted or not.
 func IsTrafficManagerProfileDeleted(ctx context.Context, k8sClient client.Client, name types.NamespacedName) {
 	gomega.Eventually(func() error {

test/e2e/e2e_test.go

@@ -35,6 +35,7 @@ var (
 	fleet          *framework.Fleet
 
 	scheme = runtime.NewScheme()
+	ctx    = context.Background()
 )
 
 func init() {
@@ -92,7 +93,6 @@ var _ = AfterSuite(func() {
 			Name: testNamespace,
 		},
 	}
-	ctx := context.Background()
 	Expect(hubCluster.Client().Delete(ctx, &ns)).Should(Succeed(), "Failed to delete namespace %s cluster %s", testNamespace, hubClusterName)
 	for _, m := range memberClusters {
 		Expect(m.Client().Delete(ctx, &ns)).Should(Succeed(), "Failed to delete namespace %s cluster %s", testNamespace, m.Name())

test/e2e/export_service_test.go

@@ -6,7 +6,6 @@ Licensed under the MIT license.
 package e2e
 
 import (
-	"context"
 	"fmt"
 	"io"
 	"net/http"
@@ -35,14 +34,10 @@ const (
 
 var _ = Describe("Test exporting service", func() {
 	var (
-		ctx context.Context
-
 		wm *framework.WorkloadManager
 	)
 
 	BeforeEach(func() {
-		ctx = context.Background()
-
 		wm = framework.NewWorkloadManager(fleet)
 
 		By("Deploying workload")

test/e2e/framework/workload_manager.go

@@ -21,6 +21,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/retry"
+	"k8s.io/utils/ptr"
 
 	fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1"
 	"go.goms.io/fleet-networking/pkg/common/uniquename"
@@ -113,7 +114,7 @@ func (wm *WorkloadManager) ServiceExport() fleetnetv1alpha1.ServiceExport {
 	}
 }
 
-// ServiceExport returns the MultiClusterService definition from pre-defined service name and namespace.
+// MultiClusterService returns the MultiClusterService definition from pre-defined service name and namespace.
 func (wm *WorkloadManager) MultiClusterService() fleetnetv1alpha1.MultiClusterService {
 	return fleetnetv1alpha1.MultiClusterService{
 		ObjectMeta: metav1.ObjectMeta{
@@ -128,6 +129,40 @@ func (wm *WorkloadManager) MultiClusterService() fleetnetv1alpha1.MultiClusterSe
 	}
 }
 
+// TrafficManagerProfile returns the TrafficManagerProfile definition from pre-defined service name and namespace.
+func (wm *WorkloadManager) TrafficManagerProfile() fleetnetv1alpha1.TrafficManagerProfile {
+	return fleetnetv1alpha1.TrafficManagerProfile{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: wm.namespace,
+			Name:      wm.service.Name, // use the service name as the profile name
+		},
+		Spec: fleetnetv1alpha1.TrafficManagerProfileSpec{
+			MonitorConfig: &fleetnetv1alpha1.MonitorConfig{
+				Port: ptr.To(int64(80)),
+			},
+		},
+	}
+}
+
+// TrafficManagerBackend returns the TrafficManagerBackend definition from pre-defined service name and namespace.
+func (wm *WorkloadManager) TrafficManagerBackend() fleetnetv1alpha1.TrafficManagerBackend {
+	return fleetnetv1alpha1.TrafficManagerBackend{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: wm.namespace,
+			Name:      wm.service.Name, // use the service name as the endpoint name
+		},
+		Spec: fleetnetv1alpha1.TrafficManagerBackendSpec{
+			Profile: fleetnetv1alpha1.TrafficManagerProfileRef{
+				Name: wm.service.Name,
+			},
+			Backend: fleetnetv1alpha1.TrafficManagerBackendRef{
+				Name: wm.service.Name,
+			},
+			Weight: ptr.To(int64(100)),
+		},
+	}
+}
+
 // Deployment returns an deployment definition base on the cluster name.
 func (wm *WorkloadManager) Deployment(clusterName string) *appsv1.Deployment {
 	deployment := wm.deploymentTemplate
@@ -161,7 +196,7 @@ func (wm *WorkloadManager) DeployWorkload(ctx context.Context) error {
 	return nil
 }
 
-// DeployWorkload deletes workload(deployment and its service) from member clusters.
+// RemoveWorkload deletes workload(deployment and its service) from member clusters.
 func (wm *WorkloadManager) RemoveWorkload(ctx context.Context) error {
 	for _, m := range wm.Fleet.MemberClusters() {
 		deploymentDef := wm.Deployment(m.Name())

test/e2e/join_test.go

@@ -6,8 +6,6 @@ Licensed under the MIT license.
 package e2e
 
 import (
-	"context"
-
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	. "github.com/onsi/ginkgo/v2"
@@ -27,7 +25,6 @@ import (
 // When member cluster leaves, it will delete all the networking resources and cause the flaky behaviors.
 var _ = Describe("Test Join/Leave workflow", Serial, Ordered, func() {
 	var (
-		ctx                    context.Context
 		memberClusterName      = memberClusterNames[0]
 		memberClusterNamespace = "fleet-member-" + memberClusterName
 		imcKey                 = types.NamespacedName{Namespace: memberClusterNamespace, Name: memberClusterName}
@@ -46,7 +43,6 @@ var _ = Describe("Test Join/Leave workflow", Serial, Ordered, func() {
 	Context("Member cluster agents should join/leave fleet", func() {
 		BeforeEach(func() {
 			By("Creating internalMemberCluster")
-			ctx = context.Background()
 			imc = fleetv1beta1.InternalMemberCluster{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      memberClusterName,

test/e2e/traffic_manager_test.go

@@ -0,0 +1,85 @@
+/*
+Copyright (c) Microsoft Corporation.
+Licensed under the MIT license.
+*/
+package e2e
+
+import (
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	fleetnetv1alpha1 "go.goms.io/fleet-networking/api/v1alpha1"
+	"go.goms.io/fleet-networking/test/common/trafficmanager/validator"
+	"go.goms.io/fleet-networking/test/e2e/framework"
+)
+
+var (
+	enabled = os.Getenv("ENABLE_TRAFFIC_MANAGER") == "true"
+)
+
+var _ = Describe("Test exporting service via Azure traffic manager", func() {
+	var wm *framework.WorkloadManager
+	var profile fleetnetv1alpha1.TrafficManagerProfile
+	var hubClient client.Client
+	//var dnsName string
+
+	BeforeEach(func() {
+		if !enabled {
+			Skip("Skipping setting up when traffic manager is not enabled")
+		}
+
+		wm = framework.NewWorkloadManager(fleet)
+		hubClient = wm.Fleet.HubCluster().Client()
+
+		By("Deploying workload")
+		Expect(wm.DeployWorkload(ctx)).Should(Succeed(), "Failed to deploy workloads")
+
+		By("Creating trafficManagerProfile")
+		profile = wm.TrafficManagerProfile()
+		Expect(hubClient.Create(ctx, &profile)).Should(Succeed(), "Failed to creat the trafficManagerProfile")
+
+		By("Validating the trafficManagerProfile status")
+		validator.ValidateIfTrafficManagerProfileIsProgrammed(ctx, hubClient, types.NamespacedName{Namespace: profile.Namespace, Name: profile.Name})
+	})
+
+	AfterEach(func() {
+		if !enabled {
+			Skip("Skipping deleting when traffic manager is not enabled")
+		}
+
+		By("Removing workload")
+		Expect(wm.RemoveWorkload(ctx)).Should(Succeed())
+
+		By("Deleting trafficManagerProfile")
+		Expect(hubClient.Delete(ctx, &profile)).Should(Succeed(), "Failed to delete the trafficManagerProfile")
+
+		By("Validating trafficManagerProfile is deleted")
+		validator.IsTrafficManagerProfileDeleted(ctx, hubClient, types.NamespacedName{Namespace: profile.Namespace, Name: profile.Name})
+	})
+
+	Context("Test invalid trafficManagerBackend (invalid serviceImport)", Ordered, func() {
+		var backend fleetnetv1alpha1.TrafficManagerBackend
+		var name types.NamespacedName
+		BeforeAll(func() {
+			By("Creating trafficManagerBackend")
+			backend = wm.TrafficManagerBackend()
+			name = types.NamespacedName{Namespace: backend.Namespace, Name: backend.Name}
+			Expect(hubClient.Create(ctx, &backend)).Should(Succeed(), "Failed to create the trafficManagerBackend")
+		})
+
+		AfterAll(func() {
+			By("Deleting trafficManagerBackend")
+			Expect(hubClient.Delete(ctx, &backend)).Should(Succeed(), "Failed to delete the trafficManagerBackend")
+			validator.IsTrafficManagerBackendDeleted(ctx, hubClient, name)
+		})
+
+		It("Validating the trafficManagerBackend status", func() {
+			status := validator.ValidateTrafficManagerBackendIfAcceptedAndIgnoringEndpointName(ctx, hubClient, name, nil)
+			validator.ValidateTrafficManagerBackendStatusAndIgnoringEndpointNameConsistently(ctx, hubClient, name, status)
+		})
+	})
+})