add CRP eviction E2E

cmd/hubagent/workload/setup.go

@@ -28,6 +28,7 @@ import (
 	"go.goms.io/fleet/pkg/controllers/clusterinventory/clusterprofile"
 	"go.goms.io/fleet/pkg/controllers/clusterresourcebindingwatcher"
 	"go.goms.io/fleet/pkg/controllers/clusterresourceplacement"
+	"go.goms.io/fleet/pkg/controllers/clusterresourceplacementeviction"
 	"go.goms.io/fleet/pkg/controllers/clusterresourceplacementwatcher"
 	"go.goms.io/fleet/pkg/controllers/clusterschedulingpolicysnapshot"
 	"go.goms.io/fleet/pkg/controllers/memberclusterplacement"
@@ -206,6 +207,14 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager,
 			return err
 		}
 
+		klog.Info("Setting up cluster resource placement eviction controller")
+		if err := (&clusterresourceplacementeviction.Reconciler{
+			Client: mgr.GetClient(),
+		}).SetupWithManager(mgr); err != nil {
+			klog.ErrorS(err, "Unable to set up cluster resource placement eviction controller")
+			return err
+		}
+
 		// Set up the work generator
 		klog.Info("Setting up work generator")
 		if err := (&workgenerator.Reconciler{

test/e2e/actuals_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/ginkgo/v2"
+	placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -890,6 +891,16 @@ func crpRemovedActual(crpName string) func() error {
 	}
 }
 
+func crpEvictionRemovedActual(crpEvictionName string) func() error {
+	return func() error {
+		if err := hubClient.Get(ctx, types.NamespacedName{Name: crpEvictionName}, &placementv1alpha1.ClusterResourcePlacementEviction{}); !errors.IsNotFound(err) {
+			return fmt.Errorf("CRP eviction still exists or an unexpected error occurred: %w", err)
+		}
+
+		return nil
+	}
+}
+
 func validateCRPSnapshotRevisions(crpName string, wantPolicySnapshotRevision, wantResourceSnapshotRevision int) error {
 	matchingLabels := client.MatchingLabels{placementv1beta1.CRPTrackingLabel: crpName}
 

test/e2e/placement_eviction_test.go

@@ -0,0 +1,83 @@
+package e2e
+
+import (
+	"fmt"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	placementv1alpha1 "go.goms.io/fleet/apis/placement/v1alpha1"
+	placementv1beta1 "go.goms.io/fleet/apis/placement/v1beta1"
+	"go.goms.io/fleet/test/e2e/framework"
+)
+
+var _ = Describe("ClusterResourcePlacement eviction of bound binding", Ordered, Serial, func() {
+	crpName := fmt.Sprintf(crpNameTemplate, GinkgoParallelProcess())
+	crpEvictionName := fmt.Sprintf(crpEvictionNameTemplate, GinkgoParallelProcess())
+	taintClusterNames := []string{memberCluster1EastProdName}
+	selectedClusterNames1 := []string{memberCluster1EastProdName, memberCluster2EastCanaryName, memberCluster3WestProdName}
+	selectedClusterNames2 := []string{memberCluster2EastCanaryName, memberCluster3WestProdName}
+
+	BeforeAll(func() {
+		By("creating work resources")
+		createWorkResources()
+
+		// Create the CRP.
+		crp := &placementv1beta1.ClusterResourcePlacement{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: crpName,
+				// Add a custom finalizer; this would allow us to better observe
+				// the behavior of the controllers.
+				Finalizers: []string{customDeletionBlockerFinalizer},
+			},
+			Spec: placementv1beta1.ClusterResourcePlacementSpec{
+				ResourceSelectors: workResourceSelector(),
+			},
+		}
+		By(fmt.Sprintf("creating placement %s", crpName))
+		Expect(hubClient.Create(ctx, crp)).To(Succeed(), "Failed to create CRP %s", crpName)
+	})
+
+	AfterAll(func() {
+		// Remove taint from member cluster 1,2.
+		removeTaintsFromMemberClusters(taintClusterNames)
+		ensureCRPEvictionDeletion(crpEvictionName)
+		ensureCRPAndRelatedResourcesDeletion(crpName, allMemberClusters)
+	})
+
+	It("should update cluster resource placement status as expected", func() {
+		crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), selectedClusterNames1, nil, "0")
+		Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
+	})
+
+	It("add taint to member cluster 1", func() {
+		addTaintsToMemberClusters(taintClusterNames, buildTaints(taintClusterNames))
+	})
+
+	It("create cluster resource placement eviction targeting member cluster 1", func() {
+		crpe := &placementv1alpha1.ClusterResourcePlacementEviction{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: crpEvictionName,
+			},
+			Spec: placementv1alpha1.PlacementEvictionSpec{
+				PlacementName: crpName,
+				ClusterName:   memberCluster1EastProdName,
+			},
+		}
+		Expect(hubClient.Create(ctx, crpe)).To(Succeed(), "Failed to create CRP eviction %s", crpe.Name)
+	})
+
+	It("should ensure no resources exist on evicted member cluster with taint", func() {
+		unSelectedClusters := []*framework.Cluster{memberCluster1EastProd}
+		for _, cluster := range unSelectedClusters {
+			resourceRemovedActual := workNamespaceRemovedFromClusterActual(cluster)
+			Eventually(resourceRemovedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to check if resources doesn't exist on member cluster")
+		}
+	})
+
+	It("should update cluster resource placement status as expected", func() {
+		crpStatusUpdatedActual := crpStatusUpdatedActual(workResourceIdentifiers(), selectedClusterNames2, nil, "0")
+		Eventually(crpStatusUpdatedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "Failed to update cluster resource placement status as expected")
+	})
+})

test/e2e/resources_test.go

@@ -33,6 +33,7 @@ const (
 	internalServiceExportNameTemplate = "ise-%d"
 	internalServiceImportNameTemplate = "isi-%d"
 	endpointSliceExportNameTemplate   = "ep-%d"
+	crpEvictionNameTemplate           = "crpe-%d"
 
 	customDeletionBlockerFinalizer = "custom-deletion-blocker-finalizer"
 	workNamespaceLabelName         = "process"

test/e2e/utils_test.go

@@ -917,6 +917,17 @@ func ensureCRPAndRelatedResourcesDeletion(crpName string, memberClusters []*fram
 	cleanupWorkResources()
 }
 
+func ensureCRPEvictionDeletion(crpEvictionName string) {
+	crpe := &placementv1alpha1.ClusterResourcePlacementEviction{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: crpEvictionName,
+		},
+	}
+	Expect(hubClient.Delete(ctx, crpe)).Should(SatisfyAny(Succeed(), utils.NotFoundMatcher{}), "Failed to delete CRP eviction")
+	removedActual := crpEvictionRemovedActual(crpEvictionName)
+	Eventually(removedActual, eventuallyDuration, eventuallyInterval).Should(Succeed(), "CRP eviction still exists")
+}
+
 // verifyWorkPropagationAndMarkAsAvailable verifies that works derived from a specific CPR have been created
 // for a specific cluster, and marks these works in the specific member cluster's
 // reserved namespace as applied and available.