Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add e2e for migration rollback feature #5609

Merged
merged 1 commit into from
Oct 26, 2024
Merged

add e2e for migration rollback feature #5609

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions test/e2e/framework/work.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
/*
Copyright 2024 The Karmada Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
"context"
"fmt"

"github.com/onsi/gomega"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/klog/v2"

karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
)

// WaitForWorkToDisappear waiting for work to disappear util timeout
func WaitForWorkToDisappear(client karmada.Interface, namespace, name string) {
klog.Infof("Waiting for work(%s/%s) to disappear", namespace, name)
gomega.Eventually(func() error {
_, err := client.WorkV1alpha1().Works(namespace).Get(context.TODO(), name, metav1.GetOptions{})
if err == nil {
return fmt.Errorf("work(%s/%s) still exist", namespace, name)
}
if err != nil && !apierrors.IsNotFound(err) {
return fmt.Errorf("failed to get work(%s/%s), err: %w", namespace, name, err)
}
return nil
}, pollTimeout, pollInterval).ShouldNot(gomega.HaveOccurred())
}
209 changes: 185 additions & 24 deletions test/e2e/seamless_migration_test.go → test/e2e/migration_and_rollback_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,17 +30,30 @@ import (
"k8s.io/apimachinery/pkg/util/rand"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
"k8s.io/utils/ptr"

policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
workv1alpha2 "github.com/karmada-io/karmada/pkg/apis/work/v1alpha2"
"github.com/karmada-io/karmada/pkg/util"
"github.com/karmada-io/karmada/pkg/util/names"
"github.com/karmada-io/karmada/test/e2e/framework"
"github.com/karmada-io/karmada/test/helper"
)

var _ = ginkgo.Describe("Seamless migration testing", func() {
var _ = ginkgo.Describe("Seamless migration and rollback testing", func() {
var member1 string
var member1Client kubernetes.Interface
karmadaLabels := []string{workv1alpha2.ResourceBindingPermanentIDLabel, workv1alpha2.WorkPermanentIDLabel, util.ManagedByKarmadaLabel}
wulemao marked this conversation as resolved.
Show resolved Hide resolved
karmadaAnnotations := []string{
workv1alpha2.ManagedAnnotation,
workv1alpha2.ManagedLabels,
workv1alpha2.ResourceBindingNamespaceAnnotationKey,
workv1alpha2.ResourceBindingNameAnnotationKey,
workv1alpha2.ResourceTemplateUIDAnnotation,
workv1alpha2.ResourceTemplateGenerationAnnotationKey,
workv1alpha2.WorkNameAnnotation,
workv1alpha2.WorkNamespaceAnnotation,
}

ginkgo.BeforeEach(func() {
member1 = framework.ClusterNames()[0]
Expand All @@ -51,7 +64,7 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
ginkgo.Context("Test migrate namespaced resource: Deployment", func() {
var deployment *appsv1.Deployment
var propagationPolicy *policyv1alpha1.PropagationPolicy
var bindingName string
var bindingName, workName, workNamespace string

ginkgo.BeforeEach(func() {
deployment = helper.NewDeployment(testNamespace, deploymentNamePrefix+rand.String(RandomStrLength))
Expand All @@ -65,6 +78,8 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}},
})
bindingName = names.GenerateBindingName(deployment.Kind, deployment.Name)
workName = names.GenerateWorkName(deployment.Kind, deployment.Name, deployment.Namespace)
workNamespace = names.GenerateExecutionSpaceName(member1)
})

ginkgo.BeforeEach(func() {
Expand All @@ -76,14 +91,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
framework.CreatePropagationPolicy(karmadaClient, propagationPolicy)

ginkgo.DeferCleanup(func() {
// Delete Deployment in karmada control plane
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
// Delete Deployment in member cluster
framework.RemoveDeployment(member1Client, deployment.Namespace, deployment.Name)
wulemao marked this conversation as resolved.
Show resolved Hide resolved
// Delete PropagationPolicy in karmada control plane
framework.RemovePropagationPolicy(karmadaClient, propagationPolicy.Namespace, propagationPolicy.Name)

// Verify Deployment in member cluster will be deleted automatically after promotion since it has been deleted from Karmada
klog.Infof("Waiting for Deployment deleted from cluster(%s)", member1)
framework.WaitDeploymentDisappearOnCluster(member1, testNamespace, deployment.Name)
})
})

Expand All @@ -103,9 +114,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
})

// Step 2, Update PropagationPolicy in karmada control plane with conflictResolution=Overwrite
// Step 2, Update PropagationPolicy in karmada control plane with conflictResolution=Overwrite and preserveResourcesOnDeletion=true
ginkgo.By(fmt.Sprintf("Update PropagationPolicy %s in karmada control plane with conflictResolution=Overwrite", propagationPolicy.Name), func() {
propagationPolicy.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite
propagationPolicy.Spec.PreserveResourcesOnDeletion = ptr.To[bool](true)
framework.UpdatePropagationPolicyWithSpec(karmadaClient, propagationPolicy.Namespace, propagationPolicy.Name, propagationPolicy.Spec)
})

Expand All @@ -122,14 +134,32 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
gomega.Expect(items[0].Applied).Should(gomega.BeTrue())
gomega.Expect(items[0].Health).Should(gomega.Equal(workv1alpha2.ResourceHealthy))
})

// Step 4, Delete resource template and check whether member cluster resource is preserved
ginkgo.By("Delete resource template and check whether member cluster resource is preserved", func() {
// Delete Deployment in karmada control plane
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)

// Wait for work deleted
framework.WaitForWorkToDisappear(karmadaClient, workNamespace, workName)

// Check member cluster resource is preserved
framework.WaitDeploymentPresentOnClusterFitWith(member1, deployment.Namespace, deployment.Name,
func(memberDeploy *appsv1.Deployment) bool {
// ensure resource exist in member cluster while related labels/annotations is cleared
return allFlagsCleared(memberDeploy.Labels, karmadaLabels) &&
allFlagsCleared(memberDeploy.Annotations, karmadaAnnotations)
})

})
})
})

ginkgo.Context("Test migrate cluster resources: ClusterRole", func() {
var clusterRoleName string
var clusterRole *rbacv1.ClusterRole
var cpp *policyv1alpha1.ClusterPropagationPolicy
var bindingName string
var bindingName, workName, workNamespace string

ginkgo.BeforeEach(func() {
clusterRoleName = clusterRoleNamePrefix + rand.String(RandomStrLength)
Expand All @@ -151,7 +181,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}},
})
cpp.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite
cpp.Spec.PreserveResourcesOnDeletion = ptr.To[bool](true)
bindingName = names.GenerateBindingName(clusterRole.Kind, clusterRole.Name)
workName = names.GenerateWorkName(clusterRole.Kind, clusterRole.Name, clusterRole.Namespace)
workNamespace = names.GenerateExecutionSpaceName(member1)
})

ginkgo.BeforeEach(func() {
Expand All @@ -163,14 +196,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
framework.CreateClusterPropagationPolicy(karmadaClient, cpp)

ginkgo.DeferCleanup(func() {
// Delete ClusterRole in karmada control plane
framework.RemoveClusterRole(kubeClient, clusterRoleName)
// Delete ClusterRole in member cluster
framework.RemoveClusterRole(member1Client, clusterRoleName)
wulemao marked this conversation as resolved.
Show resolved Hide resolved
// Delete ClusterPropagationPolicy in karmada control plane
framework.RemoveClusterPropagationPolicy(karmadaClient, cpp.Name)

// Verify ClusterRole in member cluster will be deleted automatically after promotion since it has been deleted from Karmada
klog.Infof("Waiting for ClusterRole deleted from cluster(%s)", member1)
framework.WaitClusterRoleDisappearOnCluster(member1, clusterRoleName)
})
})

Expand All @@ -187,14 +216,31 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
return e1 == nil && e2 == nil && len(binding.Status.AggregatedStatus) > 0 && binding.Status.AggregatedStatus[0].Applied
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
})

ginkgo.By("Delete resource template and check whether member cluster resource is preserved", func() {
// Delete ClusterRole in karmada control plane
framework.RemoveClusterRole(kubeClient, clusterRole.Name)

// Wait for work deleted
framework.WaitForWorkToDisappear(karmadaClient, workNamespace, workName)

// Check member cluster resource is preserved
framework.WaitClusterRolePresentOnClusterFitWith(member1, clusterRole.Name,
func(memberClusterRole *rbacv1.ClusterRole) bool {
// ensure resource exist in member cluster while related labels/annotations is cleared
return allFlagsCleared(memberClusterRole.Labels, karmadaLabels) &&
allFlagsCleared(memberClusterRole.Annotations, karmadaAnnotations)
wulemao marked this conversation as resolved.
Show resolved Hide resolved
})

})
})
})

ginkgo.Context("Test migrate namespaced resource: Service (NodePort)", func() {
var serviceName string
var service *corev1.Service
var pp *policyv1alpha1.PropagationPolicy
var bindingName string
var bindingName, workName, workNamespace string

ginkgo.BeforeEach(func() {
serviceName = serviceNamePrefix + rand.String(RandomStrLength)
Expand All @@ -209,7 +255,10 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}},
})
pp.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite
pp.Spec.PreserveResourcesOnDeletion = ptr.To[bool](true)
bindingName = names.GenerateBindingName(service.Kind, service.Name)
workName = names.GenerateWorkName(service.Kind, service.Name, service.Namespace)
workNamespace = names.GenerateExecutionSpaceName(member1)
})

ginkgo.BeforeEach(func() {
Expand All @@ -221,19 +270,15 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
framework.CreatePropagationPolicy(karmadaClient, pp)

ginkgo.DeferCleanup(func() {
// Delete Service in karmada control plane
framework.RemoveService(kubeClient, testNamespace, serviceName)
// Delete Service in member cluster
wulemao marked this conversation as resolved.
Show resolved Hide resolved
framework.RemoveService(member1Client, testNamespace, serviceName)
// Delete PropagationPolicy in karmada control plane
framework.RemovePropagationPolicy(karmadaClient, testNamespace, pp.Name)

// Verify Service in member cluster will be deleted automatically after promotion since it has been deleted from Karmada
klog.Infof("Waiting for Service deleted from cluster(%s)", member1)
framework.WaitServiceDisappearOnCluster(member1, testNamespace, serviceName)
})
})

ginkgo.It("Verify migrate a Service from member cluster", func() {
ginkgo.By(fmt.Sprintf("Verify PropagationPolicy %s got Applied by overwriting conflict resource", bindingName), func() {
ginkgo.By(fmt.Sprintf("Verify ResourceBinding %s got Applied by overwriting conflict resource", bindingName), func() {
klog.Infof("Waiting to verify ResourceBinding %s got Applied by overwriting conflict resource", bindingName)
gomega.Eventually(func() bool {
framework.WaitServicePresentOnClusterFitWith(member1, testNamespace, serviceName, func(*corev1.Service) bool {
Expand All @@ -245,6 +290,122 @@ var _ = ginkgo.Describe("Seamless migration testing", func() {
return e1 == nil && e2 == nil && len(binding.Status.AggregatedStatus) > 0 && binding.Status.AggregatedStatus[0].Applied
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
})

ginkgo.By("Delete resource template and check whether member cluster resource is preserved", func() {
// Delete Service in karmada control plane
framework.RemoveService(kubeClient, service.Namespace, service.Name)

// Wait for work deleted
framework.WaitForWorkToDisappear(karmadaClient, workNamespace, workName)

// Check member cluster resource is preserved
framework.WaitServicePresentOnClusterFitWith(member1, service.Namespace, service.Name,
func(memberService *corev1.Service) bool {
// ensure resource exist in member cluster while related labels/annotations is cleared
return allFlagsCleared(memberService.Labels, karmadaLabels) &&
allFlagsCleared(memberService.Annotations, karmadaAnnotations)
})
})
})
})

ginkgo.Context("Test migrate dependent resource", func() {
var secret *corev1.Secret
var volume []corev1.Volume
var deployment *appsv1.Deployment
var propagationPolicy *policyv1alpha1.PropagationPolicy
var bindingName, workName, workNamespace string

ginkgo.BeforeEach(func() {
secret = helper.NewSecret(testNamespace, secretNamePrefix+rand.String(RandomStrLength), map[string][]byte{"test": []byte("test")})
volume = []corev1.Volume{{
Name: secret.Name,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secret.Name,
},
},
}}
deployment = helper.NewDeploymentWithVolumes(testNamespace, deploymentNamePrefix+rand.String(RandomStrLength), volume)
propagationPolicy = helper.NewPropagationPolicy(deployment.Namespace, deployment.Name, []policyv1alpha1.ResourceSelector{
{
APIVersion: deployment.APIVersion,
Kind: deployment.Kind,
Name: deployment.Name,
},
}, policyv1alpha1.Placement{
ClusterAffinity: &policyv1alpha1.ClusterAffinity{ClusterNames: []string{member1}},
})
propagationPolicy.Spec.PropagateDeps = true
propagationPolicy.Spec.ConflictResolution = policyv1alpha1.ConflictOverwrite
propagationPolicy.Spec.PreserveResourcesOnDeletion = ptr.To[bool](true)
bindingName = names.GenerateBindingName(secret.Kind, secret.Name)
workName = names.GenerateWorkName(secret.Kind, secret.Name, secret.Namespace)
workNamespace = names.GenerateExecutionSpaceName(member1)
})

ginkgo.BeforeEach(func() {
// Create Deployment in member1 cluster
framework.CreateDeployment(member1Client, deployment)
// Create Secret in member1 cluster
framework.CreateSecret(member1Client, secret)
// Create Deployment in karmada control plane
framework.CreateDeployment(kubeClient, deployment)
// Create Secret in karmada control plane
framework.CreateSecret(kubeClient, secret)
// Create PropagationPolicy in karmada control plane without conflictResolution field
framework.CreatePropagationPolicy(karmadaClient, propagationPolicy)

ginkgo.DeferCleanup(func() {
// Delete Deployment in control plane and member cluster
framework.RemoveDeployment(kubeClient, deployment.Namespace, deployment.Name)
framework.RemoveDeployment(member1Client, deployment.Namespace, deployment.Name)
// Delete Secret in member cluster
framework.RemoveSecret(member1Client, secret.Namespace, secret.Name)
// Delete PropagationPolicy in karmada control plane
framework.RemovePropagationPolicy(karmadaClient, propagationPolicy.Namespace, propagationPolicy.Name)
})
})

ginkgo.It("Verify migrate a dependent secret from member cluster", func() {

ginkgo.By(fmt.Sprintf("Verify ResourceBinding %s got Applied by overwriting conflict resource", bindingName), func() {
klog.Infof("Waiting to verify ResourceBinding %s got Applied by overwriting conflict resource", bindingName)
gomega.Eventually(func() bool {
framework.WaitSecretPresentOnClusterFitWith(member1, testNamespace, secret.Name, func(*corev1.Secret) bool {
return true
})
_, e1 := kubeClient.CoreV1().Secrets(testNamespace).Get(context.TODO(), secret.Name, metav1.GetOptions{})
binding, e2 := karmadaClient.WorkV1alpha2().ResourceBindings(testNamespace).Get(context.TODO(), bindingName, metav1.GetOptions{})

return e1 == nil && e2 == nil && len(binding.Status.AggregatedStatus) > 0 && binding.Status.AggregatedStatus[0].Applied
}, pollTimeout, pollInterval).Should(gomega.Equal(true))
})

ginkgo.By("Delete dependent secret template and check whether member cluster secret is preserved", func() {
// Delete dependent secret in karmada control plane
framework.RemoveSecret(kubeClient, secret.Namespace, secret.Name)

// Wait for work deleted
framework.WaitForWorkToDisappear(karmadaClient, workNamespace, workName)

// Check member cluster secret is preserved
framework.WaitSecretPresentOnClusterFitWith(member1, secret.Namespace, secret.Name,
func(memberSecret *corev1.Secret) bool {
// ensure resource exist in member cluster while related labels/annotations is cleared
return allFlagsCleared(memberSecret.Labels, karmadaLabels) &&
allFlagsCleared(memberSecret.Annotations, karmadaAnnotations)
})
})
})
})
})

func allFlagsCleared(flags map[string]string, checkingKeys []string) bool {
for _, key := range checkingKeys {
if _, exist := flags[key]; exist {
return false
}
}
return true
}