From 83095712f16687e45267310ea7c34db6093f64b8 Mon Sep 17 00:00:00 2001 From: shaloulcy Date: Mon, 25 Sep 2023 17:54:54 +0800 Subject: [PATCH] e2e: add basic quota e2e cases (#1685) Signed-off-by: chuanyun.lcy Co-authored-by: chuanyun.lcy --- .../plugins/elasticquota/quota_handler.go | 2 + test/e2e/quota/multi_tree.go | 35 ++- test/e2e/quota/quota.go | 193 ++++++++++++++++ test/e2e/quota/quota_guaranteed.go | 212 ++++++++++++++++++ 4 files changed, 439 insertions(+), 3 deletions(-) create mode 100644 test/e2e/quota/quota.go create mode 100644 test/e2e/quota/quota_guaranteed.go diff --git a/pkg/scheduler/plugins/elasticquota/quota_handler.go b/pkg/scheduler/plugins/elasticquota/quota_handler.go index 76e61f072..86c816ae5 100644 --- a/pkg/scheduler/plugins/elasticquota/quota_handler.go +++ b/pkg/scheduler/plugins/elasticquota/quota_handler.go @@ -247,7 +247,9 @@ func (g *Plugin) handlerQuotaWhenRoot(quota *schedulerv1alpha1.ElasticQuota, mgr var delta corev1.ResourceList if isDelete { delta = quotav1.Subtract(corev1.ResourceList{}, totalResource) + g.quotaManagerLock.Lock() delete(g.groupQuotaManagersForQuotaTree, mgr.GetTreeID()) + g.quotaManagerLock.Unlock() } else { delta = mgr.SetTotalResourceForTree(totalResource) } diff --git a/test/e2e/quota/multi_tree.go b/test/e2e/quota/multi_tree.go index 3dbb49018..3adcf8c87 100644 --- a/test/e2e/quota/multi_tree.go +++ b/test/e2e/quota/multi_tree.go @@ -117,6 +117,7 @@ var _ = SIGDescribe("multi-quota-tree", func() { framework.Logf("root quota has tree id: %v, is root: %v", rootQuota1.Labels[extension.LabelQuotaTreeID], rootQuota1.Labels[extension.LabelQuotaIsRoot]) ginkgo.By("create child quota for root quota 1") + time.Sleep(2 * time.Second) childQuota1 := &schedv1alpha1.ElasticQuota{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace.Name, @@ -204,6 +205,7 @@ var _ = SIGDescribe("multi-quota-tree", func() { framework.Logf("root quota 1 tree id: %v, root quota 2 tree id: %v", rootQuota1.Labels[extension.LabelQuotaTreeID], rootQuota2.Labels[extension.LabelQuotaTreeID]) ginkgo.By("create child quota for root quota 2") + time.Sleep(2 * time.Second) childQuota2 := &schedv1alpha1.ElasticQuota{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace.Name, @@ -272,6 +274,20 @@ var _ = SIGDescribe("multi-quota-tree", func() { _, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), childQuota3, metav1.CreateOptions{}) framework.ExpectError(err) framework.Logf("create childQuota3 failed, err: %v", err) + + ginkgo.By("clean profile/quotas") + err = f.KoordinatorClientSet.QuotaV1alpha1().ElasticQuotaProfiles(f.Namespace.Name).Delete(context.TODO(), profile1.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = f.KoordinatorClientSet.QuotaV1alpha1().ElasticQuotaProfiles(f.Namespace.Name).Delete(context.TODO(), profile2.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), childQuota1.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), childQuota2.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), profile1.Spec.QuotaName, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), profile2.Spec.QuotaName, metav1.DeleteOptions{}) + framework.ExpectNoError(err) }) framework.ConformanceIt("create profile with resource ratio", func() { @@ -326,6 +342,12 @@ var _ = SIGDescribe("multi-quota-tree", func() { gomega.Expect(quotav1.Equals(totalResource, rootQuota.Spec.Min)).Should(gomega.Equal(true)) framework.Logf("root quota min is right") framework.Logf("node %v resource: %v, root quota min: %v", nodeList.Items[0].Name, util.DumpJSON(nodeList.Items[0].Status.Allocatable), util.DumpJSON(rootQuota.Spec.Min)) + + ginkgo.By("clean profile/quotas") + err = f.KoordinatorClientSet.QuotaV1alpha1().ElasticQuotaProfiles(f.Namespace.Name).Delete(context.TODO(), profile.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), profile.Spec.QuotaName, metav1.DeleteOptions{}) + framework.ExpectNoError(err) }) framework.ConformanceIt("root quota show resource usage", func() { @@ -372,6 +394,7 @@ var _ = SIGDescribe("multi-quota-tree", func() { framework.ExpectNoError(err) ginkgo.By("create child quota") + time.Sleep(2 * time.Second) childQuota := &schedv1alpha1.ElasticQuota{ ObjectMeta: metav1.ObjectMeta{ Namespace: f.Namespace.Name, @@ -415,7 +438,7 @@ var _ = SIGDescribe("multi-quota-tree", func() { } ginkgo.By("create pod with child quota") - pod := createQuotaE2ePod(f.Namespace.Name, fmt.Sprintf("%s-quota-test-pod-1", f.Namespace.Name), childQuota.Name, podRequests) + pod := createQuotaE2EPod(f.Namespace.Name, fmt.Sprintf("%s-quota-test-pod-1", f.Namespace.Name), childQuota.Name, podRequests) f.PodClient().Create(pod) gomega.Eventually(func() bool { @@ -442,14 +465,20 @@ var _ = SIGDescribe("multi-quota-tree", func() { return false }, 60*time.Second, 1*time.Second).Should(gomega.Equal(true)) - ginkgo.By("clean pod") + ginkgo.By("clean pod/profiles/quotas") f.PodClient().DeleteSync(pod.Name, metav1.DeleteOptions{}, 5*time.Minute) + err = f.KoordinatorClientSet.QuotaV1alpha1().ElasticQuotaProfiles(f.Namespace.Name).Delete(context.TODO(), profile.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), childQuota.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), profile.Spec.QuotaName, metav1.DeleteOptions{}) + framework.ExpectNoError(err) }) }) }) -func createQuotaE2ePod(namespace, name, quotaName string, requests corev1.ResourceList) *corev1.Pod { +func createQuotaE2EPod(namespace, name, quotaName string, requests corev1.ResourceList) *corev1.Pod { pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: name, diff --git a/test/e2e/quota/quota.go b/test/e2e/quota/quota.go new file mode 100644 index 000000000..215cbfad8 --- /dev/null +++ b/test/e2e/quota/quota.go @@ -0,0 +1,193 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "context" + "fmt" + "time" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + schedclientset "sigs.k8s.io/scheduler-plugins/pkg/generated/clientset/versioned" + + "github.com/koordinator-sh/koordinator/apis/extension" + "github.com/koordinator-sh/koordinator/pkg/util" + "github.com/koordinator-sh/koordinator/test/e2e/framework" + e2enode "github.com/koordinator-sh/koordinator/test/e2e/framework/node" + e2epod "github.com/koordinator-sh/koordinator/test/e2e/framework/pod" + schedv1alpha1 "sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1" +) + +var _ = SIGDescribe("basic-quota", func() { + var nodeList *corev1.NodeList + var err error + var quotaClient schedclientset.Interface + var totalResource = corev1.ResourceList{} + + f := framework.NewDefaultFramework("basic-quota") + + ginkgo.BeforeEach(func() { + framework.Logf("get some nodes which are ready and schedulable") + nodeList, err = e2enode.GetReadySchedulableNodes(f.ClientSet) + framework.ExpectNoError(err) + + quotaClient, err = schedclientset.NewForConfig(f.ClientConfig()) + framework.ExpectNoError(err) + + // fail the test when no node available + gomega.Expect(len(nodeList.Items)).Should(gomega.BeNumerically(">", 0)) + + totalResource[corev1.ResourceCPU] = nodeList.Items[0].Status.Allocatable[corev1.ResourceCPU] + totalResource[corev1.ResourceMemory] = nodeList.Items[0].Status.Allocatable[corev1.ResourceMemory] + + cpu := totalResource[corev1.ResourceCPU] + memory := totalResource[corev1.ResourceMemory] + gomega.Expect(cpu.Value()).Should(gomega.BeNumerically(">=", 2)) + gomega.Expect(memory.Value()).Should(gomega.BeNumerically(">=", 4*1024*1024*1024)) + }) + ginkgo.AfterEach(func() {}) + + framework.KoordinatorDescribe("basic-quota", func() { + framework.ConformanceIt("the sum of child min is smaller than parent min", func() { + ginkgo.By("create parent quota") + parentQuota := &schedv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: fmt.Sprintf("%s-parent-quota", f.Namespace.Name), + Labels: map[string]string{ + QuotaE2eLabel: "true", + extension.LabelQuotaIsParent: "true", + }, + }, + Spec: schedv1alpha1.ElasticQuotaSpec{ + Min: totalResource, + Max: totalResource, + }, + } + + _, err := quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), parentQuota, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("create child quota 1, min is the 0.5*parentQuota.Min") + + childQuota1 := &schedv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: fmt.Sprintf("%s-child-quota-1", f.Namespace.Name), + Labels: map[string]string{ + QuotaE2eLabel: "true", + extension.LabelQuotaParent: parentQuota.Name, + }, + }, + Spec: schedv1alpha1.ElasticQuotaSpec{ + Min: corev1.ResourceList{ + corev1.ResourceCPU: util.MultiplyQuant(totalResource[corev1.ResourceCPU], 0.5), + corev1.ResourceMemory: util.MultiplyQuant(totalResource[corev1.ResourceMemory], 0.5), + }, + Max: totalResource, + }, + } + + _, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), childQuota1, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("create child quota 2, min is the 0.6*parentQuota.Min, failed") + childQuota2 := &schedv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: fmt.Sprintf("%s-child-quota-2", f.Namespace.Name), + Labels: map[string]string{ + QuotaE2eLabel: "true", + extension.LabelQuotaParent: parentQuota.Name, + }, + }, + Spec: schedv1alpha1.ElasticQuotaSpec{ + Min: corev1.ResourceList{ + corev1.ResourceCPU: util.MultiplyQuant(totalResource[corev1.ResourceCPU], 0.6), + corev1.ResourceMemory: util.MultiplyQuant(totalResource[corev1.ResourceMemory], 0.6), + }, + Max: totalResource, + }, + } + + _, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), childQuota2, metav1.CreateOptions{}) + framework.ExpectError(err) + framework.Logf("failed create childQuota2, err: %v", err) + + ginkgo.By("create child quota 2, min is the 0.5*parentQuota.Min, success") + childQuota2.Spec.Min = corev1.ResourceList{ + corev1.ResourceCPU: util.MultiplyQuant(totalResource[corev1.ResourceCPU], 0.5), + corev1.ResourceMemory: util.MultiplyQuant(totalResource[corev1.ResourceMemory], 0.5), + } + + _, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), childQuota2, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), childQuota1.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), childQuota2.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), parentQuota.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + }) + + framework.ConformanceIt("check the quota max", func() { + + requests := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + } + + ginkgo.By("create basic quota") + basicQuota := &schedv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: fmt.Sprintf("%s-basic-quota", f.Namespace.Name), + Labels: map[string]string{ + QuotaE2eLabel: "true", + }, + }, + Spec: schedv1alpha1.ElasticQuotaSpec{ + Max: requests, + }, + } + + _, err := quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), basicQuota, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + pod1 := createQuotaE2EPod(f.Namespace.Name, "basic-pod-1", basicQuota.Name, requests) + _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod1, metav1.CreateOptions{}) + framework.ExpectNoError(err) + err = e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, pod1.Name, pod1.Namespace, framework.PodStartTimeout) + framework.ExpectNoError(err) + + pod2 := createQuotaE2EPod(f.Namespace.Name, "basic-pod-2", basicQuota.Name, requests) + _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod1, metav1.CreateOptions{}) + framework.ExpectError(err) + framework.Logf("create pod %v failed, err: %v", pod2.Name, err) + + ginkgo.By("clean pods and elasticquotas") + f.PodClient().DeleteSync(pod1.Name, metav1.DeleteOptions{}, 5*time.Minute) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), basicQuota.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + }) + }) +}) diff --git a/test/e2e/quota/quota_guaranteed.go b/test/e2e/quota/quota_guaranteed.go new file mode 100644 index 000000000..f5c1247e9 --- /dev/null +++ b/test/e2e/quota/quota_guaranteed.go @@ -0,0 +1,212 @@ +/* +Copyright 2022 The Koordinator Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quota + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/onsi/ginkgo" + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + quotav1 "k8s.io/apiserver/pkg/quota/v1" + apiv1 "k8s.io/kubernetes/pkg/api/v1/pod" + schedclientset "sigs.k8s.io/scheduler-plugins/pkg/generated/clientset/versioned" + + "github.com/koordinator-sh/koordinator/apis/extension" + "github.com/koordinator-sh/koordinator/pkg/util" + "github.com/koordinator-sh/koordinator/test/e2e/framework" + e2enode "github.com/koordinator-sh/koordinator/test/e2e/framework/node" + e2epod "github.com/koordinator-sh/koordinator/test/e2e/framework/pod" + schedv1alpha1 "sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1" +) + +var _ = SIGDescribe("quota-guaranteed", func() { + var nodeList *corev1.NodeList + var err error + var quotaClient schedclientset.Interface + var totalResource = corev1.ResourceList{} + + f := framework.NewDefaultFramework("quota-guaranteed") + + ginkgo.BeforeEach(func() { + framework.Logf("get some nodes which are ready and schedulable") + nodeList, err = e2enode.GetReadySchedulableNodes(f.ClientSet) + framework.ExpectNoError(err) + + quotaClient, err = schedclientset.NewForConfig(f.ClientConfig()) + framework.ExpectNoError(err) + + // fail the test when no node available + gomega.Expect(len(nodeList.Items)).Should(gomega.BeNumerically(">", 0)) + + totalResource[corev1.ResourceCPU] = nodeList.Items[0].Status.Allocatable[corev1.ResourceCPU] + totalResource[corev1.ResourceMemory] = nodeList.Items[0].Status.Allocatable[corev1.ResourceMemory] + + cpu := totalResource[corev1.ResourceCPU] + memory := totalResource[corev1.ResourceMemory] + gomega.Expect(cpu.Value()).Should(gomega.BeNumerically(">=", 2)) + gomega.Expect(memory.Value()).Should(gomega.BeNumerically(">=", 4*1024*1024*1024)) + }) + ginkgo.AfterEach(func() {}) + + framework.KoordinatorDescribe("quota-guaranteed", func() { + framework.ConformanceIt("quota guaranteed", func() { + ginkgo.By("create parent quota") + parentQuota := &schedv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: fmt.Sprintf("%s-parent-quota", f.Namespace.Name), + Labels: map[string]string{ + QuotaE2eLabel: "true", + extension.LabelQuotaIsParent: "true", + }, + }, + Spec: schedv1alpha1.ElasticQuotaSpec{ + Min: totalResource, + Max: totalResource, + }, + } + + _, err := quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), parentQuota, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("create child quota 1, min is zero") + + childQuota1 := &schedv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: fmt.Sprintf("%s-child-quota-1", f.Namespace.Name), + Labels: map[string]string{ + QuotaE2eLabel: "true", + extension.LabelQuotaParent: parentQuota.Name, + }, + }, + Spec: schedv1alpha1.ElasticQuotaSpec{ + Max: totalResource, + }, + } + + _, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), childQuota1, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + podRequests := corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + } + + ginkgo.By("create child quota 2, min is totalRequests-podRequests") + + childQuota2 := &schedv1alpha1.ElasticQuota{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: f.Namespace.Name, + Name: fmt.Sprintf("%s-child-quota-2", f.Namespace.Name), + Labels: map[string]string{ + QuotaE2eLabel: "true", + extension.LabelQuotaParent: parentQuota.Name, + }, + }, + Spec: schedv1alpha1.ElasticQuotaSpec{ + Min: quotav1.Subtract(totalResource, podRequests), + Max: totalResource, + }, + } + + _, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Create(context.TODO(), childQuota2, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + ginkgo.By("create pod1 with child quota 1, success") + pod1 := createQuotaE2EPod(f.Namespace.Name, "basic-pod-1", childQuota1.Name, podRequests) + _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod1, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + err = e2epod.WaitTimeoutForPodReadyInNamespace(f.ClientSet, pod1.Name, pod1.Namespace, framework.PodStartTimeout) + framework.ExpectNoError(err) + + ginkgo.By("check quota runtime and guaranteed") + + quota, err := quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Get(context.TODO(), childQuota1.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + guaranteed, _ := extension.GetGuaranteed(quota) + runtime, _ := extension.GetRuntime(quota) + framework.Logf("childQuota1 runtime: %v, guaranteed: %v", util.DumpJSON(guaranteed), util.DumpJSON(runtime)) + gomega.Expect(quotav1.Equals(guaranteed, podRequests)).Should(gomega.Equal(true)) + gomega.Expect(quotav1.Equals(runtime, podRequests)).Should(gomega.Equal(true)) + + quota, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Get(context.TODO(), childQuota2.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + guaranteed, _ = extension.GetGuaranteed(quota) + runtime, _ = extension.GetRuntime(quota) + framework.Logf("childQuota2 runtime: %v, guaranteed: %v", util.DumpJSON(guaranteed), util.DumpJSON(runtime)) + gomega.Expect(quotav1.Equals(guaranteed, childQuota2.Spec.Min)).Should(gomega.Equal(true)) + gomega.Expect(quotav1.Equals(runtime, childQuota2.Spec.Min)).Should(gomega.Equal(true)) + + quota, err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Get(context.TODO(), parentQuota.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + + guaranteed, _ = extension.GetGuaranteed(quota) + runtime, _ = extension.GetRuntime(quota) + framework.Logf("parentQuota runtime: %v, guaranteed: %v", util.DumpJSON(guaranteed), util.DumpJSON(runtime)) + gomega.Expect(quotav1.Equals(guaranteed, parentQuota.Spec.Min)).Should(gomega.Equal(true)) + gomega.Expect(quotav1.Equals(runtime, parentQuota.Spec.Min)).Should(gomega.Equal(true)) + + ginkgo.By("create pod2 with child quota 1, failed") + pod2 := createQuotaE2EPod(f.Namespace.Name, "basic-pod-2", childQuota1.Name, podRequests) + _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod2, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + time.Sleep(2 * time.Second) + pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), pod2.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + gomega.Expect(pod.Spec.NodeName).Should(gomega.Equal("")) + _, condition := apiv1.GetPodCondition(&pod.Status, corev1.PodScheduled) + gomega.Expect(condition).ShouldNot(gomega.Equal(nil)) + gomega.Expect(condition.Status).Should(gomega.Equal(corev1.ConditionFalse)) + gomega.Expect(strings.Contains(condition.Message, "exceedDimensions")).Should(gomega.Equal(true)) + framework.Logf("pod %v/%v failed schedule, condition: %v", pod.Namespace, pod.Name, util.DumpJSON(condition)) + + ginkgo.By("create pod3 with child quota 2, success") + pod3 := createQuotaE2EPod(f.Namespace.Name, "basic-pod-3", childQuota2.Name, podRequests) + _, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Create(context.TODO(), pod3, metav1.CreateOptions{}) + framework.ExpectNoError(err) + + time.Sleep(2 * time.Second) + pod, err = f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), pod3.Name, metav1.GetOptions{}) + framework.ExpectNoError(err) + gomega.Expect(pod.Spec.NodeName).ShouldNot(gomega.Equal("")) + framework.Logf("pod %v/%v scheduled to node %v", pod.Namespace, pod.Name, pod.Spec.NodeName) + + ginkgo.By("clean pods and elasticquotas") + f.PodClient().DeleteSync(pod1.Name, metav1.DeleteOptions{}, 5*time.Minute) + f.PodClient().DeleteSync(pod2.Name, metav1.DeleteOptions{}, 5*time.Minute) + f.PodClient().DeleteSync(pod3.Name, metav1.DeleteOptions{}, 5*time.Minute) + + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), childQuota1.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), childQuota2.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + err = quotaClient.SchedulingV1alpha1().ElasticQuotas(f.Namespace.Name).Delete(context.TODO(), parentQuota.Name, metav1.DeleteOptions{}) + framework.ExpectNoError(err) + }) + }) +})