From ad0e1e45e6ae152048491a7b15863372a150cad3 Mon Sep 17 00:00:00 2001 From: shaloulcy Date: Fri, 29 Sep 2023 14:28:58 +0800 Subject: [PATCH] koord-scheduler: add decorator for quota (#1686) Signed-off-by: chuanyun.lcy Co-authored-by: chuanyun.lcy --- apis/extension/elastic_quota.go | 41 ++++++++++--------- .../profile/profile_controller.go | 40 ++++++++++++------ .../plugins/elasticquota/controller.go | 18 ++++++++ .../plugins/elasticquota/quota_handler.go | 2 +- 4 files changed, 68 insertions(+), 33 deletions(-) diff --git a/apis/extension/elastic_quota.go b/apis/extension/elastic_quota.go index be38089a1..a7f4f4a8f 100644 --- a/apis/extension/elastic_quota.go +++ b/apis/extension/elastic_quota.go @@ -27,26 +27,27 @@ import ( // RootQuotaName means quotaTree's root\head. const ( - SystemQuotaName = "koordinator-system-quota" - RootQuotaName = "koordinator-root-quota" - DefaultQuotaName = "koordinator-default-quota" - QuotaKoordinatorPrefix = "quota.scheduling.koordinator.sh" - LabelQuotaIsParent = QuotaKoordinatorPrefix + "/is-parent" - LabelQuotaParent = QuotaKoordinatorPrefix + "/parent" - LabelAllowLentResource = QuotaKoordinatorPrefix + "/allow-lent-resource" - LabelQuotaName = QuotaKoordinatorPrefix + "/name" - LabelQuotaProfile = QuotaKoordinatorPrefix + "/profile" - LabelQuotaIsRoot = QuotaKoordinatorPrefix + "/is-root" - LabelQuotaTreeID = QuotaKoordinatorPrefix + "/tree-id" - AnnotationSharedWeight = QuotaKoordinatorPrefix + "/shared-weight" - AnnotationRuntime = QuotaKoordinatorPrefix + "/runtime" - AnnotationRequest = QuotaKoordinatorPrefix + "/request" - AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request" - AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys" - AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource" - AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces" - AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed" - AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated" + SystemQuotaName = "koordinator-system-quota" + RootQuotaName = "koordinator-root-quota" + DefaultQuotaName = "koordinator-default-quota" + QuotaKoordinatorPrefix = "quota.scheduling.koordinator.sh" + LabelQuotaIsParent = QuotaKoordinatorPrefix + "/is-parent" + LabelQuotaParent = QuotaKoordinatorPrefix + "/parent" + LabelAllowLentResource = QuotaKoordinatorPrefix + "/allow-lent-resource" + LabelQuotaName = QuotaKoordinatorPrefix + "/name" + LabelQuotaProfile = QuotaKoordinatorPrefix + "/profile" + LabelQuotaIsRoot = QuotaKoordinatorPrefix + "/is-root" + LabelQuotaTreeID = QuotaKoordinatorPrefix + "/tree-id" + LabelQuotaIgnoreDefaultTree = QuotaKoordinatorPrefix + "/ignore-default-tree" + AnnotationSharedWeight = QuotaKoordinatorPrefix + "/shared-weight" + AnnotationRuntime = QuotaKoordinatorPrefix + "/runtime" + AnnotationRequest = QuotaKoordinatorPrefix + "/request" + AnnotationChildRequest = QuotaKoordinatorPrefix + "/child-request" + AnnotationResourceKeys = QuotaKoordinatorPrefix + "/resource-keys" + AnnotationTotalResource = QuotaKoordinatorPrefix + "/total-resource" + AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces" + AnnotationGuaranteed = QuotaKoordinatorPrefix + "/guaranteed" + AnnotationAllocated = QuotaKoordinatorPrefix + "/allocated" ) func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string { diff --git a/pkg/quota-controller/profile/profile_controller.go b/pkg/quota-controller/profile/profile_controller.go index 51b1a5e3f..e6176b506 100644 --- a/pkg/quota-controller/profile/profile_controller.go +++ b/pkg/quota-controller/profile/profile_controller.go @@ -54,6 +54,16 @@ const ( ReasonUpdateQuotaFailed = "UpdateQuotaFailed" ) +var resourceDecorators = []func(profile *v1alpha1.ElasticQuotaProfile, total corev1.ResourceList){ + DecorateResourceByResourceRatio, +} + +func decorateTotalResource(profile *v1alpha1.ElasticQuotaProfile, total corev1.ResourceList) { + for _, fn := range resourceDecorators { + fn(profile, total) + } +} + // QuotaProfileReconciler reconciles a QuotaProfile object type QuotaProfileReconciler struct { client.Client @@ -132,17 +142,7 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request totalResource = quotav1.Add(totalResource, node.Status.Allocatable) } - ratio := 1.0 - if profile.Spec.ResourceRatio != nil { - val, err := strconv.ParseFloat(*profile.Spec.ResourceRatio, 64) - if err == nil && val > 0 && val <= 1.0 { - ratio = val - } - } - - for resourceName, quantity := range totalResource { - totalResource[resourceName] = MultiplyQuantity(quantity, resourceName, ratio) - } + decorateTotalResource(profile, totalResource) resourceKeys := []string{"cpu", "memory"} raw, ok := profile.Annotations[extension.AnnotationResourceKeys] @@ -163,7 +163,7 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request } min[resourceName] = quantity - max[resourceName] = *resource.NewQuantity(math.MaxInt64/5, resource.DecimalSI) + max[resourceName] = *resource.NewQuantity(math.MaxInt64/2000, resource.DecimalSI) } // update min and max @@ -255,3 +255,19 @@ func hash(s string) string { h.Write([]byte(s)) return strconv.FormatUint(h.Sum64(), 10) } + +func DecorateResourceByResourceRatio(profile *v1alpha1.ElasticQuotaProfile, total corev1.ResourceList) { + if profile.Spec.ResourceRatio == nil { + return + } + + ratio := 1.0 + val, err := strconv.ParseFloat(*profile.Spec.ResourceRatio, 64) + if err == nil && val > 0 && val <= 1.0 { + ratio = val + } + + for resourceName, quantity := range total { + total[resourceName] = MultiplyQuantity(quantity, resourceName, ratio) + } +} diff --git a/pkg/scheduler/plugins/elasticquota/controller.go b/pkg/scheduler/plugins/elasticquota/controller.go index d7e91b8d0..9f82f98d0 100644 --- a/pkg/scheduler/plugins/elasticquota/controller.go +++ b/pkg/scheduler/plugins/elasticquota/controller.go @@ -30,6 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/wait" quotav1 "k8s.io/apiserver/pkg/quota/v1" "k8s.io/klog/v2" + "sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1" "sigs.k8s.io/scheduler-plugins/pkg/util" "github.com/koordinator-sh/koordinator/apis/extension" @@ -41,6 +42,14 @@ const ( ElasticQuotaControllerName = "QuotaCRDController" ) +var resourceDecorators = []func(quota *v1alpha1.ElasticQuota, resource v1.ResourceList){} + +func decorateResource(quota *v1alpha1.ElasticQuota, resource v1.ResourceList) { + for _, fn := range resourceDecorators { + fn(quota, resource) + } +} + // Controller is a controller that update elastic quota crd type Controller struct { plugin *Plugin @@ -101,6 +110,15 @@ func (ctrl *Controller) syncHandler() []error { return } + // decorate resources. + // It will convert unit or remove some resources which we don't want to expose. + decorateResource(eq, used) + decorateResource(eq, request) + decorateResource(eq, childRequest) + decorateResource(eq, runtime) + decorateResource(eq, guaranteed) + decorateResource(eq, allocated) + var oriRuntime, oriRequest, oriChildRequest, oriGuaranteed, oriAllocated v1.ResourceList if eq.Annotations[extension.AnnotationRequest] != "" { if err := json.Unmarshal([]byte(eq.Annotations[extension.AnnotationRequest]), &oriRequest); err != nil { diff --git a/pkg/scheduler/plugins/elasticquota/quota_handler.go b/pkg/scheduler/plugins/elasticquota/quota_handler.go index 86c816ae5..f771d6c83 100644 --- a/pkg/scheduler/plugins/elasticquota/quota_handler.go +++ b/pkg/scheduler/plugins/elasticquota/quota_handler.go @@ -254,7 +254,7 @@ func (g *Plugin) handlerQuotaWhenRoot(quota *schedulerv1alpha1.ElasticQuota, mgr delta = mgr.SetTotalResourceForTree(totalResource) } - if !quotav1.IsZero(delta) { + if !quotav1.IsZero(delta) && quota.Labels[extension.LabelQuotaIgnoreDefaultTree] != "true" { // decrease the default GroupQuotaManager resource deltaForDefault := quotav1.Subtract(corev1.ResourceList{}, delta) g.groupQuotaManager.UpdateClusterTotalResource(deltaForDefault)