koord-scheduler: add decorator for quota (#1686)

Signed-off-by: chuanyun.lcy <[email protected]> Co-authored-by: chuanyun.lcy <[email protected]>
koordinator-sh · Sep 29, 2023 · ad0e1e4 · ad0e1e4
1 parent ac77337
commit ad0e1e4
Show file tree

Hide file tree

Showing 4 changed files with 68 additions and 33 deletions.
diff --git a/apis/extension/elastic_quota.go b/apis/extension/elastic_quota.go
@@ -27,26 +27,27 @@ import (
 
 // RootQuotaName means quotaTree's root\head.
 const (
-	SystemQuotaName           = "koordinator-system-quota"
-	RootQuotaName             = "koordinator-root-quota"
-	DefaultQuotaName          = "koordinator-default-quota"
-	QuotaKoordinatorPrefix    = "quota.scheduling.koordinator.sh"
-	LabelQuotaIsParent        = QuotaKoordinatorPrefix + "/is-parent"
-	LabelQuotaParent          = QuotaKoordinatorPrefix + "/parent"
-	LabelAllowLentResource    = QuotaKoordinatorPrefix + "/allow-lent-resource"
-	LabelQuotaName            = QuotaKoordinatorPrefix + "/name"
-	LabelQuotaProfile         = QuotaKoordinatorPrefix + "/profile"
-	LabelQuotaIsRoot          = QuotaKoordinatorPrefix + "/is-root"
-	LabelQuotaTreeID          = QuotaKoordinatorPrefix + "/tree-id"
-	AnnotationSharedWeight    = QuotaKoordinatorPrefix + "/shared-weight"
-	AnnotationRuntime         = QuotaKoordinatorPrefix + "/runtime"
-	AnnotationRequest         = QuotaKoordinatorPrefix + "/request"
-	AnnotationChildRequest    = QuotaKoordinatorPrefix + "/child-request"
-	AnnotationResourceKeys    = QuotaKoordinatorPrefix + "/resource-keys"
-	AnnotationTotalResource   = QuotaKoordinatorPrefix + "/total-resource"
-	AnnotationQuotaNamespaces = QuotaKoordinatorPrefix + "/namespaces"
-	AnnotationGuaranteed      = QuotaKoordinatorPrefix + "/guaranteed"
-	AnnotationAllocated       = QuotaKoordinatorPrefix + "/allocated"
+	SystemQuotaName             = "koordinator-system-quota"
+	RootQuotaName               = "koordinator-root-quota"
+	DefaultQuotaName            = "koordinator-default-quota"
+	QuotaKoordinatorPrefix      = "quota.scheduling.koordinator.sh"
+	LabelQuotaIsParent          = QuotaKoordinatorPrefix + "/is-parent"
+	LabelQuotaParent            = QuotaKoordinatorPrefix + "/parent"
+	LabelAllowLentResource      = QuotaKoordinatorPrefix + "/allow-lent-resource"
+	LabelQuotaName              = QuotaKoordinatorPrefix + "/name"
+	LabelQuotaProfile           = QuotaKoordinatorPrefix + "/profile"
+	LabelQuotaIsRoot            = QuotaKoordinatorPrefix + "/is-root"
+	LabelQuotaTreeID            = QuotaKoordinatorPrefix + "/tree-id"
+	LabelQuotaIgnoreDefaultTree = QuotaKoordinatorPrefix + "/ignore-default-tree"
+	AnnotationSharedWeight      = QuotaKoordinatorPrefix + "/shared-weight"
+	AnnotationRuntime           = QuotaKoordinatorPrefix + "/runtime"
+	AnnotationRequest           = QuotaKoordinatorPrefix + "/request"
+	AnnotationChildRequest      = QuotaKoordinatorPrefix + "/child-request"
+	AnnotationResourceKeys      = QuotaKoordinatorPrefix + "/resource-keys"
+	AnnotationTotalResource     = QuotaKoordinatorPrefix + "/total-resource"
+	AnnotationQuotaNamespaces   = QuotaKoordinatorPrefix + "/namespaces"
+	AnnotationGuaranteed        = QuotaKoordinatorPrefix + "/guaranteed"
+	AnnotationAllocated         = QuotaKoordinatorPrefix + "/allocated"
 )
 
 func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string {

diff --git a/pkg/quota-controller/profile/profile_controller.go b/pkg/quota-controller/profile/profile_controller.go
@@ -54,6 +54,16 @@ const (
 	ReasonUpdateQuotaFailed = "UpdateQuotaFailed"
 )
 
+var resourceDecorators = []func(profile *v1alpha1.ElasticQuotaProfile, total corev1.ResourceList){
+	DecorateResourceByResourceRatio,
+}
+
+func decorateTotalResource(profile *v1alpha1.ElasticQuotaProfile, total corev1.ResourceList) {
+	for _, fn := range resourceDecorators {
+		fn(profile, total)
+	}
+}
+
 // QuotaProfileReconciler reconciles a QuotaProfile object
 type QuotaProfileReconciler struct {
 	client.Client
@@ -132,17 +142,7 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		totalResource = quotav1.Add(totalResource, node.Status.Allocatable)
 	}
 
-	ratio := 1.0
-	if profile.Spec.ResourceRatio != nil {
-		val, err := strconv.ParseFloat(*profile.Spec.ResourceRatio, 64)
-		if err == nil && val > 0 && val <= 1.0 {
-			ratio = val
-		}
-	}
-
-	for resourceName, quantity := range totalResource {
-		totalResource[resourceName] = MultiplyQuantity(quantity, resourceName, ratio)
-	}
+	decorateTotalResource(profile, totalResource)
 
 	resourceKeys := []string{"cpu", "memory"}
 	raw, ok := profile.Annotations[extension.AnnotationResourceKeys]
@@ -163,7 +163,7 @@ func (r *QuotaProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		}
 
 		min[resourceName] = quantity
-		max[resourceName] = *resource.NewQuantity(math.MaxInt64/5, resource.DecimalSI)
+		max[resourceName] = *resource.NewQuantity(math.MaxInt64/2000, resource.DecimalSI)
 	}
 
 	// update min and max
@@ -255,3 +255,19 @@ func hash(s string) string {
 	h.Write([]byte(s))
 	return strconv.FormatUint(h.Sum64(), 10)
 }
+
+func DecorateResourceByResourceRatio(profile *v1alpha1.ElasticQuotaProfile, total corev1.ResourceList) {
+	if profile.Spec.ResourceRatio == nil {
+		return
+	}
+
+	ratio := 1.0
+	val, err := strconv.ParseFloat(*profile.Spec.ResourceRatio, 64)
+	if err == nil && val > 0 && val <= 1.0 {
+		ratio = val
+	}
+
+	for resourceName, quantity := range total {
+		total[resourceName] = MultiplyQuantity(quantity, resourceName, ratio)
+	}
+}
diff --git a/pkg/scheduler/plugins/elasticquota/controller.go b/pkg/scheduler/plugins/elasticquota/controller.go
@@ -30,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	quotav1 "k8s.io/apiserver/pkg/quota/v1"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/scheduler-plugins/pkg/apis/scheduling/v1alpha1"
 	"sigs.k8s.io/scheduler-plugins/pkg/util"
 
 	"github.com/koordinator-sh/koordinator/apis/extension"
@@ -41,6 +42,14 @@ const (
 	ElasticQuotaControllerName = "QuotaCRDController"
 )
 
+var resourceDecorators = []func(quota *v1alpha1.ElasticQuota, resource v1.ResourceList){}
+
+func decorateResource(quota *v1alpha1.ElasticQuota, resource v1.ResourceList) {
+	for _, fn := range resourceDecorators {
+		fn(quota, resource)
+	}
+}
+
 // Controller is a controller that update elastic quota crd
 type Controller struct {
 	plugin *Plugin
@@ -101,6 +110,15 @@ func (ctrl *Controller) syncHandler() []error {
 				return
 			}
 
+			// decorate resources.
+			// It will convert unit or remove some resources which we don't want to expose.
+			decorateResource(eq, used)
+			decorateResource(eq, request)
+			decorateResource(eq, childRequest)
+			decorateResource(eq, runtime)
+			decorateResource(eq, guaranteed)
+			decorateResource(eq, allocated)
+
 			var oriRuntime, oriRequest, oriChildRequest, oriGuaranteed, oriAllocated v1.ResourceList
 			if eq.Annotations[extension.AnnotationRequest] != "" {
 				if err := json.Unmarshal([]byte(eq.Annotations[extension.AnnotationRequest]), &oriRequest); err != nil {

diff --git a/pkg/scheduler/plugins/elasticquota/quota_handler.go b/pkg/scheduler/plugins/elasticquota/quota_handler.go
@@ -254,7 +254,7 @@ func (g *Plugin) handlerQuotaWhenRoot(quota *schedulerv1alpha1.ElasticQuota, mgr
 			delta = mgr.SetTotalResourceForTree(totalResource)
 		}
 
-		if !quotav1.IsZero(delta) {
+		if !quotav1.IsZero(delta) && quota.Labels[extension.LabelQuotaIgnoreDefaultTree] != "true" {
 			// decrease the default GroupQuotaManager resource
 			deltaForDefault := quotav1.Subtract(corev1.ResourceList{}, delta)
 			g.groupQuotaManager.UpdateClusterTotalResource(deltaForDefault)