proposal: min-excess control for quota

apis/extension/elastic_quota.go

@@ -55,6 +55,7 @@ const (
 	AnnotationNonPreemptibleRequest = QuotaKoordinatorPrefix + "/non-preemptible-request"
 	AnnotationNonPreemptibleUsed    = QuotaKoordinatorPrefix + "/non-preemptible-used"
 	AnnotationAdmission             = QuotaKoordinatorPrefix + "/admission"
+	AnnotationMinExcess             = QuotaKoordinatorPrefix + "/min-excess"
 )
 
 func GetParentQuotaName(quota *v1alpha1.ElasticQuota) string {
@@ -218,3 +219,13 @@ func GetAdmission(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) {
 	}
 	return admission, nil
 }
+
+func GetMinExcess(quota *v1alpha1.ElasticQuota) (corev1.ResourceList, error) {
+	minExcess := corev1.ResourceList{}
+	if quota.Annotations[AnnotationMinExcess] != "" {
+		if err := json.Unmarshal([]byte(quota.Annotations[AnnotationMinExcess]), &minExcess); err != nil {
+			return minExcess, fmt.Errorf("failed to unmarshal min-excess, err=%v", err)
+		}
+	}
+	return minExcess, nil
+}

pkg/features/features.go

@@ -68,6 +68,9 @@ const (
 	// to belong to the users and will not be preempted back.
 	ElasticQuotaGuaranteeUsage featuregate.Feature = "ElasticQuotaGuaranteeUsage"
 
+	// ElasticQuotaMinExcess enable control the upper limit of the min-excess resources
+	ElasticQuotaMinExcess featuregate.Feature = "ElasticQuotaMinExcess"
+
 	// DisableDefaultQuota disable default quota.
 	DisableDefaultQuota featuregate.Feature = "DisableDefaultQuota"
 

pkg/features/scheduler_features.go

@@ -82,6 +82,7 @@ var defaultSchedulerFeatureGates = map[featuregate.Feature]featuregate.FeatureSp
 	ElasticQuotaIgnoreTerminatingPod:          {Default: false, PreRelease: featuregate.Alpha},
 	ElasticQuotaImmediateIgnoreTerminatingPod: {Default: false, PreRelease: featuregate.Alpha},
 	ElasticQuotaGuaranteeUsage:                {Default: false, PreRelease: featuregate.Alpha},
+	ElasticQuotaMinExcess:                     {Default: false, PreRelease: featuregate.Alpha},
 	DisableDefaultQuota:                       {Default: false, PreRelease: featuregate.Alpha},
 	SupportParentQuotaSubmitPod:               {Default: false, PreRelease: featuregate.Alpha},
 	LazyReservationRestore:                    {Default: false, PreRelease: featuregate.Alpha},

pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go

@@ -224,6 +224,12 @@ func (gqm *GroupQuotaManager) recursiveUpdateGroupTreeWithDeltaRequest(deltaReq,
 	}
 }
 
+type addUsedRecursiveState struct {
+	isSelfUsed         bool
+	minExcessEnabled   bool
+	minExcessUsedDelta v1.ResourceList
+}
+
 // updateGroupDeltaUsedNoLock updates the usedQuota of a node, it also updates all parent nodes
 // no need to lock gqm.hierarchyUpdateLock
 func (gqm *GroupQuotaManager) updateGroupDeltaUsedNoLock(quotaName string, delta, deltaNonPreemptibleUsed v1.ResourceList) {
@@ -234,10 +240,13 @@ func (gqm *GroupQuotaManager) updateGroupDeltaUsedNoLock(quotaName string, delta
 	}
 
 	defer gqm.scopedLockForQuotaInfo(curToAllParInfos)()
+	recursiveState := &addUsedRecursiveState{
+		minExcessEnabled: utilfeature.DefaultFeatureGate.Enabled(features.ElasticQuotaMinExcess),
+	}
 	for i := 0; i < allQuotaInfoLen; i++ {
 		quotaInfo := curToAllParInfos[i]
-		isSelfUsed := i == 0
-		quotaInfo.addUsedNonNegativeNoLock(delta, deltaNonPreemptibleUsed, isSelfUsed)
+		recursiveState.isSelfUsed = i == 0
+		quotaInfo.addUsedNonNegativeNoLock(delta, deltaNonPreemptibleUsed, recursiveState)
 	}
 
 	if utilfeature.DefaultFeatureGate.Enabled(features.ElasticQuotaGuaranteeUsage) {
@@ -1009,6 +1018,10 @@ func (gqm *GroupQuotaManager) updateQuotaInternalNoLock(newQuotaInfo, oldQuotaIn
 		gqm.doUpdateOneGroupSharedWeightNoLock(newQuotaInfo.Name, newQuotaInfo.CalculateInfo.SharedWeight)
 	}
 
+	// min-excess changed
+	if !quotav1.Equals(newQuotaInfo.CalculateInfo.MinExcess, oldQuotaInfo.CalculateInfo.MinExcess) {
+		oldQuotaInfo.setMinExcessNoLock(newQuotaInfo.CalculateInfo.MinExcess)
+	}
 }
 
 func (gqm *GroupQuotaManager) doUpdateOneGroupMaxQuotaNoLock(quotaName string, newMax v1.ResourceList) {

pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go

@@ -119,6 +119,7 @@ func TestGroupQuotaManager_UpdateQuotaInternal(t *testing.T) {
 	AddQuotaToManager(t, gqm, "test1", extension.RootQuotaName, 96, 160*GigaByte, 50, 80*GigaByte, true, false)
 
 	quota := CreateQuota("test1", extension.RootQuotaName, 64, 100*GigaByte, 50, 80*GigaByte, true, false)
+	quota.Annotations[extension.AnnotationMinExcess] = `{"cpu":10, "memory": "40Gi"}`
 	gqm.UpdateQuota(quota, false)
 	quotaInfo := gqm.quotaInfoMap["test1"]
 	assert.True(t, quotaInfo != nil)
@@ -127,11 +128,13 @@ func TestGroupQuotaManager_UpdateQuotaInternal(t *testing.T) {
 	assert.Equal(t, createResourceList(50, 80*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin)
 	assert.Equal(t, int64(64), quotaInfo.CalculateInfo.SharedWeight.Cpu().Value())
 	assert.Equal(t, int64(100*GigaByte), quotaInfo.CalculateInfo.SharedWeight.Memory().Value())
+	assert.True(t, quotav1.Equals(createResourceList(10, 40*GigaByte), quotaInfo.CalculateInfo.MinExcess))
 
 	AddQuotaToManager(t, gqm, "test2", extension.RootQuotaName, 96, 160*GigaByte, 80, 80*GigaByte, true, false)
 
 	quota = CreateQuota("test1", extension.RootQuotaName, 84, 120*GigaByte, 60, 100*GigaByte, true, false)
 	quota.Labels[extension.LabelQuotaIsParent] = "true"
+	quota.Annotations[extension.AnnotationMinExcess] = `{"cpu":30, "memory": "120Gi"}`
 	err := gqm.UpdateQuota(quota, false)
 	assert.Nil(t, err)
 	quotaInfo = gqm.quotaInfoMap["test1"]
@@ -142,6 +145,7 @@ func TestGroupQuotaManager_UpdateQuotaInternal(t *testing.T) {
 	assert.Equal(t, createResourceList(60, 100*GigaByte), quotaInfo.CalculateInfo.AutoScaleMin)
 	assert.Equal(t, int64(84), quotaInfo.CalculateInfo.SharedWeight.Cpu().Value())
 	assert.Equal(t, int64(120*GigaByte), quotaInfo.CalculateInfo.SharedWeight.Memory().Value())
+	assert.True(t, quotav1.Equals(createResourceList(30, 120*GigaByte), quotaInfo.CalculateInfo.MinExcess))
 }
 
 func TestGroupQuotaManager_UpdateQuota(t *testing.T) {
@@ -2108,3 +2112,84 @@ func TestGroupQuotaManager_ImmediateIgnoreTerminatingPod(t *testing.T) {
 	assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetRequest())
 	assert.Equal(t, createResourceList(0, 0), gqm.GetQuotaInfoByName("1").GetUsed())
 }
+
+func TestGroupQuotaManager_UpdateGroupDeltaUsedAndMinExcessUsed(t *testing.T) {
+	defer utilfeature.SetFeatureGateDuringTest(t, k8sfeature.DefaultFeatureGate,
+		features.ElasticQuotaMinExcess, true)()
+	gqm := NewGroupQuotaManagerForTest()
+
+	// quota1 Max[40, 40]  Min[20,20] Used[0,0]  MinExcess[20,20]
+	//   |-- quota2 Max[30, 30]  Min[10,10] Used[0,0]  MinExcess[10,10]
+	//   |-- quota3 Max[20, 20]  Min[5,5]   Used[0,0]  MinExcess[15,15]
+	q1 := createQuota("1", extension.RootQuotaName, 40, 40, 20, 20)
+	q1.Annotations[extension.AnnotationMinExcess] = `{"cpu":20,"memory":20}`
+	q2 := createQuota("2", "1", 30, 30, 10, 10)
+	q2.Annotations[extension.AnnotationMinExcess] = `{"cpu":10,"memory":10}`
+	q3 := createQuota("3", "1", 20, 20, 5, 5)
+	q3.Annotations[extension.AnnotationMinExcess] = `{"cpu":15,"memory":15}`
+	gqm.UpdateQuota(q1, false)
+	gqm.UpdateQuota(q2, false)
+	gqm.UpdateQuota(q3, false)
+
+	qi1 := gqm.GetQuotaInfoByName("1")
+	qi2 := gqm.GetQuotaInfoByName("2")
+	qi3 := gqm.GetQuotaInfoByName("3")
+	assert.NotNil(t, qi1)
+	assert.NotNil(t, qi2)
+	assert.NotNil(t, qi3)
+
+	// 1. quota2 used [15,20]
+	//    expected: quota2 minExcessUsed [5, 10], quota1 minExcessUsed [5,10]
+	delta := createResourceList(15, 20)
+	nonPreemptibleUsed := createResourceList(0, 0)
+	gqm.updateGroupDeltaUsedNoLock("2", delta, nonPreemptibleUsed)
+	assert.Equal(t, createResourceList(15, 20), qi1.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(5, 10), qi1.CalculateInfo.MinExcessUsed)
+	assert.Equal(t, createResourceList(15, 20), qi2.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(5, 10), qi2.CalculateInfo.MinExcessUsed)
+
+	// 2. quota3 used [10,10]
+	//    expected: quota3 minExcessUsed [5,5], quota1 minExcessUsed [10,15]
+	delta = createResourceList(10, 10)
+	gqm.updateGroupDeltaUsedNoLock("3", delta, nonPreemptibleUsed)
+	assert.Equal(t, createResourceList(10, 10), qi3.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(5, 5), qi3.CalculateInfo.MinExcessUsed)
+	assert.Equal(t, createResourceList(10, 15), qi1.CalculateInfo.MinExcessUsed)
+
+	// 3. quota2 used decreases to [5,15]
+	//    expected: quota2 minExcessUsed [0,5], quota1 minExcessUsed [5,10]
+	delta = createResourceList(-10, -5)
+	gqm.updateGroupDeltaUsedNoLock("2", delta, nonPreemptibleUsed)
+	assert.Equal(t, createResourceList(5, 15), qi2.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(0, 5), qi2.CalculateInfo.MinExcessUsed)
+	assert.Equal(t, createResourceList(5, 10), qi1.CalculateInfo.MinExcessUsed)
+
+	// 4. quota3 used decreases to [2,2]
+	//    expected: quota3 minExcessUsed [0,0], quota1 minExcessUsed [0,5]
+	delta = createResourceList(-8, -8)
+	gqm.updateGroupDeltaUsedNoLock("3", delta, nonPreemptibleUsed)
+	assert.Equal(t, createResourceList(2, 2), qi3.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(0, 0), qi3.CalculateInfo.MinExcessUsed)
+	assert.Equal(t, createResourceList(0, 5), qi1.CalculateInfo.MinExcessUsed)
+
+	// 5. quota2 used increases to [20,20]
+	//    expected: quota2 minExcessUsed [10,10], quota1 minExcessUsed [10,10]
+	delta = createResourceList(15, 5)
+	gqm.updateGroupDeltaUsedNoLock("2", delta, nonPreemptibleUsed)
+	assert.Equal(t, createResourceList(20, 20), qi2.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(10, 10), qi2.CalculateInfo.MinExcessUsed)
+	assert.Equal(t, createResourceList(10, 10), qi1.CalculateInfo.MinExcessUsed)
+
+	// 6. quota2 and quota3 decrease to [0,0]
+	//    expected: minExcessUsed [0,0] for all quotas
+	delta = createResourceList(-20, -20)
+	gqm.updateGroupDeltaUsedNoLock("2", delta, nonPreemptibleUsed)
+	delta = createResourceList(-2, -2)
+	gqm.updateGroupDeltaUsedNoLock("3", delta, nonPreemptibleUsed)
+	assert.Equal(t, createResourceList(0, 0), qi1.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(0, 0), qi2.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(0, 0), qi3.CalculateInfo.Used)
+	assert.Equal(t, createResourceList(0, 0), qi1.CalculateInfo.MinExcessUsed)
+	assert.Equal(t, createResourceList(0, 0), qi2.CalculateInfo.MinExcessUsed)
+	assert.Equal(t, createResourceList(0, 0), qi3.CalculateInfo.MinExcessUsed)
+}

pkg/scheduler/plugins/elasticquota/core/helper.go

@@ -18,6 +18,7 @@ package core
 
 import (
 	corev1 "k8s.io/api/core/v1"
+	quotav1 "k8s.io/apiserver/pkg/quota/v1"
 	k8sfeature "k8s.io/apiserver/pkg/util/feature"
 	apiresource "k8s.io/kubernetes/pkg/api/v1/resource"
 
@@ -32,3 +33,11 @@ func PodRequests(pod *corev1.Pod) (reqs corev1.ResourceList) {
 	}
 	return apiresource.PodRequests(pod, apiresource.PodResourcesOptions{})
 }
+
+func CalculateMinExcessUsedDelta(min, used, delta corev1.ResourceList) (minExcessUsedDelta corev1.ResourceList) {
+	minResourceNames := quotav1.ResourceNames(min)
+	minExcessUsed := quotav1.SubtractWithNonNegativeResult(quotav1.Mask(used, minResourceNames), min)
+	newUsed := quotav1.Add(quotav1.Mask(used, minResourceNames), quotav1.Mask(delta, minResourceNames))
+	newMinExcessUsed := quotav1.SubtractWithNonNegativeResult(quotav1.Mask(newUsed, minResourceNames), min)
+	return quotav1.Subtract(newMinExcessUsed, minExcessUsed)
+}