scheduler: delete quota inplace (#2312)

Signed-off-by: chuanyun.lcy <[email protected]> Co-authored-by: chuanyun.lcy <[email protected]>
koordinator-sh · Jan 9, 2025 · 71c9682 · 71c9682
1 parent 9f5f964
commit 71c9682
Show file tree

Hide file tree

Showing 6 changed files with 313 additions and 6 deletions.
diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager.go
@@ -388,12 +388,7 @@ func (gqm *GroupQuotaManager) UpdateQuota(quota *v1alpha1.ElasticQuota, isDelete
 
 	quotaName := quota.Name
 	if isDelete {
-		// TODO: inplace delete
-		_, exist := gqm.quotaInfoMap[quotaName]
-		if !exist {
-			return fmt.Errorf("get quota info failed, quotaName:%v", quotaName)
-		}
-		delete(gqm.quotaInfoMap, quotaName)
+		return gqm.deleteQuotaNoLock(quota)
 	} else {
 		newQuotaInfo := NewQuotaInfoFromQuota(quota)
 		// update the local quotaInfo's crd
@@ -1109,3 +1104,49 @@ func shouldBeIgnored(pod *v1.Pod) bool {
 		return time.Now().After(pod.DeletionTimestamp.Time.Add(time.Duration(*pod.DeletionGracePeriodSeconds) * time.Second))
 	}
 }
+
+func (gqm *GroupQuotaManager) deleteQuotaNoLock(quota *v1alpha1.ElasticQuota) error {
+	quotaInfo, exist := gqm.quotaInfoMap[quota.Name]
+	if !exist {
+		return fmt.Errorf("get quota info failed, quotaName:%v", quota.Name)
+	}
+	delete(gqm.quotaInfoMap, quota.Name)
+
+	// handle runtimeQuotaCalculator.
+	quotaInfo.lock.Lock()
+	defer quotaInfo.lock.Unlock()
+	if runtimeQuotaCalculator, exist := gqm.runtimeQuotaCalculatorMap[quotaInfo.ParentName]; exist {
+		runtimeQuotaCalculator.deleteOneGroup(quotaInfo)
+	}
+	delete(gqm.runtimeQuotaCalculatorMap, quota.Name)
+
+	// remove scale min.
+	gqm.scaleMinQuotaManager.remove(quotaInfo.ParentName, quotaInfo.Name)
+
+	// remove topology node.
+	delete(gqm.quotaTopoNodeMap, quota.Name)
+	if parentNode, exist := gqm.quotaTopoNodeMap[quotaInfo.ParentName]; exist {
+		delete(parentNode.childGroupQuotaInfos, quota.Name)
+	}
+
+	// update resource keys
+	gqm.updateResourceKeyNoLock()
+
+	// update request
+	deltaReq := quotav1.Subtract(v1.ResourceList{}, quotaInfo.CalculateInfo.Request)
+	deltaNonPreemptibleRequest := quotav1.Subtract(v1.ResourceList{}, quotaInfo.CalculateInfo.NonPreemptibleRequest)
+	if !quotav1.IsZero(deltaReq) || !quotav1.IsZero(deltaNonPreemptibleRequest) {
+		gqm.updateGroupDeltaRequestNoLock(quotaInfo.ParentName, deltaReq, deltaNonPreemptibleRequest, -1)
+	}
+
+	// update used
+	deltaUsed := quotav1.Subtract(v1.ResourceList{}, quotaInfo.CalculateInfo.Used)
+	deltaNonPreemptibleUsed := quotav1.Subtract(v1.ResourceList{}, quotaInfo.CalculateInfo.NonPreemptibleUsed)
+	if !quotav1.IsZero(deltaUsed) || !quotav1.IsZero(deltaNonPreemptibleUsed) {
+		gqm.updateGroupDeltaUsedNoLock(quotaInfo.ParentName, deltaUsed, deltaNonPreemptibleUsed, -1)
+	}
+
+	klog.Infof("delete quota %v for quota tree %v", quota.Name, gqm.treeID)
+
+	return nil
+}
diff --git a/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go b/pkg/scheduler/plugins/elasticquota/core/group_quota_manager_test.go
@@ -211,6 +211,74 @@ func TestGroupQuotaManager_DeleteOneGroup(t *testing.T) {
 	assert.Equal(t, 4, len(gqm.quotaInfoMap))
 }
 
+func TestGroupQuotaManager_DeleteQuotaInternal(t *testing.T) {
+	gqm := NewGroupQuotaManagerForTest()
+	gqm.UpdateClusterTotalResource(createResourceList(1000, 1000*GigaByte))
+	parent1 := AddQuotaToManager(t, gqm, "parent1", extension.RootQuotaName, 200, 200*GigaByte, 200, 200*GigaByte, true, true)
+	// test1 allow lent min
+	test1 := AddQuotaToManager(t, gqm, "test1", "parent1", 96, 160*GigaByte, 50, 80*GigaByte, true, false)
+	// test2 not allow lent min
+	test2 := AddQuotaToManager(t, gqm, "test2", "parent1", 96, 160*GigaByte, 80, 80*GigaByte, false, false)
+	assert.Equal(t, 4, len(gqm.runtimeQuotaCalculatorMap))
+	assert.Equal(t, 4, len(gqm.quotaTopoNodeMap))
+	assert.Equal(t, 6, len(gqm.quotaInfoMap))
+	assert.Equal(t, 2, len(gqm.quotaTopoNodeMap["parent1"].childGroupQuotaInfos))
+
+	parent1Info := gqm.GetQuotaInfoByName("parent1")
+	assert.Equal(t, createResourceList(80, 80*GigaByte), parent1Info.CalculateInfo.Request)
+	assert.Equal(t, createResourceList(80, 80*GigaByte), parent1Info.CalculateInfo.ChildRequest)
+
+	test1Info := gqm.GetQuotaInfoByName("test1")
+	assert.Equal(t, v1.ResourceList{}, test1Info.CalculateInfo.Request)
+	assert.Equal(t, v1.ResourceList{}, test1Info.CalculateInfo.ChildRequest)
+
+	test2Info := gqm.GetQuotaInfoByName("test2")
+	assert.Equal(t, createResourceList(80, 80*GigaByte), test2Info.CalculateInfo.Request)
+	assert.Equal(t, v1.ResourceList{}, test2Info.CalculateInfo.ChildRequest)
+
+	// delete test1 quota
+	err := gqm.deleteQuotaNoLock(test1)
+	assert.Nil(t, err)
+	test1Info = gqm.GetQuotaInfoByName("test1")
+	assert.True(t, test1Info == nil)
+
+	assert.Equal(t, 3, len(gqm.runtimeQuotaCalculatorMap))
+	assert.Equal(t, 3, len(gqm.quotaTopoNodeMap))
+	assert.Equal(t, 5, len(gqm.quotaInfoMap))
+
+	// check parent requests
+	parent1Info = gqm.GetQuotaInfoByName("parent1")
+	assert.Equal(t, createResourceList(80, 80*GigaByte), parent1Info.CalculateInfo.Request)
+	assert.Equal(t, createResourceList(80, 80*GigaByte), parent1Info.CalculateInfo.ChildRequest)
+	assert.Equal(t, 1, len(gqm.quotaTopoNodeMap["parent1"].childGroupQuotaInfos))
+
+	// delete test2 quota
+	err = gqm.deleteQuotaNoLock(test2)
+	assert.Nil(t, err)
+	test2Info = gqm.GetQuotaInfoByName("test2")
+	assert.True(t, test2Info == nil)
+
+	assert.Equal(t, 2, len(gqm.runtimeQuotaCalculatorMap))
+	assert.Equal(t, 2, len(gqm.quotaTopoNodeMap))
+	assert.Equal(t, 4, len(gqm.quotaInfoMap))
+
+	// check parent requests
+	parent1Info = gqm.GetQuotaInfoByName("parent1")
+	assert.Equal(t, createResourceList(0, 0), parent1Info.CalculateInfo.Request)
+	assert.Equal(t, createResourceList(0, 0), parent1Info.CalculateInfo.ChildRequest)
+	assert.Equal(t, 0, len(gqm.quotaTopoNodeMap["parent1"].childGroupQuotaInfos))
+
+	// delete parent1 quota
+	err = gqm.deleteQuotaNoLock(parent1)
+	assert.Nil(t, err)
+	parent1Info = gqm.GetQuotaInfoByName("parent1")
+	assert.True(t, parent1Info == nil)
+
+	assert.Equal(t, 1, len(gqm.runtimeQuotaCalculatorMap))
+	assert.Equal(t, 1, len(gqm.quotaTopoNodeMap))
+	assert.Equal(t, 3, len(gqm.quotaInfoMap))
+}
+
 func TestGroupQuotaManager_UpdateQuotaDeltaRequest(t *testing.T) {
 	gqm := NewGroupQuotaManagerForTest()
 

diff --git a/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator.go b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator.go
@@ -98,6 +98,12 @@ func (qt *quotaTree) updateGuaranteed(groupName string, guarantee int64) {
 	}
 }
 
+func (qt *quotaTree) erase(groupName string) {
+	if _, exist := qt.quotaNodes[groupName]; exist {
+		delete(qt.quotaNodes, groupName)
+	}
+}
+
 func (qt *quotaTree) find(groupName string) (bool, *quotaNode) {
 	if nodeValue, exist := qt.quotaNodes[groupName]; exist {
 		return exist, nodeValue
@@ -516,3 +522,22 @@ func createQuantity(value int64, resName v1.ResourceName) resource.Quantity {
 	}
 	return q
 }
+
+func (qtw *RuntimeQuotaCalculator) deleteOneGroup(quotaInfo *QuotaInfo) {
+	qtw.lock.Lock()
+	defer qtw.lock.Unlock()
+
+	for resKey := range qtw.resourceKeys {
+		if exist, _ := qtw.quotaTree[resKey].find(quotaInfo.Name); exist {
+			qtw.quotaTree[resKey].erase(quotaInfo.Name)
+		}
+	}
+	delete(qtw.groupReqLimit, quotaInfo.Name)
+	delete(qtw.groupGuaranteed, quotaInfo.Name)
+
+	qtw.globalRuntimeVersion++
+
+	if klog.V(5).Enabled() {
+		qtw.logQuotaInfoNoLock("deleteOneGroup finish", quotaInfo)
+	}
+}
diff --git a/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go b/pkg/scheduler/plugins/elasticquota/core/runtime_quota_calculator_test.go
@@ -376,6 +376,16 @@ func TestRuntimeQuotaCalculator_UpdateOneGroupRuntimeQuota(t *testing.T) {
 
 	qtw.updateOneGroupRuntimeQuota(test2)
 	assert.Equal(t, test2.CalculateInfo.AutoScaleMin, test2.CalculateInfo.Runtime)
+
+	// delete test1
+	// test2 max[100, 1000], min[50, 500], request[90, 900], runtime[90, 900]
+	qtw.deleteOneGroup(test1)
+	assert.Equal(t, int64(12), qtw.globalRuntimeVersion)
+	qtw.updateOneGroupRuntimeQuota(test2)
+	assert.Equal(t, createResourceList(90, 900), test2.CalculateInfo.Runtime)
+	cpu := corev1.ResourceCPU
+	assert.Equal(t, 1, len(qtw.groupReqLimit))
+	assert.Equal(t, 1, len(qtw.quotaTree[cpu].quotaNodes))
 }
 
 func TestRuntimeQuotaCalculator_UpdateOneGroupRuntimeQuota2(t *testing.T) {
@@ -436,3 +446,71 @@ func TestQuotaInfo_GetRuntime(t *testing.T) {
 		"cpu": *resource.NewQuantity(10, resource.DecimalSI),
 	})
 }
+
+func TestRuntimeQuotaCalculator_DeleteOneGroup(t *testing.T) {
+	cpu := corev1.ResourceCPU
+
+	qtw := createRuntimeQuotaCalculator()
+	totalResource := createResourceList(100, 1000)
+	qtw.setClusterTotalResource(totalResource)
+
+	// test1 max[80, 800], min[60, 600], request[0, 0], runtime[0, 0]
+	// test2 max[100, 1000], min[50, 500], request[90, 900], runtime[90, 900]
+	max := createResourceList(80, 800)
+	min := createResourceList(60, 600)
+	sharedWeight := createResourceList(1, 1)
+	test1 := createQuotaInfoWithRes("test1", max, min)
+	updateQuotaInfo(qtw, test1, max, min, sharedWeight)
+
+	max = createResourceList(100, 1000)
+	min = createResourceList(50, 500)
+	request := createResourceList(90, 900)
+	test2 := createQuotaInfoWithRes("test2", max, min)
+	test2.CalculateInfo.Request = request.DeepCopy()
+	updateQuotaInfo(qtw, test2, max, min, sharedWeight)
+
+	qtw.updateOneGroupRequest(test2)
+	qtw.updateOneGroupRuntimeQuota(test1)
+	qtw.updateOneGroupRuntimeQuota(test2)
+	assert.Equal(t, totalResource, qtw.totalResource)
+	assert.Equal(t, 2, len(qtw.quotaTree))
+	assert.Equal(t, int64(0), test1.CalculateInfo.Runtime.Name("cpu", resource.DecimalSI).Value())
+	assert.Equal(t, int64(0), test1.CalculateInfo.Runtime.Name("memory", resource.DecimalSI).Value())
+	assert.Equal(t, request, test2.CalculateInfo.Runtime)
+
+	// test1 max[80, 800], min[60, 600], request[30, 300], runtime[30, 300]
+	// test2 max[100, 1000], min[50, 500], request[90, 900], runtime[70, 700]
+	request = createResourceList(30, 300)
+	test1.CalculateInfo.Request = request.DeepCopy()
+	qtw.updateOneGroupRequest(test1)
+	qtw.updateOneGroupRuntimeQuota(test1)
+	qtw.updateOneGroupRuntimeQuota(test2)
+
+	assert.Equal(t, request, test1.CalculateInfo.Runtime)
+	assert.Equal(t, v12.Subtract(totalResource, request), test2.CalculateInfo.Runtime)
+
+	// test1 max[80, 800], min[60, 600], request[60, 600], runtime[60, 600]
+	// test2 max[100, 1000], min[50, 500], request[90, 900], runtime[50, 500]
+	request = createResourceList(60, 600)
+	test1.CalculateInfo.Request = request.DeepCopy()
+	qtw.updateOneGroupRequest(test1)
+	qtw.updateOneGroupRuntimeQuota(test1)
+
+	assert.Equal(t, request, test1.CalculateInfo.Runtime)
+
+	qtw.updateOneGroupRuntimeQuota(test2)
+	assert.Equal(t, test2.CalculateInfo.AutoScaleMin, test2.CalculateInfo.Runtime)
+
+	assert.Equal(t, int64(11), qtw.globalRuntimeVersion)
+	assert.Equal(t, 2, len(qtw.groupReqLimit))
+	assert.Equal(t, 2, len(qtw.quotaTree[cpu].quotaNodes))
+
+	// delete test1
+	// test2 max[100, 1000], min[50, 500], request[90, 900], runtime[90, 900]
+	qtw.deleteOneGroup(test1)
+	assert.Equal(t, int64(12), qtw.globalRuntimeVersion)
+	qtw.updateOneGroupRuntimeQuota(test2)
+	assert.Equal(t, createResourceList(90, 900), test2.CalculateInfo.Runtime)
+	assert.Equal(t, 1, len(qtw.groupReqLimit))
+	assert.Equal(t, 1, len(qtw.quotaTree[cpu].quotaNodes))
+}
diff --git a/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res.go b/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res.go
@@ -96,6 +96,30 @@ func (s *ScaleMinQuotaManager) update(parQuotaName, subQuotaName string, subMinQ
 	s.quotaEnableMinQuotaScaleMap[subQuotaName] = enableScaleMinQuota
 }
 
+func (s *ScaleMinQuotaManager) remove(parQuotaName, subQuotaName string) {
+	s.lock.Lock()
+	defer s.lock.Unlock()
+
+	if !s.quotaEnableMinQuotaScaleMap[subQuotaName] {
+		if _, ok := s.disableScaleSubsSumMinQuotaMap[parQuotaName]; ok {
+			s.disableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.SubtractWithNonNegativeResult(s.disableScaleSubsSumMinQuotaMap[parQuotaName],
+				s.originalMinQuotaMap[subQuotaName])
+		}
+	} else {
+		if _, ok := s.enableScaleSubsSumMinQuotaMap[parQuotaName]; ok {
+			s.enableScaleSubsSumMinQuotaMap[parQuotaName] = quotav1.SubtractWithNonNegativeResult(s.enableScaleSubsSumMinQuotaMap[parQuotaName],
+				s.originalMinQuotaMap[subQuotaName])
+		}
+	}
+
+	delete(s.originalMinQuotaMap, subQuotaName)
+	delete(s.quotaEnableMinQuotaScaleMap, subQuotaName)
+
+	if klog.V(5).Enabled() {
+		klog.Infof("ScaleMinQuotaManager remove, parQuota: %v, subQuota: %v  ", parQuotaName, subQuotaName)
+	}
+}
+
 func (s *ScaleMinQuotaManager) getScaledMinQuota(newTotalRes v1.ResourceList, parQuotaName, subQuotaName string) (bool, v1.ResourceList) {
 	s.lock.Lock()
 	defer s.lock.Unlock()

diff --git a/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res_test.go b/pkg/scheduler/plugins/elasticquota/core/scale_minquota_when_over_root_res_test.go
@@ -231,3 +231,74 @@ func TestScaleMinQuotaWhenOverRootResInfo_Update(t *testing.T) {
 		}
 	}
 }
+
+func TestScaleMinQuotaWhenOverRootResInfo_Remove(t *testing.T) {
+	info := NewScaleMinQuotaManager()
+	if len(info.enableScaleSubsSumMinQuotaMap) != 0 ||
+		len(info.disableScaleSubsSumMinQuotaMap) != 0 || len(info.originalMinQuotaMap) != 0 || len(info.quotaEnableMinQuotaScaleMap) != 0 {
+		t.Errorf("error")
+	}
+	{
+		parQuotaName := "100"
+		subQuotaName := "1"
+		subMinQuota := createResourceList(50, 50)
+		enableScaleMinQuota := false
+		info.update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota)
+
+		if len(info.enableScaleSubsSumMinQuotaMap) != 1 ||
+			len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 1 || len(info.quotaEnableMinQuotaScaleMap) != 1 {
+			t.Errorf("error")
+		}
+		if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], v1.ResourceList{}) ||
+			!quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) ||
+			!quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != false {
+			t.Error("error")
+		}
+	}
+	{
+		parQuotaName := "100"
+		subQuotaName := "2"
+		subMinQuota := createResourceList(60, 60)
+		enableScaleMinQuota := true
+		info.update(parQuotaName, subQuotaName, subMinQuota, enableScaleMinQuota)
+
+		if len(info.enableScaleSubsSumMinQuotaMap) != 1 ||
+			len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 2 || len(info.quotaEnableMinQuotaScaleMap) != 2 {
+			t.Errorf("error")
+		}
+
+		if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(60, 60)) ||
+			!quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(50, 50)) ||
+			!quotav1.Equals(info.originalMinQuotaMap["1"], createResourceList(50, 50)) || info.quotaEnableMinQuotaScaleMap["1"] != false ||
+			!quotav1.Equals(info.originalMinQuotaMap["2"], createResourceList(60, 60)) || info.quotaEnableMinQuotaScaleMap["2"] != true {
+			t.Error("error")
+		}
+	}
+	{
+		info.remove("100", "1")
+
+		if len(info.enableScaleSubsSumMinQuotaMap) != 1 ||
+			len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 1 || len(info.quotaEnableMinQuotaScaleMap) != 1 {
+			t.Errorf("error")
+		}
+
+		if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(60, 60)) ||
+			!quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) ||
+			!quotav1.Equals(info.originalMinQuotaMap["2"], createResourceList(60, 60)) || info.quotaEnableMinQuotaScaleMap["2"] != true {
+			t.Error("error")
+		}
+	}
+	{
+		info.remove("100", "2")
+
+		if len(info.enableScaleSubsSumMinQuotaMap) != 1 ||
+			len(info.disableScaleSubsSumMinQuotaMap) != 1 || len(info.originalMinQuotaMap) != 0 || len(info.quotaEnableMinQuotaScaleMap) != 0 {
+			t.Errorf("error")
+		}
+
+		if !quotav1.Equals(info.enableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) ||
+			!quotav1.Equals(info.disableScaleSubsSumMinQuotaMap["100"], createResourceList(0, 0)) {
+			t.Error("error")
+		}
+	}
+}