diff --git a/pkg/common/errors.go b/pkg/common/errors.go index 27ca0c74a..1e8334ea3 100644 --- a/pkg/common/errors.go +++ b/pkg/common/errors.go @@ -20,8 +20,10 @@ package common import "errors" -// Common errors -var ( - // InvalidQueueName returned when queue name is invalid - InvalidQueueName = errors.New("invalid queue name, max 64 characters consisting of alphanumeric characters and '-', '_', '#', '@', '/', ':' allowed") -) +// InvalidQueueName returned when queue name is invalid +var InvalidQueueName = errors.New("invalid queue name, max 64 characters consisting of alphanumeric characters and '-', '_', '#', '@', '/', ':' allowed") + +const PreemptionPreconditionsFailed = "Preemption preconditions failed" +const PreemptionDoesNotGuarantee = "Preemption queue guarantees check failed" +const PreemptionShortfall = "Preemption helped but short of resources" +const PreemptionDoesNotHelp = "Preemption does not help" diff --git a/pkg/scheduler/objects/application.go b/pkg/scheduler/objects/application.go index a7fee254a..a6840854b 100644 --- a/pkg/scheduler/objects/application.go +++ b/pkg/scheduler/objects/application.go @@ -1056,6 +1056,7 @@ func (sa *Application) tryAllocate(headRoom *resources.Resource, allowPreemption // preemption occurred, and possibly reservation return result } + request.LogAllocationFailure(common.PreemptionDoesNotHelp, true) } } request.LogAllocationFailure(NotEnoughQueueQuota, true) // error message MUST be constant! @@ -1122,6 +1123,7 @@ func (sa *Application) tryAllocate(headRoom *resources.Resource, allowPreemption return result } } + request.LogAllocationFailure(common.PreemptionDoesNotHelp, true) } } } @@ -1394,6 +1396,7 @@ func (sa *Application) tryPreemption(headRoom *resources.Resource, preemptionDel // validate prerequisites for preemption of an ask and mark ask for preemption if successful if !preemptor.CheckPreconditions() { + ask.LogAllocationFailure(common.PreemptionPreconditionsFailed, true) return nil, false } diff --git a/pkg/scheduler/objects/application_test.go b/pkg/scheduler/objects/application_test.go index 8741c8e74..097125852 100644 --- a/pkg/scheduler/objects/application_test.go +++ b/pkg/scheduler/objects/application_test.go @@ -2009,6 +2009,7 @@ func TestTryAllocatePreemptQueue(t *testing.T) { result3 := app2.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 0}), true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) assert.Assert(t, result3 == nil, "result3 not expected") assert.Assert(t, !alloc2.IsPreempted(), "alloc2 should not have been preempted") + assertAllocationLog(t, ask3) // pass the time and try again ask3.createTime = ask3.createTime.Add(-30 * time.Second) @@ -2068,28 +2069,25 @@ func TestTryAllocatePreemptNode(t *testing.T) { preemptionAttemptsRemaining := 10 // consume capacity with 'unlimited' app - result00 := app0.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 40}), true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) - assert.Assert(t, result00 != nil, "result00 expected") - alloc00 := result00.Request - assert.Assert(t, alloc00 != nil, "alloc00 expected") - alloc00.SetNodeID(result00.NodeID) - result01 := app0.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 39}), true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) - assert.Assert(t, result01 != nil, "result01 expected") - alloc01 := result01.Request - assert.Assert(t, alloc01 != nil, "alloc01 expected") - alloc01.SetNodeID(result01.NodeID) + for _, r := range []*resources.Resource{resources.NewResourceFromMap(map[string]resources.Quantity{"first": 40}), resources.NewResourceFromMap(map[string]resources.Quantity{"first": 39})} { + result0 := app0.tryAllocate(r, true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) + assert.Assert(t, result0 != nil, "result0 expected") + alloc0 := result0.Request + assert.Assert(t, alloc0 != nil, "alloc0 expected") + alloc0.SetNodeID(result0.NodeID) + } // consume remainder of space but not quota - result1 := app1.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 28}), true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) - assert.Assert(t, result1 != nil, "result1 expected") - alloc1 := result1.Request - assert.Assert(t, alloc1 != nil, "alloc1 expected") - alloc1.SetNodeID(result1.NodeID) - result2 := app1.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 23}), true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) - assert.Assert(t, result2 != nil, "result2 expected") - alloc2 := result2.Request - assert.Assert(t, alloc2 != nil, "alloc2 expected") - alloc2.SetNodeID(result2.NodeID) + allocs := make([]*Allocation, 0) + for _, r := range []*resources.Resource{resources.NewResourceFromMap(map[string]resources.Quantity{"first": 28}), resources.NewResourceFromMap(map[string]resources.Quantity{"first": 23})} { + var alloc1 *Allocation + result1 := app1.tryAllocate(r, true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) + assert.Assert(t, result1 != nil, "result1 expected") + alloc1 = result1.Request + assert.Assert(t, result1.Request != nil, "alloc1 expected") + alloc1.SetNodeID(result1.NodeID) + allocs = append(allocs, alloc1) + } // on first attempt, should see a reservation since we're after the reservation timeout ask3.createTime = ask3.createTime.Add(-10 * time.Second) @@ -2099,10 +2097,16 @@ func TestTryAllocatePreemptNode(t *testing.T) { assert.Assert(t, alloc3 != nil, "alloc3 not expected") assert.Equal(t, "node1", result3.NodeID, "wrong node assignment") assert.Equal(t, Reserved, result3.ResultType, "expected reservation") - assert.Assert(t, !alloc2.IsPreempted(), "alloc2 should not have been preempted") + assert.Assert(t, !allocs[1].IsPreempted(), "alloc2 should not have been preempted") err = node1.Reserve(app2, ask3) assert.NilError(t, err) + // preemption delay not yet passed, so preemption should fail + result3 = app2.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 18}), true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) + assert.Assert(t, result3 == nil, "result3 expected") + assert.Assert(t, !allocs[1].IsPreempted(), "alloc1 should have been preempted") + assertAllocationLog(t, ask3) + // pass the time and try again ask3.createTime = ask3.createTime.Add(-30 * time.Second) result3 = app2.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 18}), true, 30*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) @@ -2110,7 +2114,7 @@ func TestTryAllocatePreemptNode(t *testing.T) { assert.Equal(t, Reserved, result3.ResultType, "expected reservation") alloc3 = result3.Request assert.Assert(t, alloc3 != nil, "alloc3 expected") - assert.Assert(t, alloc1.IsPreempted(), "alloc1 should have been preempted") + assert.Assert(t, allocs[0].IsPreempted(), "alloc1 should have been preempted") } func TestMaxAskPriority(t *testing.T) { diff --git a/pkg/scheduler/objects/preemption.go b/pkg/scheduler/objects/preemption.go index 0f62163f4..441b12dd6 100644 --- a/pkg/scheduler/objects/preemption.go +++ b/pkg/scheduler/objects/preemption.go @@ -26,6 +26,7 @@ import ( "go.uber.org/zap" + "github.com/apache/yunikorn-core/pkg/common" "github.com/apache/yunikorn-core/pkg/common/resources" "github.com/apache/yunikorn-core/pkg/log" "github.com/apache/yunikorn-core/pkg/plugins" @@ -203,7 +204,6 @@ func (p *Preemptor) checkPreemptionQueueGuarantees() bool { } } } - return false } @@ -558,6 +558,7 @@ func (p *Preemptor) tryNodes() (string, []*Allocation, bool) { func (p *Preemptor) TryPreemption() (*AllocationResult, bool) { // validate that sufficient capacity can be freed if !p.checkPreemptionQueueGuarantees() { + p.ask.LogAllocationFailure(common.PreemptionDoesNotGuarantee, true) return nil, false } @@ -615,6 +616,7 @@ func (p *Preemptor) TryPreemption() (*AllocationResult, bool) { if p.ask.GetAllocatedResource().StrictlyGreaterThanOnlyExisting(victimsTotalResource) { // there is shortfall, so preemption doesn't help + p.ask.LogAllocationFailure(common.PreemptionShortfall, true) return nil, false } diff --git a/pkg/scheduler/objects/preemption_test.go b/pkg/scheduler/objects/preemption_test.go index cfba589e1..62a05bfd5 100644 --- a/pkg/scheduler/objects/preemption_test.go +++ b/pkg/scheduler/objects/preemption_test.go @@ -25,6 +25,7 @@ import ( "gotest.tools/v3/assert" + "github.com/apache/yunikorn-core/pkg/common" "github.com/apache/yunikorn-core/pkg/common/resources" "github.com/apache/yunikorn-core/pkg/mock" "github.com/apache/yunikorn-core/pkg/plugins" @@ -85,6 +86,13 @@ func TestCheckPreconditions(t *testing.T) { // verify that recently checked ask is disqualified ask.preemptCheckTime = time.Now() assert.Assert(t, !preemptor.CheckPreconditions(), "preconditions succeeded with recently tried ask") + getNode := func(nodeID string) *Node { + return node + } + preemptionAttemptsRemaining := 1 + result := app.tryAllocate(resources.NewResourceFromMap(map[string]resources.Quantity{"first": 2}), true, 1*time.Second, &preemptionAttemptsRemaining, iterator, iterator, getNode) + assert.Check(t, result == nil, "unexpected result") + assertAllocationLog(t, ask) ask.preemptCheckTime = time.Now().Add(-1 * time.Minute) assert.Assert(t, preemptor.CheckPreconditions(), "preconditions failed") assert.Assert(t, !preemptor.CheckPreconditions(), "preconditions succeeded on successive run") @@ -106,6 +114,7 @@ func TestCheckPreemptionQueueGuarantees(t *testing.T) { childQ1.applications[appID1] = app1 ask1 := newAllocationAsk("alloc1", appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5})) assert.NilError(t, app1.AddAllocationAsk(ask1)) + ask2 := newAllocationAsk("alloc2", appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5})) assert.NilError(t, app1.AddAllocationAsk(ask2)) alloc1 := newAllocationWithKey("alloc1", appID1, "node1", resources.NewResourceFromMap(map[string]resources.Quantity{"first": 5})) @@ -129,6 +138,9 @@ func TestCheckPreemptionQueueGuarantees(t *testing.T) { // verify too large of a resource will not succeed ask3.allocatedResource = resources.NewResourceFromMap(map[string]resources.Quantity{"first": 25}) assert.Assert(t, !preemptor.checkPreemptionQueueGuarantees(), "queue guarantees did not fail") + result, _ := preemptor.TryPreemption() + assert.Assert(t, result == nil, "no result") + assert.Equal(t, ask3.GetAllocationLog()[0].Message, common.PreemptionDoesNotGuarantee) } func TestCheckPreemptionQueueGuaranteesWithNoGuaranteedResources(t *testing.T) { @@ -178,7 +190,16 @@ func TestCheckPreemptionQueueGuaranteesWithNoGuaranteedResources(t *testing.T) { childQ2.incPendingResource(ask3.GetAllocatedResource()) headRoom := resources.NewResourceFromMap(map[string]resources.Quantity{"first": 10}) preemptor := NewPreemptor(app2, headRoom, 30*time.Second, ask3, iterator(), false) - assert.Equal(t, tt.expected, preemptor.checkPreemptionQueueGuarantees(), "unexpected resultType") + result, ok := preemptor.TryPreemption() + assert.Equal(t, tt.expected, ok, "unexpected resultType") + if tt.expected { + assert.Equal(t, result != nil, true, "unexpected resultType") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) + } else { + assert.Equal(t, result == nil, true, "unexpected resultType") + assert.Equal(t, len(ask3.GetAllocationLog()), 1) + assert.Equal(t, ask3.GetAllocationLog()[0].Message, common.PreemptionDoesNotGuarantee) + } }) } } @@ -236,6 +257,7 @@ func TestTryPreemption(t *testing.T) { assert.Equal(t, "alloc3", result.Request.GetAllocationKey(), "wrong alloc") assert.Check(t, alloc1.IsPreempted(), "alloc1 not preempted") assert.Check(t, !alloc2.IsPreempted(), "alloc2 preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemptionOnNode Test try preemption on node with simple queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -301,6 +323,7 @@ func TestTryPreemptionOnNode(t *testing.T) { assert.Equal(t, nodeID2, result.NodeID, "wrong node") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemption_NodeWithCapacityLesserThanAsk Test try preemption on node whose capacity is lesser than ask resource requirements with simple queue hierarchy. Since Node won't accommodate the ask even after preempting all allocations, there is no use in considering the node. @@ -355,6 +378,7 @@ func TestTryPreemption_NodeWithCapacityLesserThanAsk(t *testing.T) { assert.Equal(t, ok, false, "no victims found") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, !alloc2.IsPreempted(), "alloc2 preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemptionOnNodeWithOGParentAndUGPreemptor Test try preemption on node with simple queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -422,6 +446,7 @@ func TestTryPreemptionOnNodeWithOGParentAndUGPreemptor(t *testing.T) { assert.Equal(t, "alloc7", result.Request.allocationKey, "wrong alloc") assert.Equal(t, nodeID2, result.NodeID, "wrong node") assert.Check(t, node2.GetAllocation("alloc1").IsPreempted(), "alloc1 preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemptionOnQueue Test try preemption on queue with simple queue hierarchy. Since Node has enough resources to accomodate, preemption happens because of queue resource constraint. @@ -486,6 +511,7 @@ func TestTryPreemptionOnQueue(t *testing.T) { assert.Equal(t, nodeID2, result.NodeID, "wrong node") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemption_VictimsAvailable_InsufficientResource Test try preemption on queue with simple queue hierarchy. Since Node has enough resources to accomodate, preemption happens because of queue resource constraint. @@ -541,6 +567,8 @@ func TestTryPreemption_VictimsAvailable_InsufficientResource(t *testing.T) { assert.Equal(t, ok, false, "no victims found") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, !alloc2.IsPreempted(), "alloc2 preempted") + log := ask3.GetAllocationLog() + assert.Equal(t, log[0].Message, common.PreemptionShortfall) } // TestTryPreemption_VictimsOnDifferentNodes_InsufficientResource Test try preemption on queue with simple queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -606,6 +634,8 @@ func TestTryPreemption_VictimsOnDifferentNodes_InsufficientResource(t *testing.T assert.Equal(t, ok, false, "no victims found") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, !alloc2.IsPreempted(), "alloc2 preempted") + log := ask3.GetAllocationLog() + assert.Equal(t, log[0].Message, common.PreemptionShortfall) } // TestTryPreemption_VictimsAvailableOnDifferentNodes Test try preemption on queue with simple queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -670,6 +700,8 @@ func TestTryPreemption_VictimsAvailableOnDifferentNodes(t *testing.T) { assert.Equal(t, ok, false, "no victims found") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, !alloc2.IsPreempted(), "alloc2 preempted") + log := ask3.GetAllocationLog() + assert.Equal(t, log[0].Message, common.PreemptionShortfall) } // TestTryPreemption_OnQueue_VictimsOnDifferentNodes Test try preemption on queue with simple queue hierarchy. Since Node has enough resources to accomodate, preemption happens because of queue resource constraint.xw @@ -758,6 +790,7 @@ func TestTryPreemption_OnQueue_VictimsOnDifferentNodes(t *testing.T) { assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, alloc4.IsPreempted(), "alloc2 not preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemption_OnQueue_VictimsAvailable_LowerPriority Test try preemption on queue with simple queue hierarchy. Since Node has enough resources to accomodate, preemption happens because of queue resource constraint. @@ -847,6 +880,7 @@ func TestTryPreemption_OnQueue_VictimsAvailable_LowerPriority(t *testing.T) { assert.Check(t, alloc1.IsPreempted(), "alloc1 not preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, !alloc4.IsPreempted(), "alloc4 preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnPreemptorSide Test try preemption with 2 level queue hierarchy. @@ -919,6 +953,7 @@ func TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnPreemptorSide(t *test assert.Equal(t, nodeID1, alloc2.nodeID, "wrong node") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnPreemptorSide Test try preemption with 2 level queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -990,6 +1025,7 @@ func TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnPreemptorSide( assert.Equal(t, nodeID1, alloc2.nodeID, "wrong node") assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") + assert.Equal(t, len(ask3.GetAllocationLog()), 0) } // TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSides Test try preemption with 2 level queue hierarchy. @@ -1023,17 +1059,7 @@ func TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSid assert.NilError(t, err) _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil, nil) assert.NilError(t, err) - - app1 := newApplication(appID1, "default", "root.parent.parent2.child2") - app1.SetQueue(childQ2) - childQ2.applications[appID1] = app1 - app2 := newApplication(appID2, "default", "root.parent.parent2.child2") - app2.SetQueue(childQ2) - childQ2.applications[appID2] = app2 - app3 := newApplication(appID3, "default", "root.parent.parent2.child2") - app3.SetQueue(childQ2) - childQ2.applications[appID3] = app3 - + app1, app2, app3 := createVictimApplications(childQ2) for i := 5; i < 8; i++ { askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"mem": 100})) askN.createTime = time.Now().Add(-2 * time.Minute) @@ -1098,6 +1124,7 @@ func TestTryPreemption_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSid assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, alloc3.IsPreempted(), "alloc3 not preempted") + assert.Equal(t, len(ask4.GetAllocationLog()), 0) } // TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnVictimAndPreemptorSides Test try preemption with 2 level queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -1129,17 +1156,7 @@ func TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnVictimAndPreem assert.NilError(t, err) _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil, nil) assert.NilError(t, err) - - app1 := newApplication(appID1, "default", "root.parent.parent2.child2") - app1.SetQueue(childQ2) - childQ2.applications[appID1] = app1 - app2 := newApplication(appID2, "default", "root.parent.parent2.child2") - app2.SetQueue(childQ2) - childQ2.applications[appID2] = app2 - app3 := newApplication(appID3, "default", "root.parent.parent2.child2") - app3.SetQueue(childQ2) - childQ2.applications[appID3] = app3 - + app1, app2, app3 := createVictimApplications(childQ2) for i := 5; i < 8; i++ { askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"mem": 100})) askN.createTime = time.Now().Add(-2 * time.Minute) @@ -1203,6 +1220,20 @@ func TestTryPreemption_OnNode_AskResTypesDifferent_GuaranteedSetOnVictimAndPreem assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, alloc3.IsPreempted(), "alloc3 not preempted") + assert.Equal(t, len(ask4.GetAllocationLog()), 0) +} + +func createVictimApplications(childQ2 *Queue) (*Application, *Application, *Application) { + app1 := newApplication(appID1, "default", "root.parent.parent2.child2") + app1.SetQueue(childQ2) + childQ2.applications[appID1] = app1 + app2 := newApplication(appID2, "default", "root.parent.parent2.child2") + app2.SetQueue(childQ2) + childQ2.applications[appID2] = app2 + app3 := newApplication(appID3, "default", "root.parent.parent2.child2") + app3.SetQueue(childQ2) + childQ2.applications[appID3] = app3 + return app1, app2, app3 } // TestTryPreemption_AskResTypesSame_GuaranteedSetOnPreemptorSide Test try preemption with 2 level queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -1236,17 +1267,7 @@ func TestTryPreemption_AskResTypesSame_GuaranteedSetOnPreemptorSide(t *testing.T assert.NilError(t, err) _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil, nil) assert.NilError(t, err) - - app1 := newApplication(appID1, "default", "root.parent.parent2.child2") - app1.SetQueue(childQ2) - childQ2.applications[appID1] = app1 - app2 := newApplication(appID2, "default", "root.parent.parent2.child2") - app2.SetQueue(childQ2) - childQ2.applications[appID2] = app2 - app3 := newApplication(appID3, "default", "root.parent.parent2.child2") - app3.SetQueue(childQ2) - childQ2.applications[appID3] = app3 - + app1, app2, app3 := createVictimApplications(childQ2) for i := 5; i < 8; i++ { askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100})) askN.createTime = time.Now().Add(-2 * time.Minute) @@ -1311,6 +1332,7 @@ func TestTryPreemption_AskResTypesSame_GuaranteedSetOnPreemptorSide(t *testing.T assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, alloc3.IsPreempted(), "alloc3 not preempted") + assert.Equal(t, len(ask4.GetAllocationLog()), 0) } // TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnPreemptorSide Test try preemption with 2 level queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -1342,17 +1364,7 @@ func TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnPreemptorSide(t *te assert.NilError(t, err) _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil, nil) assert.NilError(t, err) - - app1 := newApplication(appID1, "default", "root.parent.parent2.child2") - app1.SetQueue(childQ2) - childQ2.applications[appID1] = app1 - app2 := newApplication(appID2, "default", "root.parent.parent2.child2") - app2.SetQueue(childQ2) - childQ2.applications[appID2] = app2 - app3 := newApplication(appID3, "default", "root.parent.parent2.child2") - app3.SetQueue(childQ2) - childQ2.applications[appID3] = app3 - + app1, app2, app3 := createVictimApplications(childQ2) for i := 5; i < 8; i++ { askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100})) askN.createTime = time.Now().Add(-2 * time.Minute) @@ -1416,6 +1428,7 @@ func TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnPreemptorSide(t *te assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, alloc3.IsPreempted(), "alloc3 not preempted") + assert.Equal(t, len(ask4.GetAllocationLog()), 0) } // TestTryPreemption_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides Test try preemption with 2 level queue hierarchy. @@ -1449,17 +1462,7 @@ func TestTryPreemption_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides(t assert.NilError(t, err) _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil, nil) assert.NilError(t, err) - - app1 := newApplication(appID1, "default", "root.parent.parent2.child2") - app1.SetQueue(childQ2) - childQ2.applications[appID1] = app1 - app2 := newApplication(appID2, "default", "root.parent.parent2.child2") - app2.SetQueue(childQ2) - childQ2.applications[appID2] = app2 - app3 := newApplication(appID3, "default", "root.parent.parent2.child2") - app3.SetQueue(childQ2) - childQ2.applications[appID3] = app3 - + app1, app2, app3 := createVictimApplications(childQ2) for i := 5; i < 8; i++ { askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100})) askN.createTime = time.Now().Add(-2 * time.Minute) @@ -1525,6 +1528,7 @@ func TestTryPreemption_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides(t assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, alloc3.IsPreempted(), "alloc3 not preempted") + assert.Equal(t, len(ask4.GetAllocationLog()), 0) } // TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorSides Test try preemption with 2 level queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. @@ -1556,17 +1560,7 @@ func TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorS assert.NilError(t, err) _, err = createManagedQueueGuaranteed(parentQ2, "child3", false, nil, nil) assert.NilError(t, err) - - app1 := newApplication(appID1, "default", "root.parent.parent2.child2") - app1.SetQueue(childQ2) - childQ2.applications[appID1] = app1 - app2 := newApplication(appID2, "default", "root.parent.parent2.child2") - app2.SetQueue(childQ2) - childQ2.applications[appID2] = app2 - app3 := newApplication(appID3, "default", "root.parent.parent2.child2") - app3.SetQueue(childQ2) - childQ2.applications[appID3] = app3 - + app1, app2, app3 := createVictimApplications(childQ2) for i := 5; i < 8; i++ { askN := newAllocationAsk(alloc+strconv.Itoa(i), appID1, resources.NewResourceFromMap(map[string]resources.Quantity{"gpu": 100})) askN.createTime = time.Now().Add(-2 * time.Minute) @@ -1631,6 +1625,7 @@ func TestTryPreemption_OnNode_AskResTypesSame_GuaranteedSetOnVictimAndPreemptorS assert.Check(t, !alloc1.IsPreempted(), "alloc1 preempted") assert.Check(t, alloc2.IsPreempted(), "alloc2 not preempted") assert.Check(t, alloc3.IsPreempted(), "alloc3 not preempted") + assert.Equal(t, len(ask4.GetAllocationLog()), 0) } // TestTryPreemption_OnNode_UGParent_With_UGPreemptorChild_GNotSetOnVictimChild_As_Siblings Test try preemption with 2 level queue hierarchy. Since Node doesn't have enough resources to accomodate, preemption happens because of node resource constraint. diff --git a/pkg/scheduler/objects/utilities_test.go b/pkg/scheduler/objects/utilities_test.go index 18e19d337..d12d644ec 100644 --- a/pkg/scheduler/objects/utilities_test.go +++ b/pkg/scheduler/objects/utilities_test.go @@ -26,6 +26,7 @@ import ( "github.com/google/btree" "gotest.tools/v3/assert" + "github.com/apache/yunikorn-core/pkg/common" "github.com/apache/yunikorn-core/pkg/common/configs" "github.com/apache/yunikorn-core/pkg/common/resources" "github.com/apache/yunikorn-core/pkg/common/security" @@ -292,6 +293,21 @@ func assertUserResourcesAndGroupResources(t *testing.T, userGroup security.UserG assert.Equal(t, resources.Equals(groupResource, expectedGroupResources), true) } +func assertAllocationLog(t *testing.T, ask *Allocation) { + log := ask.GetAllocationLog() + preemptionPreconditionsFailed := false + PreemptionDoesNotHelp := false + for _, l := range log { + if l.Message == common.PreemptionPreconditionsFailed { + preemptionPreconditionsFailed = true + } else if l.Message == common.PreemptionDoesNotHelp { + PreemptionDoesNotHelp = true + } + } + assert.Assert(t, preemptionPreconditionsFailed) + assert.Assert(t, PreemptionDoesNotHelp) +} + func getNodeIteratorFn(nodes ...*Node) func() NodeIterator { tree := btree.New(7) for _, node := range nodes {