From 9bea9006654ed95414b7d4e3dda07f41e84efe81 Mon Sep 17 00:00:00 2001
From: Maximilian Pass <22845248+mpass99@users.noreply.github.com>
Date: Thu, 5 Sep 2024 12:26:04 +0200
Subject: [PATCH] Fix prohibiting Nomad Job recreation with delay in between

by not purging a job once its allocation stops (and maybe would be replaced).
---
 internal/nomad/event_stream_handling_test.go | 26 ++++++++++++++++++++
 internal/nomad/nomad.go                      |  3 ++-
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/internal/nomad/event_stream_handling_test.go b/internal/nomad/event_stream_handling_test.go
index 296102df..08f4a3d0 100644
--- a/internal/nomad/event_stream_handling_test.go
+++ b/internal/nomad/event_stream_handling_test.go
@@ -495,6 +495,32 @@ func (s *MainTestSuite) TestAPIClient_WatchAllocationsReturnsErrorOnUnexpectedEO
 	s.Equal(1, eventsProcessed)
 }
 
+func (s *MainTestSuite) TestAPIClient_WatchAllocationsReturnsLocalDestroyReasonOnStoppingAllocation() {
+	pendingAllocation := createRecentAllocation(structs.AllocClientStatusPending, structs.AllocDesiredStatusRun)
+	startedAllocation := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusRun)
+	stoppingAllocation := createRecentAllocation(structs.AllocClientStatusRunning, structs.AllocDesiredStatusStop)
+	events := nomadApi.Events{Events: []nomadApi.Event{
+		eventForAllocation(s.T(), pendingAllocation),
+		eventForAllocation(s.T(), startedAllocation),
+		eventForAllocation(s.T(), stoppingAllocation),
+	}}
+
+	callbackCalled := false
+	callbacks := &AllocationProcessing{
+		OnNew: func(_ context.Context, _ *nomadApi.Allocation, _ time.Duration) {},
+		OnDeleted: func(_ context.Context, jobID string, reason error) bool {
+			callbackCalled = true
+			s.Equal(tests.DefaultRunnerID, jobID)
+			s.ErrorIs(reason, ErrLocalDestruction)
+			return false
+		},
+	}
+
+	_, err := runAllocationWatching(s, []*nomadApi.Events{&events}, callbacks)
+	s.Require().NoError(err)
+	s.True(callbackCalled)
+}
+
 func (s *MainTestSuite) TestAPIClient_WatchAllocationsCanHandleMigration() {
 	jobID := "10-0331c7d8-03c1-11ef-b832-fa163e7afdf8"
 	a1ID := "84a734a1-5573-6116-5678-86060ce4c479"
diff --git a/internal/nomad/nomad.go b/internal/nomad/nomad.go
index cfbb5ca4..affe3088 100644
--- a/internal/nomad/nomad.go
+++ b/internal/nomad/nomad.go
@@ -33,7 +33,8 @@ var (
 		"%w correctly but rescheduled", ErrAllocationStopped)
 	// ErrAllocationCompleted is for reporting the reason for the stopped allocation.
 	// We do not consider it as an error but add it anyway for a complete reporting.
-	ErrAllocationCompleted RunnerDeletedReason = errors.New("the allocation completed")
+	// It is a ErrLocalDestruction because another allocation might be replacing the allocation in the same job.
+	ErrAllocationCompleted RunnerDeletedReason = fmt.Errorf("the allocation completed: %w", ErrLocalDestruction)
 )
 
 type RunnerDeletedReason error