Fixing race condition related to closing stale job

If the thread to check for stale jobs is triggered right as an
evaluation is completing, it will close the job before results could be
got resulting in the job "failing"

src/wres/server/EvaluationService.java

@@ -392,9 +392,6 @@ public Response createEvaluation( String projectDeclaration,
         // Sets up stream redirect to avoid missing log statements
         streamRedirectSetup();
 
-        // Start a timer to prevent abandoned jobs from blocking the queue
-        evaluationTimeoutThread();
-
         updateSystemSettingsIfNeeded( host, name, port );
 
         evaluationResponse = startEvaluation( projectDeclaration, projectId );
@@ -790,6 +787,15 @@ private void updateStatus( EvaluationStatusOuterClass.EvaluationStatus evaluatio
             persistInformation( id, evaluationMetadata );
         }
         EVALUATION_STAGE.set( evaluationStatus );
+
+        // Creates a timeout thread to verify that a project does not stay in one of our "transition" states (OPEN/COMPLETED)
+        if ( evaluationStatus.equals( COMPLETED ) || evaluationStatus.equals( OPENED ) )
+        {
+            // OPENED jobs cover the step of server setup before automatically going to ONGOING
+            // COMPLETED jobs cover evaluations that have finished, but results have not been retrieved
+            // After results have been retrieved the user closes the job or it is done automatically and results are lost
+            evaluationTimeoutThread();
+        }
     }
 
     /**
@@ -937,23 +943,20 @@ private static void close()
     private static void evaluationTimeoutThread()
     {
         Runnable timeoutRunnable = () -> {
-            while ( !EVALUATION_STAGE.get().equals( CLOSED ) )
+            try
             {
-                try
+                Thread.sleep( ONE_MINUTE_IN_MILLISECONDS );
+                if ( EVALUATION_STAGE.get().equals( OPENED ) || EVALUATION_STAGE.get().equals( COMPLETED ) )
                 {
-                    Thread.sleep( ONE_MINUTE_IN_MILLISECONDS );
-                    if ( EVALUATION_STAGE.get().equals( OPENED ) || EVALUATION_STAGE.get().equals( COMPLETED ) )
-                    {
-                        LOGGER.info( "Timeout thread closing project" );
-                        close();
-                    }
-                }
-                catch ( InterruptedException interruptedException )
-                {
-                    LOGGER.info( "Evaluation was closed so thread was interrupted.", interruptedException );
-                    Thread.currentThread().interrupt();
+                    LOGGER.warn( "There was not an expected status transition; timeout thread closing project in status {}", EVALUATION_STAGE.get() );
+                    close();
                 }
             }
+            catch ( InterruptedException interruptedException )
+            {
+                LOGGER.info( "Evaluation was closed so thread was interrupted.", interruptedException );
+                Thread.currentThread().interrupt();
+            }
         };
 
         timeoutThread = new Thread( timeoutRunnable );