From 52e565060d2c57f540377374020606a44607bdb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Fran=C3=A7ois=20Caud?=
 <82875602+frcaud@users.noreply.github.com>
Date: Fri, 1 Mar 2024 11:43:20 +0100
Subject: [PATCH] AWS: add a cmd to find error in _training_successful (#591)

---
 doc/whats_new/v0.11.rst            | 1 +
 ramp-engine/ramp_engine/aws/api.py | 5 +++++
 2 files changed, 6 insertions(+)

diff --git a/doc/whats_new/v0.11.rst b/doc/whats_new/v0.11.rst
index 26608510..2470d90a 100644
--- a/doc/whats_new/v0.11.rst
+++ b/doc/whats_new/v0.11.rst
@@ -10,4 +10,5 @@ Changelog
 `ramp-database`
 ...............
 
+- AWS runner: better detection of training errors :pr:`591`
 - Switch to fetching starting kit repos via HTTP rather than using git clone to avoid being blocked by Github :pr:`592`.
diff --git a/ramp-engine/ramp_engine/aws/api.py b/ramp-engine/ramp_engine/aws/api.py
index e273365e..85b3289d 100644
--- a/ramp-engine/ramp_engine/aws/api.py
+++ b/ramp-engine/ramp_engine/aws/api.py
@@ -834,6 +834,11 @@ def _training_successful(config, instance_id, submission_name, actual_nb_folds=N
 
     cmd = "find {}|egrep 'fold.*/y_pred_test.npz'|wc -l".format(folder)
     nb_test_files = int(_run(config, instance_id, cmd, return_output=True))
+
+    cmd = f"find {folder}|egrep 'fold.*/error.txt'|wc -l"
+    n_error = int(_run(config, instance_id, cmd, return_output=True))
+    if n_error > 0:
+        return False
     if actual_nb_folds is not None:
         return nb_folds == nb_train_files == nb_test_files == actual_nb_folds
     else: