AWS: add a cmd to find error in _training_successful (#591)

paris-saclay-cds · Mar 1, 2024 · 52e5650 · 52e5650
1 parent a924439
commit 52e5650
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/doc/whats_new/v0.11.rst b/doc/whats_new/v0.11.rst
@@ -10,4 +10,5 @@ Changelog
 `ramp-database`
 ...............
 
+- AWS runner: better detection of training errors :pr:`591`
 - Switch to fetching starting kit repos via HTTP rather than using git clone to avoid being blocked by Github :pr:`592`.
diff --git a/ramp-engine/ramp_engine/aws/api.py b/ramp-engine/ramp_engine/aws/api.py
@@ -834,6 +834,11 @@ def _training_successful(config, instance_id, submission_name, actual_nb_folds=N
 
     cmd = "find {}|egrep 'fold.*/y_pred_test.npz'|wc -l".format(folder)
     nb_test_files = int(_run(config, instance_id, cmd, return_output=True))
+
+    cmd = f"find {folder}|egrep 'fold.*/error.txt'|wc -l"
+    n_error = int(_run(config, instance_id, cmd, return_output=True))
+    if n_error > 0:
+        return False
     if actual_nb_folds is not None:
         return nb_folds == nb_train_files == nb_test_files == actual_nb_folds
     else: