From a3c2c6411622a80fa0e00996416a0ec073e69a7c Mon Sep 17 00:00:00 2001 From: liamcli Date: Wed, 17 Mar 2021 01:36:07 -0700 Subject: [PATCH] ci: print trial logs for timed out experiments (#2053) --- e2e_tests/tests/experiment/experiment.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e_tests/tests/experiment/experiment.py b/e2e_tests/tests/experiment/experiment.py index 9cd1db64bb4..2b5b1450ff4 100644 --- a/e2e_tests/tests/experiment/experiment.py +++ b/e2e_tests/tests/experiment/experiment.py @@ -150,6 +150,9 @@ def wait_for_experiment_state( time.sleep(1) else: + if target_state == "COMPLETED": + cancel_experiment(experiment_id) + report_failed_experiment(experiment_id, "CANCELED") pytest.fail( "Experiment did not reach target state {} after {} seconds".format( target_state, max_wait_secs @@ -367,15 +370,16 @@ def report_failed_experiment(experiment_id: int, state: str) -> None: trials = experiment_trials(experiment_id) active_trials = [t for t in trials if t["state"] == "ACTIVE"] error_trials = [t for t in trials if t["state"] == "ERROR"] + canceled_trials = [t for t in trials if t["state"] == "CANCELED"] print( - "Experiment {}: {} trials, {} active trials, {} failed trials".format( - experiment_id, len(trials), len(active_trials), len(error_trials) + "Experiment {}: {} trials, {} active trials, {} failed trials, {} canceled trials".format( + experiment_id, len(trials), len(active_trials), len(error_trials), len(canceled_trials) ), file=sys.stderr, ) - for trial in error_trials: + for trial in error_trials + canceled_trials: print_trial_logs(trial["id"])