Add crashed-pruning test

tests/test_03_state.py

@@ -2,6 +2,7 @@
 from pathlib import Path
 import zmq
 import time
+import psutil
 
 from tomato import ketchup, tomato
 from .utils import wait_until_tomato_running, wait_until_ketchup_status
@@ -20,6 +21,7 @@ def test_recover_queued_jobs(datadir, start_tomato_daemon, stop_tomato_daemon):
     ketchup.submit(payload="dummy_random_5_2.yml", jobname="job-2", **kwargs)
     tomato.stop(**kwargs)
     assert os.path.exists("tomato_state_12345.pkl")
+
     tomato.start(**kwargs, appdir=Path(), logdir=Path(), verbosity=0)
     assert wait_until_tomato_running(port=PORT, timeout=WAIT)
     ret = tomato.status(**kwargs, with_data=True)
@@ -38,6 +40,7 @@ def test_recover_running_jobs(datadir, start_tomato_daemon, stop_tomato_daemon):
     wait_until_ketchup_status(jobid=1, status="r", port=PORT, timeout=WAIT)
     tomato.stop(**kwargs)
     assert os.path.exists("tomato_state_12345.pkl")
+
     tomato.start(**kwargs, appdir=Path(), logdir=Path(), verbosity=0)
     assert wait_until_tomato_running(port=PORT, timeout=WAIT)
     ret = tomato.status(**kwargs, with_data=True)
@@ -57,7 +60,9 @@ def test_recover_waiting_jobs(datadir, start_tomato_daemon, stop_tomato_daemon):
     wait_until_ketchup_status(jobid=1, status="r", port=PORT, timeout=WAIT)
     tomato.stop(**kwargs)
     assert os.path.exists("tomato_state_12345.pkl")
+
     time.sleep(10)
+
     tomato.start(**kwargs, appdir=Path(), logdir=Path(), verbosity=0)
     assert wait_until_tomato_running(port=PORT, timeout=WAIT)
     ret = tomato.status(**kwargs, with_data=True)
@@ -66,3 +71,29 @@ def test_recover_waiting_jobs(datadir, start_tomato_daemon, stop_tomato_daemon):
     assert len(ret.data.jobs) == 1
     assert ret.data.nextjob == 2
     assert ret.data.jobs[1].status == "c"
+
+
+def test_prune_crashed_jobs(datadir, start_tomato_daemon, stop_tomato_daemon):
+    assert wait_until_tomato_running(port=PORT, timeout=WAIT)
+    os.chdir(datadir)
+    ketchup.submit(payload="dummy_random_30_1.yml", jobname="job-1", **kwargs)
+    tomato.pipeline_load(**kwargs, pipeline="dummy-5", sampleid="dummy_random_30_1")
+    tomato.pipeline_ready(**kwargs, pipeline="dummy-5")
+    wait_until_ketchup_status(jobid=1, status="r", port=PORT, timeout=WAIT)
+    ret = tomato.status(**kwargs, with_data=True)
+    print(f"{ret=}")
+    tomato.stop(**kwargs)
+    assert os.path.exists("tomato_state_12345.pkl")
+
+    proc = psutil.Process(pid=ret.data.jobs[1].pid)
+    proc.terminate()
+    time.sleep(5)
+
+    tomato.start(**kwargs, appdir=Path(), logdir=Path(), verbosity=0)
+    assert wait_until_tomato_running(port=PORT, timeout=WAIT)
+    ret = tomato.status(**kwargs, with_data=True)
+    print(f"{ret=}")
+    assert ret.success
+    assert len(ret.data.jobs) == 1
+    assert ret.data.nextjob == 2
+    assert ret.data.jobs[1].status == "ce"

tests/utils.py

@@ -40,7 +40,6 @@ def wait_until_tomato_running(port: int, timeout: int):
         data = yaml.safe_load(ret.stdout)
         if data["success"]:
             return True
-        print(f"{data=}")
         time.sleep(timeout / 20000)
     return False
 
@@ -56,6 +55,5 @@ def wait_until_ketchup_status(jobid: int, status: str, port: int, timeout: int):
         data = yaml.safe_load(ret.stdout)["data"]
         if data[jobid]["status"] == status:
             return True
-        print(f"{data=}")
         time.sleep(timeout / 20000)
     return False