From 2983d2f3a89f86e357ec0f5eade011ba7e4c8110 Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Mon, 3 Jul 2023 16:04:47 +0200 Subject: [PATCH 1/2] Clean singularity cache before each run --- tests/test_containers_gpu.py | 15 +++++++++++++++ tests/test_singularity_containers.py | 25 +++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/tests/test_containers_gpu.py b/tests/test_containers_gpu.py index f9eb15a..1c3f443 100644 --- a/tests/test_containers_gpu.py +++ b/tests/test_containers_gpu.py @@ -12,6 +12,11 @@ DOCKER_SINGULARITY = "singularity" # "docker" +def clean_singularity_cache(): + print("Cleaning singularity cache") + os.system("singularity cache clean --force") + + def generate_run_kwargs(): test_recording, _ = se.toy_example( duration=30, @@ -53,26 +58,36 @@ def run_kwargs(work_dir): def test_kilosort2(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_kilosort2(output_folder="kilosort2", **run_kwargs) print(sorting) def test_kilosort2_5(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_kilosort2_5(output_folder="kilosort2_5", **run_kwargs) print(sorting) def test_kilosort3(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_kilosort3(output_folder="kilosort3", **run_kwargs) print(sorting) def test_yass(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_yass(output_folder="yass", **run_kwargs) print(sorting) def test_pykilosort(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_pykilosort(output_folder="pykilosort", **run_kwargs) print(sorting) diff --git a/tests/test_singularity_containers.py b/tests/test_singularity_containers.py index c3373a2..6474339 100644 --- a/tests/test_singularity_containers.py +++ b/tests/test_singularity_containers.py @@ -11,6 +11,10 @@ # test docker or singularity DOCKER_SINGULARITY = "singularity" # "docker" +def clean_singularity_cache(): + print("Cleaning singularity cache") + os.system("singularity cache clean --force") + def generate_run_kwargs(): test_recording, _ = se.toy_example( @@ -54,45 +58,66 @@ def run_kwargs(work_dir): def test_spykingcircus(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_spykingcircus(output_folder="spykingcircus", **run_kwargs) print(sorting) def test_mountainsort4(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_mountainsort4(output_folder="mountainsort4", **run_kwargs) print(sorting) def test_tridesclous(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_tridesclous(output_folder="tridesclous", **run_kwargs) print(sorting) def test_klusta(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() + recording = run_kwargs["recording"] + recording.extra_requirements.append("pandas") + run_kwargs["recording"] = recording sorting = ss.run_klusta(output_folder="klusta", **run_kwargs) print(sorting) def test_ironclust(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_ironclust(output_folder="ironclust", fGpu=False, **run_kwargs) print(sorting) def test_waveclus(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_waveclus(output_folder="waveclus", **run_kwargs) print(sorting) def test_hdsort(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_hdsort(output_folder="hdsort", **run_kwargs) print(sorting) def test_kilosort1(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() sorting = ss.run_kilosort(output_folder="kilosort", useGPU=False, **run_kwargs) print(sorting) def test_combinato(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() rec = run_kwargs['recording'] channels = rec.get_channel_ids()[0:1] rec_one_channel = rec.channel_slice(channels) From 28df0ac441eee985d037b856b947cadd31b7cd8e Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Tue, 11 Jul 2023 14:53:10 +0200 Subject: [PATCH 2/2] Update run_* functions and skip klusta and YASS --- tests/test_containers_gpu.py | 16 ++++++------- tests/test_singularity_containers.py | 36 ++++++++++++++-------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tests/test_containers_gpu.py b/tests/test_containers_gpu.py index 7e74f81..1ec19f5 100644 --- a/tests/test_containers_gpu.py +++ b/tests/test_containers_gpu.py @@ -60,35 +60,35 @@ def run_kwargs(work_dir): def test_kilosort2(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_kilosort2(output_folder="kilosort2", **run_kwargs) + sorting = ss.run_sorter("kilosort2", output_folder="kilosort2", **run_kwargs) print(sorting) def test_kilosort2_5(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_kilosort2_5(output_folder="kilosort2_5", **run_kwargs) + sorting = ss.run_sorter("kilosort2_5", output_folder="kilosort2_5", **run_kwargs) print(sorting) def test_kilosort3(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_kilosort3(output_folder="kilosort3", **run_kwargs) + sorting = ss.run_sorter("kilosort3", output_folder="kilosort3", **run_kwargs) print(sorting) -def test_yass(run_kwargs): +def test_pykilosort(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_yass(output_folder="yass", **run_kwargs) + sorting = ss.run_sorter("pykilosort", output_folder="pykilosort", **run_kwargs) print(sorting) - -def test_pykilosort(run_kwargs): +@pytest.mark.skip(reason="Legcay sorter: requires Python=3.7") +def test_yass(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_pykilosort(output_folder="pykilosort", **run_kwargs) + sorting = ss.run_sorter("yass", output_folder="yass", **run_kwargs) print(sorting) if __name__ == "__main__": diff --git a/tests/test_singularity_containers.py b/tests/test_singularity_containers.py index 6474339..9b2fc0b 100644 --- a/tests/test_singularity_containers.py +++ b/tests/test_singularity_containers.py @@ -60,59 +60,49 @@ def run_kwargs(work_dir): def test_spykingcircus(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_spykingcircus(output_folder="spykingcircus", **run_kwargs) + sorting = ss.run_sorter("spykingcircus", output_folder="spykingcircus", **run_kwargs) print(sorting) def test_mountainsort4(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_mountainsort4(output_folder="mountainsort4", **run_kwargs) + sorting = ss.run_sorter("mountainsort4", output_folder="mountainsort4", **run_kwargs) print(sorting) def test_tridesclous(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_tridesclous(output_folder="tridesclous", **run_kwargs) - print(sorting) - - -def test_klusta(run_kwargs): - if DOCKER_SINGULARITY == "singularity": - clean_singularity_cache() - recording = run_kwargs["recording"] - recording.extra_requirements.append("pandas") - run_kwargs["recording"] = recording - sorting = ss.run_klusta(output_folder="klusta", **run_kwargs) + sorting = ss.run_sorter("tridesclous", output_folder="tridesclous", **run_kwargs) print(sorting) def test_ironclust(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_ironclust(output_folder="ironclust", fGpu=False, **run_kwargs) + sorting = ss.run_sorter("ironclust", output_folder="ironclust", fGpu=False, **run_kwargs) print(sorting) def test_waveclus(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_waveclus(output_folder="waveclus", **run_kwargs) + sorting = ss.run_sorter("waveclus", output_folder="waveclus", **run_kwargs) print(sorting) def test_hdsort(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_hdsort(output_folder="hdsort", **run_kwargs) + sorting = ss.run_sorter("hdsort", output_folder="hdsort", **run_kwargs) print(sorting) def test_kilosort1(run_kwargs): if DOCKER_SINGULARITY == "singularity": clean_singularity_cache() - sorting = ss.run_kilosort(output_folder="kilosort", useGPU=False, **run_kwargs) + sorting = ss.run_sorter("kilosort", output_folder="kilosort", useGPU=False, **run_kwargs) print(sorting) def test_combinato(run_kwargs): @@ -122,7 +112,17 @@ def test_combinato(run_kwargs): channels = rec.get_channel_ids()[0:1] rec_one_channel = rec.channel_slice(channels) run_kwargs['recording'] = rec_one_channel - sorting = ss.run_combinato(output_folder='combinato', **run_kwargs) + sorting = ss.run_sorter("combinato", output_folder='combinato', **run_kwargs) + print(sorting) + +@pytest.mark.skip(reason="Legcay sorter: requires Python=3.7") +def test_klusta(run_kwargs): + if DOCKER_SINGULARITY == "singularity": + clean_singularity_cache() + recording = run_kwargs["recording"] + recording.extra_requirements.append("pandas") + run_kwargs["recording"] = recording + sorting = ss.run_sorter("klusta", output_folder="klusta", **run_kwargs) print(sorting) if __name__ == "__main__":